Simple Pinecone setup with LlamaIndex + Eval¶

In this example you will create a simple Llama Index RAG application and create the vector store in Pinecone. You'll also set up evaluation and logging with TruLens.

Setup¶

Install dependencies¶

Let's install some of the dependencies for this notebook if we don't have them already

In [ ]:

# !pip install trulens trulens-apps-llamaindex trulens-providers-openai llama_index==0.10.11 llama-index-readers-pinecone pinecone-client==3.0.3 nltk>=3.8.1 html2text>=2020.1.16

# !pip install trulens trulens-apps-llamaindex trulens-providers-openai llama_index==0.10.11 llama-index-readers-pinecone pinecone-client==3.0.3 nltk>=3.8.1 html2text>=2020.1.16

Add API keys¶

For this quickstart, you will need Open AI and Huggingface keys

In [ ]:

import os

os.environ["OPENAI_API_KEY"] = "..."
os.environ["PINECONE_API_KEY"] = "..."
os.environ["PINECONE_ENVIRONMENT"] = "..."

import os os.environ["OPENAI_API_KEY"] = "..." os.environ["PINECONE_API_KEY"] = "..." os.environ["PINECONE_ENVIRONMENT"] = "..."

Import from LlamaIndex and TruLens¶

In [ ]:

from llama_index.core import VectorStoreIndex
from llama_index.core.storage.storage_context import StorageContext
from llama_index.legacy import ServiceContext
from llama_index.llms.openai import OpenAI
from llama_index.readers.web import SimpleWebPageReader
from llama_index.vector_stores.pinecone import PineconeVectorStore
import pinecone
from trulens.core import Feedback
from trulens.core import TruSession
from trulens.apps.llamaindex import TruLlama
from trulens.providers.openai import OpenAI as fOpenAI

session = TruSession()

from llama_index.core import VectorStoreIndex from llama_index.core.storage.storage_context import StorageContext from llama_index.legacy import ServiceContext from llama_index.llms.openai import OpenAI from llama_index.readers.web import SimpleWebPageReader from llama_index.vector_stores.pinecone import PineconeVectorStore import pinecone from trulens.core import Feedback from trulens.core import TruSession from trulens.apps.llamaindex import TruLlama from trulens.providers.openai import OpenAI as fOpenAI session = TruSession()

First we need to load documents. We can use SimpleWebPageReader¶

In [ ]:

# load documents
documents = SimpleWebPageReader(html_to_text=True).load_data(
    ["http://paulgraham.com/worked.html"]
)

# load documents documents = SimpleWebPageReader(html_to_text=True).load_data( ["http://paulgraham.com/worked.html"] )

Next we can create the vector store in pinecone.

In [ ]:

index_name = "paulgraham-essay"

# find API key in console at app.pinecone.io
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
# find ENV (cloud region) next to API key in console
PINECONE_ENVIRONMENT = os.getenv("PINECONE_ENVIRONMENT")

# initialize pinecone
pinecone.init(api_key=PINECONE_API_KEY, environment=PINECONE_ENVIRONMENT)

index_name = "paulgraham-essay" # find API key in console at app.pinecone.io PINECONE_API_KEY = os.getenv("PINECONE_API_KEY") # find ENV (cloud region) next to API key in console PINECONE_ENVIRONMENT = os.getenv("PINECONE_ENVIRONMENT") # initialize pinecone pinecone.init(api_key=PINECONE_API_KEY, environment=PINECONE_ENVIRONMENT)

In [ ]:

# create the index
pinecone.create_index(name=index_name, dimension=1536)

# set vector store as pinecone
vector_store = PineconeVectorStore(
    index_name=index_name, environment=os.environ["PINECONE_ENVIRONMENT"]
)

# create the index pinecone.create_index(name=index_name, dimension=1536) # set vector store as pinecone vector_store = PineconeVectorStore( index_name=index_name, environment=os.environ["PINECONE_ENVIRONMENT"] )

In [ ]:

# set storage context
storage_context = StorageContext.from_defaults(vector_store=vector_store)

# set service context
llm = OpenAI(temperature=0, model="gpt-3.5-turbo")
service_context = ServiceContext.from_defaults(llm=llm)

# create index from documents
index = VectorStoreIndex.from_documents(
    documents,
    storage_context=storage_context,
    service_context=service_context,
)

# set storage context storage_context = StorageContext.from_defaults(vector_store=vector_store) # set service context llm = OpenAI(temperature=0, model="gpt-3.5-turbo") service_context = ServiceContext.from_defaults(llm=llm) # create index from documents index = VectorStoreIndex.from_documents( documents, storage_context=storage_context, service_context=service_context, )

After creating the index, we can initilaize our query engine.¶

In [ ]:

query_engine = index.as_query_engine()

query_engine = index.as_query_engine()

Now we can set the engine up for evaluation and tracking¶

In [ ]:

import numpy as np

# Initialize OpenAI-based feedback function collection class:
provider = fOpenAI()

# Define groundedness
f_groundedness = (
    Feedback(
        provider.groundedness_measure_with_cot_reasons, name="Groundedness"
    )
    .on(
        TruLlama.select_context().collect()  # context
    )
    .on_output()
)

# Question/answer relevance between overall question and answer.
f_answer_relevance = Feedback(
    provider.relevance_with_cot_reasons, name="Answer Relevance"
).on_input_output()

# Question/statement relevance between question and each context chunk.
f_context_relevance = (
    Feedback(
        provider.context_relevance_with_cot_reasons, name="Context Relevance"
    )
    .on_input()
    .on(TruLlama.select_context())
    .aggregate(np.mean)
)

import numpy as np # Initialize OpenAI-based feedback function collection class: provider = fOpenAI() # Define groundedness f_groundedness = ( Feedback( provider.groundedness_measure_with_cot_reasons, name="Groundedness" ) .on( TruLlama.select_context().collect() # context ) .on_output() ) # Question/answer relevance between overall question and answer. f_answer_relevance = Feedback( provider.relevance_with_cot_reasons, name="Answer Relevance" ).on_input_output() # Question/statement relevance between question and each context chunk. f_context_relevance = ( Feedback( provider.context_relevance_with_cot_reasons, name="Context Relevance" ) .on_input() .on(TruLlama.select_context()) .aggregate(np.mean) )

Instrument query engine for logging with TruLens¶

In [ ]:

tru_query_engine_recorder = TruLlama(
    query_engine,
    app_name="LlamaIndex_App",
    app_version="1",
    feedbacks=[f_groundedness, f_answer_relevance, f_context_relevance],
)

tru_query_engine_recorder = TruLlama( query_engine, app_name="LlamaIndex_App", app_version="1", feedbacks=[f_groundedness, f_answer_relevance, f_context_relevance], )

In [ ]:

# Instrumented query engine can operate as a context manager:
with tru_query_engine_recorder as recording:
    llm_response = query_engine.query("What did the author do growing up?")
    print(llm_response)

# Instrumented query engine can operate as a context manager: with tru_query_engine_recorder as recording: llm_response = query_engine.query("What did the author do growing up?") print(llm_response)

Explore in a Dashboard¶

In [ ]:

from trulens.dashboard import run_dashboard

run_dashboard(session)  # open a local streamlit app to explore

# stop_dashboard(session) # stop if needed

from trulens.dashboard import run_dashboard run_dashboard(session) # open a local streamlit app to explore # stop_dashboard(session) # stop if needed

Or view results directly in your notebook¶

In [ ]:

session.get_records_and_feedback()[0]

session.get_records_and_feedback()[0]