RAG Integration

# Ingestion Pipeline
Documents → Chunking → Embedding → Pinecone
   PDF         Split text     OpenAI        Store vectors
   HTML        ~500 tokens    ada-002       + metadata

# Query Pipeline
Question → Embed → Search → Context → LLM → Answer
  User text    Vector    Pinecone   Top-k docs   GPT-4    Grounded
                         top_k=5    + question    Claude   response

from pinecone import Pinecone, ServerlessSpec
from openai import OpenAI
import os

# Initialize clients
pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
openai_client = OpenAI()

# Create index (if not exists)
if "rag-index" not in [idx.name for idx in pc.list_indexes()]:
    pc.create_index(
        name="rag-index",
        dimension=1536,
        metric="cosine",
        spec=ServerlessSpec(cloud="aws", region="us-east-1")
    )

index = pc.Index("rag-index")

def chunk_text(text, chunk_size=500, overlap=50):
    """Split text into overlapping chunks."""
    words = text.split()
    chunks = []
    for i in range(0, len(words), chunk_size - overlap):
        chunk = " ".join(words[i:i + chunk_size])
        chunks.append(chunk)
    return chunks

# Process documents
documents = [
    {"id": "doc1", "text": "Long document text...", "source": "guide.pdf"},
    {"id": "doc2", "text": "Another document...", "source": "faq.md"},
]

vectors = []
for doc in documents:
    chunks = chunk_text(doc["text"])
    for i, chunk in enumerate(chunks):
        embedding = get_embedding(chunk)
        vectors.append({
            "id": f"{doc['id']}-chunk-{i}",
            "values": embedding,
            "metadata": {
                "text": chunk,
                "source": doc["source"],
                "chunk_index": i
            }
        })

# Upsert in batches
for i in range(0, len(vectors), 100):
    index.upsert(vectors=vectors[i:i+100])

def rag_query(question, top_k=5):
    """Answer a question using RAG with Pinecone."""

    # 1. Embed the question
    query_embedding = get_embedding(question)

    # 2. Search Pinecone for relevant chunks
    results = index.query(
        vector=query_embedding,
        top_k=top_k,
        include_metadata=True
    )

    # 3. Build context from retrieved chunks
    context = "\n\n".join([
        match["metadata"]["text"]
        for match in results["matches"]
    ])

    # 4. Generate answer with LLM
    response = openai_client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content":
                "Answer based on the provided context. "
                "If the answer is not in the context, say so."},
            {"role": "user", "content":
                f"Context:\n{context}\n\nQuestion: {question}"}
        ]
    )

    return {
        "answer": response.choices[0].message.content,
        "sources": [m["metadata"]["source"] for m in results["matches"]]
    }

# Use the RAG pipeline
result = rag_query("What is deep learning?")
print(result["answer"])
print("Sources:", result["sources"])

from langchain_pinecone import PineconeVectorStore
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain.chains import RetrievalQA
from langchain.text_splitter import RecursiveCharacterTextSplitter

# Create vector store
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = PineconeVectorStore(
    index_name="rag-index",
    embedding=embeddings
)

# Add documents
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000, chunk_overlap=200
)
docs = splitter.create_documents(["Your document text..."])
vectorstore.add_documents(docs)

# Create RAG chain
llm = ChatOpenAI(model="gpt-4o")
qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    retriever=vectorstore.as_retriever(search_kwargs={"k": 5}),
    return_source_documents=True
)

# Query
result = qa_chain.invoke("What is deep learning?")
print(result["result"])