Troubleshooting

Quick Diagnostic Sequence

# 1. Does the answer exist in any chunk? (0 = ingestion problem)
print(sum(1 for c in all_chunks if "keyword" in c.page_content.lower()), "chunks match")
# 2. Are the right chunks retrieved?
for doc, score in vectorstore.similarity_search_with_score(query, k=5):
    print(f"{score:.4f} | {doc.metadata.get('source')} | {doc.page_content[:80]}")
# 3. Does the LLM use the context?
chain.verbose = True; chain.invoke({"question": query})

Retrieval Returns Irrelevant Chunks

from langchain.retrievers import BM25Retriever, EnsembleRetriever
from sentence_transformers import CrossEncoder

ensemble = EnsembleRetriever(
    retrievers=[BM25Retriever.from_documents(docs, k=5),
                vectorstore.as_retriever(search_kwargs={"k": 5})],
    weights=[0.4, 0.6],
)

reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L6-v2")
def rerank(query, candidates, top_n=3):
    scores = reranker.predict([(query, d.page_content) for d in candidates])
    return [d for _, d in sorted(zip(scores, candidates), reverse=True)[:top_n]]

Answers Ignore Retrieved Context

SYSTEM = "Answer ONLY from context. If absent say 'I don't have enough information.'\nContext: {context}"

import tiktoken
enc = tiktoken.encoding_for_model("gpt-4o")
def trim_to_budget(chunks, system, question, limit=120_000):
    used = len(enc.encode(system + question)); out = []
    for c in chunks:
        cost = len(enc.encode(c.page_content))
        if used + cost > limit: break
        out.append(c); used += cost
    return out

Hallucination Despite Retrieval

from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy
from datasets import Dataset
# faithfulness < 0.85 → claims not grounded in retrieved context
result = evaluate(
    Dataset.from_dict({"question": [q], "answer": [a], "contexts": [[d.page_content for d in docs]]}),
    metrics=[faithfulness, answer_relevancy])

import hashlib
def dedup(chunks):
    seen, out = set(), []
    for c in chunks:
        h = hashlib.md5(c.page_content.strip().encode()).hexdigest()
        if h not in seen: seen.add(h); out.append(c)
    return out

Chunk Boundaries Split Key Facts

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain.retrievers import ParentDocumentRetriever
from langchain.storage import InMemoryStore

# Search small child chunks; return their larger parent passages
retriever = ParentDocumentRetriever(
    vectorstore=vectorstore, docstore=InMemoryStore(),
    child_splitter=RecursiveCharacterTextSplitter(chunk_size=200),
    parent_splitter=RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=128),
)
retriever.add_documents(docs)

Slow Queries at Scale

# pgvector — run once after bulk load
# CREATE INDEX ON document_chunks
# USING hnsw (embedding vector_cosine_ops) WITH (m=16, ef_construction=64);

# Qdrant — set HNSW at collection creation
# hnsw_config=HnswConfigDiff(m=16, ef_construct=100)

Stale Indexes After Updates

from langchain_text_splitters import RecursiveCharacterTextSplitter
import json, hashlib, pathlib

def upsert(vs, doc_id, text, source):
    vs._collection.delete(where={"doc_id": doc_id})   # Chroma delete-before-insert
    splitter = RecursiveCharacterTextSplitter(chunk_size=512, chunk_overlap=128)
    vs.add_documents(splitter.create_documents([text], metadatas=[{"doc_id": doc_id, "source": source}]))

def sync_dir(doc_dir, vs, mf="manifest.json"):
    p = pathlib.Path(mf); m = json.loads(p.read_text()) if p.exists() else {}
    for path in pathlib.Path(doc_dir).rglob("*.md"):
        text = path.read_text(); key = str(path)
        h = hashlib.sha256(text.encode()).hexdigest()
        if m.get(key) != h: upsert(vs, key, text, key); m[key] = h
    p.write_text(json.dumps(m, indent=2))

Store embedding_model in chunk metadata. When you change models, build a new collection in parallel, validate recall, then switch traffic — no in-place upgrade exists.

Evaluation Metrics at a Glance

Related References

• RAG & Knowledge Systems — Course Landing
• Glossary
• Cheat Sheet
• Vector DB Reference
• Ground Rules