LangSmith

# LangSmith Tracing Setup
import os

# Enable tracing with environment variables
os.environ["LANGCHAIN_TRACING_V2"] = "true"
os.environ["LANGCHAIN_API_KEY"] = "ls__your_api_key"
os.environ["LANGCHAIN_PROJECT"] = "my-project"  # Optional: organize by project

# That's it! All LangChain calls are now traced automatically.
# Every chain.invoke(), model.invoke(), retriever.invoke() is logged.

# View traces at: https://smith.langchain.com

# What tracing captures:
tracing_captures = {
    "Input/Output": "Full prompt and response for every LLM call",
    "Latency": "Time taken for each step and total chain execution",
    "Token usage": "Input tokens, output tokens, and cost",
    "Tool calls": "Which tools were called, with what arguments, and results",
    "Errors": "Full stack traces for any failures",
    "Nested runs": "Parent-child relationships between chain steps",
    "Metadata": "Custom tags, session IDs, user IDs"
}

# Add custom metadata to traces
from langchain_core.tracers.context import tracing_v2_enabled

with tracing_v2_enabled(project_name="my-project", tags=["production"]):
    result = chain.invoke({"question": "What is LangSmith?"})

# Or add metadata per-call
result = chain.invoke(
    {"question": "What is LangSmith?"},
    config={"metadata": {"user_id": "user-123", "session_id": "sess-456"}}
)

# LangSmith Datasets and Evaluation
from langsmith import Client

client = Client()

# Create a dataset
dataset = client.create_dataset(
    dataset_name="qa-test-set",
    description="QA evaluation dataset for RAG pipeline"
)

# Add examples to the dataset
examples = [
    {
        "inputs": {"question": "What is LangChain?"},
        "outputs": {"answer": "LangChain is a framework for building LLM applications."}
    },
    {
        "inputs": {"question": "What is RAG?"},
        "outputs": {"answer": "RAG is retrieval-augmented generation."}
    }
]

for example in examples:
    client.create_example(
        inputs=example["inputs"],
        outputs=example["outputs"],
        dataset_id=dataset.id
    )

# Run evaluation
from langsmith.evaluation import evaluate

def predict(inputs: dict) -> dict:
    """Run your chain on the input."""
    result = rag_chain.invoke(inputs["question"])
    return {"answer": result}

# Custom evaluator
def correctness(run, example) -> dict:
    """Check if the prediction matches the reference."""
    prediction = run.outputs["answer"]
    reference = example.outputs["answer"]
    score = 1 if reference.lower() in prediction.lower() else 0
    return {"key": "correctness", "score": score}

# Run evaluation
results = evaluate(
    predict,
    data="qa-test-set",
    evaluators=[correctness],
    experiment_prefix="rag-v1"
)

# Built-in evaluators you should know:
from langsmith.evaluation import LangChainStringEvaluator

evaluators = {
    "criteria": "LLM-based evaluation against criteria (correctness, helpfulness)",
    "embedding_distance": "Cosine distance between prediction and reference embeddings",
    "string_distance": "Edit distance between prediction and reference strings",
    "exact_match": "Exact string match between prediction and reference"
}

# Annotation Queues - Human review of LLM outputs
from langsmith import Client

client = Client()

# Create an annotation queue
queue = client.create_annotation_queue(
    name="review-queue",
    description="Queue for reviewing RAG responses"
)

# Add runs to the queue for review
# (This is typically done through the LangSmith UI or programmatically)

# Annotation queue use cases:
annotation_uses = {
    "Quality review": "Human reviewers score LLM outputs for quality",
    "Dataset curation": "Approved examples are added to evaluation datasets",
    "Feedback loop": "Use human feedback to improve prompts and chains",
    "Compliance": "Review outputs for safety, accuracy, and policy compliance",
    "Edge case collection": "Identify and catalog failure cases"
}

# LangSmith features overview:
langsmith_features = {
    "Tracing": "Automatic logging of all LLM calls, chains, and tools",
    "Datasets": "Create test sets for systematic evaluation",
    "Evaluation": "Run automated evaluations with custom metrics",
    "Annotation Queues": "Human review workflows for quality assurance",
    "Monitoring": "Track latency, cost, error rates in production",
    "Prompt Hub": "Version and manage prompt templates",
    "Playground": "Test prompts interactively in the browser"
}

# LangSmith Prompt Hub - Version and manage prompts
from langchain import hub

# Pull a prompt from the Hub
prompt = hub.pull("rlm/rag-prompt")

# Push a prompt to the Hub
from langchain_core.prompts import ChatPromptTemplate

my_prompt = ChatPromptTemplate.from_messages([
    ("system", "You are a helpful assistant. Answer based on context:\n{context}"),
    ("human", "{question}")
])

hub.push("my-org/rag-prompt-v2", my_prompt)

# Prompt versioning benefits:
prompt_versioning = {
    "Version control": "Track changes to prompts over time",
    "A/B testing": "Compare prompt versions with evaluation",
    "Rollback": "Revert to a previous prompt version if quality drops",
    "Collaboration": "Team members can share and review prompts",
    "Audit trail": "See who changed what and when"
}