LangSmith：把一次 RAG 请求变成可比较的 Trace 和实验

python -m pip install "langsmith>=0.3.13"
export LANGSMITH_TRACING=true
export LANGSMITH_API_KEY="replace-with-your-key"
export LANGSMITH_PROJECT="stage07-rag"

from langsmith import traceable


@traceable(run_type="retriever", tags=["hybrid-v1"])
def retrieve(question: str) -> list[str]:
    return ["关键告警必须立即通知负责人。"]


@traceable(name="rag-answer", metadata={"prompt_version": "answer-v2"})
def answer(inputs: dict) -> dict:
    docs = retrieve(inputs["question"])
    return {"answer": f"负责人 [S1]", "sources": ["S1"], "docs": docs}


print(answer({"question": "关键告警通知谁？"}))

from langsmith import Client, evaluate

client = Client()
dataset_name = "stage07-rag-regression"
if not client.has_dataset(dataset_name=dataset_name):
    dataset = client.create_dataset(dataset_name=dataset_name)
    client.create_examples(
        inputs=[{"question": "关键告警通知谁？"}],
        outputs=[{"required": "负责人"}],
        dataset_id=dataset.id,
    )


def contains_required(run, example) -> dict:
    required = example.outputs["required"]
    return {"key": "contains_required", "score": int(required in run.outputs["answer"])}


evaluate(
    answer,
    data=dataset_name,
    evaluators=[contains_required],
    experiment_prefix="hybrid-v1-answer-v2",
    metadata={"embedding": "embed-v1", "splitter": "recursive-500-80"},
)