RAG 评估指南：怎么知道你的 RAG 好不好

from datasets import Dataset
from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall,
)

# 每条数据需要：问题、生成的答案、检索到的上下文、（可选）标准答案
eval_data = {
    "question": [
        "What is the return policy?",
        "How do I reset my password?",
        "What payment methods are accepted?",
    ],
    "answer": [
        "You can return items within 30 days of purchase.",
        "Click 'Forgot Password' on the login page and follow the email instructions.",
        "We accept Visa, Mastercard, and PayPal.",
    ],
    "contexts": [
        ["Our return policy allows returns within 30 days. Items must be unused."],
        ["To reset your password, click 'Forgot Password' and check your email."],
        ["Accepted payment methods: Visa, Mastercard, American Express, PayPal."],
    ],
    "ground_truth": [
        "Items can be returned within 30 days if unused.",
        "Use the 'Forgot Password' link and follow email instructions.",
        "Visa, Mastercard, American Express, and PayPal are accepted.",
    ],
}

dataset = Dataset.from_dict(eval_data)

result = evaluate(
    dataset,
    metrics=[faithfulness, answer_relevancy, context_precision, context_recall],
)

print(result)
# 输出示例：
# {'faithfulness': 0.92, 'answer_relevancy': 0.88,
#  'context_precision': 0.85, 'context_recall': 0.79}

from ragas.testset.generator import TestsetGenerator
from ragas.testset.evolutions import simple, reasoning, multi_context

generator = TestsetGenerator.with_openai()
testset = generator.generate_with_langchain_docs(
    documents,       # 你的知识库文档
    test_size=50,    # 生成 50 个问答对
    distributions={
        simple: 0.5,         # 50% 简单问题
        reasoning: 0.3,      # 30% 推理问题
        multi_context: 0.2,  # 20% 多文档问题
    },
)

sampled_queries = (
    db.query(ProductionTrace)
    .filter(ProductionTrace.date >= "2024-01-01")
    .order_by(func.random())
    .limit(100)
    .all()
)

for q in sampled_queries:
    print(f"Q: {q.user_query}")
    print(f"A: {q.model_response}")
    print("---")

# evaluate_rag.py — 在 CI 中运行
import sys
from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy

THRESHOLDS = {
    "faithfulness": 0.80,
    "answer_relevancy": 0.75,
}

def run_evaluation(dataset):
    result = evaluate(dataset, metrics=[faithfulness, answer_relevancy])

    failed = []
    for metric, threshold in THRESHOLDS.items():
        score = result[metric]
        status = "✅" if score >= threshold else "❌"
        print(f"{status} {metric}: {score:.3f} (threshold: {threshold})")
        if score < threshold:
            failed.append(metric)

    if failed:
        print(f"\nFailed metrics: {failed}")
        sys.exit(1)  # CI 失败

if __name__ == "__main__":
    dataset = load_eval_dataset("eval_dataset.json")
    run_evaluation(dataset)

# .github/workflows/rag-eval.yml
name: RAG Evaluation
on:
  push:
    paths:
      - "rag/**"
      - "embeddings/**"
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run RAG evaluation
        run: python evaluate_rag.py
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}