📊RAGAS Değerlendirme Framework'ü

RAGAS (Retrieval-Augmented Generation Assessment), RAG sistemlerinin kalitesini ölçmek için kullanılan standart değerlendirme framework'üdür. Komtaş'ta RAGAS, her RAG pipeline'ı için zorunlu kalite kapısı olarak kullanılmaktadır.

4.1 RAGAS Metrikleri — Detaylı Kılavuz

RAGAS Python Implementasyonu

from ragas import evaluate
from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_precision,
    context_recall,
    context_relevancy,
    answer_correctness
)
from ragas.llms import LangchainLLMWrapper
from ragas.embeddings import LangchainEmbeddingsWrapper
from langchain_anthropic import ChatAnthropic
from langchain_openai import OpenAIEmbeddings
from datasets import Dataset
import pandas as pd

# Model ve embedding konfigürasyonu
llm = LangchainLLMWrapper(ChatAnthropic(model="claude-opus-4-6"))
embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings(model="text-embedding-3-small"))

# Test dataset oluşturma
def create_ragas_dataset(qa_pairs: list[dict]) -> Dataset:
    """
    qa_pairs: [{"question": "...", "answer": "...", "contexts": [...], "ground_truth": "..."}]
    """
    return Dataset.from_dict({
        "question": [p["question"] for p in qa_pairs],
        "answer": [p["answer"] for p in qa_pairs],
        "contexts": [p["contexts"] for p in qa_pairs],
        "ground_truth": [p["ground_truth"] for p in qa_pairs]
    })

# Değerlendirme çalıştırma
def run_ragas_evaluation(dataset: Dataset) -> dict:
    metrics = [
        faithfulness,
        answer_relevancy,
        context_precision,
        context_recall,
        context_relevancy,
        answer_correctness
    ]
    
    result = evaluate(
        dataset=dataset,
        metrics=metrics,
        llm=llm,
        embeddings=embeddings,
        raise_exceptions=False
    )
    
    return result.to_pandas()

# Eşik kontrolü
THRESHOLDS = {
    "faithfulness": 0.85,
    "answer_relevancy": 0.80,
    "context_precision": 0.75,
    "context_recall": 0.80,
    "context_relevancy": 0.70,
    "answer_correctness": 0.80
}

def check_quality_gate(scores: dict) -> tuple[bool, list[str]]:
    """Kalite kapısı kontrolü — CI/CD pipeline için"""
    failures = []
    for metric, threshold in THRESHOLDS.items():
        if scores.get(metric, 0) < threshold:
            failures.append(
                f"{metric}: {scores[metric]:.3f} < {threshold} (BAŞARISIZ)"
            )
    return len(failures) == 0, failures

# Kullanım
if __name__ == "__main__":
    dataset = create_ragas_dataset(load_test_cases("tests/rag_testcases.json"))
    scores_df = run_ragas_evaluation(dataset)
    avg_scores = scores_df.mean().to_dict()
    
    passed, failures = check_quality_gate(avg_scores)
    
    if not passed:
        print("❌ Kalite Kapısı BAŞARISIZ:")
        for f in failures: print(f"  - {f}")
        exit(1)  # CI/CD pipeline'ı durdur
    else:
        print("✅ Tüm RAGAS metrikleri eşiklerin üzerinde")

4.2 CI/CD Entegrasyonu

# .github/workflows/rag_quality.yml
name: RAG Quality Gate

on:
  pull_request:
    paths:
      - 'rag/**'
      - 'prompts/**'
      - 'embeddings/**'

jobs:
  ragas-evaluation:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      
      - name: Install dependencies
        run: pip install ragas langchain-anthropic datasets
      
      - name: Run RAGAS evaluation
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          python scripts/run_ragas_evaluation.py \
            --testset tests/rag_testcases.json \
            --output reports/ragas_scores.json
      
      - name: Check quality gates
        run: python scripts/check_quality_gates.py reports/ragas_scores.json
      
      - name: Upload RAGAS report
        uses: actions/upload-artifact@v3
        with:
          name: ragas-report
          path: reports/ragas_scores.json
      
      - name: Comment PR with scores
        uses: actions/github-script@v6
        with:
          script: |
            const scores = require('./reports/ragas_scores.json');
            const body = \`## RAGAS Değerlendirme Sonuçları
            | Metrik | Skor | Eşik | Durum |
            |--------|------|------|-------|
            \${Object.entries(scores).map(([k,v]) => 
              \`| \${k} | \${v.toFixed(3)} | \${THRESHOLDS[k]} | \${v >= THRESHOLDS[k] ? '✅' : '❌'} |\`
            ).join('\n')}\`;
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body
            });

4.3 Langfuse Entegrasyonu

from langfuse import Langfuse
from langfuse.decorators import observe, langfuse_context
from ragas import evaluate

langfuse = Langfuse(
    public_key=os.getenv("LANGFUSE_PUBLIC_KEY"),
    secret_key=os.getenv("LANGFUSE_SECRET_KEY"),
    host=os.getenv("LANGFUSE_HOST", "http://langfuse:3000")
)

@observe(name="rag-pipeline-evaluation")
def evaluate_and_trace(question: str, answer: str, contexts: list, ground_truth: str):
    """RAG değerlendirmesini Langfuse'a trace et"""
    
    # RAGAS değerlendirmesi
    dataset = create_ragas_dataset([{
        "question": question,
        "answer": answer,
        "contexts": contexts,
        "ground_truth": ground_truth
    }])
    
    scores = run_ragas_evaluation(dataset)
    
    # Langfuse'a skor gönder
    trace_id = langfuse_context.get_current_trace_id()
    
    for metric_name, score_value in scores.items():
        langfuse.score(
            trace_id=trace_id,
            name=f"ragas_{metric_name}",
            value=float(score_value),
            comment=f"RAGAS {metric_name} skoru"
        )
    
    return scores

4.4 Uygulama Politikaları ve Senaryolar