|
1 | 1 | import argparse |
2 | 2 | import logging |
3 | 3 | import os |
| 4 | +import re |
4 | 5 | from pathlib import Path |
5 | 6 | from typing import Any |
6 | 7 |
|
7 | 8 | import azure.identity |
8 | 9 | from dotenv import load_dotenv |
9 | 10 | from evaltools.eval.evaluate import run_evaluate_from_config |
| 11 | +from evaltools.eval.evaluate_metrics import register_metric |
| 12 | +from evaltools.eval.evaluate_metrics.base_metric import BaseMetric |
10 | 13 | from rich.logging import RichHandler |
11 | 14 |
|
12 | 15 | logger = logging.getLogger("ragapp") |
13 | 16 |
|
14 | 17 |
|
| 18 | +class CitationsMatchedMetric(BaseMetric): |
| 19 | + METRIC_NAME = "citations_matched" |
| 20 | + |
| 21 | + @classmethod |
| 22 | + def evaluator_fn(cls, **kwargs): |
| 23 | + def citations_overlap(*, response, ground_truth, **kwargs): |
| 24 | + if response is None: |
| 25 | + logger.warning("Received response of None, can't compute citation_match metric. Setting to -1.") |
| 26 | + return {cls.METRIC_NAME: -1} |
| 27 | + truth_citations = set(re.findall(r"\[(\d+)\]", ground_truth)) |
| 28 | + response_citations = set(re.findall(r"\[(\d+)\]", response)) |
| 29 | + # Count the percentage of citations that are present in the response |
| 30 | + num_citations = len(truth_citations) |
| 31 | + num_matched_citations = len(truth_citations.intersection(response_citations)) |
| 32 | + return {cls.METRIC_NAME: num_matched_citations / num_citations} |
| 33 | + |
| 34 | + return citations_overlap |
| 35 | + |
| 36 | + @classmethod |
| 37 | + def get_aggregate_stats(cls, df): |
| 38 | + df = df[df[cls.METRIC_NAME] != -1] |
| 39 | + return {"mean": round(df[cls.METRIC_NAME].mean(), 2)} |
| 40 | + |
| 41 | + |
15 | 42 | def get_openai_config() -> dict: |
16 | 43 | openai_config: dict[str, Any] |
17 | 44 | if os.environ.get("OPENAI_CHAT_HOST") == "azure": |
@@ -60,6 +87,7 @@ def get_openai_config() -> dict: |
60 | 87 |
|
61 | 88 | openai_config = get_openai_config() |
62 | 89 |
|
| 90 | + register_metric(CitationsMatchedMetric) |
63 | 91 | run_evaluate_from_config( |
64 | 92 | working_dir=Path(__file__).parent, |
65 | 93 | config_path="eval_config.json", |
|
0 commit comments