fix(backend): 落实 PR #11 第二轮评审意见
- Benchmark 容量淘汰只删终态 run,满容量且全活动时返回 BENCHMARK_CAPACITY_EXCEEDED - 创建 run 前校验索引兼容性(BENCHMARK_INDEX_INCOMPATIBLE) - 取消 run 补发 RunCancelled 终止事件;失败分支脱敏(BENCHMARK_RUN_FAILED) - 失败样本计入汇总分母,报告输出 total/successful/failed/failure_rate - load_dataset 按文件名隔离无关损坏文件,顶层非对象拒绝 - FTS score_threshold 先于计数/分页,total 与 items 一致 - Benchmark SSE 支持 Last-Event-ID 游标 - 移除 Agent Benchmark 501 占位接口 - 同步第二阶段接口契约与开发说明文档 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -1,12 +1,13 @@
|
||||
"""RAG Benchmark Runner:调用检索引擎对数据集逐 Case 求值并聚合指标。
|
||||
|
||||
只读操作,直接复用 app.retrieval.engine 的 search(),不旁路检索链路。指标按
|
||||
(mode, case, repeat) 逐样本计算,再按 mode 聚合;失败样本保留在报告中但不计入汇总,
|
||||
避免异常样本污染指标。
|
||||
(mode, case, repeat) 逐样本计算,再按 mode 聚合;失败样本按零分计入质量指标分母,
|
||||
避免把执行失败误判为检索质量(同时保留 total/successful/failed/failure_rate)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
@@ -22,6 +23,8 @@ from app.contracts import (
|
||||
)
|
||||
from app.retrieval.engine import engine
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BenchmarkCancelled(Exception):
|
||||
"""运行在 Case 之间被取消时抛出,用于中断后台执行并标记 cancelled。"""
|
||||
@@ -75,12 +78,19 @@ async def _evaluate_one(
|
||||
response = await engine.search(search_request)
|
||||
latency_ms = (time.perf_counter() - start) * 1000.0
|
||||
except Exception as exc: # 单个样本失败不中断整个 Benchmark
|
||||
# 详细异常只进日志,公开响应只带项目错误码与安全消息,避免泄露路径/SQL 等敏感信息
|
||||
logger.warning(
|
||||
"RAG case evaluation failed: case=%s mode=%s", case.case_id, mode.value,
|
||||
exc_info=exc,
|
||||
)
|
||||
return RAGCaseResult(
|
||||
case_id=case.case_id,
|
||||
mode=mode,
|
||||
repeat=repeat,
|
||||
latency_ms=(time.perf_counter() - start) * 1000.0,
|
||||
error=str(exc),
|
||||
citation_applicable=case.citation_required,
|
||||
error="RAG case evaluation failed.",
|
||||
error_code="BENCHMARK_CASE_EVALUATION_FAILED",
|
||||
)
|
||||
|
||||
retrieved_note_ids = [item.note_id for item in response.items]
|
||||
@@ -107,19 +117,27 @@ async def _evaluate_one(
|
||||
|
||||
def _aggregate(cases: list[RAGCaseResult], mode: SearchMode) -> RAGMetrics:
|
||||
samples = [c for c in cases if c.mode == mode]
|
||||
ok = [c for c in samples if c.error is None]
|
||||
if not ok:
|
||||
total = len(samples)
|
||||
failed = sum(1 for c in samples if c.error is not None)
|
||||
successful = total - failed
|
||||
if total == 0:
|
||||
return RAGMetrics()
|
||||
|
||||
latencies = [c.latency_ms for c in ok]
|
||||
# citation_hit_rate 只统计声明了 expected_block_ids 的样本
|
||||
citation_samples = [c for c in ok if c.citation_applicable]
|
||||
# 延迟只统计成功样本;失败样本按零分计入质量指标分母,避免汇总虚高
|
||||
latencies = [c.latency_ms for c in samples if c.error is None]
|
||||
citation_samples = [c for c in samples if c.citation_applicable]
|
||||
return RAGMetrics(
|
||||
hit_at_1=m.mean([1.0 if c.hit_at_1 else 0.0 for c in ok]),
|
||||
hit_at_5=m.mean([1.0 if c.hit_at_5 else 0.0 for c in ok]),
|
||||
recall_at_k=m.mean([c.recall for c in ok]),
|
||||
mrr=m.mean([c.reciprocal_rank for c in ok]),
|
||||
citation_hit_rate=m.mean([1.0 if c.citation_hit else 0.0 for c in citation_samples]),
|
||||
hit_at_1=m.mean([1.0 if (c.error is None and c.hit_at_1) else 0.0 for c in samples]),
|
||||
hit_at_5=m.mean([1.0 if (c.error is None and c.hit_at_5) else 0.0 for c in samples]),
|
||||
recall_at_k=m.mean([c.recall if c.error is None else 0.0 for c in samples]),
|
||||
mrr=m.mean([c.reciprocal_rank if c.error is None else 0.0 for c in samples]),
|
||||
citation_hit_rate=m.mean(
|
||||
[1.0 if (c.error is None and c.citation_hit) else 0.0 for c in citation_samples]
|
||||
),
|
||||
p50_latency_ms=m.percentile(latencies, 50.0),
|
||||
p95_latency_ms=m.percentile(latencies, 95.0),
|
||||
total_cases=total,
|
||||
successful_cases=successful,
|
||||
failed_cases=failed,
|
||||
failure_rate=failed / total,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user