fix(backend): 落实 PR #9 评审意见

- 检索调优参数(rrf_k/rerank/rerank_candidates/score_threshold)透传到引擎实际执行
- Recall 去重，避免同一 Note 多 Block 重复导致 Recall 超 1
- RAG 运行改为后台异步执行：创建即 queued + 202，支持取消与 SSE 实时事件
- 数据集元数据校验，坏文件隔离跳过；citation_required 语义修正
- modes 空/重复校验；配置快照记录模型版本与索引元信息

Co-Authored-By: Claude Code <noreply@anthropic.com>

backend/app/benchmarks/datasets.py

@@ -11,7 +11,7 @@ import json
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from pydantic import ValidationError
+from pydantic import BaseModel, Field, ValidationError
 
 from app.config import get_settings
 from app.contracts import (
@@ -34,6 +34,20 @@ class RAGDataset:
     content_hash: str = ""
 
 
+class _DatasetMeta(BaseModel):
+    """Dataset 元数据的最小校验模型。
+
+    list_datasets 用它逐文件校验元信息字段结构，把「合法 JSON 但字段类型错误」
+    （如 cases: 42）这类损坏文件隔离掉，而不是让 len() 抛 TypeError 拖垮整个列表。
+    """
+
+    dataset_id: str = Field(min_length=1)
+    kind: str = ""
+    version: str = ""
+    description: str = ""
+    cases: list = Field(default_factory=list)
+
+
 def _datasets_dir() -> Path:
     return get_settings().benchmark_datasets_path
 
@@ -109,6 +123,14 @@ def _dataset_from_raw(raw: dict, raw_bytes: bytes, kind: BenchmarkKind) -> RAGDa
                 f"Dataset case '{parsed.case_id}' must declare expected_note_ids or expected_block_ids.",
                 {"dataset_id": dataset_id, "case_id": parsed.case_id},
             )
+        # citation_required=true 时必须声明 expected_block_ids，否则无法计算 Citation Hit Rate
+        if parsed.citation_required and not parsed.expected_block_ids:
+            raise ApiError(
+                422,
+                "BENCHMARK_DATASET_INVALID",
+                f"Dataset case '{parsed.case_id}' requires expected_block_ids when citation_required is true.",
+                {"dataset_id": dataset_id, "case_id": parsed.case_id},
+            )
         cases.append(parsed)
 
     return RAGDataset(
@@ -124,23 +146,25 @@ def _dataset_from_raw(raw: dict, raw_bytes: bytes, kind: BenchmarkKind) -> RAGDa
 def list_datasets(kind: BenchmarkKind) -> list[BenchmarkDatasetInfo]:
     """枚举受控目录下指定 kind 的数据集元信息（不含 Case 内容）。
 
-    个别文件损坏时跳过而非整体失败，保证列表接口健壮；损坏细节由 load_dataset 抛出。
+    逐文件用 _DatasetMeta 校验元信息字段结构，单个损坏文件隔离跳过而非整体失败，
+    保证列表接口健壮；损坏细节由 load_dataset 抛出。
     """
     infos: list[BenchmarkDatasetInfo] = []
     for path in _dataset_files():
         try:
             raw, raw_bytes = _read_json(path)
-        except ApiError:
+            meta = _DatasetMeta.model_validate(raw)
+        except (ApiError, ValidationError):
             continue
-        if raw.get("kind", kind.value) != kind.value:
+        if meta.kind not in ("", kind.value):
             continue
         infos.append(
             BenchmarkDatasetInfo(
-                dataset_id=raw.get("dataset_id", path.stem),
+                dataset_id=meta.dataset_id,
                 kind=kind,
-                version=str(raw.get("version", "")),
-                description=str(raw.get("description", "")),
-                case_count=len(raw.get("cases", [])),
+                version=meta.version,
+                description=meta.description,
+                case_count=len(meta.cases),
                 content_hash=_content_hash(raw_bytes),
             )
         )

backend/app/benchmarks/metrics.py

@@ -13,11 +13,14 @@ def hit_at_k(retrieved: list[str], expected: set[str], k: int) -> bool:
 
 
 def recall_at_k(retrieved: list[str], expected: set[str], k: int) -> float:
-    """前 k 个结果召回的期望 id 占比；期望为空时视为 0。"""
+    """前 k 个结果召回的期望 id 占比；期望为空时视为 0。
+
+    结果先去重：检索结果是 Block 级，同一 Note 可能经多个 Block 重复出现，
+    直接逐项计数会把同一 Note 算多次、导致 Recall 超过 1。
+    """
     if not expected:
         return 0.0
-    hits = sum(1 for item in retrieved[:k] if item in expected)
-    return hits / len(expected)
+    return len(set(retrieved[:k]) & expected) / len(expected)
 
 
 def reciprocal_rank(retrieved: list[str], expected: set[str]) -> float:

backend/app/benchmarks/rag.py

@@ -23,14 +23,20 @@ from app.contracts import (
 from app.retrieval.engine import engine
 
 
+class BenchmarkCancelled(Exception):
+    """运行在 Case 之间被取消时抛出，用于中断后台执行并标记 cancelled。"""
+
+
 async def run_rag(
     dataset: RAGDataset,
     request: RAGRunRequest,
     on_case: Callable[[RAGCaseResult, int, int], None] | None = None,
+    should_cancel: Callable[[], bool] | None = None,
 ) -> tuple[dict[str, RAGMetrics], list[RAGCaseResult]]:
     """执行 RAG Benchmark，返回 (按 mode 聚合的指标, 全部逐样本结果)。
 
     on_case 在每个样本求值完成后回调 (result, done, total)，供上层更新进度与事件。
+    should_cancel 在每个样本开始前被检查；返回 True 时抛出 BenchmarkCancelled 中断运行。
     """
     total = len(request.modes) * len(dataset.cases) * request.repeat
     done = 0
@@ -39,6 +45,8 @@ async def run_rag(
     for mode in request.modes:
         for case in dataset.cases:
             for repeat in range(request.repeat):
+                if should_cancel is not None and should_cancel():
+                    raise BenchmarkCancelled()
                 result = await _evaluate_one(case, mode, request, repeat)
                 results.append(result)
                 done += 1
@@ -57,6 +65,10 @@ async def _evaluate_one(
         mode=mode,
         limit=request.retrieval.top_k,
         include_snippet=False,
+        rrf_k=request.retrieval.rrf_k,
+        rerank=request.retrieval.rerank,
+        rerank_candidates=request.retrieval.rerank_candidates,
+        score_threshold=request.retrieval.score_threshold,
     )
     start = time.perf_counter()
     try:
@@ -89,7 +101,7 @@ async def _evaluate_one(
         recall=m.recall_at_k(retrieved_note_ids, expected_notes, k),
         reciprocal_rank=m.reciprocal_rank(retrieved_note_ids, expected_notes),
         citation_hit=m.citation_hit(retrieved_block_ids, expected_blocks),
-        citation_applicable=bool(case.expected_block_ids),
+        citation_applicable=case.citation_required,
     )
 
 

backend/app/benchmarks/service.py

@@ -1,19 +1,22 @@
 """Benchmark 服务：运行注册表、配置快照与报告组装。
 
-MVP 阶段运行是同步的（与 index_service 一致）：POST 创建后立即执行完并返回
-completed 的 BenchmarkRun。运行记录、事件与报告暂存内存（_runs/_events/_reports），
-不持久化到 SQLite；后续接入异步任务队列时再落库。
+RAG Benchmark 采用「创建即返回 queued、后台 Task 异步执行」的模式（与 index_service
+的 rebuild 一致）：POST 创建后立即返回 202 queued 的 BenchmarkRun，由受管 asyncio.Task
+在后台逐 Case 求值，进度与事件实时写入内存注册表，供 SSE 订阅。运行记录、事件与报告
+暂存内存（_runs/_events/_reports），不持久化到 SQLite；后续接入异步任务队列时再落库。
 """
 
 from __future__ import annotations
 
+import asyncio
 import sys
 from datetime import datetime, timezone
 from uuid import uuid4
 
+from app import repository
 from app.benchmarks import datasets
 from app.benchmarks.datasets import RAGDataset
-from app.benchmarks.rag import run_rag
+from app.benchmarks.rag import BenchmarkCancelled, run_rag
 from app.config import get_settings
 from app.contracts import (
     BenchmarkEvent,
@@ -32,6 +35,9 @@ from app.retrieval.engine import engine
 _runs: dict[str, BenchmarkRun] = {}
 _events: dict[str, list[BenchmarkEvent]] = {}
 _reports: dict[str, BenchmarkReport] = {}
+_tasks: dict[str, asyncio.Task] = {}
+_subscribers: dict[str, list[asyncio.Queue[BenchmarkEvent]]] = {}
+_cancel_flags: dict[str, asyncio.Event] = {}
 MAX_RUNS = 100
 
 
@@ -46,6 +52,9 @@ def _remember(run: BenchmarkRun) -> None:
         _runs.pop(oldest, None)
         _events.pop(oldest, None)
         _reports.pop(oldest, None)
+        _tasks.pop(oldest, None)
+        _subscribers.pop(oldest, None)
+        _cancel_flags.pop(oldest, None)
 
 
 def _config_snapshot(request: RAGRunRequest, dataset: RAGDataset) -> dict:
@@ -58,8 +67,16 @@ def _config_snapshot(request: RAGRunRequest, dataset: RAGDataset) -> dict:
         "modes": [m.value for m in request.modes],
         "retrieval": request.retrieval.model_dump(),
         "repeat": request.repeat,
-        "embedding": {"model_id": engine.embedding.model_id, "dim": engine.embedding.dim},
-        "reranker": {"model_id": engine.reranker.model_id},
+        "embedding": {
+            "model_id": engine.embedding.model_id,
+            "version": engine.embedding.version,
+            "dim": engine.embedding.dim,
+        },
+        "reranker": {
+            "model_id": engine.reranker.model_id,
+            "version": engine.reranker.version,
+        },
+        "index_meta": repository.get_index_meta(),
         "app": {"version": settings.version, "environment": settings.environment},
         "python": sys.version.split()[0],
         "metadata": request.metadata,
@@ -67,7 +84,7 @@ def _config_snapshot(request: RAGRunRequest, dataset: RAGDataset) -> dict:
 
 
 async def create_rag_run(request: RAGRunRequest) -> BenchmarkRun:
-    """创建并同步执行一次 RAG Benchmark，返回 completed 的 BenchmarkRun。"""
+    """创建一次 RAG Benchmark，立即返回 queued 的 BenchmarkRun，由后台 Task 执行。"""
     dataset = datasets.load_dataset(request.dataset_id, BenchmarkKind.rag)
     run_id = "benchmark_" + uuid4().hex[:12]
     snapshot = _config_snapshot(request, dataset)
@@ -77,24 +94,41 @@ async def create_rag_run(request: RAGRunRequest) -> BenchmarkRun:
         kind=BenchmarkKind.rag,
         dataset_id=dataset.dataset_id,
         dataset_hash=dataset.content_hash,
-        status=BenchmarkStatus.running,
+        status=BenchmarkStatus.queued,
         progress=0.0,
         config_snapshot=snapshot,
         created_at=_now(),
-        started_at=_now(),
     )
     _remember(run)
     _events[run_id] = []
+    _subscribers[run_id] = []
+    _cancel_flags[run_id] = asyncio.Event()
+    _tasks[run_id] = asyncio.create_task(_execute_rag(run_id, request, dataset, snapshot))
+    return run
+
+
+async def _execute_rag(
+    run_id: str, request: RAGRunRequest, dataset: RAGDataset, snapshot: dict
+) -> None:
+    """后台执行 RAG Benchmark，实时更新进度/事件，结束后写入报告并关闭订阅。"""
+    cancel_event = _cancel_flags[run_id]
 
     def emit(event_type: BenchmarkEventType, data: dict) -> None:
         sequence = len(_events[run_id])
-        _events[run_id].append(
-            BenchmarkEvent(
-                event=event_type, run_id=run_id, sequence=sequence,
-                data=data, timestamp=_now(),
-            )
+        event = BenchmarkEvent(
+            event=event_type, run_id=run_id, sequence=sequence, data=data, timestamp=_now()
         )
+        _events[run_id].append(event)
+        for queue in _subscribers.get(run_id, []):
+            queue.put_nowait(event)
 
+    def finish() -> None:
+        _subscribers.pop(run_id, None)
+        _cancel_flags.pop(run_id, None)
+
+    _runs[run_id] = _runs[run_id].model_copy(
+        update={"status": BenchmarkStatus.running, "started_at": _now()}
+    )
     emit(
         BenchmarkEventType.run_started,
         {"dataset_id": dataset.dataset_id, "modes": [m.value for m in request.modes]},
@@ -107,8 +141,31 @@ async def create_rag_run(request: RAGRunRequest) -> BenchmarkRun:
         emit(BenchmarkEventType.case_completed, result.model_dump(mode="json"))
 
     try:
-        metrics_by_mode, results = await run_rag(dataset, request, on_case=on_case)
-    except Exception as exc:
+        metrics_by_mode, results = await run_rag(
+            dataset,
+            request,
+            on_case=on_case,
+            should_cancel=cancel_event.is_set,
+        )
+    except BenchmarkCancelled:
+        _runs[run_id] = _runs[run_id].model_copy(
+            update={
+                "status": BenchmarkStatus.cancelled,
+                "progress": 1.0,
+                "completed_at": _now(),
+            }
+        )
+        _reports[run_id] = BenchmarkReport(
+            run_id=run_id,
+            kind=BenchmarkKind.rag,
+            dataset_id=dataset.dataset_id,
+            dataset_hash=dataset.content_hash,
+            status=BenchmarkStatus.cancelled,
+            config_snapshot=snapshot,
+        )
+        finish()
+        return
+    except Exception as exc:  # 单次运行失败不拖垮服务，记录错误后结束
         _runs[run_id] = _runs[run_id].model_copy(
             update={
                 "status": BenchmarkStatus.failed,
@@ -127,7 +184,8 @@ async def create_rag_run(request: RAGRunRequest) -> BenchmarkRun:
             config_snapshot=snapshot,
             error=str(exc),
         )
-        raise ApiError(500, "BENCHMARK_RUN_FAILED", str(exc), {"run_id": run_id}) from exc
+        finish()
+        return
 
     metrics = {mode: m.model_dump() for mode, m in metrics_by_mode.items()}
     _runs[run_id] = _runs[run_id].model_copy(
@@ -149,7 +207,7 @@ async def create_rag_run(request: RAGRunRequest) -> BenchmarkRun:
         metrics=metrics,
         cases=results,
     )
-    return _runs[run_id]
+    finish()
 
 
 def list_runs(
@@ -181,13 +239,38 @@ def get_events(run_id: str) -> list[BenchmarkEvent]:
 
 
 def cancel_run(run_id: str) -> BenchmarkRun | None:
-    """取消运行：同步 MVP 下运行通常已结束，仅对仍在排队/运行的记录置为 cancelled。"""
+    """取消运行：对 queued/running 设置取消标志，后台 Task 在 Case 边界检查后置为 cancelled。"""
     run = _runs.get(run_id)
     if run is None:
         return None
     if run.status in (BenchmarkStatus.queued, BenchmarkStatus.running):
-        run = run.model_copy(
-            update={"status": BenchmarkStatus.cancelled, "completed_at": _now()}
-        )
-        _runs[run_id] = run
+        _cancel_flags[run_id].set()
     return run
+
+
+def subscribe(run_id: str) -> asyncio.Queue[BenchmarkEvent] | None:
+    """订阅运行事件流；运行已结束（completed/failed/cancelled）时返回 None。"""
+    run = _runs.get(run_id)
+    if run is None or run.status in (
+        BenchmarkStatus.completed,
+        BenchmarkStatus.failed,
+        BenchmarkStatus.cancelled,
+    ):
+        return None
+    queue: asyncio.Queue[BenchmarkEvent] = asyncio.Queue()
+    _subscribers.setdefault(run_id, []).append(queue)
+    return queue
+
+
+def unsubscribe(run_id: str, queue: asyncio.Queue[BenchmarkEvent]) -> None:
+    subscribers = _subscribers.get(run_id)
+    if subscribers and queue in subscribers:
+        subscribers.remove(queue)
+
+
+async def wait_for_run(run_id: str) -> BenchmarkRun:
+    """等待后台任务结束（测试/轮询用）；无任务时直接返回当前状态。"""
+    task = _tasks.get(run_id)
+    if task is not None:
+        await task
+    return _runs.get(run_id)