fix(backend): 落实 PR #11 第二轮评审意见

- Benchmark 容量淘汰只删终态 run，满容量且全活动时返回 BENCHMARK_CAPACITY_EXCEEDED - 创建 run 前校验索引兼容性（BENCHMARK_INDEX_INCOMPATIBLE） - 取消 run 补发 RunCancelled 终止事件；失败分支脱敏（BENCHMARK_RUN_FAILED） - 失败样本计入汇总分母，报告输出 total/successful/failed/failure_rate - load_dataset 按文件名隔离无关损坏文件，顶层非对象拒绝 - FTS score_threshold 先于计数/分页，total 与 items 一致 - Benchmark SSE 支持 Last-Event-ID 游标 - 移除 Agent Benchmark 501 占位接口 - 同步第二阶段接口契约与开发说明文档 Co-Authored-By: Claude Code <noreply@anthropic.com>
2026-09-03 22:20:11 +08:00
parent c6cde2500b
commit fcc601fcf3
11 changed files with 378 additions and 69 deletions
@@ -17,7 +17,13 @@ from pydantic import ValidationError

 from app.benchmarks import datasets, metrics as m, service
 from app.config import get_settings
-from app.contracts import BenchmarkKind, RAGRunRequest, SearchMode
+from app.contracts import (
+    BenchmarkKind,
+    BenchmarkRun,
+    BenchmarkStatus,
+    RAGRunRequest,
+    SearchMode,
+)
 from app.errors import ApiError


@@ -323,3 +329,113 @@ def test_benchmark_run_not_found_raises() -> None:
    with pytest.raises(ApiError) as exc:
        asyncio.run(routes.get_benchmark_run("benchmark_missing"))
    assert exc.value.code == "BENCHMARK_RUN_NOT_FOUND"
+
+
+# --------------------------------------------------------------------------- #
+# 审阅回归：索引兼容 / 容量 / 失败样本 / 取消事件 / 数据集隔离
+# --------------------------------------------------------------------------- #
+def test_create_rag_run_requires_built_index() -> None:
+    # 空索引（无已索引 block）会让所有模式得到全 0 指标，应在创建时拒绝而非跑出误导结果
+    _write_dataset("empty-index-v1", [{"case_id": "x", "query": "q", "expected_note_ids": ["n"]}])
+    with pytest.raises(ApiError) as exc:
+        asyncio.run(
+            service.create_rag_run(
+                RAGRunRequest(dataset_id="empty-index-v1", modes=[SearchMode.fts])
+            )
+        )
+    assert exc.value.status_code == 409
+    assert exc.value.code == "BENCHMARK_INDEX_INCOMPATIBLE"
+
+
+def test_capacity_exceeded_when_all_runs_active(monkeypatch) -> None:
+    # 满容量且全为活动（非终态）run 时，无法淘汰，应拒绝创建而非删掉正在运行的 run
+    _, _, case = _single_note_case()
+    _write_dataset("capacity-v1", [case])
+
+    monkeypatch.setattr(service, "MAX_RUNS", 1)
+    fake_id = "benchmark_fake_active"
+    service._runs[fake_id] = BenchmarkRun(
+        run_id=fake_id,
+        kind=BenchmarkKind.rag,
+        dataset_id="capacity-v1",
+        dataset_hash="sha256:fake",
+        status=BenchmarkStatus.queued,
+        created_at=service._now(),
+    )
+    try:
+        with pytest.raises(ApiError) as exc:
+            asyncio.run(
+                service.create_rag_run(
+                    RAGRunRequest(dataset_id="capacity-v1", modes=[SearchMode.fts])
+                )
+            )
+        assert exc.value.status_code == 429
+        assert exc.value.code == "BENCHMARK_CAPACITY_EXCEEDED"
+    finally:
+        service._runs.pop(fake_id, None)
+
+
+def test_failed_samples_counted_as_zero_in_aggregate() -> None:
+    from app.benchmarks import rag as rag_module
+    from app.contracts import RAGCaseResult
+
+    cases = [
+        RAGCaseResult(
+            case_id="ok", mode=SearchMode.fts, repeat=0, latency_ms=10.0,
+            hit_at_1=True, recall=1.0, reciprocal_rank=1.0,
+            citation_hit=True, citation_applicable=True,
+        ),
+        RAGCaseResult(
+            case_id="boom", mode=SearchMode.fts, repeat=0, latency_ms=0.0,
+            error="RAG case evaluation failed.",
+            error_code="BENCHMARK_CASE_EVALUATION_FAILED",
+        ),
+    ]
+    metrics = rag_module._aggregate(cases, SearchMode.fts)
+
+    assert metrics.total_cases == 2
+    assert metrics.successful_cases == 1
+    assert metrics.failed_cases == 1
+    assert metrics.failure_rate == 0.5
+    # 失败样本按零分计入质量指标分母，汇总不虚高
+    assert metrics.hit_at_1 == 0.5
+    assert metrics.recall_at_k == 0.5
+    # 延迟只统计成功样本
+    assert metrics.p50_latency_ms == 10.0
+
+
+def test_cancel_emits_run_cancelled_event() -> None:
+    _, _, case = _single_note_case()
+    _write_dataset("cancel-event-v1", [case])
+
+    async def _scenario():
+        run = await service.create_rag_run(
+            RAGRunRequest(dataset_id="cancel-event-v1", modes=[SearchMode.fts])
+        )
+        service.cancel_run(run.run_id)
+        return await service.wait_for_run(run.run_id)
+
+    run = asyncio.run(_scenario())
+    assert run.status.value == "cancelled"
+    events = service.get_events(run.run_id)
+    assert events[-1].event.value == "RunCancelled"
+
+
+def test_load_dataset_ignores_corrupted_unrelated_files() -> None:
+    # 无关文件损坏（非法 JSON / 顶层非对象）不应阻断目标数据集加载
+    directory = get_settings().benchmark_datasets_path
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / "broken.json").write_text("{ not valid json", encoding="utf-8")
+    (directory / "array.json").write_text('["a", "b"]', encoding="utf-8")
+    _write_dataset("ok-v1", [{"case_id": "x", "query": "q", "expected_note_ids": ["n"]}])
+
+    dataset = datasets.load_dataset("ok-v1", BenchmarkKind.rag)
+    assert dataset.dataset_id == "ok-v1"
+    assert len(dataset.cases) == 1
+
+
+def test_load_dataset_top_level_must_be_object() -> None:
+    _write_raw("array-top", ["a", "b"])
+    with pytest.raises(ApiError) as exc:
+        datasets.load_dataset("array-top", BenchmarkKind.rag)
+    assert exc.value.code == "BENCHMARK_DATASET_INVALID"
@@ -436,6 +436,44 @@ def test_fts_pagination_is_not_truncated_at_one_thousand(vault) -> None:
    assert len(response.items) == 10


+def test_fts_score_threshold_filters_before_total(vault) -> None:
+    """score_threshold 先于计数与分页生效：total 反映过滤后数量，与 items 一致。
+
+    高阈值过滤掉全部结果时 total==0 且 items 为空，杜绝「空页但 total>0」的
+    不一致（审阅 P2-7）。
+    """
+    from app.retrieval.engine import engine
+    from app.services import note_service
+
+    # 10 个 block，含「目标」次数递增，bm25 分数各异，min-max 归一化后分数落在 [0,1]
+    markdown = "\n\n".join(f"{'目标' * i} 分隔内容" for i in range(1, 11))
+    asyncio.run(
+        note_service.create_note(title="阈值过滤", markdown=markdown, folder="", tags=[])
+    )
+
+    all_hits = asyncio.run(
+        engine.search(
+            SearchRequest(query="目标", mode=SearchMode.fts, limit=20, score_threshold=0.0)
+        )
+    )
+    filtered = asyncio.run(
+        engine.search(
+            SearchRequest(query="目标", mode=SearchMode.fts, limit=20, score_threshold=0.5)
+        )
+    )
+    none = asyncio.run(
+        engine.search(
+            SearchRequest(query="目标", mode=SearchMode.fts, limit=20, score_threshold=2.0)
+        )
+    )
+
+    assert all_hits.page.total >= 10
+    assert 0 < filtered.page.total < all_hits.page.total  # 阈值过滤掉部分而非全部
+    assert filtered.page.total == len(filtered.items)
+    assert none.page.total == 0
+    assert none.items == []
+
+
 # --------------------------------------------------------------------------- #
 # 审阅回归：PATCH tags 语义 / 向量-块一致性 / 过滤漏召回 / rebuild 语义与回滚
 # --------------------------------------------------------------------------- #