"""Benchmark Dataset 注册：从受控目录加载 JSON 数据集并校验。 Dataset 只能来自配置目录（settings.benchmark_datasets_path），API 不接受调用方提交 任意文件路径。目录不存在或为空时按「无数据集」处理，不报错。 """ from __future__ import annotations import hashlib import json from dataclasses import dataclass, field from pathlib import Path from pydantic import BaseModel, Field, ValidationError from app.config import get_settings from app.contracts import ( BenchmarkDatasetInfo, BenchmarkKind, RAGDatasetCase, AgentDatasetCase, ) from app.errors import ApiError @dataclass class RAGDataset: """内存中的 RAG / Agent 数据集：元信息 + 已校验的 Case 列表 + 内容哈希。""" dataset_id: str kind: BenchmarkKind version: str description: str cases: list[RAGDatasetCase | AgentDatasetCase] = field(default_factory=list) content_hash: str = "" class _DatasetMeta(BaseModel): """Dataset 元数据的最小校验模型。 list_datasets 用它逐文件校验元信息字段结构，把「合法 JSON 但字段类型错误」 （如 cases: 42）这类损坏文件隔离掉，而不是让 len() 抛 TypeError 拖垮整个列表。 """ dataset_id: str = Field(min_length=1) kind: str = "" version: str = "" description: str = "" cases: list = Field(default_factory=list) def _datasets_dir() -> Path: return get_settings().benchmark_datasets_path def _dataset_files() -> list[Path]: directory = _datasets_dir() if not directory.is_dir(): return [] return sorted(directory.glob("*.json")) def _content_hash(raw: bytes) -> str: return "sha256:" + hashlib.sha256(raw).hexdigest() def _read_json(path: Path) -> tuple[dict, bytes]: """读取并解析 JSON 文件，返回 (dict, 原始字节)；非法 JSON 抛 BENCHMARK_DATASET_INVALID。""" try: raw_bytes = path.read_bytes() return json.loads(raw_bytes.decode("utf-8")), raw_bytes except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", f"Dataset file is not valid JSON: {path.name}", {"path": str(path)}, ) from exc def _dataset_from_raw(raw: dict, raw_bytes: bytes, kind: BenchmarkKind) -> RAGDataset: """把单个数据集 JSON 解析为 RAGDataset，非法结构抛 BENCHMARK_DATASET_INVALID。""" dataset_id = raw.get("dataset_id") if not isinstance(dataset_id, str) or not dataset_id: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", "Dataset must declare a non-empty string 'dataset_id'.", {}, ) file_kind = raw.get("kind", kind.value) if file_kind != kind.value: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", f"Dataset kind mismatch: expected '{kind.value}', got '{file_kind}'.", {"dataset_id": dataset_id}, ) raw_cases = raw.get("cases") if not isinstance(raw_cases, list) or not raw_cases: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", "Dataset 'cases' must be a non-empty list.", {"dataset_id": dataset_id}, ) cases: list[RAGDatasetCase | AgentDatasetCase] = [] for index, case in enumerate(raw_cases): try: parsed = (AgentDatasetCase if kind == BenchmarkKind.agent else RAGDatasetCase).model_validate(case) except ValidationError as exc: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", f"Dataset case #{index} is invalid.", {"dataset_id": dataset_id, "case_index": index, "errors": exc.errors()}, ) from exc if kind == BenchmarkKind.agent: if not (parsed.expected_tools or parsed.output_contains or parsed.citation_required or parsed.tasks_created is not None): raise ApiError(422, 'BENCHMARK_DATASET_INVALID', 'Agent case requires objective expectations.') if any(tool.name not in parsed.allowed_tools for tool in parsed.expected_tools): raise ApiError(422, 'BENCHMARK_DATASET_INVALID', 'Expected tools must be allowed.') cases.append(parsed) continue # 每个 Case 至少要声明一个期望 id，否则无法计算命中/召回 if not parsed.expected_note_ids and not parsed.expected_block_ids: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", f"Dataset case '{parsed.case_id}' must declare expected_note_ids or expected_block_ids.", {"dataset_id": dataset_id, "case_id": parsed.case_id}, ) # citation_required=true 时必须声明 expected_block_ids，否则无法计算 Citation Hit Rate if parsed.citation_required and not parsed.expected_block_ids: raise ApiError( 422, "BENCHMARK_DATASET_INVALID", f"Dataset case '{parsed.case_id}' requires expected_block_ids when citation_required is true.", {"dataset_id": dataset_id, "case_id": parsed.case_id}, ) cases.append(parsed) if len(cases) > 100 or len({c.case_id for c in cases}) != len(cases): raise ApiError(422, 'BENCHMARK_DATASET_INVALID', 'Dataset case IDs must be unique; maximum 100 cases.') return RAGDataset( dataset_id=dataset_id, kind=kind, version=str(raw.get("version", "")), description=str(raw.get("description", "")), cases=cases, content_hash=_content_hash(raw_bytes), ) def list_datasets(kind: BenchmarkKind) -> list[BenchmarkDatasetInfo]: """枚举受控目录下指定 kind 的数据集元信息（不含 Case 内容）。 逐文件用 _DatasetMeta 校验元信息字段结构，单个损坏文件隔离跳过而非整体失败， 保证列表接口健壮；损坏细节由 load_dataset 抛出。 """ infos: list[BenchmarkDatasetInfo] = [] for path in _dataset_files(): try: raw, raw_bytes = _read_json(path) meta = _DatasetMeta.model_validate(raw) except (ApiError, ValidationError): continue if meta.kind not in ("", kind.value): continue infos.append( BenchmarkDatasetInfo( dataset_id=meta.dataset_id, kind=kind, version=meta.version, description=meta.description, case_count=len(meta.cases), content_hash=_content_hash(raw_bytes), ) ) return infos def load_dataset(dataset_id: str, kind: BenchmarkKind) -> RAGDataset: """按文件名加载并校验数据集；找不到抛 BENCHMARK_DATASET_NOT_FOUND。 只读取与请求 dataset_id 同名的文件（{dataset_id}.json），无关文件的损坏（JSON 语法 错误、UTF-8 解码错误、顶层非对象）不会阻断目标数据集加载；只有目标文件本身损坏 才抛 BENCHMARK_DATASET_INVALID。按现有文件 stem 精确匹配，不拼接调用方传入的路径。 """ for path in _dataset_files(): if path.stem != dataset_id: continue raw, raw_bytes = _read_json(path) if not isinstance(raw, dict): raise ApiError( 422, "BENCHMARK_DATASET_INVALID", "Dataset top-level must be a JSON object.", {"dataset_id": dataset_id, "path": path.name}, ) return _dataset_from_raw(raw, raw_bytes, kind) raise ApiError( 404, "BENCHMARK_DATASET_NOT_FOUND", f"Benchmark dataset does not exist: {dataset_id}", {"dataset_id": dataset_id, "kind": kind.value}, )