fix: rebase embedding evaluation on current retrieval docs

2026-10-16 13:13:21 +08:00
parent d0efc90155 97c9ea2cb4
commit ca55923559
7 changed files with 629 additions and 0 deletions
@@ -0,0 +1,165 @@
+"""Optional local embeddings; importing this module never loads model weights or PyTorch."""
+
+import hashlib
+import json
+import math
+import re
+from dataclasses import asdict, dataclass
+from typing import Protocol
+
+QUERY_INSTRUCTION = "为这个句子生成表示以用于检索相关文章："
+
+
+class EmbeddingError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class EmbeddingSpec:
+    model: str
+    revision: str
+    dimension: int
+    token_limit: int = 512
+    query_instruction: str = QUERY_INSTRUCTION
+    implementation: str = "hf-cls-l2-cpu-fp32-v1"
+    checkpoint_format: str = "safetensors"
+
+    def __post_init__(self):
+        if not isinstance(self.revision, str) or not re.fullmatch(r"[0-9a-f]{40}", self.revision):
+            raise EmbeddingError("模型必须固定到完整 revision SHA")
+        if (
+            type(self.dimension) is not int
+            or type(self.token_limit) is not int
+            or self.dimension < 1
+            or self.token_limit < 2
+        ):
+            raise EmbeddingError("向量维度和 token 上限无效")
+        if (
+            not isinstance(self.model, str)
+            or not self.model
+            or not isinstance(self.query_instruction, str)
+        ):
+            raise EmbeddingError("模型名称或查询指令无效")
+        if self.checkpoint_format not in {"safetensors", "pytorch-weights-only"}:
+            raise EmbeddingError("不支持的权重格式")
+
+    @property
+    def space_id(self) -> str:
+        return hashlib.sha256(
+            json.dumps(asdict(self), sort_keys=True, ensure_ascii=False).encode()
+        ).hexdigest()
+
+
+SMALL_ZH = EmbeddingSpec("BAAI/bge-small-zh-v1.5", "7999e1d3359715c523056ef9478215996d62a620", 512)
+BASE_ZH = EmbeddingSpec(
+    "BAAI/bge-base-zh-v1.5",
+    "f03589ceff5aac7111bd60cfc7d497ca17ecac65",
+    768,
+    checkpoint_format="pytorch-weights-only",
+)
+LOCAL_MODELS = {"small": SMALL_ZH, "base": BASE_ZH}
+
+
+class EmbeddingProvider(Protocol):
+    spec: EmbeddingSpec
+
+    def encode(self, texts: list[str], *, query: bool = False) -> list[list[float]]: ...
+
+
+def validate_vectors(vectors: list[list[float]], count: int, spec: EmbeddingSpec):
+    if len(vectors) != count:
+        raise EmbeddingError("模型返回数量与输入不一致")
+    for vector in vectors:
+        try:
+            finite = all(type(x) in {float, int} and math.isfinite(x) for x in vector)
+        except (TypeError, OverflowError):
+            finite = False
+        if len(vector) != spec.dimension or not finite:
+            raise EmbeddingError("模型返回维度错误或非有限数值")
+        if not math.isclose(sum(x * x for x in vector), 1, abs_tol=1e-4):
+            raise EmbeddingError("模型输出没有正确 L2 归一化")
+
+
+class LocalEmbeddingProvider:
+    def __init__(
+        self,
+        spec: EmbeddingSpec,
+        *,
+        batch_size: int = 8,
+        threads: int = 2,
+        cache_dir=None,
+        offline: bool = True,
+    ):
+        if (
+            type(batch_size) is not int
+            or type(threads) is not int
+            or not 1 <= batch_size <= 32
+            or not 1 <= threads <= 4
+        ):
+            raise EmbeddingError("CPU batch_size 必须为 1–32，threads 必须为 1–4")
+        import torch
+        from huggingface_hub import snapshot_download
+        from transformers import AutoModel, AutoTokenizer
+
+        torch.set_num_threads(threads)
+        self.spec, self.batch_size, self.torch = spec, batch_size, torch
+        path = snapshot_download(
+            repo_id=spec.model,
+            revision=spec.revision,
+            cache_dir=cache_dir,
+            local_files_only=offline,
+            token=False,
+            allow_patterns=[
+                "config.json",
+                "model.safetensors"
+                if spec.checkpoint_format == "safetensors"
+                else "pytorch_model.bin",
+                "tokenizer.json",
+                "tokenizer_config.json",
+                "special_tokens_map.json",
+                "vocab.txt",
+                "added_tokens.json",
+            ],
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            path, local_files_only=True, trust_remote_code=False
+        )
+        self.model = (
+            AutoModel.from_pretrained(
+                path,
+                local_files_only=True,
+                trust_remote_code=False,
+                use_safetensors=spec.checkpoint_format == "safetensors",
+                weights_only=True,
+                dtype=torch.float32,
+            )
+            .to("cpu")
+            .eval()
+        )
+        if self.model.config.hidden_size != spec.dimension:
+            raise EmbeddingError("模型实际维度与配置不一致")
+        if self.tokenizer.model_max_length != spec.token_limit:
+            raise EmbeddingError("模型 tokenizer 上限与配置不一致")
+
+    def encode(self, texts: list[str], *, query: bool = False) -> list[list[float]]:
+        if not isinstance(texts, list) or any(
+            not isinstance(text, str) or not text.strip() or len(text) > 10_000 for text in texts
+        ):
+            raise EmbeddingError("每条输入须为非空文本且最多 10000 字符")
+        torch = self.torch
+        vectors = []
+        for offset in range(0, len(texts), self.batch_size):
+            batch = texts[offset : offset + self.batch_size]
+            if query:
+                batch = [self.spec.query_instruction + text for text in batch]
+            encoded = self.tokenizer(
+                batch, padding=True, truncation=False, return_tensors="pt", verbose=False
+            )
+            if encoded["input_ids"].shape[1] > self.spec.token_limit:
+                raise EmbeddingError("输入超出模型 token 上限；请重新切分，禁止静默截断")
+            with torch.inference_mode():
+                features = self.model(**encoded).last_hidden_state[:, 0]
+                normalized = torch.nn.functional.normalize(features, p=2, dim=1)
+            vectors.extend(normalized.tolist())
+        validate_vectors(vectors, len(texts), self.spec)
+        return vectors
@@ -0,0 +1 @@
+"""Offline, synthetic local-model evaluation; never writes the application database."""
@@ -0,0 +1,221 @@
+"""Reproducible CPU smoke evaluation; no application database or remote inference service."""
+
+import argparse
+import hashlib
+import importlib.metadata
+import json
+import math
+import platform
+import statistics
+import sys
+from dataclasses import asdict
+from pathlib import Path
+from time import perf_counter
+
+from app.services.embedding import LOCAL_MODELS, EmbeddingError, LocalEmbeddingProvider
+
+
+def load_dataset(path: Path) -> tuple[dict, str]:
+    with path.open("rb") as source:
+        raw = source.read(2_000_001)
+    if len(raw) > 2_000_000:
+        raise ValueError("评估输入不得超过 2MB")
+    data = json.loads(raw)
+    if not isinstance(data, dict):
+        raise ValueError("评估输入须为 JSON 对象")
+    documents, queries = data["documents"], data["queries"]
+    if not isinstance(documents, list) or not isinstance(queries, list):
+        raise ValueError("documents 与 queries 须为列表")
+    if not 1 <= len(documents) <= 1000 or not 1 <= len(queries) <= 1000:
+        raise ValueError("文档和问题各须为 1–1000 条")
+    for rows in [documents, queries]:
+        if any(not isinstance(row, dict) for row in rows):
+            raise ValueError("每条记录须为对象")
+        ids = [row["id"] for row in rows]
+        if any(not isinstance(x, str) or not x or len(x) > 100 for x in ids) or len(
+            set(ids)
+        ) != len(ids):
+            raise ValueError("每组 ID 必须非空、唯一且最多 100 字符")
+        if any(
+            not isinstance(row["text"], str) or not row["text"].strip() or len(row["text"]) > 10000
+            for row in rows
+        ):
+            raise ValueError("每条文本须为非空且最多 10000 字符")
+    ids = {row["id"] for row in documents}
+    for query in queries:
+        relevant = query["relevant_ids"]
+        if (
+            not isinstance(relevant, list)
+            or not relevant
+            or any(not isinstance(x, str) for x in relevant)
+            or len(set(relevant)) != len(relevant)
+            or not set(relevant) <= ids
+        ):
+            raise ValueError("相关文档标注必须非空、唯一并引用现有文档")
+    if not isinstance(data["dataset_id"], str) or not isinstance(data["purpose"], str):
+        raise ValueError("必须填写数据集 ID 与用途")
+    return data, hashlib.sha256(raw).hexdigest()
+
+
+def rank_and_measure(documents, queries, document_vectors, query_vectors, *, top_k=3):
+    if not documents or not queries or top_k < 1:
+        raise ValueError("评估须有文档、问题且 top_k 为正数")
+    if len(document_vectors) != len(documents) or len(query_vectors) != len(queries):
+        raise ValueError("评估向量数量与输入不一致")
+    rows = []
+    for query, vector in zip(queries, query_vectors, strict=True):
+        ranked = sorted(
+            [
+                (doc["id"], math.fsum(x * y for x, y in zip(vector, embedding, strict=True)))
+                for doc, embedding in zip(documents, document_vectors, strict=True)
+            ],
+            key=lambda pair: (-pair[1], pair[0]),
+        )
+        relevant = set(query["relevant_ids"])
+        first = next(
+            (rank for rank, (id_, _) in enumerate(ranked[:top_k], 1) if id_ in relevant), None
+        )
+        rows.append(
+            {
+                "query_id": query["id"],
+                "top_results": [
+                    {"id": id_, "score": round(score, 8)} for id_, score in ranked[:top_k]
+                ],
+                "hit_at_1": int(ranked[0][0] in relevant),
+                "recall_at_k": len({id_ for id_, _ in ranked[:top_k]} & relevant) / len(relevant),
+                "reciprocal_rank_at_k": 1 / first if first else 0,
+            }
+        )
+    return {
+        "top_k": top_k,
+        "hit_at_1": statistics.mean(row["hit_at_1"] for row in rows),
+        "recall_at_k": statistics.mean(row["recall_at_k"] for row in rows),
+        "mrr_at_k": statistics.mean(row["reciprocal_rank_at_k"] for row in rows),
+        "per_query": rows,
+    }
+
+
+def measure(call, repeats):
+    call()  # Warm this exact input shape before taking the recorded samples.
+    timings = []
+    result = None
+    for _ in range(repeats):
+        start = perf_counter()
+        result = call()
+        timings.append((perf_counter() - start) * 1000)
+    return result, {
+        "samples_ms": [round(value, 3) for value in timings],
+        "median_ms": round(statistics.median(timings), 3),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="本地 CPU Embedding 实测；合成结果不能用于论文结论"
+    )
+    parser.add_argument("--model", choices=LOCAL_MODELS, required=True)
+    parser.add_argument(
+        "--dataset", type=Path, default=Path(__file__).with_name("synthetic_zh.json")
+    )
+    parser.add_argument("--batch-size", type=int, default=8)
+    parser.add_argument("--threads", type=int, default=2)
+    parser.add_argument("--repeats", type=int, default=5)
+    parser.add_argument("--cache-dir", type=Path)
+    parser.add_argument(
+        "--download", action="store_true", help="允许下载固定 revision 的公开模型文件"
+    )
+    parser.add_argument("--output", type=Path)
+    args = parser.parse_args()
+    if sys.platform != "linux":
+        parser.error("当前 RSS 测量仅支持 Linux；Provider 可在其他支持的平台单独使用")
+    import resource
+
+    if not 1 <= args.repeats <= 20:
+        parser.error("repeats 必须为 1–20")
+    dataset, digest = load_dataset(args.dataset)
+    print("初始化固定版本的 CPU 模型。", file=sys.stderr, flush=True)
+    started = perf_counter()
+    provider = LocalEmbeddingProvider(
+        LOCAL_MODELS[args.model],
+        batch_size=args.batch_size,
+        threads=args.threads,
+        cache_dir=args.cache_dir,
+        offline=not args.download,
+    )
+    initialization_seconds = perf_counter() - started
+    print("模型就绪，执行预热和编码测量。", file=sys.stderr, flush=True)
+    provider.encode(["预热测试"], query=True)
+    corpus = [row["text"] for row in dataset["documents"]]
+    queries = [row["text"] for row in dataset["queries"]]
+    documents, corpus_time = measure(lambda: provider.encode(corpus), args.repeats)
+    vectors, query_time = measure(lambda: provider.encode(queries, query=True), args.repeats)
+    _, single_time = measure(lambda: provider.encode([queries[0]], query=True), args.repeats)
+    long_text = "合成测试资料仅用于测量本地编码资源，不表示任何真实景区信息。" * 12
+    _, long_time = measure(lambda: provider.encode([long_text] * args.batch_size), args.repeats)
+    try:
+        provider.encode(["超长测试" * 500])
+    except EmbeddingError:
+        length_guard = True
+    else:
+        raise AssertionError("超长输入未被拒绝")
+    import torch
+
+    report = {
+        "schema_version": 1,
+        "evaluation_kind": "synthetic_smoke"
+        if dataset["purpose"] == "synthetic_smoke_not_domain_benchmark"
+        else "provided_dataset_not_independently_verified",
+        "dataset": {
+            "id": dataset["dataset_id"],
+            "purpose": dataset["purpose"],
+            "sha256": digest,
+            "documents": len(corpus),
+            "queries": len(queries),
+        },
+        "embedding_spec": asdict(provider.spec),
+        "space_id": provider.spec.space_id,
+        "runtime": {
+            "python": platform.python_version(),
+            "torch": torch.__version__,
+            "transformers": importlib.metadata.version("transformers"),
+            "huggingface_hub": importlib.metadata.version("huggingface_hub"),
+            "numpy": importlib.metadata.version("numpy"),
+            "tokenizers": importlib.metadata.version("tokenizers"),
+            "safetensors": importlib.metadata.version("safetensors"),
+            "device": "cpu",
+            "threads": torch.get_num_threads(),
+            "interop_threads": torch.get_num_interop_threads(),
+            "batch_size": args.batch_size,
+            "repeats": args.repeats,
+            "warmup_per_shape": 1,
+        },
+        "initialization_seconds": round(initialization_seconds, 3),
+        "timings": {
+            "corpus_encode": corpus_time,
+            "query_batch_encode": query_time,
+            "single_query_encode": single_time,
+            "long_text_batch_encode": long_time,
+        },
+        "long_text_characters": len(long_text),
+        "long_text_tokens": len(provider.tokenizer(long_text)["input_ids"]),
+        "peak_process_rss_mib": round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 1),
+        "token_overflow_rejected": length_guard,
+        "retrieval": rank_and_measure(dataset["documents"], dataset["queries"], documents, vectors),
+        "limitations": [
+            "合成小样本仅检查流程，不能代表那拉提领域质量或正式实验分数",
+            "计时为本机预热后的编码，包含分词/补齐/归一化/结果转换，不含 API、数据库与网络请求",
+            "峰值 RSS 是整个独立进程的高水位，包含解释器和模型；初始化包含缓存检查/下载与首次模型加载",
+        ],
+    }
+    encoded = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        args.output.write_text(encoded, encoding="utf-8")
+        args.output.chmod(0o600)
+        print("评估完成，报告已写入指定本机位置。", file=sys.stderr)
+    else:
+        print(encoded, end="")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,30 @@
+{
+  "dataset_id": "synthetic-zh-travel-smoke-v1",
+  "purpose": "synthetic_smoke_not_domain_benchmark",
+  "documents": [
+    {"id":"d01","text":"【合成测试】示例园区的停车场设有接驳车，步行游客可换乘到游客中心。"},
+    {"id":"d02","text":"【合成测试】示例停车区域按车辆停留时段管理，离场时到出口办理停车手续。"},
+    {"id":"d03","text":"【合成测试】示例展馆提供轮椅通道和电梯，行动不便的访客可从无障碍入口进入。"},
+    {"id":"d04","text":"【合成测试】示例演出因降雨取消，已购票观众可沿原购票渠道申请退款。"},
+    {"id":"d05","text":"【合成测试】示例红色文化展厅设有预约讲解，访客可提前登记并选择导览时段。"},
+    {"id":"d06","text":"【合成测试】示例资料管理员核查发布机构、来源链接和更新时间，只有审核通过的资料才能成为问答依据。"},
+    {"id":"d07","text":"【合成测试】示例民宿在前台办理入住，旅客需要出示有效身份证件并确认预订信息。"},
+    {"id":"d08","text":"【合成测试】示例住宿提供早餐，有素食需求的客人可以提前向接待人员说明。"},
+    {"id":"d09","text":"【合成测试】示例山间步道遇到恶劣天气暂停通行，游客应查看关闭提示并选择室内活动。"},
+    {"id":"d10","text":"【合成测试】示例游客服务台统一接收捡到的物品，并登记失物线索协助寻找主人。"},
+    {"id":"d11","text":"【合成测试】示例停车区设有电动车充电点，司机可按现场设备说明补充电量。"},
+    {"id":"d12","text":"【合成测试】示例场馆的开放安排以入口公告为准，参观者应提前确认当天是否接待。"}
+  ],
+  "queries": [
+    {"id":"q01","text":"没有开车的人怎样从停车区去游客中心？","relevant_ids":["d01"]},
+    {"id":"q02","text":"同行者坐轮椅，可以从哪里进入展馆？","relevant_ids":["d03"]},
+    {"id":"q03","text":"演出被雨取消，已经付的钱能退吗？","relevant_ids":["d04"]},
+    {"id":"q04","text":"想听展厅里的故事讲解，如何安排导览？","relevant_ids":["d05"]},
+    {"id":"q05","text":"怎么确保问答使用的旅游介绍有可靠出处？","relevant_ids":["d06"]},
+    {"id":"q06","text":"到了住宿地点办理入住需要准备什么证件？","relevant_ids":["d07"]},
+    {"id":"q07","text":"早饭不吃肉，应该提前找谁说明？","relevant_ids":["d08"]},
+    {"id":"q08","text":"天气恶劣时还能在山路上游玩吗？","relevant_ids":["d09"]},
+    {"id":"q09","text":"在路上捡到游客遗落的东西应交到哪里？","relevant_ids":["d10"]},
+    {"id":"q10","text":"电动汽车快没电了，到哪儿补充电量？","relevant_ids":["d11"]}
+  ]
+}
@@ -0,0 +1,8 @@
+# Optional Linux CPU evaluation environment; not required by FastAPI startup.
+# Install the CPU torch wheel from its official index first; see the evaluation guide.
+torch==2.14.1+cpu
+transformers==5.20.0
+huggingface-hub==2.2.0
+numpy==2.4.6
+tokenizers==0.23.3
+safetensors==0.8.0
@@ -0,0 +1,127 @@
+"""Lightweight contract/metric tests do not require model packages or network access."""
+
+import json
+from dataclasses import FrozenInstanceError, replace
+from pathlib import Path
+
+import pytest
+
+from app.services.embedding import (
+    SMALL_ZH,
+    EmbeddingError,
+    EmbeddingSpec,
+    LocalEmbeddingProvider,
+    validate_vectors,
+)
+from evaluation.local_embeddings import load_dataset, rank_and_measure
+
+
+def test_model_space_identity_is_immutable_and_includes_semantic_configuration():
+    assert SMALL_ZH.space_id == replace(SMALL_ZH).space_id
+    for change in [
+        {"revision": "a" * 40},
+        {"dimension": 768},
+        {"query_instruction": ""},
+        {"token_limit": 256},
+        {"implementation": "another-pooling"},
+    ]:
+        assert replace(SMALL_ZH, **change).space_id != SMALL_ZH.space_id
+    with pytest.raises(FrozenInstanceError):
+        SMALL_ZH.dimension = 768
+
+
+@pytest.mark.parametrize("revision", ["main", "v1.5", "a" * 39, None])
+def test_model_requires_full_revision(revision):
+    with pytest.raises(EmbeddingError):
+        EmbeddingSpec("example", revision, 2)
+
+
+@pytest.mark.parametrize(
+    "vectors,count",
+    [
+        ([[1.0, 0.0]], 2),
+        ([[1.0]], 1),
+        ([[0.0, 0.0]], 1),
+        ([[float("nan"), 0.0]], 1),
+        ([[float("inf"), 0.0]], 1),
+        ([["invalid", 0.0]], 1),
+        ([[True, 0.0]], 1),
+    ],
+)
+def test_invalid_vectors_cannot_pass_contract(vectors, count):
+    spec = EmbeddingSpec("example", "a" * 40, 2)
+    with pytest.raises(EmbeddingError):
+        validate_vectors(vectors, count, spec)
+    validate_vectors([[1.0, 0.0]], 1, spec)
+
+
+def test_token_overflow_is_rejected_before_any_model_execution():
+    provider = object.__new__(LocalEmbeddingProvider)
+    provider.spec = SMALL_ZH
+    provider.batch_size = 8
+    provider.torch = None
+
+    class Shape:
+        shape = (1, 513)
+
+    provider.tokenizer = lambda *args, **kwargs: {"input_ids": Shape()}
+    # There is intentionally no model: exceeding the budget must never execute one.
+    with pytest.raises(EmbeddingError, match="token"):
+        provider.encode(["占位文本"])
+
+
+def test_ranking_metrics_match_hand_calculation_and_ties_are_stable():
+    docs = [{"id": id_} for id_ in ["a", "b", "c"]]
+    queries = [{"id": "q1", "relevant_ids": ["b", "c"]}, {"id": "q2", "relevant_ids": ["c"]}]
+    result = rank_and_measure(docs, queries, [[1, 0], [0, 1], [-1, 0]], [[1, 1], [-1, 0]], top_k=2)
+    assert result["hit_at_1"] == 0.5
+    assert result["recall_at_k"] == 0.75
+    assert result["mrr_at_k"] == 0.75
+    assert [x["id"] for x in result["per_query"][0]["top_results"]] == ["a", "b"]
+
+
+def test_dataset_validation_rejects_unknown_labels_and_duplicate_ids(tmp_path):
+    data = {
+        "dataset_id": "placeholder",
+        "purpose": "synthetic_smoke_not_domain_benchmark",
+        "documents": [{"id": "d", "text": "占位"}],
+        "queries": [{"id": "q", "text": "测试", "relevant_ids": ["missing"]}],
+    }
+    path = tmp_path / "dataset.json"
+    path.write_text(json.dumps(data))
+    with pytest.raises(ValueError):
+        load_dataset(path)
+    data["queries"][0]["relevant_ids"] = ["d"]
+    data["documents"].append({"id": "d", "text": "重复"})
+    path.write_text(json.dumps(data))
+    with pytest.raises(ValueError):
+        load_dataset(path)
+
+
+def test_bundled_dataset_is_explicitly_synthetic_and_reproducible():
+    path = Path(__file__).resolve().parents[1] / "evaluation" / "synthetic_zh.json"
+    dataset, digest = load_dataset(path)
+    assert dataset["purpose"] == "synthetic_smoke_not_domain_benchmark"
+    assert all(row["text"].startswith("【合成测试】") for row in dataset["documents"])
+    assert load_dataset(path)[1] == digest
+
+
+@pytest.mark.parametrize(
+    "options", [{"batch_size": 0}, {"batch_size": 33}, {"threads": 5}, {"batch_size": 1.5}]
+)
+def test_invalid_resource_options_fail_without_loading_optional_dependencies(options):
+    with pytest.raises(EmbeddingError):
+        LocalEmbeddingProvider(SMALL_ZH, **options)
+
+
+@pytest.mark.parametrize("checkpoint", ["pickle", "automatic-fallback"])
+def test_unsupported_checkpoint_format_is_rejected(checkpoint):
+    with pytest.raises(EmbeddingError):
+        replace(SMALL_ZH, checkpoint_format=checkpoint)
+
+
+@pytest.mark.parametrize("texts", [None, "不能逐字编码的字符串", ("占位",), [None], ["   "]])
+def test_input_shape_and_text_are_validated_before_encoding(texts):
+    provider = object.__new__(LocalEmbeddingProvider)
+    with pytest.raises(EmbeddingError):
+        provider.encode(texts)
@@ -0,0 +1,77 @@
+# T07：本地 Embedding 与 CPU 资源评估
+
+本阶段从 `dev` 独立开发，交付可替换 Provider 和可复现的本地 CPU 评估工具。它不依赖待审 PR #6/#7/#8 的接口或数据库迁移；后续向量检索集成再组合这些基础。普通后端启动及普通自动化测试均不导入 PyTorch、不加载模型，也不要求模型下载。
+
+## 固定配置与边界
+
+| CLI 名称 | 官方模型 | 固定 revision | 维度 | Token 上限 | 权重格式 |
+|---|---|---|---:|---:|---|
+| small | BAAI/bge-small-zh-v1.5 | `7999e1d3359715c523056ef9478215996d62a620` | 512 | 512 | Safetensors |
+| base | BAAI/bge-base-zh-v1.5 | `f03589ceff5aac7111bd60cfc7d497ca17ecac65` | 768 | 512 | PyTorch，仅权重读取 |
+
+两者使用 CLS 输出、L2 归一化、CPU FP32；查询添加官方检索指令，正文不添加该指令。`EmbeddingSpec` 冻结模型/revision/维度/token 上限/查询指令/实现版本/权重格式，由规范 JSON 生成 `space_id`。不同空间必须分开保存和查询；目前只提供身份契约，数据库隔离约束尚待向量阶段实现。
+
+模型仓库均标注 MIT。基础版该修订只提供 `pytorch_model.bin`，显式使用 Transformers 的 `weights_only=True` 读取；不会自动回退未知格式或开启远程模型代码。公开模型下载使用 `token=False`，不要求或读取用户保存的 Hugging Face 凭据。默认只用本机缓存，CLI `--download` 才允许下载固定修订。
+
+`LocalEmbeddingProvider` 按批处理，默认 batch=8、CPU 线程=2；显式限制 batch 1–32、线程 1–4。输入先按真实 tokenizer 计数，包含查询指令和特殊 token；超过模型上限明确报错，不静默截断。输出检查数量、维度、有限数值和 L2 范数。字符切分长度不等于 token 数，后续接入 chunks 时须据此重新检查。
+
+## 输入与指标
+
+项目当前资料台账只有未核验的占位行，没有可用于正式选型的领域数据。本阶段自带 `backend/evaluation/synthetic_zh.json`，含 12 条明确标注“合成测试”的示例文本及 10 个问题，ID/相关文档标注在运行前固定。这些句子不描述那拉提真实事实，不写入应用数据库。
+
+评估保存输入文件 SHA-256、模型完整配置与空间 ID、运行库版本、实际命中 ID/分数、命中率、Recall@3 和 MRR@3。排序采用归一化向量内积，同分按文档 ID；数值不是正确率或置信度。命中率检查首位相关文档；Recall@k 为前 k 命中相关文档数/标注相关文档总数；MRR@k 为首个相关命中的倒数，未命中为 0。
+
+每项计时先按同一输入形状预热一次，然后记录 5 次及中位数：12 条语料编码、10 个查询批量编码、单个查询、8 条长文本批量编码。长文本均为相同的合成句子重复形成，360 字符/362 token，仅测试资源。包含分词、补齐、前向、归一化和结果转换；不包含网络、数据库、HTTP 或整个 RAG 延迟。初始化时间单独记录，可能包含缓存检查/下载与首次加载，不用于模型推理速度比较。Linux 峰值 RSS 是整个独立进程高水位，包含解释器、运行库和模型，不能当作仅权重大小。
+
+## 本机实测
+
+运行环境：Linux、Python 3.11.17、4 个逻辑 CPU、内存总计约 9.7 GiB，检查时可用约 5.0 GiB；GPU 驱动不可用，本轮仅用 CPU。PyTorch `2.14.1+cpu`、Transformers `5.20.0`、huggingface-hub `2.2.0`、NumPy `2.4.6`；每个模型用独立进程测量，线程 2、batch 8、每项 5 次。未记录用户名、主机名或本机个人路径。
+
+输入文件 SHA-256：`9f4762a10dc1502b871c84ccedf9fbbdbd93067b8318476a77c78aa14789b243`。
+
+| 模型 | 12 条正文批编码中位数 | 10 个查询批编码中位数 | 单查询中位数 | 8 条长文批编码中位数 | 峰值进程 RSS | 合成 Hit@1 / Recall@3 / MRR@3 |
+|---|---:|---:|---:|---:|---:|---|
+| small | 98.708 ms | 72.877 ms | 8.927 ms | 457.944 ms | 531.2 MiB | 0.90 / 0.90 / 0.90 |
+| base | 582.315 ms | 416.287 ms | 52.912 ms | 3016.157 ms | 865.5 MiB | 1.00 / 1.00 / 1.00 |
+
+两个模型在该负载下都能使用本机 CPU。small 在合成问题 q05（出处可靠性）把住宿条目 d07 排在首位，目标资料审核条目 d06 未进入前三；base 将 d06 排首位。这里保留失败结果，不修改数据以提高分数。两个模型的相似度分布不同，不能共用固定分数阈值。
+
+建议：资源优先的后端开发验证可先使用 small，保留 base 作对照。正式默认模型暂不冻结，先在已核验语料与独立问题集上比较相关性、无依据情形和实际端到端延迟。当前未测 GPU、并发服务、最大 token 负载或完整索引构建。
+
+这些是合成小样本的功能与资源结果，不是项目正式实验分数。不能据此断言真实文旅问答质量或生产尾延迟。应先用资料负责人核验后的语料及独立问题集复测，再固定默认模型；不能通过修改测试题迎合已观察的排名。
+
+## 复现（Linux，仓库根目录）
+
+可使用 `uv` 创建独立 Python 3.11 环境；或以等价 venv/pip 命令安装。依赖独立于 `requirements.txt`，不把 GPU 依赖或模型加入默认 API 启动流程。
+
+```bash
+task_eval_root=$(git rev-parse --absolute-git-dir)
+test -x "$task_eval_root/embedding-eval-venv/bin/python" || \
+  uv venv --python 3.11 "$task_eval_root/embedding-eval-venv"
+uv pip install --python "$task_eval_root/embedding-eval-venv/bin/python" \
+  --index-url https://download.pytorch.org/whl/cpu torch==2.14.1+cpu
+uv pip install --python "$task_eval_root/embedding-eval-venv/bin/python" \
+  -r backend/requirements-embedding-cpu.txt
+export HF_HUB_DISABLE_TELEMETRY=1 HF_HUB_DISABLE_IMPLICIT_TOKEN=1
+export HF_HUB_DISABLE_PROGRESS_BARS=1 TOKENIZERS_PARALLELISM=false
+export HF_HUB_DISABLE_XET=1
+(
+  cd backend
+  "$task_eval_root/embedding-eval-venv/bin/python" -m evaluation.local_embeddings \
+    --model small --download --cache-dir "$task_eval_root/embedding-model-cache" \
+    --output "$task_eval_root/embedding-evaluation/small.json"
+  "$task_eval_root/embedding-eval-venv/bin/python" -m evaluation.local_embeddings \
+    --model base --download --cache-dir "$task_eval_root/embedding-model-cache" \
+    --output "$task_eval_root/embedding-evaluation/base.json"
+)
+```
+
+后续离线复跑去掉 `--download`。可用 `--dataset` 指定同结构数据，文档/问题各最多 1000 条、文件最多 2MB；拒绝重复 ID、未知或重复的相关文档标注。报告不含输入文本，但实际问题/标签可能仍敏感，原始报告、运行输出、环境及模型缓存都应只留在本机 Git 元数据目录，不提交。参数错误或模型失败不会生成伪造的成功结果。
+
+普通后端验证仍是在 `backend/` 执行 `ruff check .`、`ruff format --check .`、`pytest -q`。轻量测试验证模型身份、非法输出、资源参数、token 拒绝、输入标注及手算排序指标，不需要模型包或联网；真实模型执行由上述独立 CLI 验证。
+
+## 下一步
+
+将 Provider 接入已发布的文本版本，完成 pgvector 存储与严格空间隔离、向量检索及 RRF，再使用真实审核语料评测。模型编码耗时应在发布锁外运行，发布前重新核对目录/语料版本；过期或失败时保留旧版本。当前没有新增向量数据库字段、检索 API 或论文结论。
+
+依据：[BGE small 官方模型说明](https://huggingface.co/BAAI/bge-small-zh-v1.5)、[BGE base 官方文件](https://huggingface.co/BAAI/bge-base-zh-v1.5/tree/f03589ceff5aac7111bd60cfc7d497ca17ecac65)、[Transformers 模型加载参数](https://huggingface.co/docs/transformers/main_classes/model#transformers.PreTrainedModel.from_pretrained)、[PyTorch CPU 安装](https://pytorch.org/get-started/locally/)、[Hugging Face 固定修订下载](https://huggingface.co/docs/huggingface_hub/guides/download)。
				`@@ -0,0 +1 @@`
				`"""Offline, synthetic local-model evaluation; never writes the application database."""`