feat: evaluate pinned local embedding models on CPU

backend/tests/test_embedding.py

@@ -0,0 +1,120 @@
+"""Lightweight contract/metric tests do not require model packages or network access."""
+
+import json
+from dataclasses import FrozenInstanceError, replace
+from pathlib import Path
+
+import pytest
+
+from app.services.embedding import (
+    SMALL_ZH,
+    EmbeddingError,
+    EmbeddingSpec,
+    LocalEmbeddingProvider,
+    validate_vectors,
+)
+from evaluation.local_embeddings import load_dataset, rank_and_measure
+
+
+def test_model_space_identity_is_immutable_and_includes_semantic_configuration():
+    assert SMALL_ZH.space_id == replace(SMALL_ZH).space_id
+    for change in [
+        {"revision": "a" * 40},
+        {"dimension": 768},
+        {"query_instruction": ""},
+        {"token_limit": 256},
+        {"implementation": "another-pooling"},
+    ]:
+        assert replace(SMALL_ZH, **change).space_id != SMALL_ZH.space_id
+    with pytest.raises(FrozenInstanceError):
+        SMALL_ZH.dimension = 768
+
+
+@pytest.mark.parametrize("revision", ["main", "v1.5", "a" * 39, None])
+def test_model_requires_full_revision(revision):
+    with pytest.raises(EmbeddingError):
+        EmbeddingSpec("example", revision, 2)
+
+
+@pytest.mark.parametrize(
+    "vectors,count",
+    [
+        ([[1.0, 0.0]], 2),
+        ([[1.0]], 1),
+        ([[0.0, 0.0]], 1),
+        ([[float("nan"), 0.0]], 1),
+        ([[float("inf"), 0.0]], 1),
+        ([["invalid", 0.0]], 1),
+        ([[True, 0.0]], 1),
+    ],
+)
+def test_invalid_vectors_cannot_pass_contract(vectors, count):
+    spec = EmbeddingSpec("example", "a" * 40, 2)
+    with pytest.raises(EmbeddingError):
+        validate_vectors(vectors, count, spec)
+    validate_vectors([[1.0, 0.0]], 1, spec)
+
+
+def test_token_overflow_is_rejected_before_any_model_execution():
+    provider = object.__new__(LocalEmbeddingProvider)
+    provider.spec = SMALL_ZH
+    provider.batch_size = 8
+    provider.torch = None
+
+    class Shape:
+        shape = (1, 513)
+
+    provider.tokenizer = lambda *args, **kwargs: {"input_ids": Shape()}
+    # There is intentionally no model: exceeding the budget must never execute one.
+    with pytest.raises(EmbeddingError, match="token"):
+        provider.encode(["占位文本"])
+
+
+def test_ranking_metrics_match_hand_calculation_and_ties_are_stable():
+    docs = [{"id": id_} for id_ in ["a", "b", "c"]]
+    queries = [{"id": "q1", "relevant_ids": ["b", "c"]}, {"id": "q2", "relevant_ids": ["c"]}]
+    result = rank_and_measure(docs, queries, [[1, 0], [0, 1], [-1, 0]], [[1, 1], [-1, 0]], top_k=2)
+    assert result["hit_at_1"] == 0.5
+    assert result["recall_at_k"] == 0.75
+    assert result["mrr_at_k"] == 0.75
+    assert [x["id"] for x in result["per_query"][0]["top_results"]] == ["a", "b"]
+
+
+def test_dataset_validation_rejects_unknown_labels_and_duplicate_ids(tmp_path):
+    data = {
+        "dataset_id": "placeholder",
+        "purpose": "synthetic_smoke_not_domain_benchmark",
+        "documents": [{"id": "d", "text": "占位"}],
+        "queries": [{"id": "q", "text": "测试", "relevant_ids": ["missing"]}],
+    }
+    path = tmp_path / "dataset.json"
+    path.write_text(json.dumps(data))
+    with pytest.raises(ValueError):
+        load_dataset(path)
+    data["queries"][0]["relevant_ids"] = ["d"]
+    data["documents"].append({"id": "d", "text": "重复"})
+    path.write_text(json.dumps(data))
+    with pytest.raises(ValueError):
+        load_dataset(path)
+
+
+def test_bundled_dataset_is_explicitly_synthetic_and_reproducible():
+    path = Path(__file__).resolve().parents[1] / "evaluation" / "synthetic_zh.json"
+    dataset, digest = load_dataset(path)
+    assert dataset["purpose"] == "synthetic_smoke_not_domain_benchmark"
+    assert all(row["text"].startswith("【合成测试】") for row in dataset["documents"])
+    assert load_dataset(path)[1] == digest
+
+
+@pytest.mark.parametrize(
+    "options", [{"batch_size": 0}, {"batch_size": 33}, {"threads": 5}, {"batch_size": 1.5}]
+)
+def test_invalid_resource_options_fail_without_loading_optional_dependencies(options):
+    with pytest.raises(EmbeddingError):
+        LocalEmbeddingProvider(SMALL_ZH, **options)
+
+
+@pytest.mark.parametrize("checkpoint", ["pickle", "automatic-fallback"])
+def test_unsupported_checkpoint_format_is_rejected(checkpoint):
+    with pytest.raises(EmbeddingError):
+        replace(SMALL_ZH, checkpoint_format=checkpoint)