"""Lightweight contract/metric tests do not require model packages or network access.""" import json from dataclasses import FrozenInstanceError, replace from pathlib import Path import pytest from app.services.embedding import ( SMALL_ZH, EmbeddingError, EmbeddingSpec, LocalEmbeddingProvider, validate_vectors, ) from evaluation.local_embeddings import load_dataset, rank_and_measure def test_model_space_identity_is_immutable_and_includes_semantic_configuration(): assert SMALL_ZH.space_id == replace(SMALL_ZH).space_id for change in [ {"revision": "a" * 40}, {"dimension": 768}, {"query_instruction": ""}, {"token_limit": 256}, {"implementation": "another-pooling"}, ]: assert replace(SMALL_ZH, **change).space_id != SMALL_ZH.space_id with pytest.raises(FrozenInstanceError): SMALL_ZH.dimension = 768 @pytest.mark.parametrize("revision", ["main", "v1.5", "a" * 39, None]) def test_model_requires_full_revision(revision): with pytest.raises(EmbeddingError): EmbeddingSpec("example", revision, 2) @pytest.mark.parametrize( "vectors,count", [ ([[1.0, 0.0]], 2), ([[1.0]], 1), ([[0.0, 0.0]], 1), ([[float("nan"), 0.0]], 1), ([[float("inf"), 0.0]], 1), ([["invalid", 0.0]], 1), ([[True, 0.0]], 1), ], ) def test_invalid_vectors_cannot_pass_contract(vectors, count): spec = EmbeddingSpec("example", "a" * 40, 2) with pytest.raises(EmbeddingError): validate_vectors(vectors, count, spec) validate_vectors([[1.0, 0.0]], 1, spec) def test_token_overflow_is_rejected_before_any_model_execution(): provider = object.__new__(LocalEmbeddingProvider) provider.spec = SMALL_ZH provider.batch_size = 8 provider.torch = None class Shape: shape = (1, 513) provider.tokenizer = lambda *args, **kwargs: {"input_ids": Shape()} # There is intentionally no model: exceeding the budget must never execute one. with pytest.raises(EmbeddingError, match="token"): provider.encode(["占位文本"]) def test_ranking_metrics_match_hand_calculation_and_ties_are_stable(): docs = [{"id": id_} for id_ in ["a", "b", "c"]] queries = [{"id": "q1", "relevant_ids": ["b", "c"]}, {"id": "q2", "relevant_ids": ["c"]}] result = rank_and_measure(docs, queries, [[1, 0], [0, 1], [-1, 0]], [[1, 1], [-1, 0]], top_k=2) assert result["hit_at_1"] == 0.5 assert result["recall_at_k"] == 0.75 assert result["mrr_at_k"] == 0.75 assert [x["id"] for x in result["per_query"][0]["top_results"]] == ["a", "b"] def test_dataset_validation_rejects_unknown_labels_and_duplicate_ids(tmp_path): data = { "dataset_id": "placeholder", "purpose": "synthetic_smoke_not_domain_benchmark", "documents": [{"id": "d", "text": "占位"}], "queries": [{"id": "q", "text": "测试", "relevant_ids": ["missing"]}], } path = tmp_path / "dataset.json" path.write_text(json.dumps(data)) with pytest.raises(ValueError): load_dataset(path) data["queries"][0]["relevant_ids"] = ["d"] data["documents"].append({"id": "d", "text": "重复"}) path.write_text(json.dumps(data)) with pytest.raises(ValueError): load_dataset(path) def test_bundled_dataset_is_explicitly_synthetic_and_reproducible(): path = Path(__file__).resolve().parents[1] / "evaluation" / "synthetic_zh.json" dataset, digest = load_dataset(path) assert dataset["purpose"] == "synthetic_smoke_not_domain_benchmark" assert all(row["text"].startswith("【合成测试】") for row in dataset["documents"]) assert load_dataset(path)[1] == digest @pytest.mark.parametrize( "options", [{"batch_size": 0}, {"batch_size": 33}, {"threads": 5}, {"batch_size": 1.5}] ) def test_invalid_resource_options_fail_without_loading_optional_dependencies(options): with pytest.raises(EmbeddingError): LocalEmbeddingProvider(SMALL_ZH, **options) @pytest.mark.parametrize("checkpoint", ["pickle", "automatic-fallback"]) def test_unsupported_checkpoint_format_is_rejected(checkpoint): with pytest.raises(EmbeddingError): replace(SMALL_ZH, checkpoint_format=checkpoint)