diff --git a/backend/app/database/migrations.py b/backend/app/database/migrations.py index c41f8e0..5fd1bd8 100644 --- a/backend/app/database/migrations.py +++ b/backend/app/database/migrations.py @@ -127,6 +127,10 @@ MIGRATIONS: list[str] = [ query TEXT NOT NULL UNIQUE ); """, + # v6: persist each block's embedding policy for partitioned retrieval. + """ + ALTER TABLE blocks ADD COLUMN embedding_local_only INTEGER NOT NULL DEFAULT 0; + """, ] diff --git a/backend/app/retrieval/routed_vectors.py b/backend/app/retrieval/routed_vectors.py index 666526c..f259d3a 100644 --- a/backend/app/retrieval/routed_vectors.py +++ b/backend/app/retrieval/routed_vectors.py @@ -22,6 +22,7 @@ from app.database.db import connect, transaction from app.errors import ApiError from app.retrieval.vectorstore import VectorHit from app.retrieval.provenance import record_embedding +from app.retrieval.hybrid import rrf_fuse logger = logging.getLogger(__name__) @@ -167,9 +168,18 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa Read coverage and vectors together so concurrent note updates cannot produce an apparently complete subset. Never fill missing remote hits with local hits. """ + if accept_local: + conn = connect() + try: + policies = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")} + finally: + conn.close() + if True in policies: + return await _search_partitioned(query, policies, top_k=top_k, strict=strict) batch = await embed_remote([query], accept_local=accept_local, strict=strict) if batch is None: return None + record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions}) try: conn = connect() @@ -221,3 +231,55 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa "Embedding 已可用，但当前模型的向量索引缺失、不完整或已失效。请在「设置 → 索引与模型」中重建全部索引。", {"model_id": batch.space_id, "dimensions": batch.dimensions, "source": batch.source}) from exc return None + + +async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, strict: bool): + """Embed per policy; rank each space independently and fuse ranks, not vectors.""" + batches = {} + for policy in sorted(policies): + batch = await embed_remote([query], accept_local=True, strict=strict, local_only=policy) + if batch is None: + return None + batches[policy] = batch + conn = connect() + try: + with transaction(conn): + # Query vectors are ready before opening the single read snapshot. + current = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")} + if current != policies: + raise ValueError("embedding policies changed while querying") + ranked = [] + for policy, batch in batches.items(): + rows = conn.execute( + "SELECT b.block_id,r.vector FROM blocks b LEFT JOIN routed_block_vectors r " + "ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? " + "WHERE b.embedding_local_only=? ORDER BY b.block_id", + (batch.space_id, batch.dimensions, int(policy)), + ) + def hits(): + for row in rows: + if row['vector'] is None: + raise ValueError("incomplete policy coverage") + vector = _unit_vector(json.loads(row['vector']), batch.dimensions) + score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector)) + yield VectorHit(id=row['block_id'], score=max(0.0, min(1.0, score))) + try: + ranked.append(heapq.nlargest(top_k, hits(), key=lambda hit: hit.score)) + finally: + rows.close() + spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions, + "local_only": policy} for policy, b in batches.items()] + record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source, + spaces=spaces, fallback_reason=None) + if len(ranked) == 1: + return ranked[0] + fused = rrf_fuse([[hit.id for hit in group] for group in ranked]) + return [VectorHit(id=key, score=score) for key, score in + sorted(fused.items(), key=lambda item: (-item[1], item[0]))[:top_k]] + except Exception as exc: + record_embedding(source="unavailable", fallback_reason="REMOTE_INDEX_UNAVAILABLE") + if strict: + raise ApiError(409, "SEMANTIC_INDEX_UNAVAILABLE", "部分索引分区缺失或已失效，请重建全部索引。") from exc + return None + finally: + conn.close() diff --git a/backend/app/services/index_service.py b/backend/app/services/index_service.py index 7291cd2..4c72c92 100644 --- a/backend/app/services/index_service.py +++ b/backend/app/services/index_service.py @@ -85,7 +85,7 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob: )) try: prepared_notes = [] - semantic_space = None + semantic_spaces = {} for rel, folder, markdown, created, updated in docs: parsed = parse_note( markdown=markdown, file_path=rel, folder=folder, tags=None, @@ -97,9 +97,10 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob: if batch is None: raise ApiError(503, "EMBEDDING_UNAVAILABLE", "Embedding 未生成向量，重建已停止，原索引已保留。") space = (batch.space_id, batch.dimensions) - if semantic_space is not None and semantic_space != space: + policy = parsed.embedding_local_only + if policy in semantic_spaces and semantic_spaces[policy] != space: raise ApiError(409, "EMBEDDING_SPACE_CHANGED", "重建期间 Embedding 模型发生切换，原索引已保留，请待模型服务稳定后重试。") - semantic_space = space + semantic_spaces[policy] = space prepared_notes.append((parsed, prepared)) # All network/model awaits precede the transaction. The concrete SQLite # methods below complete synchronously despite their async interfaces. @@ -114,12 +115,12 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob: await vector_store.clear(conn=conn) for parsed, prepared in prepared_notes: await index_note(parsed, prepared=prepared, conn=conn) - if semantic_space is not None: + for policy, space in semantic_spaces.items(): exists = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='routed_block_vectors'").fetchone() missing = not exists or conn.execute( "SELECT 1 FROM blocks b LEFT JOIN routed_block_vectors r " "ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? " - "WHERE r.block_id IS NULL LIMIT 1", semantic_space, + "WHERE b.embedding_local_only=? AND r.block_id IS NULL LIMIT 1", (*space, int(policy)), ).fetchone() if missing: raise ApiError(500, "SEMANTIC_INDEX_WRITE_FAILED", "向量索引写入失败，原索引已保留，请检查数据库和磁盘状态。") diff --git a/backend/app/services/note_service.py b/backend/app/services/note_service.py index 3c7da5c..a4f4eeb 100644 --- a/backend/app/services/note_service.py +++ b/backend/app/services/note_service.py @@ -118,6 +118,8 @@ async def index_note( blocks=parsed.blocks, ) old_ids = set(old_block_ids) + conn.execute("UPDATE blocks SET embedding_local_only=? WHERE note_id=?", + (int(parsed.embedding_local_only), parsed.note_id)) new_ids = {block.block_id for block in parsed.blocks} stale_ids = [bid for bid in old_ids if bid not in new_ids] if stale_ids: diff --git a/backend/tests/test_routed_retrieval.py b/backend/tests/test_routed_retrieval.py index 06e66c6..d830c7e 100644 --- a/backend/tests/test_routed_retrieval.py +++ b/backend/tests/test_routed_retrieval.py @@ -537,3 +537,85 @@ def test_real_embedding_rebuild_failure_preserves_index(runtime, production_engi def test_empty_vault_vector_search_returns_empty(runtime, production_engine): assert asyncio.run(production_engine.search(request())).items == [] + + +@pytest.fixture +def policy_runtime(monkeypatch): + class PolicyRuntime: + fallback = False + calls = [] + async def embed(self, texts, *, local_only=False): + self.calls.append((list(texts), local_only)) + local = local_only or self.fallback + dim = 3 if local else 2 + return SimpleNamespace(source='local' if local else 'api', model_id='local-space' if local else 'api-space', + dimensions=dim, vectors=[[1.0] + [0.0] * (dim - 1) for _ in texts], + fallback_reason='PROVIDER_TIMEOUT' if self.fallback and not local_only else None) + runtime = PolicyRuntime() + monkeypatch.setattr(routed_vectors, 'get_model_routing', lambda: runtime) + return runtime + + +async def seed_policies(): + normal = await note_service.create_note(title='Normal', markdown='apple public', folder=None, tags=[]) + private = await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[]) + return normal, private + + +@pytest.mark.parametrize('fallback', [False, True]) +def test_mixed_policy_rebuild_and_retrieval(policy_runtime, production_engine, fallback): + policy_runtime.fallback = fallback + async def scenario(): + notes = await seed_policies() + await index_service.rebuild(IndexRebuildRequest()) + for mode in (SearchMode.vector, SearchMode.hybrid): + result = await production_engine.search(SearchRequest(query='apple', mode=mode)) + assert {item.note_id for item in result.items} == {note.note_id for note in notes} + for texts, local_only in policy_runtime.calls: + if any('private' in text for text in texts): + assert local_only + if not fallback: + assert {r[0] for r in rows('SELECT DISTINCT space_id FROM routed_block_vectors')} == {'api-space', 'local-space'} + asyncio.run(scenario()) + + +def test_local_only_vault_never_requests_api_for_search(policy_runtime, production_engine): + async def scenario(): + await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[]) + await index_service.rebuild(IndexRebuildRequest()) + assert (await production_engine.search(request())).items + assert all(local_only for _, local_only in policy_runtime.calls) + asyncio.run(scenario()) + + +def test_partition_storage_failure_rolls_back_all_partitions(policy_runtime, production_engine, monkeypatch): + from app.errors import ApiError + async def scenario(): + await seed_policies() + before = [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')] + original = routed_vectors.store_remote + def fail_local(conn, ids, batch): + if batch.source != 'local': + original(conn, ids, batch) + monkeypatch.setattr(routed_vectors, 'store_remote', fail_local) + with pytest.raises(ApiError) as error: + await index_service.rebuild(IndexRebuildRequest()) + assert error.value.code == 'SEMANTIC_INDEX_WRITE_FAILED' + assert [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')] == before + asyncio.run(scenario()) + + +def test_missing_partition_does_not_silently_return_partial_hits(policy_runtime, production_engine): + from app.errors import ApiError + async def scenario(): + await seed_policies() + conn = connect() + try: + conn.execute("DELETE FROM routed_block_vectors WHERE space_id='local-space'") + finally: + conn.close() + with pytest.raises(ApiError) as error: + await production_engine.search(request()) + assert error.value.code == 'SEMANTIC_INDEX_UNAVAILABLE' + assert (await production_engine.search(request(SearchMode.hybrid))).items + asyncio.run(scenario()) diff --git a/docs/development/多模态管线与模型运行开发说明.md b/docs/development/多模态管线与模型运行开发说明.md index 5d2a832..2af4461 100644 --- a/docs/development/多模态管线与模型运行开发说明.md +++ b/docs/development/多模态管线与模型运行开发说明.md @@ -97,7 +97,7 @@ POST /api/providers/request-preview 不联网，隐藏正文/文件且不包含 Windows 热重载不支持异步子进程时使用线程管道兼容路径。本地 Embedding 进入推理前冻结模型与设备配置，向量空间标识来自同一快照；普通 API 成功及失败回退策略保持不变。 -仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter，索引与后续重建跳过远程 Embedding。它只约束索引，不是通用的笔记联网权限；旧导出笔记需人工补标记。本地和 API 空间不混用，无法完整覆盖时保留全文检索能力。 +仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter，索引与后续重建跳过远程 Embedding。它只约束索引，不是通用的笔记联网权限；旧导出笔记需人工补标记。SQLite v6 将策略保存到 Block，重建按普通/仅本地策略分别校验空间和覆盖，统一事务提交。查询在各空间内排序后用 RRF 合并排名，缺少分区时保留全文检索降级。升级已有库后重建一次以同步策略。 搜索记录保存在应用 SQLite，使用 `/api/search/history` GET/DELETE 读取和清空。聊天已接入真实知识库上下文与 Citation。详细原因和验证见[阶段 F 问题与解决方案](../retrospectives/阶段F-Embedding与知识库问题与解决方案.md)。 diff --git a/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md b/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md index 55f0759..9a7e732 100644 --- a/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md +++ b/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md @@ -23,6 +23,7 @@ | F-05 | 聊天忽略 `use_rag` | 没有知识库内容 | 真实 Block 上下文与来源事件 | | F-06 | 本地转写导出未传递限制 | 正文可能发送给远程 Embedding | 持久化本地索引标记 | | F-07 | 推理结束才读取当前模型标识 | 向量与空间错配 | 冻结模型、revision 和设备配置 | +| F-08 | 将不同处理策略误判为配置漂移 | 普通与仅本地笔记共存时不能重建 | 按策略校验覆盖，独立检索并融合排名 | ## 3. F-01 / F-03：模型可用不等于索引可用 @@ -108,11 +109,18 @@ embedding_local_only: true | 本地限定索引，存在 API | 不请求 API、不解析远程凭据 | | 本地限定后再发普通请求 | API 仍可调用，不泄漏临时限制 | | 推理期间修改设置 | 当前向量与身份一致，下一请求采用新设置 | -| 空间混用或索引覆盖不完整 | 严格重建拒绝提交，不混合不同向量空间 | +| 普通与仅本地笔记共存 | 分区重建，各自空间内检索，再融合排名 | +| 同一策略内空间漂移或覆盖不完整 | 严格重建拒绝提交，整体回滚 | -本地限定笔记与远程索引并存时，现有完整覆盖检查可能使纯向量查询提示不完整、重建拒绝混合空间；混合检索仍可退到全文检索。不能为获得完整远程索引而绕过本地标记。 +### F-08：混合处理策略的重建与检索 -提交审阅时已用隔离数据库复现：一篇普通 API 笔记与一篇本地限定笔记共存，配置未变化，全量重建仍返回 `EMBEDDING_SPACE_CHANGED`。此项列为合并阻碍，尚未修复；后续需区分预期的处理策略差异和实际配置漂移，并明确各向量空间的索引覆盖范围。 +提交审阅时用隔离数据库复现：一篇普通 API 笔记与一篇本地限定笔记共存，配置未变化，全量重建仍返回 `EMBEDDING_SPACE_CHANGED`。原因是重建将全部笔记约束到一个空间，查询也要求单个空间覆盖全部 Block，未区分处理策略。 + +本轮追加 SQLite v6，为 Block 保存 `embedding_local_only` 策略。重建分别检查普通和仅本地策略的空间一致性及完整覆盖，仍在同一事务提交；同一策略内模型改变、缺失向量或存储失败仍整体回滚。正常 API 回退不受跨策略差异影响。 + +查询按策略生成对应查询向量，在同一个数据库快照中检查两个分区。各分区独立计算相似度，再用 RRF 融合排名，不直接比较不同模型的向量或余弦分数。仅含本地限定笔记时，查询也不请求远程 Embedding；分区失效时纯向量明确报错，混合查询仍可退到全文。 + +已有数据库升级后应重建一次索引，将 Vault 中的策略标记同步至 Block。新增和更新笔记自动同步。回归覆盖混合策略、全部本地回退、仅本地查询、单分区缺失和跨分区写入失败回滚；此项合并阻碍已修复。 本轮在 `backend/` 执行： @@ -126,4 +134,6 @@ embedding_local_only: true ## 9. 工程经验 +F-08 修复后完整后端回归：495 项通过；新增 5 个用例覆盖混合策略重建与查询、全部本地回退、仅本地查询不访问 API、跨分区回滚和不完整分区。现有同策略空间漂移拒绝用例仍通过。 + 区分配置、权重安装、推理运行、索引覆盖四种状态；按用户实际启动方式验证；跨异步边界冻结身份；持久化处理限制；增加限制时也验证普通 API 回退没有被破坏。