fix(retrieval): 按索引策略重建并融合跨空间检索

backend/app/database/migrations.py

@@ -127,6 +127,10 @@ MIGRATIONS: list[str] = [
         query TEXT NOT NULL UNIQUE
     );
     """,
+    # v6: persist each block's embedding policy for partitioned retrieval.
+    """
+    ALTER TABLE blocks ADD COLUMN embedding_local_only INTEGER NOT NULL DEFAULT 0;
+    """,
 ]
 
 

backend/app/retrieval/routed_vectors.py

@@ -22,6 +22,7 @@ from app.database.db import connect, transaction
 from app.errors import ApiError
 from app.retrieval.vectorstore import VectorHit
 from app.retrieval.provenance import record_embedding
+from app.retrieval.hybrid import rrf_fuse
 
 logger = logging.getLogger(__name__)
 
@@ -167,9 +168,18 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
     Read coverage and vectors together so concurrent note updates cannot produce
     an apparently complete subset. Never fill missing remote hits with local hits.
     """
+    if accept_local:
+        conn = connect()
+        try:
+            policies = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")}
+        finally:
+            conn.close()
+        if True in policies:
+            return await _search_partitioned(query, policies, top_k=top_k, strict=strict)
     batch = await embed_remote([query], accept_local=accept_local, strict=strict)
     if batch is None:
         return None
+
     record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions})
     try:
         conn = connect()
@@ -221,3 +231,55 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
                            "Embedding 已可用，但当前模型的向量索引缺失、不完整或已失效。请在「设置 → 索引与模型」中重建全部索引。",
                            {"model_id": batch.space_id, "dimensions": batch.dimensions, "source": batch.source}) from exc
         return None
+
+
+async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, strict: bool):
+    """Embed per policy; rank each space independently and fuse ranks, not vectors."""
+    batches = {}
+    for policy in sorted(policies):
+        batch = await embed_remote([query], accept_local=True, strict=strict, local_only=policy)
+        if batch is None:
+            return None
+        batches[policy] = batch
+    conn = connect()
+    try:
+        with transaction(conn):
+            # Query vectors are ready before opening the single read snapshot.
+            current = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")}
+            if current != policies:
+                raise ValueError("embedding policies changed while querying")
+            ranked = []
+            for policy, batch in batches.items():
+                rows = conn.execute(
+                    "SELECT b.block_id,r.vector FROM blocks b LEFT JOIN routed_block_vectors r "
+                    "ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
+                    "WHERE b.embedding_local_only=? ORDER BY b.block_id",
+                    (batch.space_id, batch.dimensions, int(policy)),
+                )
+                def hits():
+                    for row in rows:
+                        if row['vector'] is None:
+                            raise ValueError("incomplete policy coverage")
+                        vector = _unit_vector(json.loads(row['vector']), batch.dimensions)
+                        score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
+                        yield VectorHit(id=row['block_id'], score=max(0.0, min(1.0, score)))
+                try:
+                    ranked.append(heapq.nlargest(top_k, hits(), key=lambda hit: hit.score))
+                finally:
+                    rows.close()
+            spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions,
+                       "local_only": policy} for policy, b in batches.items()]
+            record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source,
+                             spaces=spaces, fallback_reason=None)
+            if len(ranked) == 1:
+                return ranked[0]
+            fused = rrf_fuse([[hit.id for hit in group] for group in ranked])
+            return [VectorHit(id=key, score=score) for key, score in
+                    sorted(fused.items(), key=lambda item: (-item[1], item[0]))[:top_k]]
+    except Exception as exc:
+        record_embedding(source="unavailable", fallback_reason="REMOTE_INDEX_UNAVAILABLE")
+        if strict:
+            raise ApiError(409, "SEMANTIC_INDEX_UNAVAILABLE", "部分索引分区缺失或已失效，请重建全部索引。") from exc
+        return None
+    finally:
+        conn.close()

backend/app/services/index_service.py

@@ -85,7 +85,7 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
     ))
     try:
         prepared_notes = []
-        semantic_space = None
+        semantic_spaces = {}
         for rel, folder, markdown, created, updated in docs:
             parsed = parse_note(
                 markdown=markdown, file_path=rel, folder=folder, tags=None,
@@ -97,9 +97,10 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
                 if batch is None:
                     raise ApiError(503, "EMBEDDING_UNAVAILABLE", "Embedding 未生成向量，重建已停止，原索引已保留。")
                 space = (batch.space_id, batch.dimensions)
-                if semantic_space is not None and semantic_space != space:
+                policy = parsed.embedding_local_only
+                if policy in semantic_spaces and semantic_spaces[policy] != space:
                     raise ApiError(409, "EMBEDDING_SPACE_CHANGED", "重建期间 Embedding 模型发生切换，原索引已保留，请待模型服务稳定后重试。")
-                semantic_space = space
+                semantic_spaces[policy] = space
             prepared_notes.append((parsed, prepared))
         # All network/model awaits precede the transaction. The concrete SQLite
         # methods below complete synchronously despite their async interfaces.
@@ -114,12 +115,12 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
                 await vector_store.clear(conn=conn)
                 for parsed, prepared in prepared_notes:
                     await index_note(parsed, prepared=prepared, conn=conn)
-                if semantic_space is not None:
+                for policy, space in semantic_spaces.items():
                     exists = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='routed_block_vectors'").fetchone()
                     missing = not exists or conn.execute(
                         "SELECT 1 FROM blocks b LEFT JOIN routed_block_vectors r "
                         "ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
-                        "WHERE r.block_id IS NULL LIMIT 1", semantic_space,
+                        "WHERE b.embedding_local_only=? AND r.block_id IS NULL LIMIT 1", (*space, int(policy)),
                     ).fetchone()
                     if missing:
                         raise ApiError(500, "SEMANTIC_INDEX_WRITE_FAILED", "向量索引写入失败，原索引已保留，请检查数据库和磁盘状态。")

backend/app/services/note_service.py

@@ -118,6 +118,8 @@ async def index_note(
                 blocks=parsed.blocks,
             )
             old_ids = set(old_block_ids)
+            conn.execute("UPDATE blocks SET embedding_local_only=? WHERE note_id=?",
+                         (int(parsed.embedding_local_only), parsed.note_id))
             new_ids = {block.block_id for block in parsed.blocks}
             stale_ids = [bid for bid in old_ids if bid not in new_ids]
             if stale_ids:

backend/tests/test_routed_retrieval.py

@@ -537,3 +537,85 @@ def test_real_embedding_rebuild_failure_preserves_index(runtime, production_engi
 
 def test_empty_vault_vector_search_returns_empty(runtime, production_engine):
     assert asyncio.run(production_engine.search(request())).items == []
+
+
+@pytest.fixture
+def policy_runtime(monkeypatch):
+    class PolicyRuntime:
+        fallback = False
+        calls = []
+        async def embed(self, texts, *, local_only=False):
+            self.calls.append((list(texts), local_only))
+            local = local_only or self.fallback
+            dim = 3 if local else 2
+            return SimpleNamespace(source='local' if local else 'api', model_id='local-space' if local else 'api-space',
+                                   dimensions=dim, vectors=[[1.0] + [0.0] * (dim - 1) for _ in texts],
+                                   fallback_reason='PROVIDER_TIMEOUT' if self.fallback and not local_only else None)
+    runtime = PolicyRuntime()
+    monkeypatch.setattr(routed_vectors, 'get_model_routing', lambda: runtime)
+    return runtime
+
+
+async def seed_policies():
+    normal = await note_service.create_note(title='Normal', markdown='apple public', folder=None, tags=[])
+    private = await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[])
+    return normal, private
+
+
+@pytest.mark.parametrize('fallback', [False, True])
+def test_mixed_policy_rebuild_and_retrieval(policy_runtime, production_engine, fallback):
+    policy_runtime.fallback = fallback
+    async def scenario():
+        notes = await seed_policies()
+        await index_service.rebuild(IndexRebuildRequest())
+        for mode in (SearchMode.vector, SearchMode.hybrid):
+            result = await production_engine.search(SearchRequest(query='apple', mode=mode))
+            assert {item.note_id for item in result.items} == {note.note_id for note in notes}
+        for texts, local_only in policy_runtime.calls:
+            if any('private' in text for text in texts):
+                assert local_only
+        if not fallback:
+            assert {r[0] for r in rows('SELECT DISTINCT space_id FROM routed_block_vectors')} == {'api-space', 'local-space'}
+    asyncio.run(scenario())
+
+
+def test_local_only_vault_never_requests_api_for_search(policy_runtime, production_engine):
+    async def scenario():
+        await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[])
+        await index_service.rebuild(IndexRebuildRequest())
+        assert (await production_engine.search(request())).items
+        assert all(local_only for _, local_only in policy_runtime.calls)
+    asyncio.run(scenario())
+
+
+def test_partition_storage_failure_rolls_back_all_partitions(policy_runtime, production_engine, monkeypatch):
+    from app.errors import ApiError
+    async def scenario():
+        await seed_policies()
+        before = [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')]
+        original = routed_vectors.store_remote
+        def fail_local(conn, ids, batch):
+            if batch.source != 'local':
+                original(conn, ids, batch)
+        monkeypatch.setattr(routed_vectors, 'store_remote', fail_local)
+        with pytest.raises(ApiError) as error:
+            await index_service.rebuild(IndexRebuildRequest())
+        assert error.value.code == 'SEMANTIC_INDEX_WRITE_FAILED'
+        assert [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')] == before
+    asyncio.run(scenario())
+
+
+def test_missing_partition_does_not_silently_return_partial_hits(policy_runtime, production_engine):
+    from app.errors import ApiError
+    async def scenario():
+        await seed_policies()
+        conn = connect()
+        try:
+            conn.execute("DELETE FROM routed_block_vectors WHERE space_id='local-space'")
+        finally:
+            conn.close()
+        with pytest.raises(ApiError) as error:
+            await production_engine.search(request())
+        assert error.value.code == 'SEMANTIC_INDEX_UNAVAILABLE'
+        assert (await production_engine.search(request(SearchMode.hybrid))).items
+    asyncio.run(scenario())

docs/development/多模态管线与模型运行开发说明.md

@@ -97,7 +97,7 @@ POST /api/providers/request-preview 不联网，隐藏正文/文件且不包含
 
 Windows 热重载不支持异步子进程时使用线程管道兼容路径。本地 Embedding 进入推理前冻结模型与设备配置，向量空间标识来自同一快照；普通 API 成功及失败回退策略保持不变。
 
-仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter，索引与后续重建跳过远程 Embedding。它只约束索引，不是通用的笔记联网权限；旧导出笔记需人工补标记。本地和 API 空间不混用，无法完整覆盖时保留全文检索能力。
+仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter，索引与后续重建跳过远程 Embedding。它只约束索引，不是通用的笔记联网权限；旧导出笔记需人工补标记。SQLite v6 将策略保存到 Block，重建按普通/仅本地策略分别校验空间和覆盖，统一事务提交。查询在各空间内排序后用 RRF 合并排名，缺少分区时保留全文检索降级。升级已有库后重建一次以同步策略。
 
 搜索记录保存在应用 SQLite，使用 `/api/search/history` GET/DELETE 读取和清空。聊天已接入真实知识库上下文与 Citation。详细原因和验证见[阶段 F 问题与解决方案](../retrospectives/阶段F-Embedding与知识库问题与解决方案.md)。
 

docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md

@@ -23,6 +23,7 @@
 | F-05 | 聊天忽略 `use_rag` | 没有知识库内容 | 真实 Block 上下文与来源事件 |
 | F-06 | 本地转写导出未传递限制 | 正文可能发送给远程 Embedding | 持久化本地索引标记 |
 | F-07 | 推理结束才读取当前模型标识 | 向量与空间错配 | 冻结模型、revision 和设备配置 |
+| F-08 | 将不同处理策略误判为配置漂移 | 普通与仅本地笔记共存时不能重建 | 按策略校验覆盖，独立检索并融合排名 |
 
 ## 3. F-01 / F-03：模型可用不等于索引可用
 
@@ -108,11 +109,18 @@ embedding_local_only: true
 | 本地限定索引，存在 API | 不请求 API、不解析远程凭据 |
 | 本地限定后再发普通请求 | API 仍可调用，不泄漏临时限制 |
 | 推理期间修改设置 | 当前向量与身份一致，下一请求采用新设置 |
-| 空间混用或索引覆盖不完整 | 严格重建拒绝提交，不混合不同向量空间 |
+| 普通与仅本地笔记共存 | 分区重建，各自空间内检索，再融合排名 |
+| 同一策略内空间漂移或覆盖不完整 | 严格重建拒绝提交，整体回滚 |
 
-本地限定笔记与远程索引并存时，现有完整覆盖检查可能使纯向量查询提示不完整、重建拒绝混合空间；混合检索仍可退到全文检索。不能为获得完整远程索引而绕过本地标记。
+### F-08：混合处理策略的重建与检索
 
-提交审阅时已用隔离数据库复现：一篇普通 API 笔记与一篇本地限定笔记共存，配置未变化，全量重建仍返回 `EMBEDDING_SPACE_CHANGED`。此项列为合并阻碍，尚未修复；后续需区分预期的处理策略差异和实际配置漂移，并明确各向量空间的索引覆盖范围。
+提交审阅时用隔离数据库复现：一篇普通 API 笔记与一篇本地限定笔记共存，配置未变化，全量重建仍返回 `EMBEDDING_SPACE_CHANGED`。原因是重建将全部笔记约束到一个空间，查询也要求单个空间覆盖全部 Block，未区分处理策略。
+
+本轮追加 SQLite v6，为 Block 保存 `embedding_local_only` 策略。重建分别检查普通和仅本地策略的空间一致性及完整覆盖，仍在同一事务提交；同一策略内模型改变、缺失向量或存储失败仍整体回滚。正常 API 回退不受跨策略差异影响。
+
+查询按策略生成对应查询向量，在同一个数据库快照中检查两个分区。各分区独立计算相似度，再用 RRF 融合排名，不直接比较不同模型的向量或余弦分数。仅含本地限定笔记时，查询也不请求远程 Embedding；分区失效时纯向量明确报错，混合查询仍可退到全文。
+
+已有数据库升级后应重建一次索引，将 Vault 中的策略标记同步至 Block。新增和更新笔记自动同步。回归覆盖混合策略、全部本地回退、仅本地查询、单分区缺失和跨分区写入失败回滚；此项合并阻碍已修复。
 
 本轮在 `backend/` 执行：
 
@@ -126,4 +134,6 @@ embedding_local_only: true
 
 ## 9. 工程经验
 
+F-08 修复后完整后端回归：495 项通过；新增 5 个用例覆盖混合策略重建与查询、全部本地回退、仅本地查询不访问 API、跨分区回滚和不完整分区。现有同策略空间漂移拒绝用例仍通过。
+
 区分配置、权重安装、推理运行、索引覆盖四种状态；按用户实际启动方式验证；跨异步边界冻结身份；持久化处理限制；增加限制时也验证普通 API 回退没有被破坏。