Feat/multimodal pipeline #19

Merged
Kronecker merged 9 commits from feat/multimodal-pipeline into main 2026-09-04 20:17:03 +08:00
7 changed files with 170 additions and 9 deletions
Showing only changes of commit 78dd774bce - Show all commits
+4
View File
@@ -127,6 +127,10 @@ MIGRATIONS: list[str] = [
query TEXT NOT NULL UNIQUE
);
""",
# v6: persist each block's embedding policy for partitioned retrieval.
"""
ALTER TABLE blocks ADD COLUMN embedding_local_only INTEGER NOT NULL DEFAULT 0;
""",
]
+62
View File
@@ -22,6 +22,7 @@ from app.database.db import connect, transaction
from app.errors import ApiError
from app.retrieval.vectorstore import VectorHit
from app.retrieval.provenance import record_embedding
from app.retrieval.hybrid import rrf_fuse
logger = logging.getLogger(__name__)
@@ -167,9 +168,18 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
Read coverage and vectors together so concurrent note updates cannot produce
an apparently complete subset. Never fill missing remote hits with local hits.
"""
if accept_local:
conn = connect()
try:
policies = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")}
finally:
conn.close()
if True in policies:
return await _search_partitioned(query, policies, top_k=top_k, strict=strict)
batch = await embed_remote([query], accept_local=accept_local, strict=strict)
if batch is None:
return None
record_embedding(attempted_space={"model_id": batch.space_id, "dimensions": batch.dimensions})
try:
conn = connect()
@@ -221,3 +231,55 @@ async def search_remote(query: str, *, top_k: int, accept_local=False, strict=Fa
"Embedding 已可用,但当前模型的向量索引缺失、不完整或已失效。请在「设置 → 索引与模型」中重建全部索引。",
{"model_id": batch.space_id, "dimensions": batch.dimensions, "source": batch.source}) from exc
return None
async def _search_partitioned(query: str, policies: set[bool], *, top_k: int, strict: bool):
"""Embed per policy; rank each space independently and fuse ranks, not vectors."""
batches = {}
for policy in sorted(policies):
batch = await embed_remote([query], accept_local=True, strict=strict, local_only=policy)
if batch is None:
return None
batches[policy] = batch
conn = connect()
try:
with transaction(conn):
# Query vectors are ready before opening the single read snapshot.
current = {bool(row[0]) for row in conn.execute("SELECT DISTINCT embedding_local_only FROM blocks")}
if current != policies:
raise ValueError("embedding policies changed while querying")
ranked = []
for policy, batch in batches.items():
rows = conn.execute(
"SELECT b.block_id,r.vector FROM blocks b LEFT JOIN routed_block_vectors r "
"ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
"WHERE b.embedding_local_only=? ORDER BY b.block_id",
(batch.space_id, batch.dimensions, int(policy)),
)
def hits():
for row in rows:
if row['vector'] is None:
raise ValueError("incomplete policy coverage")
vector = _unit_vector(json.loads(row['vector']), batch.dimensions)
score = math.fsum(a * b for a, b in zip(batch.vectors[0], vector))
yield VectorHit(id=row['block_id'], score=max(0.0, min(1.0, score)))
try:
ranked.append(heapq.nlargest(top_k, hits(), key=lambda hit: hit.score))
finally:
rows.close()
spaces = [{"source": b.source, "model_id": b.space_id, "dimensions": b.dimensions,
"local_only": policy} for policy, b in batches.items()]
record_embedding(source="mixed" if len({b.source for b in batches.values()}) > 1 else batch.source,
spaces=spaces, fallback_reason=None)
if len(ranked) == 1:
return ranked[0]
fused = rrf_fuse([[hit.id for hit in group] for group in ranked])
return [VectorHit(id=key, score=score) for key, score in
sorted(fused.items(), key=lambda item: (-item[1], item[0]))[:top_k]]
except Exception as exc:
record_embedding(source="unavailable", fallback_reason="REMOTE_INDEX_UNAVAILABLE")
if strict:
raise ApiError(409, "SEMANTIC_INDEX_UNAVAILABLE", "部分索引分区缺失或已失效,请重建全部索引。") from exc
return None
finally:
conn.close()
+6 -5
View File
@@ -85,7 +85,7 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
))
try:
prepared_notes = []
semantic_space = None
semantic_spaces = {}
for rel, folder, markdown, created, updated in docs:
parsed = parse_note(
markdown=markdown, file_path=rel, folder=folder, tags=None,
@@ -97,9 +97,10 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
if batch is None:
raise ApiError(503, "EMBEDDING_UNAVAILABLE", "Embedding 未生成向量,重建已停止,原索引已保留。")
space = (batch.space_id, batch.dimensions)
if semantic_space is not None and semantic_space != space:
policy = parsed.embedding_local_only
if policy in semantic_spaces and semantic_spaces[policy] != space:
raise ApiError(409, "EMBEDDING_SPACE_CHANGED", "重建期间 Embedding 模型发生切换,原索引已保留,请待模型服务稳定后重试。")
semantic_space = space
semantic_spaces[policy] = space
prepared_notes.append((parsed, prepared))
# All network/model awaits precede the transaction. The concrete SQLite
# methods below complete synchronously despite their async interfaces.
@@ -114,12 +115,12 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
await vector_store.clear(conn=conn)
for parsed, prepared in prepared_notes:
await index_note(parsed, prepared=prepared, conn=conn)
if semantic_space is not None:
for policy, space in semantic_spaces.items():
exists = conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='routed_block_vectors'").fetchone()
missing = not exists or conn.execute(
"SELECT 1 FROM blocks b LEFT JOIN routed_block_vectors r "
"ON r.block_id=b.block_id AND r.space_id=? AND r.dimensions=? "
"WHERE r.block_id IS NULL LIMIT 1", semantic_space,
"WHERE b.embedding_local_only=? AND r.block_id IS NULL LIMIT 1", (*space, int(policy)),
).fetchone()
if missing:
raise ApiError(500, "SEMANTIC_INDEX_WRITE_FAILED", "向量索引写入失败,原索引已保留,请检查数据库和磁盘状态。")
+2
View File
@@ -118,6 +118,8 @@ async def index_note(
blocks=parsed.blocks,
)
old_ids = set(old_block_ids)
conn.execute("UPDATE blocks SET embedding_local_only=? WHERE note_id=?",
(int(parsed.embedding_local_only), parsed.note_id))
new_ids = {block.block_id for block in parsed.blocks}
stale_ids = [bid for bid in old_ids if bid not in new_ids]
if stale_ids:
+82
View File
@@ -537,3 +537,85 @@ def test_real_embedding_rebuild_failure_preserves_index(runtime, production_engi
def test_empty_vault_vector_search_returns_empty(runtime, production_engine):
assert asyncio.run(production_engine.search(request())).items == []
@pytest.fixture
def policy_runtime(monkeypatch):
class PolicyRuntime:
fallback = False
calls = []
async def embed(self, texts, *, local_only=False):
self.calls.append((list(texts), local_only))
local = local_only or self.fallback
dim = 3 if local else 2
return SimpleNamespace(source='local' if local else 'api', model_id='local-space' if local else 'api-space',
dimensions=dim, vectors=[[1.0] + [0.0] * (dim - 1) for _ in texts],
fallback_reason='PROVIDER_TIMEOUT' if self.fallback and not local_only else None)
runtime = PolicyRuntime()
monkeypatch.setattr(routed_vectors, 'get_model_routing', lambda: runtime)
return runtime
async def seed_policies():
normal = await note_service.create_note(title='Normal', markdown='apple public', folder=None, tags=[])
private = await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[])
return normal, private
@pytest.mark.parametrize('fallback', [False, True])
def test_mixed_policy_rebuild_and_retrieval(policy_runtime, production_engine, fallback):
policy_runtime.fallback = fallback
async def scenario():
notes = await seed_policies()
await index_service.rebuild(IndexRebuildRequest())
for mode in (SearchMode.vector, SearchMode.hybrid):
result = await production_engine.search(SearchRequest(query='apple', mode=mode))
assert {item.note_id for item in result.items} == {note.note_id for note in notes}
for texts, local_only in policy_runtime.calls:
if any('private' in text for text in texts):
assert local_only
if not fallback:
assert {r[0] for r in rows('SELECT DISTINCT space_id FROM routed_block_vectors')} == {'api-space', 'local-space'}
asyncio.run(scenario())
def test_local_only_vault_never_requests_api_for_search(policy_runtime, production_engine):
async def scenario():
await note_service.create_note(title='Private', markdown='---\nembedding_local_only: true\n---\napple private', folder=None, tags=[])
await index_service.rebuild(IndexRebuildRequest())
assert (await production_engine.search(request())).items
assert all(local_only for _, local_only in policy_runtime.calls)
asyncio.run(scenario())
def test_partition_storage_failure_rolls_back_all_partitions(policy_runtime, production_engine, monkeypatch):
from app.errors import ApiError
async def scenario():
await seed_policies()
before = [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')]
original = routed_vectors.store_remote
def fail_local(conn, ids, batch):
if batch.source != 'local':
original(conn, ids, batch)
monkeypatch.setattr(routed_vectors, 'store_remote', fail_local)
with pytest.raises(ApiError) as error:
await index_service.rebuild(IndexRebuildRequest())
assert error.value.code == 'SEMANTIC_INDEX_WRITE_FAILED'
assert [tuple(row) for row in rows('SELECT * FROM routed_block_vectors ORDER BY block_id')] == before
asyncio.run(scenario())
def test_missing_partition_does_not_silently_return_partial_hits(policy_runtime, production_engine):
from app.errors import ApiError
async def scenario():
await seed_policies()
conn = connect()
try:
conn.execute("DELETE FROM routed_block_vectors WHERE space_id='local-space'")
finally:
conn.close()
with pytest.raises(ApiError) as error:
await production_engine.search(request())
assert error.value.code == 'SEMANTIC_INDEX_UNAVAILABLE'
assert (await production_engine.search(request(SearchMode.hybrid))).items
asyncio.run(scenario())
@@ -97,7 +97,7 @@ POST /api/providers/request-preview 不联网,隐藏正文/文件且不包含
Windows 热重载不支持异步子进程时使用线程管道兼容路径。本地 Embedding 进入推理前冻结模型与设备配置,向量空间标识来自同一快照;普通 API 成功及失败回退策略保持不变。
仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter,索引与后续重建跳过远程 Embedding。它只约束索引,不是通用的笔记联网权限;旧导出笔记需人工补标记。本地和 API 空间不混用,无法完整覆盖时保留全文检索能力
仅本地转写新生成的笔记增加 `embedding_local_only: true` frontmatter,索引与后续重建跳过远程 Embedding。它只约束索引,不是通用的笔记联网权限;旧导出笔记需人工补标记。SQLite v6 将策略保存到 Block,重建按普通/仅本地策略分别校验空间和覆盖,统一事务提交。查询在各空间内排序后用 RRF 合并排名,缺少分区时保留全文检索降级。升级已有库后重建一次以同步策略
搜索记录保存在应用 SQLite,使用 `/api/search/history` GET/DELETE 读取和清空。聊天已接入真实知识库上下文与 Citation。详细原因和验证见[阶段 F 问题与解决方案](../retrospectives/阶段F-Embedding与知识库问题与解决方案.md)。
@@ -23,6 +23,7 @@
| F-05 | 聊天忽略 `use_rag` | 没有知识库内容 | 真实 Block 上下文与来源事件 |
| F-06 | 本地转写导出未传递限制 | 正文可能发送给远程 Embedding | 持久化本地索引标记 |
| F-07 | 推理结束才读取当前模型标识 | 向量与空间错配 | 冻结模型、revision 和设备配置 |
| F-08 | 将不同处理策略误判为配置漂移 | 普通与仅本地笔记共存时不能重建 | 按策略校验覆盖,独立检索并融合排名 |
## 3. F-01 / F-03:模型可用不等于索引可用
@@ -108,11 +109,18 @@ embedding_local_only: true
| 本地限定索引,存在 API | 不请求 API、不解析远程凭据 |
| 本地限定后再发普通请求 | API 仍可调用,不泄漏临时限制 |
| 推理期间修改设置 | 当前向量与身份一致,下一请求采用新设置 |
| 空间混用或索引覆盖不完整 | 严格重建拒绝提交,不混合不同向量空间 |
| 普通与仅本地笔记共存 | 分区重建,各自空间内检索,再融合排名 |
| 同一策略内空间漂移或覆盖不完整 | 严格重建拒绝提交,整体回滚 |
本地限定笔记与远程索引并存时,现有完整覆盖检查可能使纯向量查询提示不完整、重建拒绝混合空间;混合检索仍可退到全文检索。不能为获得完整远程索引而绕过本地标记。
### F-08:混合处理策略的重建与检索
提交审阅时用隔离数据库复现:一篇普通 API 笔记与一篇本地限定笔记共存,配置未变化,全量重建仍返回 `EMBEDDING_SPACE_CHANGED`此项列为合并阻碍,尚未修复;后续需区分预期的处理策略差异和实际配置漂移,并明确各向量空间的索引覆盖范围
提交审阅时用隔离数据库复现:一篇普通 API 笔记与一篇本地限定笔记共存,配置未变化,全量重建仍返回 `EMBEDDING_SPACE_CHANGED`原因是重建将全部笔记约束到一个空间,查询也要求单个空间覆盖全部 Block,未区分处理策略
本轮追加 SQLite v6,为 Block 保存 `embedding_local_only` 策略。重建分别检查普通和仅本地策略的空间一致性及完整覆盖,仍在同一事务提交;同一策略内模型改变、缺失向量或存储失败仍整体回滚。正常 API 回退不受跨策略差异影响。
查询按策略生成对应查询向量,在同一个数据库快照中检查两个分区。各分区独立计算相似度,再用 RRF 融合排名,不直接比较不同模型的向量或余弦分数。仅含本地限定笔记时,查询也不请求远程 Embedding;分区失效时纯向量明确报错,混合查询仍可退到全文。
已有数据库升级后应重建一次索引,将 Vault 中的策略标记同步至 Block。新增和更新笔记自动同步。回归覆盖混合策略、全部本地回退、仅本地查询、单分区缺失和跨分区写入失败回滚;此项合并阻碍已修复。
本轮在 `backend/` 执行:
@@ -126,4 +134,6 @@ embedding_local_only: true
## 9. 工程经验
F-08 修复后完整后端回归:495 项通过;新增 5 个用例覆盖混合策略重建与查询、全部本地回退、仅本地查询不访问 API、跨分区回滚和不完整分区。现有同策略空间漂移拒绝用例仍通过。
区分配置、权重安装、推理运行、索引覆盖四种状态;按用户实际启动方式验证;跨异步边界冻结身份;持久化处理限制;增加限制时也验证普通 API 回退没有被破坏。