fix(retrieval): 修复原子性、过滤漏召回、tags 语义与 rebuild 回滚

- 元数据 + 向量单事务提交,避免 PATCH 半提交(审阅 #2)
- vectorstore upsert 改 delete-then-insert 幂等,支持共享 conn
- FTS 取全量 + 过滤 oversample,修复 metadata 过滤漏召回(审阅 #4)
- PATCH tags 区分 None/[]/非空:保留/清空/替换(审阅 #5)
- rebuild 拒绝增量 scope/note_ids,扫描先行 + 失败回滚旧索引(审阅 #6)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
yxx
2026-08-27 22:48:55 +08:00
co-authored by Claude
parent 87717450fd
commit 6cf531f2a8
7 changed files with 275 additions and 120 deletions
+48 -18
View File
@@ -6,6 +6,7 @@ MVP 阶段重建是同步的(数据量小),完成后直接返回 completed
from __future__ import annotations
import shutil
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
@@ -13,6 +14,7 @@ from uuid import uuid4
from app import repository
from app.config import get_settings
from app.contracts import IndexJob, IndexRebuildRequest, IndexStatus
from app.errors import ApiError
from app.knowledge.parser import parse_note
from app.services.note_service import index_note
from app.retrieval.vectorstore import SqliteVecStore
@@ -22,10 +24,13 @@ vector_store = SqliteVecStore()
_jobs: dict[str, IndexJob] = {}
def _scan_vault() -> list[tuple[str, str, str]]:
"""扫描 Vault 下所有 Markdown,返回 (rel_path, folder, markdown)。"""
def _scan_vault() -> list[tuple[str, str, str, datetime, datetime]]:
"""扫描 Vault 下所有 Markdown,返回 (rel_path, folder, markdown, created, updated)。
先读入内存:若文件读取失败,rebuild 尚未清空旧索引,不会造成数据损失。
"""
vault = get_settings().vault_path
result: list[tuple[str, str, str]] = []
result: list[tuple[str, str, str, datetime, datetime]] = []
if not vault.exists():
return result
for path in sorted(vault.rglob("*.md")):
@@ -33,27 +38,52 @@ def _scan_vault() -> list[tuple[str, str, str]]:
folder = path.relative_to(vault).parent.as_posix()
if folder == ".":
folder = ""
result.append((rel, folder, path.read_text(encoding="utf-8")))
stat = path.stat()
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
result.append((rel, folder, path.read_text(encoding="utf-8"), created, updated))
return result
async def rebuild(request: IndexRebuildRequest) -> IndexJob:
job_id = "job_" + uuid4().hex[:12]
# MVPscopeall/notes/vectors)与 note_ids 增量暂不区分,统一全量重建
repository.clear_all()
await vector_store.clear()
vault = get_settings().vault_path
for rel, folder, markdown in _scan_vault():
path = vault / rel
stat = path.stat()
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
parsed = parse_note(
markdown=markdown, file_path=rel, folder=folder, tags=None,
created_at=created, updated_at=updated,
# 增量重建(scope != all 或指定 note_ids)尚未实现,明确拒绝而非静默全量重建
if request.scope != "all" or request.note_ids:
raise ApiError(
400,
"UNSUPPORTED_SCOPE",
"only full rebuild (scope='all' with empty note_ids) is supported",
{"scope": request.scope, "note_ids": request.note_ids},
)
await index_note(parsed)
# 先扫描到内存(失败不会清旧索引),再快照旧库用于失败回滚
docs = _scan_vault()
settings = get_settings()
backup_path = settings.db_path.with_suffix(".db.bak") if settings.db_path.exists() else None
if backup_path is not None:
shutil.copy2(settings.db_path, backup_path)
try:
repository.clear_all()
await vector_store.clear()
for rel, folder, markdown, created, updated in docs:
parsed = parse_note(
markdown=markdown, file_path=rel, folder=folder, tags=None,
created_at=created, updated_at=updated,
)
await index_note(parsed)
except BaseException:
# 重建失败:恢复旧索引,避免留下半成品;记录 failed 任务后向上抛
if backup_path is not None and backup_path.exists():
shutil.copy2(backup_path, settings.db_path)
_jobs[job_id] = IndexJob(
job_id=job_id, status="failed", scope=request.scope,
created_at=datetime.now(timezone.utc),
)
raise
finally:
if backup_path is not None:
backup_path.unlink(missing_ok=True)
job = IndexJob(job_id=job_id, status="completed", scope=request.scope, created_at=datetime.now(timezone.utc))
_jobs[job_id] = job