fix(retrieval): 修复原子性、过滤漏召回、tags 语义与 rebuild 回滚
- 元数据 + 向量单事务提交,避免 PATCH 半提交(审阅 #2) - vectorstore upsert 改 delete-then-insert 幂等,支持共享 conn - FTS 取全量 + 过滤 oversample,修复 metadata 过滤漏召回(审阅 #4) - PATCH tags 区分 None/[]/非空:保留/清空/替换(审阅 #5) - rebuild 拒绝增量 scope/note_ids,扫描先行 + 失败回滚旧索引(审阅 #6) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -6,6 +6,7 @@ MVP 阶段重建是同步的(数据量小),完成后直接返回 completed
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
@@ -13,6 +14,7 @@ from uuid import uuid4
|
||||
from app import repository
|
||||
from app.config import get_settings
|
||||
from app.contracts import IndexJob, IndexRebuildRequest, IndexStatus
|
||||
from app.errors import ApiError
|
||||
from app.knowledge.parser import parse_note
|
||||
from app.services.note_service import index_note
|
||||
from app.retrieval.vectorstore import SqliteVecStore
|
||||
@@ -22,10 +24,13 @@ vector_store = SqliteVecStore()
|
||||
_jobs: dict[str, IndexJob] = {}
|
||||
|
||||
|
||||
def _scan_vault() -> list[tuple[str, str, str]]:
|
||||
"""扫描 Vault 下所有 Markdown,返回 (rel_path, folder, markdown)。"""
|
||||
def _scan_vault() -> list[tuple[str, str, str, datetime, datetime]]:
|
||||
"""扫描 Vault 下所有 Markdown,返回 (rel_path, folder, markdown, created, updated)。
|
||||
|
||||
先读入内存:若文件读取失败,rebuild 尚未清空旧索引,不会造成数据损失。
|
||||
"""
|
||||
vault = get_settings().vault_path
|
||||
result: list[tuple[str, str, str]] = []
|
||||
result: list[tuple[str, str, str, datetime, datetime]] = []
|
||||
if not vault.exists():
|
||||
return result
|
||||
for path in sorted(vault.rglob("*.md")):
|
||||
@@ -33,27 +38,52 @@ def _scan_vault() -> list[tuple[str, str, str]]:
|
||||
folder = path.relative_to(vault).parent.as_posix()
|
||||
if folder == ".":
|
||||
folder = ""
|
||||
result.append((rel, folder, path.read_text(encoding="utf-8")))
|
||||
stat = path.stat()
|
||||
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
|
||||
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
result.append((rel, folder, path.read_text(encoding="utf-8"), created, updated))
|
||||
return result
|
||||
|
||||
|
||||
async def rebuild(request: IndexRebuildRequest) -> IndexJob:
|
||||
job_id = "job_" + uuid4().hex[:12]
|
||||
# MVP:scope(all/notes/vectors)与 note_ids 增量暂不区分,统一全量重建
|
||||
repository.clear_all()
|
||||
await vector_store.clear()
|
||||
|
||||
vault = get_settings().vault_path
|
||||
for rel, folder, markdown in _scan_vault():
|
||||
path = vault / rel
|
||||
stat = path.stat()
|
||||
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
|
||||
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
parsed = parse_note(
|
||||
markdown=markdown, file_path=rel, folder=folder, tags=None,
|
||||
created_at=created, updated_at=updated,
|
||||
# 增量重建(scope != all 或指定 note_ids)尚未实现,明确拒绝而非静默全量重建
|
||||
if request.scope != "all" or request.note_ids:
|
||||
raise ApiError(
|
||||
400,
|
||||
"UNSUPPORTED_SCOPE",
|
||||
"only full rebuild (scope='all' with empty note_ids) is supported",
|
||||
{"scope": request.scope, "note_ids": request.note_ids},
|
||||
)
|
||||
await index_note(parsed)
|
||||
|
||||
# 先扫描到内存(失败不会清旧索引),再快照旧库用于失败回滚
|
||||
docs = _scan_vault()
|
||||
settings = get_settings()
|
||||
backup_path = settings.db_path.with_suffix(".db.bak") if settings.db_path.exists() else None
|
||||
if backup_path is not None:
|
||||
shutil.copy2(settings.db_path, backup_path)
|
||||
|
||||
try:
|
||||
repository.clear_all()
|
||||
await vector_store.clear()
|
||||
for rel, folder, markdown, created, updated in docs:
|
||||
parsed = parse_note(
|
||||
markdown=markdown, file_path=rel, folder=folder, tags=None,
|
||||
created_at=created, updated_at=updated,
|
||||
)
|
||||
await index_note(parsed)
|
||||
except BaseException:
|
||||
# 重建失败:恢复旧索引,避免留下半成品;记录 failed 任务后向上抛
|
||||
if backup_path is not None and backup_path.exists():
|
||||
shutil.copy2(backup_path, settings.db_path)
|
||||
_jobs[job_id] = IndexJob(
|
||||
job_id=job_id, status="failed", scope=request.scope,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if backup_path is not None:
|
||||
backup_path.unlink(missing_ok=True)
|
||||
|
||||
job = IndexJob(job_id=job_id, status="completed", scope=request.scope, created_at=datetime.now(timezone.utc))
|
||||
_jobs[job_id] = job
|
||||
|
||||
Reference in New Issue
Block a user