fix(retrieval): 保证删除一致性并完善 FTS 分页
This commit is contained in:
@@ -120,20 +120,24 @@ def replace_note_metadata(
|
||||
return old_block_ids
|
||||
|
||||
|
||||
def delete_note(note_id: str) -> list[str]:
|
||||
def delete_note(
|
||||
note_id: str, *, conn: sqlite3.Connection | None = None
|
||||
) -> list[str]:
|
||||
"""删除笔记及其 Block、FTS5 索引;返回被删除的 block_id 供向量层清理。"""
|
||||
conn = connect()
|
||||
owns = conn is None
|
||||
conn = conn or connect()
|
||||
try:
|
||||
block_ids = [
|
||||
row["block_id"]
|
||||
for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,))
|
||||
]
|
||||
with transaction(conn):
|
||||
with transaction(conn) if owns else nullcontext():
|
||||
conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,))
|
||||
conn.execute("DELETE FROM notes WHERE note_id = ?", (note_id,)) # blocks 级联删除
|
||||
return block_ids
|
||||
finally:
|
||||
conn.close()
|
||||
if owns:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_note_record(note_id: str) -> NoteRecord | None:
|
||||
@@ -215,6 +219,81 @@ def fts_search(match: str, limit: int = 100) -> list[FtsHit]:
|
||||
conn.close()
|
||||
|
||||
|
||||
def fts_search_page(
|
||||
*,
|
||||
match: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
folders: list[str],
|
||||
note_ids: list[str],
|
||||
tags: list[str],
|
||||
created_from: datetime | None,
|
||||
created_to: datetime | None,
|
||||
updated_from: datetime | None,
|
||||
updated_to: datetime | None,
|
||||
) -> tuple[list[FtsHit], int]:
|
||||
"""执行带元数据过滤的 FTS 精确分页,并返回过滤后的完整命中数。"""
|
||||
where = ["blocks_fts MATCH ?"]
|
||||
params: list[object] = [match]
|
||||
|
||||
def add_in(column: str, values: list[str]) -> None:
|
||||
if not values:
|
||||
return
|
||||
placeholders = ",".join("?" * len(values))
|
||||
where.append(f"{column} IN ({placeholders})")
|
||||
params.extend(values)
|
||||
|
||||
add_in("n.folder", folders)
|
||||
add_in("n.note_id", note_ids)
|
||||
if tags:
|
||||
placeholders = ",".join("?" * len(tags))
|
||||
where.append(
|
||||
f"EXISTS (SELECT 1 FROM json_each(n.tags) AS tag WHERE tag.value IN ({placeholders}))"
|
||||
)
|
||||
params.extend(tags)
|
||||
|
||||
for column, lower, upper in (
|
||||
("n.created_at", created_from, created_to),
|
||||
("n.updated_at", updated_from, updated_to),
|
||||
):
|
||||
if lower is not None:
|
||||
where.append(f"julianday({column}) >= julianday(?)")
|
||||
params.append(_iso(lower))
|
||||
if upper is not None:
|
||||
where.append(f"julianday({column}) <= julianday(?)")
|
||||
params.append(_iso(upper))
|
||||
|
||||
from_sql = """
|
||||
FROM blocks_fts
|
||||
JOIN blocks AS b ON b.block_id = blocks_fts.block_id
|
||||
JOIN notes AS n ON n.note_id = b.note_id
|
||||
"""
|
||||
where_sql = " AND ".join(where)
|
||||
|
||||
conn = connect()
|
||||
try:
|
||||
total = conn.execute(
|
||||
f"SELECT COUNT(*) {from_sql} WHERE {where_sql}", params
|
||||
).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT blocks_fts.block_id, blocks_fts.note_id, bm25(blocks_fts) AS rank
|
||||
{from_sql}
|
||||
WHERE {where_sql}
|
||||
ORDER BY rank
|
||||
LIMIT ? OFFSET ?
|
||||
""",
|
||||
[*params, limit, offset],
|
||||
).fetchall()
|
||||
return (
|
||||
[FtsHit(block_id=row["block_id"], note_id=row["note_id"], bm25=row["rank"])
|
||||
for row in rows],
|
||||
total,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_block_hits(block_ids: list[str]) -> list[BlockHit]:
|
||||
if not block_ids:
|
||||
return []
|
||||
|
||||
@@ -31,8 +31,6 @@ CANDIDATE_POOL = 50
|
||||
MAX_CANDIDATE_POOL = 200
|
||||
# 带 metadata 过滤时放大召回倍数,缓解「先截断候选池再过滤」造成的漏召回
|
||||
OVERSCAN_FACTOR = 4
|
||||
# FTS 一次性取全量命中上限:保证 fts 模式 total 准确、过滤不漏召回;超出则截断
|
||||
FTS_FETCH_LIMIT = 1000
|
||||
|
||||
|
||||
class RetrievalEngine:
|
||||
@@ -47,6 +45,9 @@ class RetrievalEngine:
|
||||
self.vector_store = vector_store
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
if request.mode == SearchMode.fts:
|
||||
return self._search_fts(request)
|
||||
|
||||
has_filters = bool(
|
||||
request.folders or request.note_ids or request.tags
|
||||
or request.created_from or request.created_to
|
||||
@@ -67,8 +68,7 @@ class RetrievalEngine:
|
||||
if request.mode in (SearchMode.fts, SearchMode.hybrid):
|
||||
match = match_query(request.query)
|
||||
if match:
|
||||
fts_limit = FTS_FETCH_LIMIT if request.mode == SearchMode.fts else recall
|
||||
fts_hits = repository.fts_search(match, fts_limit)
|
||||
fts_hits = repository.fts_search(match, recall)
|
||||
fts_ranked = [h.block_id for h in fts_hits]
|
||||
# bm25 越小越相关,取反后统一为「越大越相关」
|
||||
fts_scores = {h.block_id: -h.bm25 for h in fts_hits}
|
||||
@@ -125,6 +125,43 @@ class RetrievalEngine:
|
||||
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
def _search_fts(self, request: SearchRequest) -> SearchResponse:
|
||||
"""FTS 专用路径:过滤、COUNT 与分页全部在 SQLite 中完成。"""
|
||||
match = match_query(request.query)
|
||||
if not match:
|
||||
return self._empty(request)
|
||||
|
||||
fts_hits, total = repository.fts_search_page(
|
||||
match=match,
|
||||
limit=request.limit,
|
||||
offset=request.offset,
|
||||
folders=request.folders,
|
||||
note_ids=request.note_ids,
|
||||
tags=request.tags,
|
||||
created_from=request.created_from,
|
||||
created_to=request.created_to,
|
||||
updated_from=request.updated_from,
|
||||
updated_to=request.updated_to,
|
||||
)
|
||||
if not fts_hits:
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
hits = {h.block_id: h for h in repository.get_block_hits([hit.block_id for hit in fts_hits])}
|
||||
ordered = normalize_scores(
|
||||
[(hit.block_id, -hit.bm25) for hit in fts_hits if hit.block_id in hits]
|
||||
)
|
||||
items = [self._build_result(hits[block_id], request, score) for block_id, score in ordered]
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
items=items,
|
||||
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
def _matches(self, hit: BlockHit, request: SearchRequest) -> bool:
|
||||
if request.folders and hit.folder not in request.folders:
|
||||
return False
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app import repository
|
||||
from app.config import get_settings
|
||||
@@ -216,9 +217,30 @@ async def delete_note(note_id: str) -> bool:
|
||||
record = repository.get_note_record(note_id)
|
||||
if record is None:
|
||||
return False
|
||||
block_ids = repository.delete_note(note_id)
|
||||
await vector_store.delete(block_ids)
|
||||
_delete_markdown(record.file_path)
|
||||
|
||||
path = _abs_path(record.file_path)
|
||||
tombstone = path.with_name(f".{path.name}.{uuid4().hex}.deleting") if path.exists() else None
|
||||
if tombstone is not None:
|
||||
path.replace(tombstone)
|
||||
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
block_ids = repository.delete_note(note_id, conn=conn)
|
||||
await vector_store.delete(block_ids, conn=conn)
|
||||
except BaseException:
|
||||
if tombstone is not None and tombstone.exists():
|
||||
tombstone.replace(path)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if tombstone is not None:
|
||||
# 数据库已提交后,tombstone 即不再属于 Vault;清理失败不应把成功删除报告为失败。
|
||||
try:
|
||||
tombstone.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user