diff --git a/backend/app/repository.py b/backend/app/repository.py index c02d2e1..cd3fbdd 100644 --- a/backend/app/repository.py +++ b/backend/app/repository.py @@ -120,20 +120,24 @@ def replace_note_metadata( return old_block_ids -def delete_note(note_id: str) -> list[str]: +def delete_note( + note_id: str, *, conn: sqlite3.Connection | None = None +) -> list[str]: """删除笔记及其 Block、FTS5 索引；返回被删除的 block_id 供向量层清理。""" - conn = connect() + owns = conn is None + conn = conn or connect() try: block_ids = [ row["block_id"] for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,)) ] - with transaction(conn): + with transaction(conn) if owns else nullcontext(): conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,)) conn.execute("DELETE FROM notes WHERE note_id = ?", (note_id,)) # blocks 级联删除 return block_ids finally: - conn.close() + if owns: + conn.close() def get_note_record(note_id: str) -> NoteRecord | None: @@ -215,6 +219,81 @@ def fts_search(match: str, limit: int = 100) -> list[FtsHit]: conn.close() +def fts_search_page( + *, + match: str, + limit: int, + offset: int, + folders: list[str], + note_ids: list[str], + tags: list[str], + created_from: datetime | None, + created_to: datetime | None, + updated_from: datetime | None, + updated_to: datetime | None, +) -> tuple[list[FtsHit], int]: + """执行带元数据过滤的 FTS 精确分页，并返回过滤后的完整命中数。""" + where = ["blocks_fts MATCH ?"] + params: list[object] = [match] + + def add_in(column: str, values: list[str]) -> None: + if not values: + return + placeholders = ",".join("?" * len(values)) + where.append(f"{column} IN ({placeholders})") + params.extend(values) + + add_in("n.folder", folders) + add_in("n.note_id", note_ids) + if tags: + placeholders = ",".join("?" * len(tags)) + where.append( + f"EXISTS (SELECT 1 FROM json_each(n.tags) AS tag WHERE tag.value IN ({placeholders}))" + ) + params.extend(tags) + + for column, lower, upper in ( + ("n.created_at", created_from, created_to), + ("n.updated_at", updated_from, updated_to), + ): + if lower is not None: + where.append(f"julianday({column}) >= julianday(?)") + params.append(_iso(lower)) + if upper is not None: + where.append(f"julianday({column}) <= julianday(?)") + params.append(_iso(upper)) + + from_sql = """ + FROM blocks_fts + JOIN blocks AS b ON b.block_id = blocks_fts.block_id + JOIN notes AS n ON n.note_id = b.note_id + """ + where_sql = " AND ".join(where) + + conn = connect() + try: + total = conn.execute( + f"SELECT COUNT(*) {from_sql} WHERE {where_sql}", params + ).fetchone()[0] + rows = conn.execute( + f""" + SELECT blocks_fts.block_id, blocks_fts.note_id, bm25(blocks_fts) AS rank + {from_sql} + WHERE {where_sql} + ORDER BY rank + LIMIT ? OFFSET ? + """, + [*params, limit, offset], + ).fetchall() + return ( + [FtsHit(block_id=row["block_id"], note_id=row["note_id"], bm25=row["rank"]) + for row in rows], + total, + ) + finally: + conn.close() + + def get_block_hits(block_ids: list[str]) -> list[BlockHit]: if not block_ids: return [] diff --git a/backend/app/retrieval/engine.py b/backend/app/retrieval/engine.py index 09bffb2..432edc4 100644 --- a/backend/app/retrieval/engine.py +++ b/backend/app/retrieval/engine.py @@ -31,8 +31,6 @@ CANDIDATE_POOL = 50 MAX_CANDIDATE_POOL = 200 # 带 metadata 过滤时放大召回倍数，缓解「先截断候选池再过滤」造成的漏召回 OVERSCAN_FACTOR = 4 -# FTS 一次性取全量命中上限：保证 fts 模式 total 准确、过滤不漏召回；超出则截断 -FTS_FETCH_LIMIT = 1000 class RetrievalEngine: @@ -47,6 +45,9 @@ class RetrievalEngine: self.vector_store = vector_store async def search(self, request: SearchRequest) -> SearchResponse: + if request.mode == SearchMode.fts: + return self._search_fts(request) + has_filters = bool( request.folders or request.note_ids or request.tags or request.created_from or request.created_to @@ -67,8 +68,7 @@ class RetrievalEngine: if request.mode in (SearchMode.fts, SearchMode.hybrid): match = match_query(request.query) if match: - fts_limit = FTS_FETCH_LIMIT if request.mode == SearchMode.fts else recall - fts_hits = repository.fts_search(match, fts_limit) + fts_hits = repository.fts_search(match, recall) fts_ranked = [h.block_id for h in fts_hits] # bm25 越小越相关，取反后统一为「越大越相关」 fts_scores = {h.block_id: -h.bm25 for h in fts_hits} @@ -125,6 +125,43 @@ class RetrievalEngine: page=PageMeta(total=total, limit=request.limit, offset=request.offset), ) + def _search_fts(self, request: SearchRequest) -> SearchResponse: + """FTS 专用路径：过滤、COUNT 与分页全部在 SQLite 中完成。""" + match = match_query(request.query) + if not match: + return self._empty(request) + + fts_hits, total = repository.fts_search_page( + match=match, + limit=request.limit, + offset=request.offset, + folders=request.folders, + note_ids=request.note_ids, + tags=request.tags, + created_from=request.created_from, + created_to=request.created_to, + updated_from=request.updated_from, + updated_to=request.updated_to, + ) + if not fts_hits: + return SearchResponse( + query=request.query, + mode=request.mode, + page=PageMeta(total=total, limit=request.limit, offset=request.offset), + ) + + hits = {h.block_id: h for h in repository.get_block_hits([hit.block_id for hit in fts_hits])} + ordered = normalize_scores( + [(hit.block_id, -hit.bm25) for hit in fts_hits if hit.block_id in hits] + ) + items = [self._build_result(hits[block_id], request, score) for block_id, score in ordered] + return SearchResponse( + query=request.query, + mode=request.mode, + items=items, + page=PageMeta(total=total, limit=request.limit, offset=request.offset), + ) + def _matches(self, hit: BlockHit, request: SearchRequest) -> bool: if request.folders and hit.folder not in request.folders: return False diff --git a/backend/app/services/note_service.py b/backend/app/services/note_service.py index 51a17b3..0983eb3 100644 --- a/backend/app/services/note_service.py +++ b/backend/app/services/note_service.py @@ -9,6 +9,7 @@ from __future__ import annotations import re from datetime import datetime, timezone from pathlib import Path +from uuid import uuid4 from app import repository from app.config import get_settings @@ -216,9 +217,30 @@ async def delete_note(note_id: str) -> bool: record = repository.get_note_record(note_id) if record is None: return False - block_ids = repository.delete_note(note_id) - await vector_store.delete(block_ids) - _delete_markdown(record.file_path) + + path = _abs_path(record.file_path) + tombstone = path.with_name(f".{path.name}.{uuid4().hex}.deleting") if path.exists() else None + if tombstone is not None: + path.replace(tombstone) + + conn = connect() + try: + with transaction(conn): + block_ids = repository.delete_note(note_id, conn=conn) + await vector_store.delete(block_ids, conn=conn) + except BaseException: + if tombstone is not None and tombstone.exists(): + tombstone.replace(path) + raise + finally: + conn.close() + + if tombstone is not None: + # 数据库已提交后，tombstone 即不再属于 Vault；清理失败不应把成功删除报告为失败。 + try: + tombstone.unlink(missing_ok=True) + except OSError: + pass return True diff --git a/backend/tests/test_retrieval.py b/backend/tests/test_retrieval.py index a8ab47b..f175543 100644 --- a/backend/tests/test_retrieval.py +++ b/backend/tests/test_retrieval.py @@ -324,6 +324,35 @@ def test_create_note_rejects_existing_path_without_overwrite(vault) -> None: assert got is not None and got.markdown == "原始正文" +def test_delete_note_rolls_back_when_vector_delete_fails(vault, monkeypatch) -> None: + """向量删除失败时，笔记数据库记录和 Markdown 都恢复到删除前。""" + from app.database.db import connect + from app.services import note_service + + note = asyncio.run( + note_service.create_note(title="删除回滚", markdown="待保留正文", folder="测试", tags=[]) + ) + path = vault / note.file_path + + async def _boom(_ids, **_kwargs): + raise RuntimeError("vector delete failed") + + monkeypatch.setattr(note_service.vector_store, "delete", _boom) + with pytest.raises(RuntimeError): + asyncio.run(note_service.delete_note(note.note_id)) + + got = asyncio.run(note_service.get_note(note.note_id)) + assert got is not None and got.markdown == "待保留正文" + assert path.exists() + conn = connect() + try: + assert conn.execute( + "SELECT COUNT(*) FROM vec_blocks WHERE block_id = ?", (note.blocks[0].block_id,) + ).fetchone()[0] == 1 + finally: + conn.close() + + def test_update_removes_stale_vectors(vault) -> None: from app.database.db import connect from app.services import note_service @@ -366,6 +395,25 @@ def test_search_pagination_total_reflects_all_matches(vault) -> None: assert page2.items # 跨过旧候选池边界仍能取到结果 +def test_fts_pagination_is_not_truncated_at_one_thousand(vault) -> None: + """FTS total 与分页由数据库计算，不在第 1000 个候选处截断。""" + from app.retrieval.engine import engine + from app.services import note_service + + markdown = "\n\n".join(f"共同词 p{i}" for i in range(1010)) + asyncio.run( + note_service.create_note(title="千条分页", markdown=markdown, folder="", tags=[]) + ) + + response = asyncio.run( + engine.search( + SearchRequest(query="共同词", mode=SearchMode.fts, limit=10, offset=1000) + ) + ) + assert response.page.total == 1010 + assert len(response.items) == 10 + + # --------------------------------------------------------------------------- # # 审阅回归：PATCH tags 语义 / 向量-块一致性 / 过滤漏召回 / rebuild 语义与回滚 # --------------------------------------------------------------------------- #