feat: 按 Vault 隔离桌面检索投影与任务链接
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
"""Rebuildable per-Vault FTS projection, sourced only through the Host broker."""
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
from app import repository
|
||||
from app.database.db import connect_knowledge, transaction
|
||||
from app.knowledge.parser import parse_note
|
||||
from app.services import desktop_notes
|
||||
from app.services.coordination import vault_mutation_lock
|
||||
|
||||
|
||||
def entries():
|
||||
result, offset = [], 0
|
||||
while True:
|
||||
page = desktop_notes.call('list', offset=offset, limit=1000)
|
||||
result.extend(page['items'])
|
||||
offset += len(page['items'])
|
||||
if offset >= page['total'] or not page['items']: return result
|
||||
|
||||
|
||||
def _refresh():
|
||||
current = entries() # Always validates authorization, including when the cache is current.
|
||||
conn = connect_knowledge()
|
||||
try:
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS host_projection (file_id TEXT PRIMARY KEY, hash TEXT NOT NULL, path TEXT NOT NULL)')
|
||||
old = {row['file_id']: (row['hash'], row['path']) for row in conn.execute('SELECT * FROM host_projection')}
|
||||
changed = []
|
||||
for entry in current:
|
||||
if old.get(entry['file_id']) == (entry['hash'], entry['path']): continue
|
||||
document = desktop_notes.call('read', file_id=entry['file_id'])
|
||||
note = desktop_notes.note_from_document(document)
|
||||
parsed = parse_note(markdown=note.markdown, file_path=note.file_path,
|
||||
folder=note.file_path.rpartition('/')[0], note_id=note.note_id,
|
||||
tags=note.tags, created_at=note.created_at, updated_at=note.updated_at)
|
||||
changed.append((document, parsed))
|
||||
removed = set(old) - {entry['file_id'] for entry in current}
|
||||
# Content is verified before starting the projection transaction. No model/network IO inside.
|
||||
with transaction(conn):
|
||||
for file_id in removed:
|
||||
for block_id in repository.delete_note(file_id, conn=conn):
|
||||
conn.execute('DELETE FROM vec_blocks WHERE block_id=?', [block_id])
|
||||
conn.execute('DELETE FROM host_projection WHERE file_id=?', [file_id])
|
||||
for document, parsed in changed:
|
||||
old_ids = repository.replace_note_metadata(conn=conn, note_id=parsed.note_id, title=parsed.title,
|
||||
file_path=parsed.file_path, folder=parsed.folder, tags=parsed.tags, created_at=parsed.created_at,
|
||||
updated_at=parsed.updated_at, blocks=parsed.blocks)
|
||||
for block_id in old_ids:
|
||||
conn.execute('DELETE FROM vec_blocks WHERE block_id=?', [block_id])
|
||||
conn.execute('UPDATE blocks SET embedding_local_only=? WHERE note_id=?', (int(parsed.embedding_local_only), parsed.note_id))
|
||||
conn.execute('INSERT OR REPLACE INTO host_projection VALUES (?,?,?)', (parsed.note_id, document['hash'], parsed.file_path))
|
||||
if removed or changed:
|
||||
repository.set_index_meta({'workspace_vectors_pending': '1'}, conn=conn)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
async def refresh():
|
||||
async with vault_mutation_lock():
|
||||
work = asyncio.create_task(asyncio.to_thread(_refresh))
|
||||
# Keep the projection gate until the worker has finished even if the request is cancelled.
|
||||
cancelled = False
|
||||
while not work.done():
|
||||
try: await asyncio.shield(work)
|
||||
except asyncio.CancelledError: cancelled = True
|
||||
work.result()
|
||||
if cancelled: raise asyncio.CancelledError
|
||||
@@ -16,7 +16,7 @@ from app.contracts import IndexJob, IndexRebuildRequest, IndexStatus
|
||||
from app.errors import ApiError
|
||||
from app.knowledge.parser import parse_note
|
||||
from app.services.note_service import index_note, prepare_note_index
|
||||
from app.database.db import connect, transaction
|
||||
from app.database.db import connect_knowledge as connect, transaction
|
||||
from app.services.coordination import vault_mutation_lock
|
||||
from app.retrieval.vectorstore import SqliteVecStore
|
||||
from app.local_models.runtime import LocalEmbedding
|
||||
@@ -47,6 +47,14 @@ def _scan_vault() -> list[tuple[str, str, str, datetime, datetime]]:
|
||||
|
||||
先读入内存:若文件读取失败,rebuild 尚未清空旧索引,不会造成数据损失。
|
||||
"""
|
||||
if get_settings().environment == 'desktop':
|
||||
from app.services.desktop_projection import entries
|
||||
from app.services import desktop_notes
|
||||
result = []
|
||||
for entry in entries():
|
||||
note = desktop_notes.note_from_document(desktop_notes.call('read', file_id=entry['file_id']))
|
||||
result.append((note.file_path, note.file_path.rpartition('/')[0], note.markdown, note.created_at, note.updated_at))
|
||||
return result
|
||||
vault = get_settings().vault_path.resolve()
|
||||
result: list[tuple[str, str, str, datetime, datetime]] = []
|
||||
if not vault.exists():
|
||||
@@ -80,8 +88,12 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
|
||||
{"scope": request.scope, "note_ids": request.note_ids},
|
||||
)
|
||||
|
||||
if get_settings().environment == 'desktop':
|
||||
from app.services.desktop_projection import refresh
|
||||
await refresh()
|
||||
docs = _scan_vault()
|
||||
saved_records = {key: repository.get_note_record(key) for key in _pending_notes()}
|
||||
record_ids = [entry.note_id for entry in repository.list_note_locations()] if get_settings().environment == 'desktop' else _pending_notes()
|
||||
saved_records = {key: repository.get_note_record(key) for key in record_ids}
|
||||
saved_paths = {record.file_path: record for record in saved_records.values() if record is not None}
|
||||
|
||||
_active_job_id = job_id
|
||||
@@ -117,7 +129,8 @@ async def rebuild(request: IndexRebuildRequest) -> IndexJob:
|
||||
# All network/model awaits precede the transaction. The concrete SQLite
|
||||
# methods below complete synchronously despite their async interfaces.
|
||||
async with vault_mutation_lock():
|
||||
if _scan_vault() != docs or saved_records != {key: repository.get_note_record(key) for key in _pending_notes()}:
|
||||
current_ids = [entry.note_id for entry in repository.list_note_locations()] if get_settings().environment == 'desktop' else _pending_notes()
|
||||
if _scan_vault() != docs or saved_records != {key: repository.get_note_record(key) for key in current_ids}:
|
||||
raise ApiError(409, "INDEX_SNAPSHOT_CHANGED", "笔记在计算期间发生变化,稍后重新计算。")
|
||||
conn = connect()
|
||||
try:
|
||||
|
||||
@@ -14,7 +14,7 @@ from uuid import uuid4
|
||||
|
||||
from app import repository
|
||||
from app.contracts import Note, NoteBlock, NoteSummary
|
||||
from app.database.db import connect, transaction
|
||||
from app.database.db import connect_knowledge as connect, transaction
|
||||
from app.errors import ApiError
|
||||
from app.knowledge.parser import ParsedNote, parse_note
|
||||
from app.local_models.runtime import LocalEmbedding, background_embeddings
|
||||
|
||||
@@ -9,7 +9,7 @@ from weakref import WeakKeyDictionary
|
||||
|
||||
from app import repository
|
||||
from app.contracts import Task, TaskStatus
|
||||
from app.database.db import connect, transaction
|
||||
from app.database.db import connect_knowledge as connect, transaction
|
||||
from app.errors import ApiError
|
||||
from app.operation_logs import log_event
|
||||
|
||||
@@ -39,6 +39,13 @@ def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _prepare_note_link(note_id: str | None) -> None:
|
||||
from app.config import get_settings
|
||||
if note_id and get_settings().environment == 'desktop':
|
||||
from app.services.desktop_projection import _refresh
|
||||
_refresh()
|
||||
|
||||
|
||||
def _task_from_row(row) -> Task:
|
||||
return Task(
|
||||
task_id=row["task_id"],
|
||||
@@ -56,6 +63,7 @@ def create_task(
|
||||
*, title: str, description: str = "", note_id: str | None = None,
|
||||
due_at: datetime | None = None,
|
||||
) -> Task:
|
||||
_prepare_note_link(note_id)
|
||||
if note_id and repository.get_note_record(note_id) is None:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
||||
task_id = f"task_{uuid4().hex}"
|
||||
@@ -109,6 +117,7 @@ def update_task(task_id: str, values: dict[str, object]) -> Task:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "task not found", {"task_id": task_id})
|
||||
if "note_id" in values and values["note_id"]:
|
||||
note_id = str(values["note_id"])
|
||||
_prepare_note_link(note_id)
|
||||
if repository.get_note_record(note_id) is None:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
||||
if values.get("title") is None:
|
||||
|
||||
Reference in New Issue
Block a user