404 lines
16 KiB
Python
404 lines
16 KiB
Python
"""Note 服务：Markdown 文件读写 + 解析 + 索引编排。
|
|
|
|
Markdown 文件是笔记正文的持久化载体（Vault），SQLite/FTS5/向量是可重建索引。
|
|
本服务负责在两者之间保持一致：写文件后解析并写入元数据、FTS5 与向量。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sqlite3
|
|
from contextlib import nullcontext
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from app import repository
|
|
from app.contracts import Note, NoteBlock, NoteSummary
|
|
from app.database.db import connect, transaction
|
|
from app.errors import ApiError
|
|
from app.knowledge.parser import ParsedNote, parse_note
|
|
from app.local_models.runtime import LocalEmbedding, background_embeddings
|
|
from app.retrieval import routed_vectors
|
|
from app.retrieval.vectorstore import SqliteVecStore, VectorRecord
|
|
from app.services.coordination import serialized_vault_mutation
|
|
from app.services.vault_paths import (
|
|
normalize_entry_name,
|
|
normalize_folder,
|
|
resolve_in_vault,
|
|
safe_note_filename,
|
|
)
|
|
|
|
# 真实模型接口不在 API 进程加载权重；测试可显式替换该实例。
|
|
embedding = LocalEmbedding()
|
|
vector_store = SqliteVecStore()
|
|
|
|
|
|
def _rel_path(folder: str | None, title: str) -> tuple[str, str]:
|
|
"""由 folder + title 生成安全的相对路径，返回 (rel_path, 清洗后的 folder)。"""
|
|
clean_folder = normalize_folder(folder)
|
|
name = safe_note_filename(title)
|
|
rel = f"{clean_folder}/{name}" if clean_folder else name
|
|
return rel, clean_folder
|
|
|
|
|
|
def _read_markdown(rel_path: str) -> str:
|
|
path = resolve_in_vault(rel_path)
|
|
return path.read_text(encoding="utf-8") if path.exists() else ""
|
|
|
|
|
|
def _write_markdown(rel_path: str, markdown: str) -> None:
|
|
path = resolve_in_vault(rel_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(markdown, encoding="utf-8")
|
|
|
|
|
|
def _create_markdown(rel_path: str, markdown: str) -> None:
|
|
"""排他创建 Markdown；目标已存在时返回资源冲突，不覆盖用户文件。"""
|
|
path = resolve_in_vault(rel_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
with path.open("x", encoding="utf-8") as handle:
|
|
handle.write(markdown)
|
|
except FileExistsError as exc:
|
|
raise ApiError(
|
|
409,
|
|
"RESOURCE_CONFLICT",
|
|
"a note already exists at this path",
|
|
{"file_path": rel_path},
|
|
) from exc
|
|
|
|
|
|
def _delete_markdown(rel_path: str) -> None:
|
|
path = resolve_in_vault(rel_path)
|
|
if path.exists():
|
|
path.unlink()
|
|
|
|
|
|
PreparedIndex = tuple[list[list[float]], routed_vectors.RemoteEmbeddings | None]
|
|
|
|
|
|
@background_embeddings
|
|
async def prepare_note_index(parsed: ParsedNote, *, strict=False) -> PreparedIndex:
|
|
"""Compute vectors before opening a write transaction (including API I/O)."""
|
|
texts = [block.content for block in parsed.blocks]
|
|
if isinstance(embedding, LocalEmbedding):
|
|
# One routed invocation: API first, validated local fallback. No hash vectors.
|
|
remote = await routed_vectors.embed_remote(texts, accept_local=True, strict=strict, local_only=parsed.embedding_local_only)
|
|
return [], remote
|
|
vectors = await embedding.embed_documents(texts)
|
|
remote = await routed_vectors.embed_remote(texts, local_only=parsed.embedding_local_only)
|
|
return vectors, remote
|
|
|
|
|
|
async def index_note(
|
|
parsed: ParsedNote, *, prepared: PreparedIndex | None = None,
|
|
conn: sqlite3.Connection | None = None,
|
|
) -> None:
|
|
"""把解析结果写入元数据 + FTS5 + 向量（三层可重建索引），单事务保证原子性。
|
|
|
|
元数据与向量在同一连接、同一事务内提交，避免「新元数据已提交、向量写入失败」的
|
|
半提交状态。替换元数据时拿到旧 block_id：清理已删除/内容变化的旧向量，只为新增
|
|
block 写向量（内容未变的 block 其向量仍有效，无需重复写入）。
|
|
"""
|
|
if conn is not None and prepared is None:
|
|
raise ValueError("Prepare embeddings before supplying a write connection")
|
|
vectors, remote = prepared if prepared is not None else await prepare_note_index(parsed)
|
|
owns = conn is None
|
|
conn = conn or connect()
|
|
try:
|
|
with transaction(conn) if owns else nullcontext():
|
|
old_block_ids = repository.replace_note_metadata(
|
|
conn=conn,
|
|
note_id=parsed.note_id,
|
|
title=parsed.title,
|
|
file_path=parsed.file_path,
|
|
folder=parsed.folder,
|
|
tags=parsed.tags,
|
|
created_at=parsed.created_at,
|
|
updated_at=parsed.updated_at,
|
|
blocks=parsed.blocks,
|
|
)
|
|
old_ids = set(old_block_ids)
|
|
conn.execute("UPDATE blocks SET embedding_local_only=? WHERE note_id=?",
|
|
(int(parsed.embedding_local_only), parsed.note_id))
|
|
new_ids = {block.block_id for block in parsed.blocks}
|
|
stale_ids = [bid for bid in old_ids if bid not in new_ids]
|
|
if stale_ids:
|
|
await vector_store.delete(stale_ids, conn=conn)
|
|
missing_ids = [bid for bid in new_ids if bid not in old_ids]
|
|
records = [
|
|
VectorRecord(id=block.block_id, vector=vector)
|
|
for block, vector in zip(parsed.blocks, vectors)
|
|
if block.block_id in missing_ids
|
|
]
|
|
await vector_store.upsert(records, conn=conn)
|
|
routed_vectors.store_remote(conn, [block.block_id for block in parsed.blocks], remote)
|
|
repository.set_index_meta(
|
|
{"embedding_model": remote.space_id if remote and isinstance(embedding, LocalEmbedding) else embedding.model_id,
|
|
"embedding_dim": str(remote.dimensions if remote and isinstance(embedding, LocalEmbedding) else embedding.dim)},
|
|
conn=conn,
|
|
)
|
|
finally:
|
|
if owns:
|
|
conn.close()
|
|
|
|
|
|
@serialized_vault_mutation
|
|
async def create_note(*, title: str, markdown: str, folder: str | None, tags: list[str]) -> Note:
|
|
rel_path, clean_folder = _rel_path(folder, title)
|
|
now = datetime.now(timezone.utc)
|
|
parsed = parse_note(
|
|
# 创建时空标签视为「未显式指定」，由 frontmatter 推导（创建无「清空」语义）
|
|
markdown=markdown, file_path=rel_path, folder=clean_folder, tags=tags or None,
|
|
created_at=now, updated_at=now,
|
|
)
|
|
parsed.title = title # 显式传入的 title 优先于正文推导（与 update_note 保持一致）
|
|
if repository.get_note_record(parsed.note_id) is not None:
|
|
raise ApiError(
|
|
409,
|
|
"RESOURCE_CONFLICT",
|
|
"a note already exists at this path",
|
|
{"note_id": parsed.note_id, "file_path": rel_path},
|
|
)
|
|
_create_markdown(rel_path, markdown)
|
|
try:
|
|
await index_note(parsed)
|
|
except BaseException:
|
|
_delete_markdown(rel_path) # 索引失败时回滚，避免「文件已写、索引缺失」的部分提交
|
|
raise
|
|
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
|
|
parsed.created_at, parsed.updated_at, parsed.blocks, markdown)
|
|
|
|
|
|
async def get_note(note_id: str) -> Note | None:
|
|
from app.config import get_settings
|
|
if get_settings().environment == 'desktop':
|
|
from app.services.desktop_notes import get_note as desktop_get_note
|
|
return await desktop_get_note(note_id)
|
|
record = repository.get_note_record(note_id)
|
|
if record is None:
|
|
return None
|
|
markdown = _read_markdown(record.file_path)
|
|
return _build_note(record.note_id, record.title, record.file_path, record.tags,
|
|
record.created_at, record.updated_at, record.blocks, markdown)
|
|
|
|
|
|
@serialized_vault_mutation
|
|
async def update_note(
|
|
note_id: str, *, title: str | None = None, markdown: str | None = None, tags: list[str] | None = None, expected_content_hash: str | None = None, defer_vectors: bool = False
|
|
) -> Note:
|
|
record = repository.get_note_record(note_id)
|
|
if record is None:
|
|
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
|
|
|
old_md = _read_markdown(record.file_path)
|
|
if expected_content_hash is not None:
|
|
import hashlib
|
|
if hashlib.sha256(old_md.encode()).hexdigest() != expected_content_hash:
|
|
raise ApiError(409, "NOTE_CONTENT_CONFLICT", "笔记已被编辑，请保留现有内容或导出为新笔记。")
|
|
|
|
new_md = old_md if markdown is None else markdown
|
|
# PATCH 语义：tags=None 保持原标签；[] 清空；非空列表替换（区别于 create 的 frontmatter 推导）
|
|
effective_tags = record.tags if tags is None else tags
|
|
_write_markdown(record.file_path, new_md)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
try:
|
|
parsed = parse_note(
|
|
markdown=new_md, file_path=record.file_path, folder=record.folder, tags=effective_tags,
|
|
created_at=record.created_at, updated_at=now,
|
|
)
|
|
if title is not None:
|
|
parsed.title = title # 显式传入的 title 覆盖正文推导结果
|
|
|
|
if defer_vectors:
|
|
conn = connect()
|
|
try:
|
|
with transaction(conn):
|
|
old_ids = repository.replace_note_metadata(
|
|
conn=conn, note_id=parsed.note_id, title=parsed.title,
|
|
file_path=parsed.file_path, folder=parsed.folder, tags=parsed.tags,
|
|
created_at=parsed.created_at, updated_at=parsed.updated_at, blocks=parsed.blocks,
|
|
)
|
|
# Saved content is immediately searchable; old vectors must not describe it.
|
|
await vector_store.delete(old_ids, conn=conn)
|
|
conn.execute('UPDATE blocks SET embedding_local_only=? WHERE note_id=?',
|
|
(int(parsed.embedding_local_only), parsed.note_id))
|
|
repository.set_index_meta({f'note_vectors_pending:{parsed.note_id}': '1'}, conn=conn)
|
|
finally:
|
|
conn.close()
|
|
else:
|
|
await index_note(parsed)
|
|
except BaseException:
|
|
_write_markdown(record.file_path, old_md) # 索引失败时回滚正文，避免部分提交
|
|
raise
|
|
if defer_vectors:
|
|
from app.services import index_service
|
|
index_service.schedule_workspace_rebuild()
|
|
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
|
|
parsed.created_at, parsed.updated_at, parsed.blocks, new_md)
|
|
|
|
|
|
@serialized_vault_mutation
|
|
async def move_note(note_id: str, *, folder: str) -> Note:
|
|
record = repository.get_note_record(note_id)
|
|
if record is None:
|
|
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
|
|
|
clean_folder = normalize_folder(folder)
|
|
filename = Path(record.file_path).name
|
|
new_rel_path = f"{clean_folder}/{filename}" if clean_folder else filename
|
|
if new_rel_path == record.file_path:
|
|
note = await get_note(note_id)
|
|
assert note is not None
|
|
return note
|
|
|
|
source = resolve_in_vault(record.file_path)
|
|
target = resolve_in_vault(new_rel_path)
|
|
if not source.is_file():
|
|
raise ApiError(
|
|
409, "NOTE_FILE_MISSING", "note file is missing from the Vault",
|
|
{"note_id": note_id, "file_path": record.file_path},
|
|
)
|
|
if target.exists():
|
|
raise ApiError(
|
|
409, "RESOURCE_CONFLICT", "a note already exists at the target path",
|
|
{"note_id": note_id, "file_path": new_rel_path},
|
|
)
|
|
|
|
markdown = source.read_text(encoding="utf-8")
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
source.replace(target)
|
|
try:
|
|
parsed = parse_note(
|
|
markdown=markdown,
|
|
file_path=new_rel_path,
|
|
folder=clean_folder,
|
|
tags=record.tags,
|
|
created_at=record.created_at,
|
|
updated_at=datetime.now(timezone.utc),
|
|
note_id=record.note_id,
|
|
)
|
|
parsed.title = record.title
|
|
await index_note(parsed)
|
|
except BaseException:
|
|
target.replace(source)
|
|
raise
|
|
return _build_note(
|
|
parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
|
|
parsed.created_at, parsed.updated_at, parsed.blocks, markdown,
|
|
)
|
|
|
|
|
|
@serialized_vault_mutation
|
|
async def rename_note(note_id: str, *, file_name: str) -> Note:
|
|
"""重命名 Markdown 文件并保留 note_id、Block 与向量身份。"""
|
|
|
|
record = repository.get_note_record(note_id)
|
|
if record is None:
|
|
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
|
|
|
normalized = normalize_entry_name(file_name, markdown=True)
|
|
source = resolve_in_vault(record.file_path)
|
|
folder = normalize_folder(record.folder)
|
|
new_file_path = f"{folder}/{normalized}" if folder else normalized
|
|
target = resolve_in_vault(new_file_path)
|
|
if new_file_path == record.file_path:
|
|
note = await get_note(note_id)
|
|
assert note is not None
|
|
return note
|
|
if not source.is_file():
|
|
raise ApiError(
|
|
409,
|
|
"NOTE_FILE_MISSING",
|
|
"note file is missing from the Vault",
|
|
{"note_id": note_id, "file_path": record.file_path},
|
|
)
|
|
if target.exists():
|
|
raise ApiError(
|
|
409,
|
|
"RESOURCE_CONFLICT",
|
|
"a note already exists with the requested file name",
|
|
{"note_id": note_id, "file_path": new_file_path},
|
|
)
|
|
|
|
source.replace(target)
|
|
now = datetime.now(timezone.utc)
|
|
conn = connect()
|
|
try:
|
|
with transaction(conn):
|
|
repository.update_note_location(
|
|
conn=conn,
|
|
note_id=note_id,
|
|
title=Path(normalized).stem,
|
|
file_path=new_file_path,
|
|
folder=folder,
|
|
updated_at=now,
|
|
)
|
|
except BaseException:
|
|
target.replace(source)
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
note = await get_note(note_id)
|
|
assert note is not None
|
|
return note
|
|
|
|
|
|
@serialized_vault_mutation
|
|
async def delete_note(note_id: str) -> bool:
|
|
record = repository.get_note_record(note_id)
|
|
if record is None:
|
|
return False
|
|
|
|
path = resolve_in_vault(record.file_path)
|
|
tombstone = path.with_name(f".{path.name}.{uuid4().hex}.deleting") if path.exists() else None
|
|
if tombstone is not None:
|
|
path.replace(tombstone)
|
|
|
|
conn = connect()
|
|
try:
|
|
with transaction(conn):
|
|
block_ids = repository.delete_note(note_id, conn=conn)
|
|
await vector_store.delete(block_ids, conn=conn)
|
|
except BaseException:
|
|
if tombstone is not None and tombstone.exists():
|
|
tombstone.replace(path)
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
if tombstone is not None:
|
|
# 数据库已提交后，tombstone 即不再属于 Vault；清理失败不应把成功删除报告为失败。
|
|
try:
|
|
tombstone.unlink(missing_ok=True)
|
|
except OSError:
|
|
pass
|
|
return True
|
|
|
|
|
|
def list_notes(*, limit: int, offset: int, folder: str | None, tag: str | None) -> tuple[list[NoteSummary], int]:
|
|
from app.config import get_settings
|
|
if get_settings().environment == 'desktop':
|
|
from app.services.desktop_notes import list_notes as desktop_list_notes
|
|
return desktop_list_notes(limit=limit, offset=offset, folder=folder, tag=tag)
|
|
items, total = repository.list_note_summaries(limit=limit, offset=offset, folder=folder, tag=tag)
|
|
return [NoteSummary(**item) for item in items], total
|
|
|
|
|
|
def _build_note(
|
|
note_id: str, title: str, file_path: str, tags: list[str],
|
|
created_at: datetime, updated_at: datetime, blocks: list[NoteBlock], markdown: str,
|
|
) -> Note:
|
|
return Note(
|
|
note_id=note_id,
|
|
title=title,
|
|
file_path=file_path,
|
|
tags=tags,
|
|
created_at=created_at,
|
|
updated_at=updated_at,
|
|
markdown=markdown,
|
|
blocks=blocks,
|
|
)
|