diff --git a/backend/app/database/__init__.py b/backend/app/database/__init__.py new file mode 100644 index 0000000..47699eb --- /dev/null +++ b/backend/app/database/__init__.py @@ -0,0 +1 @@ +"""数据库访问层:连接管理、迁移与 Repository。""" diff --git a/backend/app/database/db.py b/backend/app/database/db.py new file mode 100644 index 0000000..b2de78c --- /dev/null +++ b/backend/app/database/db.py @@ -0,0 +1,49 @@ +"""SQLite 连接管理。 + +每次调用 connect() 打开一个新连接:加载 sqlite-vec 扩展、打开外键、应用迁移。 +连接由调用方负责关闭;写入通过 transaction() 上下文显式控制提交,避免隐式事务带来的 +半提交状态。 +""" + +import sqlite3 +from contextlib import contextmanager +from collections.abc import Iterator + +import sqlite_vec + +from app.config import get_settings +from app.database.migrations import migrate + + +def _load_extension(conn: sqlite3.Connection) -> None: + """在当前连接上注册 sqlite-vec 扩展,随后关闭 load_extension 开关。""" + conn.enable_load_extension(True) + try: + sqlite_vec.load(conn) + finally: + conn.enable_load_extension(False) + + +def connect() -> sqlite3.Connection: + settings = get_settings() + settings.db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(settings.db_path) + conn.row_factory = sqlite3.Row + # 关闭 Python sqlite3 的隐式事务,提交时机由 transaction() 或显式 commit 控制。 + conn.isolation_level = None + conn.execute("PRAGMA foreign_keys = ON") + _load_extension(conn) + migrate(conn) + return conn + + +@contextmanager +def transaction(conn: sqlite3.Connection) -> Iterator[None]: + """显式事务:提交成功则 COMMIT,异常则 ROLLBACK。""" + conn.execute("BEGIN") + try: + yield + conn.execute("COMMIT") + except BaseException: + conn.execute("ROLLBACK") + raise diff --git a/backend/app/database/migrations.py b/backend/app/database/migrations.py new file mode 100644 index 0000000..b79b3c5 --- /dev/null +++ b/backend/app/database/migrations.py @@ -0,0 +1,77 @@ +"""轻量 schema 迁移。 + +约定:MIGRATIONS 列表按版本号顺序排列,只增不改。每一条是一个完整的 SQL 脚本, +执行后写入 schema_migrations 记录版本。修改 Schema 时在末尾追加新脚本,禁止删旧脚本或 +依赖运行时自动删表重建(团队约定)。 +""" + +from datetime import datetime, timezone + +from app.constants import EMBEDDING_DIM + +# 每个元素对应一个版本(下标 + 1)。vec0 建表需要本连接已加载 sqlite-vec 扩展, +# 由 db.connect() 在调用 migrate 之前完成。 +MIGRATIONS: list[str] = [ + # v1: 笔记元数据 + Block + FTS5 全文索引 + 向量表 + 索引元信息 + f""" + CREATE TABLE IF NOT EXISTS notes ( + note_id TEXT PRIMARY KEY, + title TEXT NOT NULL, + file_path TEXT NOT NULL UNIQUE, + folder TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '[]', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS blocks ( + block_id TEXT PRIMARY KEY, + note_id TEXT NOT NULL REFERENCES notes(note_id) ON DELETE CASCADE, + heading_path TEXT NOT NULL DEFAULT '[]', + start_offset INTEGER NOT NULL, + end_offset INTEGER NOT NULL, + content TEXT NOT NULL, + content_hash TEXT NOT NULL, + token_count INTEGER NOT NULL, + position INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_blocks_note ON blocks(note_id, position); + + CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5( + block_id UNINDEXED, + note_id UNINDEXED, + heading_path, + content, + tokenize = 'unicode61' + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS vec_blocks USING vec0( + block_id TEXT PRIMARY KEY, + embedding float[{EMBEDDING_DIM}] + ); + + CREATE TABLE IF NOT EXISTS index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """, +] + + +def migrate(conn) -> None: + """把尚未应用的迁移脚本按序应用到给定连接。""" + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)" + ) + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + + for idx, script in enumerate(MIGRATIONS, start=1): + if idx in applied: + continue + conn.executescript(script) + conn.execute( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)", + (idx, datetime.now(timezone.utc).isoformat()), + ) + conn.commit() diff --git a/backend/app/knowledge/__init__.py b/backend/app/knowledge/__init__.py new file mode 100644 index 0000000..21f9aef --- /dev/null +++ b/backend/app/knowledge/__init__.py @@ -0,0 +1 @@ +"""Knowledge Core:Markdown 解析与 Note Block 生成。""" diff --git a/backend/app/knowledge/parser.py b/backend/app/knowledge/parser.py new file mode 100644 index 0000000..3ab4bf3 --- /dev/null +++ b/backend/app/knowledge/parser.py @@ -0,0 +1,193 @@ +"""Markdown 解析与 Note Block 切分。 + +Block 由 Markdown 文本生成:标题行独立成块(heading_path 含自身),正文按空行分段, +每块记录其在原文中的 start_offset / end_offset,用于 Citation 跳转定位。block_id 由 +(note_id, heading_path, content) 稳定派生,内容不变则 ID 稳定。 +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path + +from app.contracts import NoteBlock +from app.textutils import count_tokens + +_HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.*?)\s*$") +_FRONTMATTER_KEY_RE = re.compile(r"^([A-Za-z0-9_-]+)\s*:\s*(.*)$") + + +@dataclass +class ParsedNote: + note_id: str + title: str + file_path: str + folder: str + tags: list[str] + created_at: datetime + updated_at: datetime + blocks: list[NoteBlock] = field(default_factory=list) + + +def note_id_for_path(rel_path: str) -> str: + """由相对路径派生稳定 note_id(路径哈希而非路径本身,见团队约定「不用路径当 ID」)。 + + MVP 阶段 ID 随文件移动而变化;后续 move 流程会保留原 ID。""" + normalized = rel_path.replace("\\", "/").strip("/") + return "note_" + hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16] + + +def parse_note( + *, + markdown: str, + file_path: str, + folder: str, + tags: list[str] | None = None, + created_at: datetime, + updated_at: datetime, +) -> ParsedNote: + """解析一篇 Markdown,生成 ParsedNote(元数据 + Block 列表)。""" + note_id = note_id_for_path(file_path) + frontmatter = _extract_frontmatter(markdown) + + fallback_title = Path(file_path).stem + title = frontmatter.get("title") or _first_heading(markdown) or fallback_title + resolved_tags = list(tags) if tags else _parse_tags(frontmatter.get("tags")) + + blocks = parse_blocks(markdown, note_id) + return ParsedNote( + note_id=note_id, + title=title, + file_path=file_path, + folder=folder, + tags=resolved_tags, + created_at=created_at, + updated_at=updated_at, + blocks=blocks, + ) + + +def parse_blocks(markdown: str, note_id: str) -> list[NoteBlock]: + """把 Markdown 切成 Block,offset 相对原文(含 frontmatter)。""" + lines = _split_lines(markdown) + content_start = _content_start(markdown) + + blocks: list[NoteBlock] = [] + heading_stack: list[str] = [] + body: list[tuple[str, int]] = [] + id_counters: dict[str, int] = {} + + def make_block(path: list[str], chunk: list[tuple[str, int]]) -> None: + if not chunk: + return + content = "\n".join(line for line, _ in chunk) + start = chunk[0][1] + end = chunk[-1][1] + len(chunk[-1][0]) + block_id = _stable_block_id(note_id, path, content, id_counters) + blocks.append( + NoteBlock( + block_id=block_id, + note_id=note_id, + heading_path=list(path), + start_offset=start, + end_offset=end, + content=content, + content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest()[:16], + token_count=count_tokens(content), + ) + ) + + def flush_body() -> None: + nonlocal body + make_block(heading_stack, body) + body = [] + + for line, offset in lines: + if offset < content_start: + continue # 跳过 frontmatter 区域,但保留 offset 准确性 + + heading = _HEADING_RE.match(line) + if heading: + flush_body() + level = len(heading.group(1)) + title = heading.group(2).strip() + heading_stack = heading_stack[: level - 1] + [title] + # 标题自身作为一个 Block,便于按章节定位 + make_block(heading_stack, [(line, offset)]) + elif line.strip() == "": + flush_body() # 空行分隔段落 + else: + body.append((line, offset)) + + flush_body() + return blocks + + +def _stable_block_id(note_id: str, path: list[str], content: str, counters: dict[str, int]) -> str: + base = hashlib.sha256( + f"{note_id}\x1f{chr(31).join(path)}\x1f{content}".encode("utf-8") + ).hexdigest()[:16] + block_id = f"blk_{base}" + # 同一篇笔记内极少出现的重复段落用后缀消歧,保证唯一 + n = counters.get(block_id, 0) + counters[block_id] = n + 1 + return block_id if n == 0 else f"{block_id}_{n}" + + +def _split_lines(text: str) -> list[tuple[str, int]]: + """按行拆分并记录每行在原文中的起始字符偏移。""" + result: list[tuple[str, int]] = [] + start = 0 + for raw in text.splitlines(keepends=True): + line = raw + if line.endswith("\r\n"): + line = line[:-2] + elif line.endswith("\n") or line.endswith("\r"): + line = line[:-1] + result.append((line, start)) + start += len(raw) + return result + + +def _content_start(markdown: str) -> int: + """返回正文起始偏移:有 frontmatter 时跳过 --- 分隔块,否则为 0。""" + if markdown.startswith("---"): + end = markdown.find("\n---", 3) + if end != -1: + return end + 4 + return 0 + + +def _extract_frontmatter(markdown: str) -> dict[str, str]: + """极简 frontmatter 解析,只提取 key: value 行。""" + if not markdown.startswith("---"): + return {} + end = markdown.find("\n---", 3) + if end == -1: + return {} + meta: dict[str, str] = {} + for line in markdown[3:end].splitlines(): + m = _FRONTMATTER_KEY_RE.match(line) + if m: + meta[m.group(1).lower()] = m.group(2).strip() + return meta + + +def _first_heading(markdown: str) -> str | None: + for line in markdown.splitlines(): + m = re.match(r"^#\s+(.*?)\s*$", line) + if m and m.group(1).strip(): + return m.group(1).strip() + return None + + +def _parse_tags(raw: str | None) -> list[str]: + if not raw: + return [] + raw = raw.strip() + if raw.startswith("[") and raw.endswith("]"): + raw = raw[1:-1] + return [t.strip().strip("'\"") for t in raw.split(",") if t.strip()] diff --git a/backend/app/repository.py b/backend/app/repository.py new file mode 100644 index 0000000..3a4582c --- /dev/null +++ b/backend/app/repository.py @@ -0,0 +1,299 @@ +"""SQLite Repository:笔记元数据、Block 与 FTS5 的读写。 + +向量(vec_blocks)不在这里处理,交给 Retrieval 基础设施层的 VectorStore(见 +app/retrieval/vectorstore.py)。本层只负责 notes / blocks / blocks_fts 三张表的访问, +返回领域记录(NoteRecord / BlockHit / FtsHit),不负责业务编排。 +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime + +from app.contracts import NoteBlock +from app.database.db import connect, transaction +from app.textutils import segment + + +def _iso(dt: datetime) -> str: + return dt.isoformat() + + +def _parse_dt(value: str) -> datetime: + return datetime.fromisoformat(value) + + +@dataclass +class NoteRecord: + note_id: str + title: str + file_path: str + folder: str + tags: list[str] + created_at: datetime + updated_at: datetime + blocks: list[NoteBlock] = field(default_factory=list) + + +@dataclass +class BlockHit: + """检索时返回的完整 Block 上下文,用于组装 Citation 与 metadata 过滤。""" + + block_id: str + note_id: str + title: str + file_path: str + folder: str + heading_path: list[str] + content: str + start_offset: int + end_offset: int + tags: list[str] + created_at: datetime + updated_at: datetime + + +@dataclass +class FtsHit: + block_id: str + note_id: str + bm25: float + + +def replace_note_metadata( + *, + note_id: str, + title: str, + file_path: str, + folder: str, + tags: list[str], + created_at: datetime, + updated_at: datetime, + blocks: list[NoteBlock], +) -> None: + """整体替换一条笔记的元数据、Block 与 FTS5 索引(单事务)。""" + conn = connect() + try: + with transaction(conn): + conn.execute( + """ + INSERT INTO notes (note_id, title, file_path, folder, tags, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(note_id) DO UPDATE SET + title = excluded.title, + file_path = excluded.file_path, + folder = excluded.folder, + tags = excluded.tags, + updated_at = excluded.updated_at + """, + (note_id, title, file_path, folder, json.dumps(tags, ensure_ascii=False), + _iso(created_at), _iso(updated_at)), + ) + conn.execute("DELETE FROM blocks WHERE note_id = ?", (note_id,)) + conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,)) + for position, block in enumerate(blocks): + conn.execute( + """ + INSERT INTO blocks + (block_id, note_id, heading_path, start_offset, end_offset, + content, content_hash, token_count, position) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + (block.block_id, note_id, json.dumps(block.heading_path, ensure_ascii=False), + block.start_offset, block.end_offset, block.content, + block.content_hash, block.token_count, position), + ) + # FTS5 存分词后的可检索文本;原文仍由 blocks.content 保留用于展示 + conn.execute( + "INSERT INTO blocks_fts (block_id, note_id, heading_path, content) VALUES (?, ?, ?, ?)", + (block.block_id, note_id, segment(" ".join(block.heading_path)), segment(block.content)), + ) + finally: + conn.close() + + +def delete_note(note_id: str) -> list[str]: + """删除笔记及其 Block、FTS5 索引;返回被删除的 block_id 供向量层清理。""" + conn = connect() + try: + block_ids = [ + row["block_id"] + for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,)) + ] + with transaction(conn): + conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,)) + conn.execute("DELETE FROM notes WHERE note_id = ?", (note_id,)) # blocks 级联删除 + return block_ids + finally: + conn.close() + + +def get_note_record(note_id: str) -> NoteRecord | None: + conn = connect() + try: + row = conn.execute("SELECT * FROM notes WHERE note_id = ?", (note_id,)).fetchone() + if row is None: + return None + blocks = [ + _block_from_row(b) + for b in conn.execute("SELECT * FROM blocks WHERE note_id = ? ORDER BY position", (note_id,)) + ] + return NoteRecord( + note_id=row["note_id"], + title=row["title"], + file_path=row["file_path"], + folder=row["folder"], + tags=json.loads(row["tags"] or "[]"), + created_at=_parse_dt(row["created_at"]), + updated_at=_parse_dt(row["updated_at"]), + blocks=blocks, + ) + finally: + conn.close() + + +def list_note_summaries( + *, limit: int = 50, offset: int = 0, folder: str | None = None, tag: str | None = None +) -> tuple[list, int]: + conn = connect() + try: + where: list[str] = [] + params: list[str] = [] + if folder: + where.append("folder = ?") + params.append(folder) + if tag: + where.append("EXISTS (SELECT 1 FROM json_each(notes.tags) AS j WHERE j.value = ?)") + params.append(tag) + where_sql = ("WHERE " + " AND ".join(where)) if where else "" + + total = conn.execute(f"SELECT COUNT(*) FROM notes {where_sql}", params).fetchone()[0] + rows = conn.execute( + f"SELECT * FROM notes {where_sql} ORDER BY updated_at DESC LIMIT ? OFFSET ?", + params + [limit, offset], + ).fetchall() + + items = [ + { + "note_id": r["note_id"], + "title": r["title"], + "file_path": r["file_path"], + "tags": json.loads(r["tags"] or "[]"), + "created_at": _parse_dt(r["created_at"]), + "updated_at": _parse_dt(r["updated_at"]), + } + for r in rows + ] + return items, total + finally: + conn.close() + + +def fts_search(match: str, limit: int = 100) -> list[FtsHit]: + conn = connect() + try: + rows = conn.execute( + """ + SELECT block_id, note_id, bm25(blocks_fts) AS rank + FROM blocks_fts + WHERE blocks_fts MATCH ? + ORDER BY rank + LIMIT ? + """, + (match, limit), + ).fetchall() + return [FtsHit(block_id=r["block_id"], note_id=r["note_id"], bm25=r["rank"]) for r in rows] + finally: + conn.close() + + +def get_block_hits(block_ids: list[str]) -> list[BlockHit]: + if not block_ids: + return [] + conn = connect() + try: + placeholders = ",".join("?" * len(block_ids)) + rows = conn.execute( + f""" + SELECT b.block_id, b.note_id, b.heading_path, b.start_offset, b.end_offset, b.content, + n.title, n.file_path, n.folder, n.tags, n.created_at, n.updated_at + FROM blocks b + JOIN notes n ON n.note_id = b.note_id + WHERE b.block_id IN ({placeholders}) + """, + block_ids, + ).fetchall() + return [_block_hit_from_row(r) for r in rows] + finally: + conn.close() + + +def set_index_meta(kv: dict[str, str]) -> None: + conn = connect() + try: + with transaction(conn): + for key, value in kv.items(): + conn.execute("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)", (key, value)) + finally: + conn.close() + + +def get_index_meta() -> dict[str, str]: + conn = connect() + try: + return {r["key"]: r["value"] for r in conn.execute("SELECT key, value FROM index_meta")} + finally: + conn.close() + + +def clear_all() -> None: + """清空元数据、Block 与 FTS5(重建索引用,向量由 VectorStore.clear 处理)。""" + conn = connect() + try: + with transaction(conn): + conn.execute("DELETE FROM blocks_fts") + conn.execute("DELETE FROM blocks") + conn.execute("DELETE FROM notes") + finally: + conn.close() + + +def stats() -> dict[str, int]: + conn = connect() + try: + notes = conn.execute("SELECT COUNT(*) AS c FROM notes").fetchone()["c"] + blocks = conn.execute("SELECT COUNT(*) AS c FROM blocks").fetchone()["c"] + return {"notes": notes, "blocks": blocks} + finally: + conn.close() + + +def _block_from_row(row) -> NoteBlock: + return NoteBlock( + block_id=row["block_id"], + note_id=row["note_id"], + heading_path=json.loads(row["heading_path"] or "[]"), + start_offset=row["start_offset"], + end_offset=row["end_offset"], + content=row["content"], + content_hash=row["content_hash"], + token_count=row["token_count"], + ) + + +def _block_hit_from_row(row) -> BlockHit: + return BlockHit( + block_id=row["block_id"], + note_id=row["note_id"], + title=row["title"], + file_path=row["file_path"], + folder=row["folder"], + heading_path=json.loads(row["heading_path"] or "[]"), + content=row["content"], + start_offset=row["start_offset"], + end_offset=row["end_offset"], + tags=json.loads(row["tags"] or "[]"), + created_at=_parse_dt(row["created_at"]), + updated_at=_parse_dt(row["updated_at"]), + ) diff --git a/backend/app/textutils.py b/backend/app/textutils.py new file mode 100644 index 0000000..0f408fd --- /dev/null +++ b/backend/app/textutils.py @@ -0,0 +1,82 @@ +"""文本分词工具,供 FTS5 与轻量 Embedding 共用。 + +FTS5 默认 unicode61 分词器不切分中文(连续汉字算一个 token),导致「死锁」这类 +子串无法命中。这里把文本统一拆成 ASCII 单词 + CJK 单字 + CJK 相邻双字,写入与查询 +走同一套拆分,实现中文子串/词级召回。 +""" + +import re + +_WORD_RE = re.compile(r"[a-z0-9]+") +_CJK_RUN_RE = re.compile(r"[一-鿿]+") + + +def tokens(text: str) -> list[str]: + """返回文本的检索 token 序列(含重复)。""" + out: list[str] = [] + out.extend(_WORD_RE.findall(text.lower())) + for run in _CJK_RUN_RE.findall(text): + out.extend(run) # 单字 + out.extend(run[i : i + 2] for i in range(len(run) - 1)) # 相邻双字 + return out + + +def unique_tokens(text: str) -> list[str]: + """去重但保序的 token 列表,用于压缩查询/索引体积。""" + seen: set[str] = set() + result: list[str] = [] + for tok in tokens(text): + if tok not in seen: + seen.add(tok) + result.append(tok) + return result + + +def segment(text: str) -> str: + """把文本转成空格分隔的 token,写入 FTS5 的 content 列。""" + return " ".join(unique_tokens(text)) + + +def match_query(query: str) -> str | None: + """把用户查询转成 FTS5 MATCH 表达式(OR 连接、按 token 精确匹配)。""" + toks = unique_tokens(query) + if not toks: + return None + return " OR ".join(f'"{tok}"' for tok in toks) + + +def count_tokens(text: str) -> int: + """粗略 token 数:ASCII 单词 + CJK 单字。仅用于展示,不做精确计量。""" + return len(_WORD_RE.findall(text.lower())) + sum(len(run) for run in _CJK_RUN_RE.findall(text)) + + +def make_snippet(content: str, query: str, max_len: int = 160) -> str: + """从原文生成命中片段:优先定位较长的 query token,向前后扩展窗口。""" + toks = unique_tokens(query) + lowered = content.lower() + best = -1 + # 优先用较长的 token(双字/单词)定位,命中最准确 + for tok in toks: + if len(tok) >= 2: + idx = lowered.find(tok) + if idx != -1: + best = idx + break + if best == -1: + for tok in toks: + idx = lowered.find(tok) + if idx != -1: + best = idx + break + + if best == -1: + snippet = content + else: + start = max(0, best - max_len // 3) + end = min(len(content), best + max_len) + snippet = content[start:end] + snippet = ("…" if start > 0 else "") + snippet + ("…" if end < len(content) else "") + + if len(snippet) > max_len + 20: + snippet = snippet[:max_len] + "…" + return snippet