feat(knowledge): 实现 SQLite/FTS5 数据层与 Markdown Block 解析
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1 @@
|
||||
"""数据库访问层：连接管理、迁移与 Repository。"""
|
||||
@@ -0,0 +1,49 @@
|
||||
"""SQLite 连接管理。
|
||||
|
||||
每次调用 connect() 打开一个新连接：加载 sqlite-vec 扩展、打开外键、应用迁移。
|
||||
连接由调用方负责关闭；写入通过 transaction() 上下文显式控制提交，避免隐式事务带来的
|
||||
半提交状态。
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from collections.abc import Iterator
|
||||
|
||||
import sqlite_vec
|
||||
|
||||
from app.config import get_settings
|
||||
from app.database.migrations import migrate
|
||||
|
||||
|
||||
def _load_extension(conn: sqlite3.Connection) -> None:
|
||||
"""在当前连接上注册 sqlite-vec 扩展，随后关闭 load_extension 开关。"""
|
||||
conn.enable_load_extension(True)
|
||||
try:
|
||||
sqlite_vec.load(conn)
|
||||
finally:
|
||||
conn.enable_load_extension(False)
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
settings = get_settings()
|
||||
settings.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(settings.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
# 关闭 Python sqlite3 的隐式事务，提交时机由 transaction() 或显式 commit 控制。
|
||||
conn.isolation_level = None
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
_load_extension(conn)
|
||||
migrate(conn)
|
||||
return conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def transaction(conn: sqlite3.Connection) -> Iterator[None]:
|
||||
"""显式事务：提交成功则 COMMIT，异常则 ROLLBACK。"""
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
yield
|
||||
conn.execute("COMMIT")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
@@ -0,0 +1,77 @@
|
||||
"""轻量 schema 迁移。
|
||||
|
||||
约定：MIGRATIONS 列表按版本号顺序排列，只增不改。每一条是一个完整的 SQL 脚本，
|
||||
执行后写入 schema_migrations 记录版本。修改 Schema 时在末尾追加新脚本，禁止删旧脚本或
|
||||
依赖运行时自动删表重建（团队约定）。
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.constants import EMBEDDING_DIM
|
||||
|
||||
# 每个元素对应一个版本（下标 + 1）。vec0 建表需要本连接已加载 sqlite-vec 扩展，
|
||||
# 由 db.connect() 在调用 migrate 之前完成。
|
||||
MIGRATIONS: list[str] = [
|
||||
# v1: 笔记元数据 + Block + FTS5 全文索引 + 向量表 + 索引元信息
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
note_id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL UNIQUE,
|
||||
folder TEXT NOT NULL DEFAULT '',
|
||||
tags TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blocks (
|
||||
block_id TEXT PRIMARY KEY,
|
||||
note_id TEXT NOT NULL REFERENCES notes(note_id) ON DELETE CASCADE,
|
||||
heading_path TEXT NOT NULL DEFAULT '[]',
|
||||
start_offset INTEGER NOT NULL,
|
||||
end_offset INTEGER NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
token_count INTEGER NOT NULL,
|
||||
position INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_blocks_note ON blocks(note_id, position);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(
|
||||
block_id UNINDEXED,
|
||||
note_id UNINDEXED,
|
||||
heading_path,
|
||||
content,
|
||||
tokenize = 'unicode61'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vec_blocks USING vec0(
|
||||
block_id TEXT PRIMARY KEY,
|
||||
embedding float[{EMBEDDING_DIM}]
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS index_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
""",
|
||||
]
|
||||
|
||||
|
||||
def migrate(conn) -> None:
|
||||
"""把尚未应用的迁移脚本按序应用到给定连接。"""
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)"
|
||||
)
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
|
||||
for idx, script in enumerate(MIGRATIONS, start=1):
|
||||
if idx in applied:
|
||||
continue
|
||||
conn.executescript(script)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)",
|
||||
(idx, datetime.now(timezone.utc).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
@@ -0,0 +1 @@
|
||||
"""Knowledge Core：Markdown 解析与 Note Block 生成。"""
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Markdown 解析与 Note Block 切分。
|
||||
|
||||
Block 由 Markdown 文本生成：标题行独立成块（heading_path 含自身），正文按空行分段，
|
||||
每块记录其在原文中的 start_offset / end_offset，用于 Citation 跳转定位。block_id 由
|
||||
(note_id, heading_path, content) 稳定派生，内容不变则 ID 稳定。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.contracts import NoteBlock
|
||||
from app.textutils import count_tokens
|
||||
|
||||
_HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.*?)\s*$")
|
||||
_FRONTMATTER_KEY_RE = re.compile(r"^([A-Za-z0-9_-]+)\s*:\s*(.*)$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedNote:
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
blocks: list[NoteBlock] = field(default_factory=list)
|
||||
|
||||
|
||||
def note_id_for_path(rel_path: str) -> str:
|
||||
"""由相对路径派生稳定 note_id（路径哈希而非路径本身，见团队约定「不用路径当 ID」）。
|
||||
|
||||
MVP 阶段 ID 随文件移动而变化；后续 move 流程会保留原 ID。"""
|
||||
normalized = rel_path.replace("\\", "/").strip("/")
|
||||
return "note_" + hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def parse_note(
|
||||
*,
|
||||
markdown: str,
|
||||
file_path: str,
|
||||
folder: str,
|
||||
tags: list[str] | None = None,
|
||||
created_at: datetime,
|
||||
updated_at: datetime,
|
||||
) -> ParsedNote:
|
||||
"""解析一篇 Markdown，生成 ParsedNote（元数据 + Block 列表）。"""
|
||||
note_id = note_id_for_path(file_path)
|
||||
frontmatter = _extract_frontmatter(markdown)
|
||||
|
||||
fallback_title = Path(file_path).stem
|
||||
title = frontmatter.get("title") or _first_heading(markdown) or fallback_title
|
||||
resolved_tags = list(tags) if tags else _parse_tags(frontmatter.get("tags"))
|
||||
|
||||
blocks = parse_blocks(markdown, note_id)
|
||||
return ParsedNote(
|
||||
note_id=note_id,
|
||||
title=title,
|
||||
file_path=file_path,
|
||||
folder=folder,
|
||||
tags=resolved_tags,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
blocks=blocks,
|
||||
)
|
||||
|
||||
|
||||
def parse_blocks(markdown: str, note_id: str) -> list[NoteBlock]:
|
||||
"""把 Markdown 切成 Block，offset 相对原文（含 frontmatter）。"""
|
||||
lines = _split_lines(markdown)
|
||||
content_start = _content_start(markdown)
|
||||
|
||||
blocks: list[NoteBlock] = []
|
||||
heading_stack: list[str] = []
|
||||
body: list[tuple[str, int]] = []
|
||||
id_counters: dict[str, int] = {}
|
||||
|
||||
def make_block(path: list[str], chunk: list[tuple[str, int]]) -> None:
|
||||
if not chunk:
|
||||
return
|
||||
content = "\n".join(line for line, _ in chunk)
|
||||
start = chunk[0][1]
|
||||
end = chunk[-1][1] + len(chunk[-1][0])
|
||||
block_id = _stable_block_id(note_id, path, content, id_counters)
|
||||
blocks.append(
|
||||
NoteBlock(
|
||||
block_id=block_id,
|
||||
note_id=note_id,
|
||||
heading_path=list(path),
|
||||
start_offset=start,
|
||||
end_offset=end,
|
||||
content=content,
|
||||
content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest()[:16],
|
||||
token_count=count_tokens(content),
|
||||
)
|
||||
)
|
||||
|
||||
def flush_body() -> None:
|
||||
nonlocal body
|
||||
make_block(heading_stack, body)
|
||||
body = []
|
||||
|
||||
for line, offset in lines:
|
||||
if offset < content_start:
|
||||
continue # 跳过 frontmatter 区域，但保留 offset 准确性
|
||||
|
||||
heading = _HEADING_RE.match(line)
|
||||
if heading:
|
||||
flush_body()
|
||||
level = len(heading.group(1))
|
||||
title = heading.group(2).strip()
|
||||
heading_stack = heading_stack[: level - 1] + [title]
|
||||
# 标题自身作为一个 Block，便于按章节定位
|
||||
make_block(heading_stack, [(line, offset)])
|
||||
elif line.strip() == "":
|
||||
flush_body() # 空行分隔段落
|
||||
else:
|
||||
body.append((line, offset))
|
||||
|
||||
flush_body()
|
||||
return blocks
|
||||
|
||||
|
||||
def _stable_block_id(note_id: str, path: list[str], content: str, counters: dict[str, int]) -> str:
|
||||
base = hashlib.sha256(
|
||||
f"{note_id}\x1f{chr(31).join(path)}\x1f{content}".encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
block_id = f"blk_{base}"
|
||||
# 同一篇笔记内极少出现的重复段落用后缀消歧，保证唯一
|
||||
n = counters.get(block_id, 0)
|
||||
counters[block_id] = n + 1
|
||||
return block_id if n == 0 else f"{block_id}_{n}"
|
||||
|
||||
|
||||
def _split_lines(text: str) -> list[tuple[str, int]]:
|
||||
"""按行拆分并记录每行在原文中的起始字符偏移。"""
|
||||
result: list[tuple[str, int]] = []
|
||||
start = 0
|
||||
for raw in text.splitlines(keepends=True):
|
||||
line = raw
|
||||
if line.endswith("\r\n"):
|
||||
line = line[:-2]
|
||||
elif line.endswith("\n") or line.endswith("\r"):
|
||||
line = line[:-1]
|
||||
result.append((line, start))
|
||||
start += len(raw)
|
||||
return result
|
||||
|
||||
|
||||
def _content_start(markdown: str) -> int:
|
||||
"""返回正文起始偏移：有 frontmatter 时跳过 --- 分隔块，否则为 0。"""
|
||||
if markdown.startswith("---"):
|
||||
end = markdown.find("\n---", 3)
|
||||
if end != -1:
|
||||
return end + 4
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_frontmatter(markdown: str) -> dict[str, str]:
|
||||
"""极简 frontmatter 解析，只提取 key: value 行。"""
|
||||
if not markdown.startswith("---"):
|
||||
return {}
|
||||
end = markdown.find("\n---", 3)
|
||||
if end == -1:
|
||||
return {}
|
||||
meta: dict[str, str] = {}
|
||||
for line in markdown[3:end].splitlines():
|
||||
m = _FRONTMATTER_KEY_RE.match(line)
|
||||
if m:
|
||||
meta[m.group(1).lower()] = m.group(2).strip()
|
||||
return meta
|
||||
|
||||
|
||||
def _first_heading(markdown: str) -> str | None:
|
||||
for line in markdown.splitlines():
|
||||
m = re.match(r"^#\s+(.*?)\s*$", line)
|
||||
if m and m.group(1).strip():
|
||||
return m.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _parse_tags(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
raw = raw.strip()
|
||||
if raw.startswith("[") and raw.endswith("]"):
|
||||
raw = raw[1:-1]
|
||||
return [t.strip().strip("'\"") for t in raw.split(",") if t.strip()]
|
||||
@@ -0,0 +1,299 @@
|
||||
"""SQLite Repository：笔记元数据、Block 与 FTS5 的读写。
|
||||
|
||||
向量（vec_blocks）不在这里处理，交给 Retrieval 基础设施层的 VectorStore（见
|
||||
app/retrieval/vectorstore.py）。本层只负责 notes / blocks / blocks_fts 三张表的访问，
|
||||
返回领域记录（NoteRecord / BlockHit / FtsHit），不负责业务编排。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from app.contracts import NoteBlock
|
||||
from app.database.db import connect, transaction
|
||||
from app.textutils import segment
|
||||
|
||||
|
||||
def _iso(dt: datetime) -> str:
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
def _parse_dt(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NoteRecord:
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
blocks: list[NoteBlock] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlockHit:
|
||||
"""检索时返回的完整 Block 上下文，用于组装 Citation 与 metadata 过滤。"""
|
||||
|
||||
block_id: str
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
heading_path: list[str]
|
||||
content: str
|
||||
start_offset: int
|
||||
end_offset: int
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class FtsHit:
|
||||
block_id: str
|
||||
note_id: str
|
||||
bm25: float
|
||||
|
||||
|
||||
def replace_note_metadata(
|
||||
*,
|
||||
note_id: str,
|
||||
title: str,
|
||||
file_path: str,
|
||||
folder: str,
|
||||
tags: list[str],
|
||||
created_at: datetime,
|
||||
updated_at: datetime,
|
||||
blocks: list[NoteBlock],
|
||||
) -> None:
|
||||
"""整体替换一条笔记的元数据、Block 与 FTS5 索引（单事务）。"""
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO notes (note_id, title, file_path, folder, tags, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(note_id) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
file_path = excluded.file_path,
|
||||
folder = excluded.folder,
|
||||
tags = excluded.tags,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(note_id, title, file_path, folder, json.dumps(tags, ensure_ascii=False),
|
||||
_iso(created_at), _iso(updated_at)),
|
||||
)
|
||||
conn.execute("DELETE FROM blocks WHERE note_id = ?", (note_id,))
|
||||
conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,))
|
||||
for position, block in enumerate(blocks):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO blocks
|
||||
(block_id, note_id, heading_path, start_offset, end_offset,
|
||||
content, content_hash, token_count, position)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(block.block_id, note_id, json.dumps(block.heading_path, ensure_ascii=False),
|
||||
block.start_offset, block.end_offset, block.content,
|
||||
block.content_hash, block.token_count, position),
|
||||
)
|
||||
# FTS5 存分词后的可检索文本；原文仍由 blocks.content 保留用于展示
|
||||
conn.execute(
|
||||
"INSERT INTO blocks_fts (block_id, note_id, heading_path, content) VALUES (?, ?, ?, ?)",
|
||||
(block.block_id, note_id, segment(" ".join(block.heading_path)), segment(block.content)),
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def delete_note(note_id: str) -> list[str]:
|
||||
"""删除笔记及其 Block、FTS5 索引；返回被删除的 block_id 供向量层清理。"""
|
||||
conn = connect()
|
||||
try:
|
||||
block_ids = [
|
||||
row["block_id"]
|
||||
for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,))
|
||||
]
|
||||
with transaction(conn):
|
||||
conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,))
|
||||
conn.execute("DELETE FROM notes WHERE note_id = ?", (note_id,)) # blocks 级联删除
|
||||
return block_ids
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_note_record(note_id: str) -> NoteRecord | None:
|
||||
conn = connect()
|
||||
try:
|
||||
row = conn.execute("SELECT * FROM notes WHERE note_id = ?", (note_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
blocks = [
|
||||
_block_from_row(b)
|
||||
for b in conn.execute("SELECT * FROM blocks WHERE note_id = ? ORDER BY position", (note_id,))
|
||||
]
|
||||
return NoteRecord(
|
||||
note_id=row["note_id"],
|
||||
title=row["title"],
|
||||
file_path=row["file_path"],
|
||||
folder=row["folder"],
|
||||
tags=json.loads(row["tags"] or "[]"),
|
||||
created_at=_parse_dt(row["created_at"]),
|
||||
updated_at=_parse_dt(row["updated_at"]),
|
||||
blocks=blocks,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_note_summaries(
|
||||
*, limit: int = 50, offset: int = 0, folder: str | None = None, tag: str | None = None
|
||||
) -> tuple[list, int]:
|
||||
conn = connect()
|
||||
try:
|
||||
where: list[str] = []
|
||||
params: list[str] = []
|
||||
if folder:
|
||||
where.append("folder = ?")
|
||||
params.append(folder)
|
||||
if tag:
|
||||
where.append("EXISTS (SELECT 1 FROM json_each(notes.tags) AS j WHERE j.value = ?)")
|
||||
params.append(tag)
|
||||
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||
|
||||
total = conn.execute(f"SELECT COUNT(*) FROM notes {where_sql}", params).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM notes {where_sql} ORDER BY updated_at DESC LIMIT ? OFFSET ?",
|
||||
params + [limit, offset],
|
||||
).fetchall()
|
||||
|
||||
items = [
|
||||
{
|
||||
"note_id": r["note_id"],
|
||||
"title": r["title"],
|
||||
"file_path": r["file_path"],
|
||||
"tags": json.loads(r["tags"] or "[]"),
|
||||
"created_at": _parse_dt(r["created_at"]),
|
||||
"updated_at": _parse_dt(r["updated_at"]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
return items, total
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def fts_search(match: str, limit: int = 100) -> list[FtsHit]:
|
||||
conn = connect()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT block_id, note_id, bm25(blocks_fts) AS rank
|
||||
FROM blocks_fts
|
||||
WHERE blocks_fts MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
""",
|
||||
(match, limit),
|
||||
).fetchall()
|
||||
return [FtsHit(block_id=r["block_id"], note_id=r["note_id"], bm25=r["rank"]) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_block_hits(block_ids: list[str]) -> list[BlockHit]:
|
||||
if not block_ids:
|
||||
return []
|
||||
conn = connect()
|
||||
try:
|
||||
placeholders = ",".join("?" * len(block_ids))
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT b.block_id, b.note_id, b.heading_path, b.start_offset, b.end_offset, b.content,
|
||||
n.title, n.file_path, n.folder, n.tags, n.created_at, n.updated_at
|
||||
FROM blocks b
|
||||
JOIN notes n ON n.note_id = b.note_id
|
||||
WHERE b.block_id IN ({placeholders})
|
||||
""",
|
||||
block_ids,
|
||||
).fetchall()
|
||||
return [_block_hit_from_row(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def set_index_meta(kv: dict[str, str]) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
for key, value in kv.items():
|
||||
conn.execute("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)", (key, value))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_index_meta() -> dict[str, str]:
|
||||
conn = connect()
|
||||
try:
|
||||
return {r["key"]: r["value"] for r in conn.execute("SELECT key, value FROM index_meta")}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def clear_all() -> None:
|
||||
"""清空元数据、Block 与 FTS5（重建索引用，向量由 VectorStore.clear 处理）。"""
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute("DELETE FROM blocks_fts")
|
||||
conn.execute("DELETE FROM blocks")
|
||||
conn.execute("DELETE FROM notes")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def stats() -> dict[str, int]:
|
||||
conn = connect()
|
||||
try:
|
||||
notes = conn.execute("SELECT COUNT(*) AS c FROM notes").fetchone()["c"]
|
||||
blocks = conn.execute("SELECT COUNT(*) AS c FROM blocks").fetchone()["c"]
|
||||
return {"notes": notes, "blocks": blocks}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _block_from_row(row) -> NoteBlock:
|
||||
return NoteBlock(
|
||||
block_id=row["block_id"],
|
||||
note_id=row["note_id"],
|
||||
heading_path=json.loads(row["heading_path"] or "[]"),
|
||||
start_offset=row["start_offset"],
|
||||
end_offset=row["end_offset"],
|
||||
content=row["content"],
|
||||
content_hash=row["content_hash"],
|
||||
token_count=row["token_count"],
|
||||
)
|
||||
|
||||
|
||||
def _block_hit_from_row(row) -> BlockHit:
|
||||
return BlockHit(
|
||||
block_id=row["block_id"],
|
||||
note_id=row["note_id"],
|
||||
title=row["title"],
|
||||
file_path=row["file_path"],
|
||||
folder=row["folder"],
|
||||
heading_path=json.loads(row["heading_path"] or "[]"),
|
||||
content=row["content"],
|
||||
start_offset=row["start_offset"],
|
||||
end_offset=row["end_offset"],
|
||||
tags=json.loads(row["tags"] or "[]"),
|
||||
created_at=_parse_dt(row["created_at"]),
|
||||
updated_at=_parse_dt(row["updated_at"]),
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""文本分词工具，供 FTS5 与轻量 Embedding 共用。
|
||||
|
||||
FTS5 默认 unicode61 分词器不切分中文（连续汉字算一个 token），导致「死锁」这类
|
||||
子串无法命中。这里把文本统一拆成 ASCII 单词 + CJK 单字 + CJK 相邻双字，写入与查询
|
||||
走同一套拆分，实现中文子串/词级召回。
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
_CJK_RUN_RE = re.compile(r"[一-鿿]+")
|
||||
|
||||
|
||||
def tokens(text: str) -> list[str]:
|
||||
"""返回文本的检索 token 序列（含重复）。"""
|
||||
out: list[str] = []
|
||||
out.extend(_WORD_RE.findall(text.lower()))
|
||||
for run in _CJK_RUN_RE.findall(text):
|
||||
out.extend(run) # 单字
|
||||
out.extend(run[i : i + 2] for i in range(len(run) - 1)) # 相邻双字
|
||||
return out
|
||||
|
||||
|
||||
def unique_tokens(text: str) -> list[str]:
|
||||
"""去重但保序的 token 列表，用于压缩查询/索引体积。"""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for tok in tokens(text):
|
||||
if tok not in seen:
|
||||
seen.add(tok)
|
||||
result.append(tok)
|
||||
return result
|
||||
|
||||
|
||||
def segment(text: str) -> str:
|
||||
"""把文本转成空格分隔的 token，写入 FTS5 的 content 列。"""
|
||||
return " ".join(unique_tokens(text))
|
||||
|
||||
|
||||
def match_query(query: str) -> str | None:
|
||||
"""把用户查询转成 FTS5 MATCH 表达式（OR 连接、按 token 精确匹配）。"""
|
||||
toks = unique_tokens(query)
|
||||
if not toks:
|
||||
return None
|
||||
return " OR ".join(f'"{tok}"' for tok in toks)
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""粗略 token 数：ASCII 单词 + CJK 单字。仅用于展示，不做精确计量。"""
|
||||
return len(_WORD_RE.findall(text.lower())) + sum(len(run) for run in _CJK_RUN_RE.findall(text))
|
||||
|
||||
|
||||
def make_snippet(content: str, query: str, max_len: int = 160) -> str:
|
||||
"""从原文生成命中片段：优先定位较长的 query token，向前后扩展窗口。"""
|
||||
toks = unique_tokens(query)
|
||||
lowered = content.lower()
|
||||
best = -1
|
||||
# 优先用较长的 token（双字/单词）定位，命中最准确
|
||||
for tok in toks:
|
||||
if len(tok) >= 2:
|
||||
idx = lowered.find(tok)
|
||||
if idx != -1:
|
||||
best = idx
|
||||
break
|
||||
if best == -1:
|
||||
for tok in toks:
|
||||
idx = lowered.find(tok)
|
||||
if idx != -1:
|
||||
best = idx
|
||||
break
|
||||
|
||||
if best == -1:
|
||||
snippet = content
|
||||
else:
|
||||
start = max(0, best - max_len // 3)
|
||||
end = min(len(content), best + max_len)
|
||||
snippet = content[start:end]
|
||||
snippet = ("…" if start > 0 else "") + snippet + ("…" if end < len(content) else "")
|
||||
|
||||
if len(snippet) > max_len + 20:
|
||||
snippet = snippet[:max_len] + "…"
|
||||
return snippet
|
||||
Reference in New Issue
Block a user