Merge branch 'feat/knowledge-retrieval-core'
This commit is contained in:
@@ -11,6 +11,8 @@ backend/.pytest_cache/
|
||||
backend/*.egg-info/
|
||||
backend/**/__pycache__/
|
||||
backend/.env
|
||||
# 运行期生成的 SQLite 索引（vault 下的 Markdown 测试数据需提交）
|
||||
backend/data/*.db*
|
||||
|
||||
# Editors and operating systems
|
||||
.idea/
|
||||
|
||||
@@ -17,3 +17,5 @@ uv run uvicorn app.main:app --reload --host 127.0.0.1 --port 8000
|
||||
团队接口清单见 `../docs/后端接口契约-开发版.md`，机器可读契约以运行时的 `/openapi.json` 为准。
|
||||
|
||||
AI Core 与 Agent Core 的模块边界、Mock Provider 和 Tool Calling 调试方式见 `../docs/AI-Core与Agent-Core开发说明.md`。
|
||||
|
||||
Knowledge Core 与 Retrieval Core 的模块边界、数据模型、接口与检索流程见 `../docs/Knowledge与Retrieval-Core开发说明.md`。
|
||||
|
||||
+16
-1
@@ -1,25 +1,40 @@
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
# backend 目录（本文件位于 backend/app/config.py，父目录的父目录即 backend）
|
||||
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
"""应用基础配置；正式环境可通过 APP_* 环境变量覆盖。"""
|
||||
"""应用基础配置；正式环境可通过 APP_* 环境变量覆盖。
|
||||
|
||||
数据目录默认落在 backend/data 下：app.db 保存 SQLite 索引，
|
||||
vault/ 保存 Markdown 笔记。三者均可通过环境变量覆盖，便于测试与正式部署分离。
|
||||
"""
|
||||
|
||||
name: str
|
||||
version: str
|
||||
environment: str
|
||||
host: str
|
||||
port: int
|
||||
data_dir: Path
|
||||
db_path: Path
|
||||
vault_path: Path
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
data_dir = Path(os.getenv("APP_DATA_DIR", str(BACKEND_DIR / "data")))
|
||||
return Settings(
|
||||
name=os.getenv("APP_NAME", "Notes Agent AI Core"),
|
||||
version=os.getenv("APP_VERSION", "0.1.0"),
|
||||
environment=os.getenv("APP_ENVIRONMENT", "development"),
|
||||
host=os.getenv("APP_HOST", "127.0.0.1"),
|
||||
port=int(os.getenv("APP_PORT", "8000")),
|
||||
data_dir=data_dir,
|
||||
db_path=Path(os.getenv("APP_DB_PATH", str(data_dir / "app.db"))),
|
||||
vault_path=Path(os.getenv("APP_VAULT_PATH", str(data_dir / "vault"))),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
"""跨层共享的常量。
|
||||
|
||||
放在这里是为了让 database（建向量表）与 retrieval（生成向量）共享同一个维度值，
|
||||
避免二者各自硬编码导致不一致。真实 BGE-M3 接入后把维度改为 1024 并重建向量索引即可。
|
||||
"""
|
||||
|
||||
# 轻量哈希向量的维度；vec0 虚拟表建表时按此维度固定列宽。
|
||||
EMBEDDING_DIM = 128
|
||||
@@ -0,0 +1 @@
|
||||
"""数据库访问层：连接管理、迁移与 Repository。"""
|
||||
@@ -0,0 +1,49 @@
|
||||
"""SQLite 连接管理。
|
||||
|
||||
每次调用 connect() 打开一个新连接：加载 sqlite-vec 扩展、打开外键、应用迁移。
|
||||
连接由调用方负责关闭；写入通过 transaction() 上下文显式控制提交，避免隐式事务带来的
|
||||
半提交状态。
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from collections.abc import Iterator
|
||||
|
||||
import sqlite_vec
|
||||
|
||||
from app.config import get_settings
|
||||
from app.database.migrations import migrate
|
||||
|
||||
|
||||
def _load_extension(conn: sqlite3.Connection) -> None:
|
||||
"""在当前连接上注册 sqlite-vec 扩展，随后关闭 load_extension 开关。"""
|
||||
conn.enable_load_extension(True)
|
||||
try:
|
||||
sqlite_vec.load(conn)
|
||||
finally:
|
||||
conn.enable_load_extension(False)
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
settings = get_settings()
|
||||
settings.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(settings.db_path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
# 关闭 Python sqlite3 的隐式事务，提交时机由 transaction() 或显式 commit 控制。
|
||||
conn.isolation_level = None
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
_load_extension(conn)
|
||||
migrate(conn)
|
||||
return conn
|
||||
|
||||
|
||||
@contextmanager
|
||||
def transaction(conn: sqlite3.Connection) -> Iterator[None]:
|
||||
"""显式事务：提交成功则 COMMIT，异常则 ROLLBACK。"""
|
||||
conn.execute("BEGIN")
|
||||
try:
|
||||
yield
|
||||
conn.execute("COMMIT")
|
||||
except BaseException:
|
||||
conn.execute("ROLLBACK")
|
||||
raise
|
||||
@@ -0,0 +1,77 @@
|
||||
"""轻量 schema 迁移。
|
||||
|
||||
约定：MIGRATIONS 列表按版本号顺序排列，只增不改。每一条是一个完整的 SQL 脚本，
|
||||
执行后写入 schema_migrations 记录版本。修改 Schema 时在末尾追加新脚本，禁止删旧脚本或
|
||||
依赖运行时自动删表重建（团队约定）。
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.constants import EMBEDDING_DIM
|
||||
|
||||
# 每个元素对应一个版本（下标 + 1）。vec0 建表需要本连接已加载 sqlite-vec 扩展，
|
||||
# 由 db.connect() 在调用 migrate 之前完成。
|
||||
MIGRATIONS: list[str] = [
|
||||
# v1: 笔记元数据 + Block + FTS5 全文索引 + 向量表 + 索引元信息
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
note_id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL UNIQUE,
|
||||
folder TEXT NOT NULL DEFAULT '',
|
||||
tags TEXT NOT NULL DEFAULT '[]',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS blocks (
|
||||
block_id TEXT PRIMARY KEY,
|
||||
note_id TEXT NOT NULL REFERENCES notes(note_id) ON DELETE CASCADE,
|
||||
heading_path TEXT NOT NULL DEFAULT '[]',
|
||||
start_offset INTEGER NOT NULL,
|
||||
end_offset INTEGER NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
token_count INTEGER NOT NULL,
|
||||
position INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_blocks_note ON blocks(note_id, position);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(
|
||||
block_id UNINDEXED,
|
||||
note_id UNINDEXED,
|
||||
heading_path,
|
||||
content,
|
||||
tokenize = 'unicode61'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS vec_blocks USING vec0(
|
||||
block_id TEXT PRIMARY KEY,
|
||||
embedding float[{EMBEDDING_DIM}]
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS index_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
""",
|
||||
]
|
||||
|
||||
|
||||
def migrate(conn) -> None:
|
||||
"""把尚未应用的迁移脚本按序应用到给定连接。"""
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations "
|
||||
"(version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)"
|
||||
)
|
||||
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
|
||||
for idx, script in enumerate(MIGRATIONS, start=1):
|
||||
if idx in applied:
|
||||
continue
|
||||
conn.executescript(script)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)",
|
||||
(idx, datetime.now(timezone.utc).isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
@@ -0,0 +1 @@
|
||||
"""Knowledge Core：Markdown 解析与 Note Block 生成。"""
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Markdown 解析与 Note Block 切分。
|
||||
|
||||
Block 由 Markdown 文本生成：标题行独立成块（heading_path 含自身），正文按空行分段，
|
||||
每块记录其在原文中的 start_offset / end_offset，用于 Citation 跳转定位。block_id 由
|
||||
(note_id, heading_path, content) 稳定派生，内容不变则 ID 稳定。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.contracts import NoteBlock
|
||||
from app.textutils import count_tokens
|
||||
|
||||
_HEADING_RE = re.compile(r"^(#{1,6})[ \t]+(.*?)\s*$")
|
||||
_FRONTMATTER_KEY_RE = re.compile(r"^([A-Za-z0-9_-]+)\s*:\s*(.*)$")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedNote:
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
blocks: list[NoteBlock] = field(default_factory=list)
|
||||
|
||||
|
||||
def note_id_for_path(rel_path: str) -> str:
|
||||
"""由相对路径派生稳定 note_id（路径哈希而非路径本身，见团队约定「不用路径当 ID」）。
|
||||
|
||||
MVP 阶段 ID 随文件移动而变化；后续 move 流程会保留原 ID。"""
|
||||
normalized = rel_path.replace("\\", "/").strip("/")
|
||||
return "note_" + hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def parse_note(
|
||||
*,
|
||||
markdown: str,
|
||||
file_path: str,
|
||||
folder: str,
|
||||
tags: list[str] | None = None,
|
||||
created_at: datetime,
|
||||
updated_at: datetime,
|
||||
) -> ParsedNote:
|
||||
"""解析一篇 Markdown，生成 ParsedNote（元数据 + Block 列表）。"""
|
||||
note_id = note_id_for_path(file_path)
|
||||
frontmatter = _extract_frontmatter(markdown)
|
||||
|
||||
fallback_title = Path(file_path).stem
|
||||
title = frontmatter.get("title") or _first_heading(markdown) or fallback_title
|
||||
resolved_tags = list(tags) if tags is not None else _parse_tags(frontmatter.get("tags"))
|
||||
|
||||
blocks = parse_blocks(markdown, note_id)
|
||||
return ParsedNote(
|
||||
note_id=note_id,
|
||||
title=title,
|
||||
file_path=file_path,
|
||||
folder=folder,
|
||||
tags=resolved_tags,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
blocks=blocks,
|
||||
)
|
||||
|
||||
|
||||
def parse_blocks(markdown: str, note_id: str) -> list[NoteBlock]:
|
||||
"""把 Markdown 切成 Block，offset 相对原文（含 frontmatter）。"""
|
||||
lines = _split_lines(markdown)
|
||||
content_start = _content_start(markdown)
|
||||
|
||||
blocks: list[NoteBlock] = []
|
||||
heading_stack: list[str] = []
|
||||
body: list[tuple[str, int]] = []
|
||||
id_counters: dict[str, int] = {}
|
||||
|
||||
def make_block(path: list[str], chunk: list[tuple[str, int]]) -> None:
|
||||
if not chunk:
|
||||
return
|
||||
content = "\n".join(line for line, _ in chunk)
|
||||
start = chunk[0][1]
|
||||
end = chunk[-1][1] + len(chunk[-1][0])
|
||||
block_id = _stable_block_id(note_id, path, content, id_counters)
|
||||
blocks.append(
|
||||
NoteBlock(
|
||||
block_id=block_id,
|
||||
note_id=note_id,
|
||||
heading_path=list(path),
|
||||
start_offset=start,
|
||||
end_offset=end,
|
||||
content=content,
|
||||
content_hash=hashlib.sha256(content.encode("utf-8")).hexdigest()[:16],
|
||||
token_count=count_tokens(content),
|
||||
)
|
||||
)
|
||||
|
||||
def flush_body() -> None:
|
||||
nonlocal body
|
||||
make_block(heading_stack, body)
|
||||
body = []
|
||||
|
||||
for line, offset in lines:
|
||||
if offset < content_start:
|
||||
continue # 跳过 frontmatter 区域，但保留 offset 准确性
|
||||
|
||||
heading = _HEADING_RE.match(line)
|
||||
if heading:
|
||||
flush_body()
|
||||
level = len(heading.group(1))
|
||||
title = heading.group(2).strip()
|
||||
heading_stack = heading_stack[: level - 1] + [title]
|
||||
# 标题自身作为一个 Block，便于按章节定位
|
||||
make_block(heading_stack, [(line, offset)])
|
||||
elif line.strip() == "":
|
||||
flush_body() # 空行分隔段落
|
||||
else:
|
||||
body.append((line, offset))
|
||||
|
||||
flush_body()
|
||||
return blocks
|
||||
|
||||
|
||||
def _stable_block_id(note_id: str, path: list[str], content: str, counters: dict[str, int]) -> str:
|
||||
base = hashlib.sha256(
|
||||
f"{note_id}\x1f{chr(31).join(path)}\x1f{content}".encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
block_id = f"blk_{base}"
|
||||
# 同一篇笔记内极少出现的重复段落用后缀消歧，保证唯一
|
||||
n = counters.get(block_id, 0)
|
||||
counters[block_id] = n + 1
|
||||
return block_id if n == 0 else f"{block_id}_{n}"
|
||||
|
||||
|
||||
def _split_lines(text: str) -> list[tuple[str, int]]:
|
||||
"""按行拆分并记录每行在原文中的起始字符偏移。"""
|
||||
result: list[tuple[str, int]] = []
|
||||
start = 0
|
||||
for raw in text.splitlines(keepends=True):
|
||||
line = raw
|
||||
if line.endswith("\r\n"):
|
||||
line = line[:-2]
|
||||
elif line.endswith("\n") or line.endswith("\r"):
|
||||
line = line[:-1]
|
||||
result.append((line, start))
|
||||
start += len(raw)
|
||||
return result
|
||||
|
||||
|
||||
def _content_start(markdown: str) -> int:
|
||||
"""返回正文起始偏移：有 frontmatter 时跳过 --- 分隔块，否则为 0。"""
|
||||
if markdown.startswith("---"):
|
||||
end = markdown.find("\n---", 3)
|
||||
if end != -1:
|
||||
return end + 4
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_frontmatter(markdown: str) -> dict[str, str]:
|
||||
"""极简 frontmatter 解析，只提取 key: value 行。"""
|
||||
if not markdown.startswith("---"):
|
||||
return {}
|
||||
end = markdown.find("\n---", 3)
|
||||
if end == -1:
|
||||
return {}
|
||||
meta: dict[str, str] = {}
|
||||
for line in markdown[3:end].splitlines():
|
||||
m = _FRONTMATTER_KEY_RE.match(line)
|
||||
if m:
|
||||
meta[m.group(1).lower()] = m.group(2).strip()
|
||||
return meta
|
||||
|
||||
|
||||
def _first_heading(markdown: str) -> str | None:
|
||||
for line in markdown.splitlines():
|
||||
m = re.match(r"^#\s+(.*?)\s*$", line)
|
||||
if m and m.group(1).strip():
|
||||
return m.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _parse_tags(raw: str | None) -> list[str]:
|
||||
if not raw:
|
||||
return []
|
||||
raw = raw.strip()
|
||||
if raw.startswith("[") and raw.endswith("]"):
|
||||
raw = raw[1:-1]
|
||||
return [t.strip().strip("'\"") for t in raw.split(",") if t.strip()]
|
||||
@@ -0,0 +1,305 @@
|
||||
"""SQLite Repository：笔记元数据、Block 与 FTS5 的读写。
|
||||
|
||||
向量（vec_blocks）不在这里处理，交给 Retrieval 基础设施层的 VectorStore（见
|
||||
app/retrieval/vectorstore.py）。本层只负责 notes / blocks / blocks_fts 三张表的访问，
|
||||
返回领域记录（NoteRecord / BlockHit / FtsHit），不负责业务编排。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
|
||||
from app.contracts import NoteBlock
|
||||
from app.database.db import connect, transaction
|
||||
from app.textutils import segment
|
||||
|
||||
|
||||
def _iso(dt: datetime) -> str:
|
||||
return dt.isoformat()
|
||||
|
||||
|
||||
def _parse_dt(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NoteRecord:
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
blocks: list[NoteBlock] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlockHit:
|
||||
"""检索时返回的完整 Block 上下文，用于组装 Citation 与 metadata 过滤。"""
|
||||
|
||||
block_id: str
|
||||
note_id: str
|
||||
title: str
|
||||
file_path: str
|
||||
folder: str
|
||||
heading_path: list[str]
|
||||
content: str
|
||||
start_offset: int
|
||||
end_offset: int
|
||||
tags: list[str]
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
@dataclass
|
||||
class FtsHit:
|
||||
block_id: str
|
||||
note_id: str
|
||||
bm25: float
|
||||
|
||||
|
||||
def replace_note_metadata(
|
||||
*,
|
||||
conn: sqlite3.Connection,
|
||||
note_id: str,
|
||||
title: str,
|
||||
file_path: str,
|
||||
folder: str,
|
||||
tags: list[str],
|
||||
created_at: datetime,
|
||||
updated_at: datetime,
|
||||
blocks: list[NoteBlock],
|
||||
) -> list[str]:
|
||||
"""整体替换一条笔记的元数据、Block 与 FTS5 索引。
|
||||
|
||||
不在此处开启/提交事务：由调用方（index_note）在同一连接上把「元数据 + 向量」包进
|
||||
单个事务，保证原子性。返回替换前的旧 block_id 列表，供调用方清理失效向量。
|
||||
"""
|
||||
old_block_ids = [
|
||||
row["block_id"]
|
||||
for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,))
|
||||
]
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO notes (note_id, title, file_path, folder, tags, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(note_id) DO UPDATE SET
|
||||
title = excluded.title,
|
||||
file_path = excluded.file_path,
|
||||
folder = excluded.folder,
|
||||
tags = excluded.tags,
|
||||
updated_at = excluded.updated_at
|
||||
""",
|
||||
(note_id, title, file_path, folder, json.dumps(tags, ensure_ascii=False),
|
||||
_iso(created_at), _iso(updated_at)),
|
||||
)
|
||||
conn.execute("DELETE FROM blocks WHERE note_id = ?", (note_id,))
|
||||
conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,))
|
||||
for position, block in enumerate(blocks):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO blocks
|
||||
(block_id, note_id, heading_path, start_offset, end_offset,
|
||||
content, content_hash, token_count, position)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(block.block_id, note_id, json.dumps(block.heading_path, ensure_ascii=False),
|
||||
block.start_offset, block.end_offset, block.content,
|
||||
block.content_hash, block.token_count, position),
|
||||
)
|
||||
# FTS5 存分词后的可检索文本；原文仍由 blocks.content 保留用于展示
|
||||
conn.execute(
|
||||
"INSERT INTO blocks_fts (block_id, note_id, heading_path, content) VALUES (?, ?, ?, ?)",
|
||||
(block.block_id, note_id, segment(" ".join(block.heading_path)), segment(block.content)),
|
||||
)
|
||||
return old_block_ids
|
||||
|
||||
|
||||
def delete_note(note_id: str) -> list[str]:
|
||||
"""删除笔记及其 Block、FTS5 索引；返回被删除的 block_id 供向量层清理。"""
|
||||
conn = connect()
|
||||
try:
|
||||
block_ids = [
|
||||
row["block_id"]
|
||||
for row in conn.execute("SELECT block_id FROM blocks WHERE note_id = ?", (note_id,))
|
||||
]
|
||||
with transaction(conn):
|
||||
conn.execute("DELETE FROM blocks_fts WHERE note_id = ?", (note_id,))
|
||||
conn.execute("DELETE FROM notes WHERE note_id = ?", (note_id,)) # blocks 级联删除
|
||||
return block_ids
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_note_record(note_id: str) -> NoteRecord | None:
|
||||
conn = connect()
|
||||
try:
|
||||
row = conn.execute("SELECT * FROM notes WHERE note_id = ?", (note_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
blocks = [
|
||||
_block_from_row(b)
|
||||
for b in conn.execute("SELECT * FROM blocks WHERE note_id = ? ORDER BY position", (note_id,))
|
||||
]
|
||||
return NoteRecord(
|
||||
note_id=row["note_id"],
|
||||
title=row["title"],
|
||||
file_path=row["file_path"],
|
||||
folder=row["folder"],
|
||||
tags=json.loads(row["tags"] or "[]"),
|
||||
created_at=_parse_dt(row["created_at"]),
|
||||
updated_at=_parse_dt(row["updated_at"]),
|
||||
blocks=blocks,
|
||||
)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_note_summaries(
|
||||
*, limit: int = 50, offset: int = 0, folder: str | None = None, tag: str | None = None
|
||||
) -> tuple[list, int]:
|
||||
conn = connect()
|
||||
try:
|
||||
where: list[str] = []
|
||||
params: list[str] = []
|
||||
if folder:
|
||||
where.append("folder = ?")
|
||||
params.append(folder)
|
||||
if tag:
|
||||
where.append("EXISTS (SELECT 1 FROM json_each(notes.tags) AS j WHERE j.value = ?)")
|
||||
params.append(tag)
|
||||
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||
|
||||
total = conn.execute(f"SELECT COUNT(*) FROM notes {where_sql}", params).fetchone()[0]
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM notes {where_sql} ORDER BY updated_at DESC LIMIT ? OFFSET ?",
|
||||
params + [limit, offset],
|
||||
).fetchall()
|
||||
|
||||
items = [
|
||||
{
|
||||
"note_id": r["note_id"],
|
||||
"title": r["title"],
|
||||
"file_path": r["file_path"],
|
||||
"tags": json.loads(r["tags"] or "[]"),
|
||||
"created_at": _parse_dt(r["created_at"]),
|
||||
"updated_at": _parse_dt(r["updated_at"]),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
return items, total
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def fts_search(match: str, limit: int = 100) -> list[FtsHit]:
|
||||
conn = connect()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT block_id, note_id, bm25(blocks_fts) AS rank
|
||||
FROM blocks_fts
|
||||
WHERE blocks_fts MATCH ?
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
""",
|
||||
(match, limit),
|
||||
).fetchall()
|
||||
return [FtsHit(block_id=r["block_id"], note_id=r["note_id"], bm25=r["rank"]) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_block_hits(block_ids: list[str]) -> list[BlockHit]:
|
||||
if not block_ids:
|
||||
return []
|
||||
conn = connect()
|
||||
try:
|
||||
placeholders = ",".join("?" * len(block_ids))
|
||||
rows = conn.execute(
|
||||
f"""
|
||||
SELECT b.block_id, b.note_id, b.heading_path, b.start_offset, b.end_offset, b.content,
|
||||
n.title, n.file_path, n.folder, n.tags, n.created_at, n.updated_at
|
||||
FROM blocks b
|
||||
JOIN notes n ON n.note_id = b.note_id
|
||||
WHERE b.block_id IN ({placeholders})
|
||||
""",
|
||||
block_ids,
|
||||
).fetchall()
|
||||
return [_block_hit_from_row(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def set_index_meta(kv: dict[str, str]) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
for key, value in kv.items():
|
||||
conn.execute("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)", (key, value))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_index_meta() -> dict[str, str]:
|
||||
conn = connect()
|
||||
try:
|
||||
return {r["key"]: r["value"] for r in conn.execute("SELECT key, value FROM index_meta")}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def clear_all() -> None:
|
||||
"""清空元数据、Block 与 FTS5（重建索引用，向量由 VectorStore.clear 处理）。"""
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute("DELETE FROM blocks_fts")
|
||||
conn.execute("DELETE FROM blocks")
|
||||
conn.execute("DELETE FROM notes")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def stats() -> dict[str, int]:
|
||||
conn = connect()
|
||||
try:
|
||||
notes = conn.execute("SELECT COUNT(*) AS c FROM notes").fetchone()["c"]
|
||||
blocks = conn.execute("SELECT COUNT(*) AS c FROM blocks").fetchone()["c"]
|
||||
return {"notes": notes, "blocks": blocks}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _block_from_row(row) -> NoteBlock:
|
||||
return NoteBlock(
|
||||
block_id=row["block_id"],
|
||||
note_id=row["note_id"],
|
||||
heading_path=json.loads(row["heading_path"] or "[]"),
|
||||
start_offset=row["start_offset"],
|
||||
end_offset=row["end_offset"],
|
||||
content=row["content"],
|
||||
content_hash=row["content_hash"],
|
||||
token_count=row["token_count"],
|
||||
)
|
||||
|
||||
|
||||
def _block_hit_from_row(row) -> BlockHit:
|
||||
return BlockHit(
|
||||
block_id=row["block_id"],
|
||||
note_id=row["note_id"],
|
||||
title=row["title"],
|
||||
file_path=row["file_path"],
|
||||
folder=row["folder"],
|
||||
heading_path=json.loads(row["heading_path"] or "[]"),
|
||||
content=row["content"],
|
||||
start_offset=row["start_offset"],
|
||||
end_offset=row["end_offset"],
|
||||
tags=json.loads(row["tags"] or "[]"),
|
||||
created_at=_parse_dt(row["created_at"]),
|
||||
updated_at=_parse_dt(row["updated_at"]),
|
||||
)
|
||||
@@ -0,0 +1 @@
|
||||
"""Retrieval Core：Embedding、VectorStore、RRF、Reranker 与混合检索引擎。"""
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Embedding 统一接口与轻量实现。
|
||||
|
||||
真实默认是本地 BGE-M3 类模型，但第一阶段先跑通链路，这里用确定性的特征哈希向量代替。
|
||||
后续接入真实模型时实现同样的 EmbeddingProvider 接口替换即可，上层检索逻辑不变。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from app.constants import EMBEDDING_DIM
|
||||
from app.textutils import tokens
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class EmbeddingProvider(Protocol):
|
||||
"""统一 Embedding 接口（与文档一致）。"""
|
||||
|
||||
model_id: str
|
||||
dim: int
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]: ...
|
||||
async def embed_query(self, query: str) -> list[float]: ...
|
||||
|
||||
|
||||
class HashEmbeddingProvider:
|
||||
"""轻量确定性向量：特征哈希 + 符号 + L2 归一化。
|
||||
|
||||
同一文本永远得到相同向量，可离线复现、无外部依赖。向量维度为 EMBEDDING_DIM，
|
||||
与 vec_blocks 建表维度一致。
|
||||
"""
|
||||
|
||||
model_id = "hash-v1"
|
||||
dim = EMBEDDING_DIM
|
||||
|
||||
async def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
return [self._embed(text) for text in texts]
|
||||
|
||||
async def embed_query(self, query: str) -> list[float]:
|
||||
return self._embed(query)
|
||||
|
||||
def _embed(self, text: str) -> list[float]:
|
||||
vec = [0.0] * self.dim
|
||||
for tok in tokens(text):
|
||||
digest = hashlib.sha256(tok.encode("utf-8")).digest()
|
||||
index = int.from_bytes(digest[:4], "little") % self.dim
|
||||
sign = 1.0 if digest[4] % 2 == 0 else -1.0
|
||||
vec[index] += sign
|
||||
norm = math.sqrt(sum(v * v for v in vec)) or 1.0
|
||||
return [v / norm for v in vec]
|
||||
@@ -0,0 +1,183 @@
|
||||
"""混合检索引擎：编排 FTS5 / Vector / RRF / Reranker / Metadata Filter / Citation。
|
||||
|
||||
对调用方（搜索页、RAG Engine、Agent Tool）暴露统一的 search(request) -> SearchResponse。
|
||||
引擎只依赖 VectorStore / EmbeddingProvider / RerankerProvider 抽象与 Repository，
|
||||
不直接拼接 vec0 内部 SQL，也不向前端输出聊天文本。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app import repository
|
||||
from app.contracts import (
|
||||
Citation,
|
||||
PageMeta,
|
||||
SearchMode,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
SearchResult,
|
||||
)
|
||||
from app.repository import BlockHit
|
||||
from app.retrieval.embedding import EmbeddingProvider, HashEmbeddingProvider
|
||||
from app.retrieval.hybrid import normalize_scores, rrf_fuse
|
||||
from app.retrieval.reranker import LexicalReranker, RankedCandidate, RerankerProvider
|
||||
from app.retrieval.vectorstore import SqliteVecStore, VectorStore
|
||||
from app.textutils import make_snippet, match_query
|
||||
|
||||
# 每个通道的候选池大小；真实规模上来后按 Retrieval Config 调整
|
||||
CANDIDATE_POOL = 50
|
||||
# 分页窗口上限：候选池至少覆盖 offset+limit，但设上限防止超大 offset 撑爆内存
|
||||
MAX_CANDIDATE_POOL = 200
|
||||
# 带 metadata 过滤时放大召回倍数，缓解「先截断候选池再过滤」造成的漏召回
|
||||
OVERSCAN_FACTOR = 4
|
||||
# FTS 一次性取全量命中上限：保证 fts 模式 total 准确、过滤不漏召回；超出则截断
|
||||
FTS_FETCH_LIMIT = 1000
|
||||
|
||||
|
||||
class RetrievalEngine:
|
||||
def __init__(
|
||||
self,
|
||||
embedding: EmbeddingProvider,
|
||||
reranker: RerankerProvider,
|
||||
vector_store: VectorStore,
|
||||
) -> None:
|
||||
self.embedding = embedding
|
||||
self.reranker = reranker
|
||||
self.vector_store = vector_store
|
||||
|
||||
async def search(self, request: SearchRequest) -> SearchResponse:
|
||||
has_filters = bool(
|
||||
request.folders or request.note_ids or request.tags
|
||||
or request.created_from or request.created_to
|
||||
or request.updated_from or request.updated_to
|
||||
)
|
||||
# 候选池至少覆盖本次请求的 offset+limit，保证分页能取到目标页；设上限防内存失控
|
||||
window = min(request.offset + request.limit, MAX_CANDIDATE_POOL)
|
||||
pool_size = max(CANDIDATE_POOL, window)
|
||||
# 带过滤时放大召回；FTS 则一次性取全量命中（≤FTS_FETCH_LIMIT）避免截断漏召回
|
||||
recall = min(pool_size * OVERSCAN_FACTOR, MAX_CANDIDATE_POOL) if has_filters else pool_size
|
||||
|
||||
# 1. 按模式收集候选（FTS 与 Vector 各产出「按相关性降序」的 block_id 列表）
|
||||
fts_ranked: list[str] = []
|
||||
vec_ranked: list[str] = []
|
||||
fts_scores: dict[str, float] = {}
|
||||
vec_scores: dict[str, float] = {}
|
||||
|
||||
if request.mode in (SearchMode.fts, SearchMode.hybrid):
|
||||
match = match_query(request.query)
|
||||
if match:
|
||||
fts_limit = FTS_FETCH_LIMIT if request.mode == SearchMode.fts else recall
|
||||
fts_hits = repository.fts_search(match, fts_limit)
|
||||
fts_ranked = [h.block_id for h in fts_hits]
|
||||
# bm25 越小越相关，取反后统一为「越大越相关」
|
||||
fts_scores = {h.block_id: -h.bm25 for h in fts_hits}
|
||||
|
||||
if request.mode in (SearchMode.vector, SearchMode.hybrid):
|
||||
query_vec = await self.embedding.embed_query(request.query)
|
||||
vec_hits = await self.vector_store.search(query_vec, top_k=recall)
|
||||
vec_ranked = [v.id for v in vec_hits]
|
||||
vec_scores = {v.id: v.score for v in vec_hits}
|
||||
|
||||
if request.mode == SearchMode.fts:
|
||||
candidate_scores = fts_scores
|
||||
elif request.mode == SearchMode.vector:
|
||||
candidate_scores = vec_scores
|
||||
else: # hybrid：RRF 融合
|
||||
candidate_scores = rrf_fuse([fts_ranked, vec_ranked])
|
||||
|
||||
if not candidate_scores:
|
||||
return self._empty(request)
|
||||
|
||||
# 2. 取完整 Block 上下文（用于过滤、摘要与 Citation 定位）
|
||||
hits = {h.block_id: h for h in repository.get_block_hits(list(candidate_scores.keys()))}
|
||||
|
||||
# 3. Metadata Filter
|
||||
filtered = [h for h in hits.values() if self._matches(h, request)]
|
||||
if not filtered:
|
||||
return self._empty(request)
|
||||
|
||||
# 4. 排序 / 精排
|
||||
if request.mode == SearchMode.hybrid:
|
||||
candidates = [
|
||||
RankedCandidate(block_id=h.block_id, score=candidate_scores[h.block_id], text=h.content)
|
||||
for h in filtered
|
||||
]
|
||||
ranked = await self.reranker.rerank(request.query, candidates)
|
||||
ordered = [(c.block_id, c.score) for c in ranked]
|
||||
else:
|
||||
ordered = sorted(
|
||||
((h.block_id, candidate_scores[h.block_id]) for h in filtered),
|
||||
key=lambda item: -item[1],
|
||||
)
|
||||
|
||||
ordered = normalize_scores(ordered)
|
||||
|
||||
# 5. 分页：total = 过滤后候选集大小。fts 已取全量（≤FTS_FETCH_LIMIT）故为真实命中数；
|
||||
# vector/hybrid 为 KNN 候选集，无全局 total。
|
||||
total = len(ordered)
|
||||
page = ordered[request.offset : request.offset + request.limit]
|
||||
items = [self._build_result(hits[block_id], request, score) for block_id, score in page]
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
items=items,
|
||||
page=PageMeta(total=total, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
def _matches(self, hit: BlockHit, request: SearchRequest) -> bool:
|
||||
if request.folders and hit.folder not in request.folders:
|
||||
return False
|
||||
if request.note_ids and hit.note_id not in request.note_ids:
|
||||
return False
|
||||
if request.tags and not (set(hit.tags) & set(request.tags)):
|
||||
return False
|
||||
if request.created_from and _utc(hit.created_at) < _utc(request.created_from):
|
||||
return False
|
||||
if request.created_to and _utc(hit.created_at) > _utc(request.created_to):
|
||||
return False
|
||||
if request.updated_from and _utc(hit.updated_at) < _utc(request.updated_from):
|
||||
return False
|
||||
if request.updated_to and _utc(hit.updated_at) > _utc(request.updated_to):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _build_result(self, hit: BlockHit, request: SearchRequest, score: float) -> SearchResult:
|
||||
citation = Citation(
|
||||
citation_id=f"cit_{hit.block_id}",
|
||||
note_id=hit.note_id,
|
||||
block_id=hit.block_id,
|
||||
file_path=hit.file_path,
|
||||
heading_path=hit.heading_path,
|
||||
start_offset=hit.start_offset,
|
||||
end_offset=hit.end_offset,
|
||||
)
|
||||
snippet = make_snippet(hit.content, request.query) if request.include_snippet else None
|
||||
return SearchResult(
|
||||
note_id=hit.note_id,
|
||||
block_id=hit.block_id,
|
||||
title=hit.title,
|
||||
file_path=hit.file_path,
|
||||
heading_path=hit.heading_path,
|
||||
snippet=snippet,
|
||||
score=score,
|
||||
citation=citation,
|
||||
)
|
||||
|
||||
def _empty(self, request: SearchRequest) -> SearchResponse:
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
page=PageMeta(total=0, limit=request.limit, offset=request.offset),
|
||||
)
|
||||
|
||||
|
||||
def _utc(dt: datetime) -> datetime:
|
||||
"""把时间统一到 naive UTC 再比较，避免 aware/naive 混用报错。"""
|
||||
if dt.tzinfo is None:
|
||||
return dt
|
||||
return dt.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
# 默认引擎实例：轻量实现跑通链路，后续可替换真实模型实现
|
||||
engine = RetrievalEngine(HashEmbeddingProvider(), LexicalReranker(), SqliteVecStore())
|
||||
@@ -0,0 +1,25 @@
|
||||
"""RRF 排名融合与分数归一化。"""
|
||||
|
||||
|
||||
def rrf_fuse(ranked_lists: list[list[str]], k: int = 60) -> dict[str, float]:
|
||||
"""Reciprocal Rank Fusion：对多个「按相关性降序」的 block_id 列表做排名融合。
|
||||
|
||||
每个 block 的融合分 = Σ 1/(k + rank)，rank 从 1 开始。返回 block_id -> 融合分。
|
||||
"""
|
||||
scores: dict[str, float] = {}
|
||||
for ids in ranked_lists:
|
||||
for rank, block_id in enumerate(ids, start=1):
|
||||
scores[block_id] = scores.get(block_id, 0.0) + 1.0 / (k + rank)
|
||||
return scores
|
||||
|
||||
|
||||
def normalize_scores(items: list[tuple[str, float]]) -> list[tuple[str, float]]:
|
||||
"""把 (block_id, score) 列表 min-max 归一化到 [0,1]，score 越大越相关。"""
|
||||
if not items:
|
||||
return []
|
||||
values = [score for _, score in items]
|
||||
lo, hi = min(values), max(values)
|
||||
span = hi - lo
|
||||
if span == 0:
|
||||
return [(block_id, 1.0) for block_id, _ in items]
|
||||
return [(block_id, round((score - lo) / span, 6)) for block_id, score in items]
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Reranker 统一接口与轻量实现。
|
||||
|
||||
真实默认是 BGE reranker 类 Cross-Encoder，第一阶段先用词面重叠 + 原始分数加权的
|
||||
确定性精排跑通链路；后续替换实现即可。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
from app.textutils import tokens
|
||||
|
||||
|
||||
@dataclass
|
||||
class RankedCandidate:
|
||||
block_id: str
|
||||
score: float
|
||||
text: str = "" # 块正文，供轻量精排计算词面重叠
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class RerankerProvider(Protocol):
|
||||
"""统一 Reranker 接口：输入候选块，输出按相关性重排后的候选块。"""
|
||||
|
||||
model_id: str
|
||||
|
||||
async def rerank(self, query: str, candidates: list[RankedCandidate]) -> list[RankedCandidate]: ...
|
||||
|
||||
|
||||
class LexicalReranker:
|
||||
"""轻量精排：query 与块正文的词面重叠度，与归一化后的原始分数加权求和。"""
|
||||
|
||||
model_id = "lexical-v1"
|
||||
|
||||
def __init__(self, lexical_weight: float = 0.5) -> None:
|
||||
self.lexical_weight = lexical_weight
|
||||
|
||||
async def rerank(self, query: str, candidates: list[RankedCandidate]) -> list[RankedCandidate]:
|
||||
if not candidates:
|
||||
return []
|
||||
|
||||
# 把原始分数（RRF 等）归一化到 [0,1]，便于与重叠度同量纲加权
|
||||
scores = [c.score for c in candidates]
|
||||
lo, hi = min(scores), max(scores)
|
||||
span = (hi - lo) or 1.0
|
||||
|
||||
query_tokens = set(tokens(query))
|
||||
ranked: list[RankedCandidate] = []
|
||||
for c in candidates:
|
||||
norm = (c.score - lo) / span
|
||||
if query_tokens:
|
||||
overlap = len(query_tokens & set(tokens(c.text))) / len(query_tokens)
|
||||
else:
|
||||
overlap = 0.0
|
||||
final = self.lexical_weight * overlap + (1 - self.lexical_weight) * norm
|
||||
ranked.append(RankedCandidate(block_id=c.block_id, score=final, text=c.text))
|
||||
|
||||
ranked.sort(key=lambda c: c.score, reverse=True)
|
||||
return ranked
|
||||
@@ -0,0 +1,94 @@
|
||||
"""VectorStore 统一接口与 sqlite-vec 实现。
|
||||
|
||||
vec0 虚拟表返回的 distance 是欧氏距离（非平方）。入库前向量已做 L2 归一化，
|
||||
因此 distance² = 2(1-cos)，余弦相似度 = 1 - distance² / 2。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import sqlite_vec
|
||||
|
||||
from app.database.db import connect, transaction
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorRecord:
|
||||
id: str
|
||||
vector: list[float]
|
||||
|
||||
|
||||
@dataclass
|
||||
class VectorHit:
|
||||
id: str
|
||||
score: float # 余弦相似度 [0,1]
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VectorStore(Protocol):
|
||||
"""统一向量存储接口（与文档一致）。上层只依赖此抽象，不读 vec0 内部表。"""
|
||||
|
||||
async def upsert(self, records: list[VectorRecord]) -> None: ...
|
||||
async def delete(self, ids: list[str]) -> None: ...
|
||||
async def search(self, vector: list[float], *, top_k: int) -> list[VectorHit]: ...
|
||||
|
||||
|
||||
class SqliteVecStore:
|
||||
"""sqlite-vec 默认实现。"""
|
||||
|
||||
async def upsert(self, records: list[VectorRecord], *, conn: sqlite3.Connection | None = None) -> None:
|
||||
if not records:
|
||||
return
|
||||
owns = conn is None
|
||||
conn = conn or connect()
|
||||
try:
|
||||
with transaction(conn) if owns else nullcontext():
|
||||
for record in records:
|
||||
# vec0 不支持 UPDATE，采用 delete-then-insert 实现幂等 upsert，避免主键冲突
|
||||
conn.execute("DELETE FROM vec_blocks WHERE block_id = ?", (record.id,))
|
||||
conn.execute(
|
||||
"INSERT INTO vec_blocks (block_id, embedding) VALUES (?, ?)",
|
||||
(record.id, sqlite_vec.serialize_float32(record.vector)),
|
||||
)
|
||||
finally:
|
||||
if owns:
|
||||
conn.close()
|
||||
|
||||
async def delete(self, ids: list[str], *, conn: sqlite3.Connection | None = None) -> None:
|
||||
if not ids:
|
||||
return
|
||||
owns = conn is None
|
||||
conn = conn or connect()
|
||||
try:
|
||||
with transaction(conn) if owns else nullcontext():
|
||||
for bid in ids:
|
||||
conn.execute("DELETE FROM vec_blocks WHERE block_id = ?", (bid,))
|
||||
finally:
|
||||
if owns:
|
||||
conn.close()
|
||||
|
||||
async def search(self, vector: list[float], *, top_k: int) -> list[VectorHit]:
|
||||
conn = connect()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT block_id, distance FROM vec_blocks WHERE embedding MATCH ? AND k = ?",
|
||||
(sqlite_vec.serialize_float32(vector), top_k),
|
||||
).fetchall()
|
||||
return [
|
||||
VectorHit(id=row["block_id"], score=max(0.0, 1.0 - row["distance"] ** 2 / 2.0))
|
||||
for row in rows
|
||||
]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
async def clear(self) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
conn.execute("DELETE FROM vec_blocks")
|
||||
finally:
|
||||
conn.close()
|
||||
+32
-38
@@ -51,6 +51,8 @@ from app.container import container
|
||||
from app.errors import ApiError, not_implemented
|
||||
from app.providers.registry import ProviderNotFoundError
|
||||
from app.providers.factory import UnsupportedProviderError
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import index_service, note_service
|
||||
|
||||
router = APIRouter(prefix="/api")
|
||||
not_implemented_response = {501: {"model": ErrorResponse, "description": "业务服务尚未实现"}}
|
||||
@@ -108,38 +110,37 @@ async def list_notes(
|
||||
folder: str | None = None,
|
||||
tag: str | None = None,
|
||||
) -> NoteListResponse:
|
||||
return NoteListResponse(page=PageMeta(limit=limit, offset=offset))
|
||||
items, total = note_service.list_notes(limit=limit, offset=offset, folder=folder, tag=tag)
|
||||
return NoteListResponse(items=items, page=PageMeta(total=total, limit=limit, offset=offset))
|
||||
|
||||
|
||||
@router.post(
|
||||
"/notes", response_model=Note, responses=not_implemented_response, tags=["Notes"]
|
||||
)
|
||||
async def create_note(_: NoteCreateRequest) -> Note:
|
||||
not_implemented("notes.create")
|
||||
@router.post("/notes", response_model=Note, tags=["Notes"])
|
||||
async def create_note(request: NoteCreateRequest) -> Note:
|
||||
return await note_service.create_note(
|
||||
title=request.title, markdown=request.markdown, folder=request.folder, tags=request.tags
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/notes/{note_id}", response_model=Note, responses=not_implemented_response, tags=["Notes"]
|
||||
)
|
||||
@router.get("/notes/{note_id}", response_model=Note, tags=["Notes"])
|
||||
async def get_note(note_id: str) -> Note:
|
||||
not_implemented(f"notes.read:{note_id}")
|
||||
note = await note_service.get_note(note_id)
|
||||
if note is None:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
||||
return note
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/notes/{note_id}", response_model=Note, responses=not_implemented_response, tags=["Notes"]
|
||||
)
|
||||
async def update_note(note_id: str, _: NoteUpdateRequest) -> Note:
|
||||
not_implemented(f"notes.update:{note_id}")
|
||||
@router.patch("/notes/{note_id}", response_model=Note, tags=["Notes"])
|
||||
async def update_note(note_id: str, request: NoteUpdateRequest) -> Note:
|
||||
return await note_service.update_note(
|
||||
note_id, title=request.title, markdown=request.markdown, tags=request.tags
|
||||
)
|
||||
|
||||
|
||||
@router.delete(
|
||||
"/notes/{note_id}",
|
||||
response_model=OperationResponse,
|
||||
responses=not_implemented_response,
|
||||
tags=["Notes"],
|
||||
)
|
||||
@router.delete("/notes/{note_id}", response_model=OperationResponse, tags=["Notes"])
|
||||
async def delete_note(note_id: str) -> OperationResponse:
|
||||
not_implemented(f"notes.delete:{note_id}")
|
||||
if not await note_service.delete_note(note_id):
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
||||
return OperationResponse(status="completed", resource_id=note_id, message="deleted")
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -152,11 +153,7 @@ async def move_note(note_id: str, _: NoteMoveRequest) -> Note:
|
||||
# Retrieval and chat
|
||||
@router.post("/search", response_model=SearchResponse, tags=["Search"])
|
||||
async def search_notes(request: SearchRequest) -> SearchResponse:
|
||||
return SearchResponse(
|
||||
query=request.query,
|
||||
mode=request.mode,
|
||||
page=PageMeta(limit=request.limit, offset=request.offset),
|
||||
)
|
||||
return await engine.search(request)
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -576,25 +573,22 @@ async def get_transcription(job_id: str) -> TranscriptionJob:
|
||||
|
||||
@router.get("/index/status", response_model=IndexStatus, tags=["Index"])
|
||||
async def get_index_status() -> IndexStatus:
|
||||
return IndexStatus()
|
||||
return index_service.get_status()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/index/rebuild",
|
||||
response_model=IndexJob,
|
||||
status_code=202,
|
||||
responses=not_implemented_response,
|
||||
tags=["Index"],
|
||||
)
|
||||
async def rebuild_index(_: IndexRebuildRequest) -> IndexJob:
|
||||
not_implemented("index.rebuild")
|
||||
async def rebuild_index(request: IndexRebuildRequest) -> IndexJob:
|
||||
return await index_service.rebuild(request)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/index/jobs/{job_id}",
|
||||
response_model=IndexJob,
|
||||
responses=not_implemented_response,
|
||||
tags=["Index"],
|
||||
)
|
||||
@router.get("/index/jobs/{job_id}", response_model=IndexJob, tags=["Index"])
|
||||
async def get_index_job(job_id: str) -> IndexJob:
|
||||
not_implemented(f"index.jobs.read:{job_id}")
|
||||
job = index_service.get_job(job_id)
|
||||
if job is None:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "index job not found", {"job_id": job_id})
|
||||
return job
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""应用服务层：编排 Knowledge/Retrieval Core 与 Repository，供路由调用。"""
|
||||
@@ -0,0 +1,99 @@
|
||||
"""索引服务：扫描 Vault、全量重建索引、查询索引状态。
|
||||
|
||||
MVP 阶段重建是同步的（数据量小），完成后直接返回 completed 的 IndexJob。
|
||||
索引任务暂存内存（_jobs），不持久化到 SQLite；后续接入异步任务队列时再落到 index_jobs 表。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app import repository
|
||||
from app.config import get_settings
|
||||
from app.contracts import IndexJob, IndexRebuildRequest, IndexStatus
|
||||
from app.errors import ApiError
|
||||
from app.knowledge.parser import parse_note
|
||||
from app.services.note_service import index_note
|
||||
from app.retrieval.vectorstore import SqliteVecStore
|
||||
|
||||
vector_store = SqliteVecStore()
|
||||
|
||||
_jobs: dict[str, IndexJob] = {}
|
||||
|
||||
|
||||
def _scan_vault() -> list[tuple[str, str, str, datetime, datetime]]:
|
||||
"""扫描 Vault 下所有 Markdown，返回 (rel_path, folder, markdown, created, updated)。
|
||||
|
||||
先读入内存：若文件读取失败，rebuild 尚未清空旧索引，不会造成数据损失。
|
||||
"""
|
||||
vault = get_settings().vault_path
|
||||
result: list[tuple[str, str, str, datetime, datetime]] = []
|
||||
if not vault.exists():
|
||||
return result
|
||||
for path in sorted(vault.rglob("*.md")):
|
||||
rel = path.relative_to(vault).as_posix()
|
||||
folder = path.relative_to(vault).parent.as_posix()
|
||||
if folder == ".":
|
||||
folder = ""
|
||||
stat = path.stat()
|
||||
created = datetime.fromtimestamp(stat.st_ctime, tz=timezone.utc)
|
||||
updated = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)
|
||||
result.append((rel, folder, path.read_text(encoding="utf-8"), created, updated))
|
||||
return result
|
||||
|
||||
|
||||
async def rebuild(request: IndexRebuildRequest) -> IndexJob:
|
||||
job_id = "job_" + uuid4().hex[:12]
|
||||
# 增量重建（scope != all 或指定 note_ids）尚未实现，明确拒绝而非静默全量重建
|
||||
if request.scope != "all" or request.note_ids:
|
||||
raise ApiError(
|
||||
400,
|
||||
"UNSUPPORTED_SCOPE",
|
||||
"only full rebuild (scope='all' with empty note_ids) is supported",
|
||||
{"scope": request.scope, "note_ids": request.note_ids},
|
||||
)
|
||||
|
||||
# 先扫描到内存（失败不会清旧索引），再快照旧库用于失败回滚
|
||||
docs = _scan_vault()
|
||||
settings = get_settings()
|
||||
backup_path = settings.db_path.with_suffix(".db.bak") if settings.db_path.exists() else None
|
||||
if backup_path is not None:
|
||||
shutil.copy2(settings.db_path, backup_path)
|
||||
|
||||
try:
|
||||
repository.clear_all()
|
||||
await vector_store.clear()
|
||||
for rel, folder, markdown, created, updated in docs:
|
||||
parsed = parse_note(
|
||||
markdown=markdown, file_path=rel, folder=folder, tags=None,
|
||||
created_at=created, updated_at=updated,
|
||||
)
|
||||
await index_note(parsed)
|
||||
except BaseException:
|
||||
# 重建失败：恢复旧索引，避免留下半成品；记录 failed 任务后向上抛
|
||||
if backup_path is not None and backup_path.exists():
|
||||
shutil.copy2(backup_path, settings.db_path)
|
||||
_jobs[job_id] = IndexJob(
|
||||
job_id=job_id, status="failed", scope=request.scope,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if backup_path is not None:
|
||||
backup_path.unlink(missing_ok=True)
|
||||
|
||||
job = IndexJob(job_id=job_id, status="completed", scope=request.scope, created_at=datetime.now(timezone.utc))
|
||||
_jobs[job_id] = job
|
||||
return job
|
||||
|
||||
|
||||
def get_status() -> IndexStatus:
|
||||
# 同步重建、无排队任务，因此状态恒为 idle；实际索引规模可由 GET /api/notes 与搜索反映
|
||||
return IndexStatus(status="idle", pending_jobs=0)
|
||||
|
||||
|
||||
def get_job(job_id: str) -> IndexJob | None:
|
||||
return _jobs.get(job_id)
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Note 服务：Markdown 文件读写 + 解析 + 索引编排。
|
||||
|
||||
Markdown 文件是笔记正文的持久化载体（Vault），SQLite/FTS5/向量是可重建索引。
|
||||
本服务负责在两者之间保持一致：写文件后解析并写入元数据、FTS5 与向量。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from app import repository
|
||||
from app.config import get_settings
|
||||
from app.contracts import Note, NoteBlock, NoteSummary
|
||||
from app.database.db import connect, transaction
|
||||
from app.errors import ApiError
|
||||
from app.knowledge.parser import ParsedNote, parse_note
|
||||
from app.retrieval.embedding import HashEmbeddingProvider
|
||||
from app.retrieval.vectorstore import SqliteVecStore, VectorRecord
|
||||
|
||||
# 轻量实现实例（无状态，可直接复用）；接入真实模型后替换为对应 Provider
|
||||
embedding = HashEmbeddingProvider()
|
||||
vector_store = SqliteVecStore()
|
||||
|
||||
|
||||
def _vault() -> Path:
|
||||
return get_settings().vault_path
|
||||
|
||||
|
||||
def _safe_name(title: str) -> str:
|
||||
name = re.sub(r'[\\/:*?"<>|]', "_", title).strip()
|
||||
return name or "untitled"
|
||||
|
||||
|
||||
def _normalize_folder(folder: str | None) -> str:
|
||||
"""清洗 folder 为安全的相对目录，拒绝 `..`/`.`/绝对路径/盘符/空字节，防路径逃逸。"""
|
||||
if not folder:
|
||||
return ""
|
||||
if "\x00" in folder:
|
||||
raise ApiError(400, "INVALID_PATH", "folder must not contain NUL bytes", {"folder": folder})
|
||||
segments: list[str] = []
|
||||
for part in re.split(r"[\\/]+", folder):
|
||||
if part == "":
|
||||
continue
|
||||
if part in (".", ".."):
|
||||
raise ApiError(400, "INVALID_PATH", "folder must not contain '.' or '..'", {"folder": folder})
|
||||
if ":" in part:
|
||||
raise ApiError(400, "INVALID_PATH", "folder must be a relative path", {"folder": folder})
|
||||
segments.append(part)
|
||||
return "/".join(segments)
|
||||
|
||||
|
||||
def _rel_path(folder: str | None, title: str) -> tuple[str, str]:
|
||||
"""由 folder + title 生成安全的相对路径，返回 (rel_path, 清洗后的 folder)。"""
|
||||
clean_folder = _normalize_folder(folder)
|
||||
name = _safe_name(title)
|
||||
if not name.endswith(".md"):
|
||||
name += ".md"
|
||||
rel = f"{clean_folder}/{name}" if clean_folder else name
|
||||
return rel, clean_folder
|
||||
|
||||
|
||||
def _abs_path(rel_path: str) -> Path:
|
||||
"""把相对路径解析为 Vault 内的绝对路径；越界即报 400，杜绝路径逃逸。"""
|
||||
if not rel_path or "\x00" in rel_path:
|
||||
raise ApiError(400, "INVALID_PATH", "invalid file path", {"file_path": rel_path})
|
||||
root = _vault().resolve()
|
||||
candidate = (_vault() / rel_path).resolve()
|
||||
if not candidate.is_relative_to(root):
|
||||
raise ApiError(400, "INVALID_PATH", "path escapes vault", {"file_path": rel_path})
|
||||
return candidate
|
||||
|
||||
|
||||
def _read_markdown(rel_path: str) -> str:
|
||||
path = _abs_path(rel_path)
|
||||
return path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
|
||||
|
||||
def _write_markdown(rel_path: str, markdown: str) -> None:
|
||||
path = _abs_path(rel_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(markdown, encoding="utf-8")
|
||||
|
||||
|
||||
def _delete_markdown(rel_path: str) -> None:
|
||||
path = _abs_path(rel_path)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
|
||||
|
||||
async def index_note(parsed: ParsedNote) -> None:
|
||||
"""把解析结果写入元数据 + FTS5 + 向量（三层可重建索引），单事务保证原子性。
|
||||
|
||||
元数据与向量在同一连接、同一事务内提交，避免「新元数据已提交、向量写入失败」的
|
||||
半提交状态。替换元数据时拿到旧 block_id：清理已删除/内容变化的旧向量，只为新增
|
||||
block 写向量（内容未变的 block 其向量仍有效，无需重复写入）。
|
||||
"""
|
||||
vectors = await embedding.embed_documents([block.content for block in parsed.blocks])
|
||||
conn = connect()
|
||||
try:
|
||||
with transaction(conn):
|
||||
old_block_ids = repository.replace_note_metadata(
|
||||
conn=conn,
|
||||
note_id=parsed.note_id,
|
||||
title=parsed.title,
|
||||
file_path=parsed.file_path,
|
||||
folder=parsed.folder,
|
||||
tags=parsed.tags,
|
||||
created_at=parsed.created_at,
|
||||
updated_at=parsed.updated_at,
|
||||
blocks=parsed.blocks,
|
||||
)
|
||||
old_ids = set(old_block_ids)
|
||||
new_ids = {block.block_id for block in parsed.blocks}
|
||||
stale_ids = [bid for bid in old_ids if bid not in new_ids]
|
||||
if stale_ids:
|
||||
await vector_store.delete(stale_ids, conn=conn)
|
||||
missing_ids = [bid for bid in new_ids if bid not in old_ids]
|
||||
records = [
|
||||
VectorRecord(id=block.block_id, vector=vector)
|
||||
for block, vector in zip(parsed.blocks, vectors)
|
||||
if block.block_id in missing_ids
|
||||
]
|
||||
await vector_store.upsert(records, conn=conn)
|
||||
finally:
|
||||
conn.close()
|
||||
repository.set_index_meta({"embedding_model": embedding.model_id, "embedding_dim": str(embedding.dim)})
|
||||
|
||||
|
||||
async def create_note(*, title: str, markdown: str, folder: str | None, tags: list[str]) -> Note:
|
||||
rel_path, clean_folder = _rel_path(folder, title)
|
||||
now = datetime.now(timezone.utc)
|
||||
parsed = parse_note(
|
||||
# 创建时空标签视为「未显式指定」，由 frontmatter 推导（创建无「清空」语义）
|
||||
markdown=markdown, file_path=rel_path, folder=clean_folder, tags=tags or None,
|
||||
created_at=now, updated_at=now,
|
||||
)
|
||||
parsed.title = title # 显式传入的 title 优先于正文推导（与 update_note 保持一致）
|
||||
_write_markdown(rel_path, markdown)
|
||||
try:
|
||||
await index_note(parsed)
|
||||
except BaseException:
|
||||
_delete_markdown(rel_path) # 索引失败时回滚，避免「文件已写、索引缺失」的部分提交
|
||||
raise
|
||||
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
|
||||
parsed.created_at, parsed.updated_at, parsed.blocks, markdown)
|
||||
|
||||
|
||||
async def get_note(note_id: str) -> Note | None:
|
||||
record = repository.get_note_record(note_id)
|
||||
if record is None:
|
||||
return None
|
||||
markdown = _read_markdown(record.file_path)
|
||||
return _build_note(record.note_id, record.title, record.file_path, record.tags,
|
||||
record.created_at, record.updated_at, record.blocks, markdown)
|
||||
|
||||
|
||||
async def update_note(
|
||||
note_id: str, *, title: str | None = None, markdown: str | None = None, tags: list[str] | None = None
|
||||
) -> Note:
|
||||
record = repository.get_note_record(note_id)
|
||||
if record is None:
|
||||
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
|
||||
|
||||
old_md = _read_markdown(record.file_path)
|
||||
new_md = old_md if markdown is None else markdown
|
||||
# PATCH 语义：tags=None 保持原标签；[] 清空；非空列表替换（区别于 create 的 frontmatter 推导）
|
||||
effective_tags = record.tags if tags is None else tags
|
||||
_write_markdown(record.file_path, new_md)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
try:
|
||||
parsed = parse_note(
|
||||
markdown=new_md, file_path=record.file_path, folder=record.folder, tags=effective_tags,
|
||||
created_at=record.created_at, updated_at=now,
|
||||
)
|
||||
if title is not None:
|
||||
parsed.title = title # 显式传入的 title 覆盖正文推导结果
|
||||
|
||||
await index_note(parsed)
|
||||
except BaseException:
|
||||
_write_markdown(record.file_path, old_md) # 索引失败时回滚正文，避免部分提交
|
||||
raise
|
||||
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
|
||||
parsed.created_at, parsed.updated_at, parsed.blocks, new_md)
|
||||
|
||||
|
||||
async def delete_note(note_id: str) -> bool:
|
||||
record = repository.get_note_record(note_id)
|
||||
if record is None:
|
||||
return False
|
||||
block_ids = repository.delete_note(note_id)
|
||||
await vector_store.delete(block_ids)
|
||||
_delete_markdown(record.file_path)
|
||||
return True
|
||||
|
||||
|
||||
def list_notes(*, limit: int, offset: int, folder: str | None, tag: str | None) -> tuple[list[NoteSummary], int]:
|
||||
items, total = repository.list_note_summaries(limit=limit, offset=offset, folder=folder, tag=tag)
|
||||
return [NoteSummary(**item) for item in items], total
|
||||
|
||||
|
||||
def _build_note(
|
||||
note_id: str, title: str, file_path: str, tags: list[str],
|
||||
created_at: datetime, updated_at: datetime, blocks: list[NoteBlock], markdown: str,
|
||||
) -> Note:
|
||||
return Note(
|
||||
note_id=note_id,
|
||||
title=title,
|
||||
file_path=file_path,
|
||||
tags=tags,
|
||||
created_at=created_at,
|
||||
updated_at=updated_at,
|
||||
markdown=markdown,
|
||||
blocks=blocks,
|
||||
)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""文本分词工具，供 FTS5 与轻量 Embedding 共用。
|
||||
|
||||
FTS5 默认 unicode61 分词器不切分中文（连续汉字算一个 token），导致「死锁」这类
|
||||
子串无法命中。这里把文本统一拆成 ASCII 单词 + CJK 单字 + CJK 相邻双字，写入与查询
|
||||
走同一套拆分，实现中文子串/词级召回。
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
_WORD_RE = re.compile(r"[a-z0-9]+")
|
||||
_CJK_RUN_RE = re.compile(r"[一-鿿]+")
|
||||
|
||||
|
||||
def tokens(text: str) -> list[str]:
|
||||
"""返回文本的检索 token 序列（含重复）。"""
|
||||
out: list[str] = []
|
||||
out.extend(_WORD_RE.findall(text.lower()))
|
||||
for run in _CJK_RUN_RE.findall(text):
|
||||
out.extend(run) # 单字
|
||||
out.extend(run[i : i + 2] for i in range(len(run) - 1)) # 相邻双字
|
||||
return out
|
||||
|
||||
|
||||
def unique_tokens(text: str) -> list[str]:
|
||||
"""去重但保序的 token 列表，用于压缩查询/索引体积。"""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
for tok in tokens(text):
|
||||
if tok not in seen:
|
||||
seen.add(tok)
|
||||
result.append(tok)
|
||||
return result
|
||||
|
||||
|
||||
def segment(text: str) -> str:
|
||||
"""把文本转成空格分隔的 token，写入 FTS5 的 content 列。"""
|
||||
return " ".join(unique_tokens(text))
|
||||
|
||||
|
||||
def match_query(query: str) -> str | None:
|
||||
"""把用户查询转成 FTS5 MATCH 表达式（OR 连接、按 token 精确匹配）。"""
|
||||
toks = unique_tokens(query)
|
||||
if not toks:
|
||||
return None
|
||||
return " OR ".join(f'"{tok}"' for tok in toks)
|
||||
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
"""粗略 token 数：ASCII 单词 + CJK 单字。仅用于展示，不做精确计量。"""
|
||||
return len(_WORD_RE.findall(text.lower())) + sum(len(run) for run in _CJK_RUN_RE.findall(text))
|
||||
|
||||
|
||||
def make_snippet(content: str, query: str, max_len: int = 160) -> str:
|
||||
"""从原文生成命中片段：优先定位较长的 query token，向前后扩展窗口。"""
|
||||
toks = unique_tokens(query)
|
||||
lowered = content.lower()
|
||||
best = -1
|
||||
# 优先用较长的 token（双字/单词）定位，命中最准确
|
||||
for tok in toks:
|
||||
if len(tok) >= 2:
|
||||
idx = lowered.find(tok)
|
||||
if idx != -1:
|
||||
best = idx
|
||||
break
|
||||
if best == -1:
|
||||
for tok in toks:
|
||||
idx = lowered.find(tok)
|
||||
if idx != -1:
|
||||
best = idx
|
||||
break
|
||||
|
||||
if best == -1:
|
||||
snippet = content
|
||||
else:
|
||||
start = max(0, best - max_len // 3)
|
||||
end = min(len(content), best + max_len)
|
||||
snippet = content[start:end]
|
||||
snippet = ("…" if start > 0 else "") + snippet + ("…" if end < len(content) else "")
|
||||
|
||||
if len(snippet) > max_len + 20:
|
||||
snippet = snippet[:max_len] + "…"
|
||||
return snippet
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
title: RAG 检索增强与引用定位
|
||||
tags: RAG, 产品
|
||||
---
|
||||
|
||||
# RAG 概述
|
||||
|
||||
检索增强生成先检索相关文档块，再交给大模型生成回答。
|
||||
|
||||
## Citation 引用
|
||||
|
||||
每个搜索结果附带 Citation，包含文件路径与起止偏移量。
|
||||
|
||||
前端可根据偏移量跳转到笔记中的原始位置。
|
||||
|
||||
## Reranker 精排
|
||||
|
||||
粗排后使用 Reranker 对候选块重新打分，提升相关性。
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
title: 周会纪要
|
||||
tags: 会议, 日记
|
||||
---
|
||||
|
||||
# 周会纪要
|
||||
|
||||
今天讨论了三件事：索引重建、混合检索、前端联调。
|
||||
|
||||
Vector index rebuild 任务需要支持增量更新。
|
||||
|
||||
下次会议在周五，记得同步接口契约。
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
title: Python 基础语法
|
||||
tags: python, 编程
|
||||
---
|
||||
|
||||
# 变量与类型
|
||||
|
||||
Python 是动态类型语言，变量无需声明类型。
|
||||
|
||||
整数、浮点数、字符串、布尔值是四种基本类型。
|
||||
|
||||
## 列表与字典
|
||||
|
||||
列表用方括号，字典用花括号。列表推导式非常常用。
|
||||
|
||||
### 函数定义
|
||||
|
||||
使用 def 关键字定义函数，支持默认参数与关键字参数。
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
title: 向量数据库与相似度检索
|
||||
tags: 向量数据库, 检索
|
||||
---
|
||||
|
||||
# 向量数据库
|
||||
|
||||
向量数据库用于存储高维向量并支持近似最近邻检索。
|
||||
|
||||
常用相似度度量有余弦相似度与欧氏距离。
|
||||
|
||||
## sqlite-vec
|
||||
|
||||
sqlite-vec 是一个轻量的 SQLite 向量扩展，支持 vec0 虚拟表。
|
||||
|
||||
可以存储 float32 向量，并通过 KNN 查询相近向量。
|
||||
|
||||
## 混合检索
|
||||
|
||||
结合全文检索与向量检索，用 RRF 融合排序结果。
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
title: Notes Agentic 项目说明
|
||||
tags: 项目
|
||||
---
|
||||
|
||||
# 项目说明
|
||||
|
||||
这是一个 AI 笔记软件，支持 Markdown 块级索引与混合检索。
|
||||
|
||||
后端基于 FastAPI，检索使用 SQLite FTS5 与 sqlite-vec 向量检索。
|
||||
@@ -7,6 +7,7 @@ requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"fastapi>=0.116,<1.0",
|
||||
"httpx>=0.28,<1.0",
|
||||
"sqlite-vec>=0.1.9",
|
||||
"uvicorn[standard]>=0.35,<1.0",
|
||||
]
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""pytest 全局隔离：把所有测试的数据目录/数据库/Vault 重定向到临时目录。
|
||||
|
||||
这样测试不会读写真实的 backend/data（真实索引与笔记），也使得「默认库应为空」这类
|
||||
断言在任意本机状态下都确定成立——即使开发者已在本地跑过 rebuild。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import get_settings
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _isolate_data_dir(tmp_path, monkeypatch):
|
||||
data_dir = tmp_path / "data"
|
||||
monkeypatch.setenv("APP_DATA_DIR", str(data_dir))
|
||||
monkeypatch.setenv("APP_DB_PATH", str(data_dir / "app.db"))
|
||||
monkeypatch.setenv("APP_VAULT_PATH", str(tmp_path / "vault"))
|
||||
# 清除 lru 缓存，让本次测试内的 get_settings() 读到临时目录
|
||||
get_settings.cache_clear()
|
||||
yield
|
||||
get_settings.cache_clear()
|
||||
@@ -0,0 +1,435 @@
|
||||
"""Knowledge / Retrieval Core 的单元与端到端测试。
|
||||
|
||||
端到端用例通过 monkeypatch 将 APP_DATA_DIR / APP_DB_PATH / APP_VAULT_PATH 指到临时目录，
|
||||
并清理 get_settings 缓存，保证不读写 backend/data 下的真实索引，也不污染其他测试。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import get_settings
|
||||
from app.contracts import IndexRebuildRequest, SearchMode, SearchRequest
|
||||
from app.knowledge.parser import note_id_for_path, parse_note
|
||||
from app.retrieval.embedding import HashEmbeddingProvider
|
||||
from app.retrieval.hybrid import rrf_fuse
|
||||
from app.textutils import match_query, tokens
|
||||
|
||||
MD = """---
|
||||
title: 测试标题
|
||||
tags: python, 检索
|
||||
---
|
||||
|
||||
# 一级标题
|
||||
|
||||
这是第一段正文。
|
||||
|
||||
## 二级标题
|
||||
|
||||
第二段正文内容。
|
||||
"""
|
||||
|
||||
|
||||
def _dt() -> datetime:
|
||||
return datetime(2026, 8, 27, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vault():
|
||||
"""返回 conftest 全局隔离后的临时 Vault 目录，用于写入示例笔记。"""
|
||||
return get_settings().vault_path
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 单元测试
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_parse_note_extracts_frontmatter_and_blocks() -> None:
|
||||
parsed = parse_note(
|
||||
markdown=MD, file_path="编程/测试.md", folder="编程",
|
||||
tags=None, created_at=_dt(), updated_at=_dt(),
|
||||
)
|
||||
|
||||
assert parsed.title == "测试标题"
|
||||
assert parsed.tags == ["python", "检索"]
|
||||
assert parsed.note_id == note_id_for_path("编程/测试.md")
|
||||
|
||||
paths = [tuple(b.heading_path) for b in parsed.blocks]
|
||||
assert ("一级标题",) in paths
|
||||
assert ("一级标题", "二级标题") in paths
|
||||
|
||||
# 每个 Block 的偏移合法且内容非空
|
||||
for b in parsed.blocks:
|
||||
assert 0 <= b.start_offset <= b.end_offset
|
||||
assert b.content.strip()
|
||||
|
||||
|
||||
def test_block_ids_are_stable() -> None:
|
||||
p1 = parse_note(markdown=MD, file_path="编程/测试.md", folder="编程",
|
||||
tags=None, created_at=_dt(), updated_at=_dt())
|
||||
p2 = parse_note(markdown=MD, file_path="编程/测试.md", folder="编程",
|
||||
tags=None, created_at=_dt(), updated_at=_dt())
|
||||
|
||||
assert [b.block_id for b in p1.blocks] == [b.block_id for b in p2.blocks]
|
||||
# block_id 前缀符合团队约定
|
||||
assert all(b.block_id.startswith("blk_") for b in p1.blocks)
|
||||
|
||||
|
||||
def test_tokens_split_cjk_bigrams_and_match_query() -> None:
|
||||
toks = tokens("向量检索")
|
||||
assert "向" in toks and "量" in toks
|
||||
assert "向量" in toks and "检索" in toks
|
||||
|
||||
q = match_query("python 向量")
|
||||
assert '"python"' in q and '"向量"' in q
|
||||
|
||||
|
||||
def test_hash_embedding_is_deterministic_and_normalized() -> None:
|
||||
emb = HashEmbeddingProvider()
|
||||
v1 = asyncio.run(emb.embed_query("向量检索"))
|
||||
v2 = asyncio.run(emb.embed_query("向量检索"))
|
||||
|
||||
assert v1 == v2
|
||||
assert len(v1) == emb.dim == 128
|
||||
norm = sum(x * x for x in v1) ** 0.5
|
||||
assert abs(norm - 1.0) < 1e-6
|
||||
|
||||
|
||||
def test_rrf_fuse_merges_ranked_lists() -> None:
|
||||
scores = rrf_fuse([["a", "b"], ["b", "a"]])
|
||||
|
||||
assert set(scores) == {"a", "b"}
|
||||
assert scores["a"] > 0 and scores["b"] > 0
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 端到端测试（隔离环境）
|
||||
# --------------------------------------------------------------------------- #
|
||||
SAMPLE_NOTES = {
|
||||
"编程/向量.md": (
|
||||
"---\ntitle: 向量数据库\ntags: 向量, 检索\n---\n\n"
|
||||
"# 向量数据库\n\n向量数据库用于存储高维向量并支持近似最近邻检索。\n"
|
||||
),
|
||||
"编程/Python.md": (
|
||||
"---\ntitle: Python 基础\ntags: python\n---\n\n"
|
||||
"# 变量\n\nPython 是动态类型语言。\n"
|
||||
),
|
||||
"产品/RAG.md": (
|
||||
"---\ntitle: RAG 概述\ntags: RAG\n---\n\n"
|
||||
"# RAG\n\n检索增强生成先检索相关文档块。\n"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _write_vault(vault, files: dict[str, str]) -> None:
|
||||
for rel, text in files.items():
|
||||
path = vault / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def test_rebuild_indexes_vault_and_lists_notes(vault) -> None:
|
||||
from app.services import index_service, note_service
|
||||
|
||||
_write_vault(vault, SAMPLE_NOTES)
|
||||
|
||||
job = asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
assert job.status == "completed"
|
||||
|
||||
items, total = note_service.list_notes(limit=50, offset=0, folder=None, tag=None)
|
||||
assert total == 3
|
||||
assert {i.file_path for i in items} == set(SAMPLE_NOTES)
|
||||
|
||||
|
||||
def test_search_returns_citations_for_each_mode(vault) -> None:
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import index_service
|
||||
|
||||
_write_vault(vault, SAMPLE_NOTES)
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
|
||||
# FTS：中文词组召回，且返回可定位的 Citation
|
||||
fts = asyncio.run(engine.search(SearchRequest(query="向量数据库", mode=SearchMode.fts)))
|
||||
assert fts.page.total >= 1
|
||||
top = fts.items[0]
|
||||
assert top.citation.citation_id.startswith("cit_")
|
||||
assert top.citation.file_path == "编程/向量.md"
|
||||
assert top.citation.block_id == top.block_id
|
||||
|
||||
# Vector：向量召回
|
||||
vec = asyncio.run(engine.search(SearchRequest(query="向量数据库", mode=SearchMode.vector)))
|
||||
assert vec.page.total >= 1
|
||||
|
||||
# Hybrid：RRF + Reranker 融合后仍有结果
|
||||
hyb = asyncio.run(engine.search(SearchRequest(query="向量数据库", mode=SearchMode.hybrid)))
|
||||
assert hyb.page.total >= 1
|
||||
assert all(0.0 <= r.score <= 1.0 for r in hyb.items)
|
||||
|
||||
|
||||
def test_search_metadata_filters(vault) -> None:
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import index_service
|
||||
|
||||
_write_vault(vault, SAMPLE_NOTES)
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
|
||||
by_folder = asyncio.run(
|
||||
engine.search(SearchRequest(query="检索", mode=SearchMode.hybrid, folders=["产品"]))
|
||||
)
|
||||
assert by_folder.page.total >= 1
|
||||
assert all(r.file_path.startswith("产品/") for r in by_folder.items)
|
||||
|
||||
by_tag = asyncio.run(
|
||||
engine.search(SearchRequest(query="向量", mode=SearchMode.hybrid, tags=["向量"]))
|
||||
)
|
||||
assert by_tag.page.total >= 1
|
||||
assert all("向量" in r.citation.heading_path or "向量" in r.title for r in by_tag.items)
|
||||
|
||||
|
||||
def test_route_handlers_wired_to_services(vault) -> None:
|
||||
"""验证 routes.py 里 notes/search/index 端点已接入真实服务（而非 501 壳子）。"""
|
||||
from app import routes
|
||||
from app.contracts import NoteCreateRequest
|
||||
|
||||
_write_vault(vault, SAMPLE_NOTES)
|
||||
job = asyncio.run(routes.rebuild_index(IndexRebuildRequest(scope="all")))
|
||||
assert job.status == "completed"
|
||||
|
||||
notes = asyncio.run(routes.list_notes(limit=50, offset=0, folder=None, tag=None))
|
||||
assert notes.page.total == 3
|
||||
|
||||
result = asyncio.run(routes.search_notes(SearchRequest(query="向量数据库", mode=SearchMode.hybrid)))
|
||||
assert result.page.total >= 1
|
||||
assert result.items[0].citation.citation_id.startswith("cit_")
|
||||
|
||||
created = asyncio.run(
|
||||
routes.create_note(NoteCreateRequest(title="接口测试", markdown="# 接口\n\n正文。"))
|
||||
)
|
||||
assert created.title == "接口测试"
|
||||
assert asyncio.run(routes.get_note(created.note_id)).note_id == created.note_id
|
||||
|
||||
|
||||
def test_get_missing_note_raises_404(vault) -> None:
|
||||
from app import routes
|
||||
from app.errors import ApiError
|
||||
|
||||
with pytest.raises(ApiError):
|
||||
asyncio.run(routes.get_note("note_missing"))
|
||||
|
||||
|
||||
def test_note_crud_roundtrip(vault) -> None:
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import note_service
|
||||
|
||||
note = asyncio.run(
|
||||
note_service.create_note(title="新建笔记", markdown="# 标题\n\n内容。", folder="测试", tags=["测试"])
|
||||
)
|
||||
assert note.note_id.startswith("note_")
|
||||
assert note.blocks
|
||||
|
||||
got = asyncio.run(note_service.get_note(note.note_id))
|
||||
assert got is not None and got.title == "新建笔记"
|
||||
|
||||
updated = asyncio.run(
|
||||
note_service.update_note(note.note_id, title="改名", markdown="# 新标题\n\n检索内容。")
|
||||
)
|
||||
assert updated.title == "改名"
|
||||
assert updated.note_id == note.note_id # 更新不改变 ID
|
||||
|
||||
# 更新后可检索到新内容
|
||||
resp = asyncio.run(engine.search(SearchRequest(query="检索内容", mode=SearchMode.fts)))
|
||||
assert any(r.note_id == note.note_id for r in resp.items)
|
||||
|
||||
assert asyncio.run(note_service.delete_note(note.note_id)) is True
|
||||
assert asyncio.run(note_service.get_note(note.note_id)) is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 审阅回归：路径逃逸 / 部分提交回滚 / 失效向量 / 搜索分页
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("folder", ["../../outside", "..", "..\\..\\etc", "C:\\Windows", "a/../b"])
|
||||
def test_create_note_rejects_path_traversal(vault, folder) -> None:
|
||||
from app.errors import ApiError
|
||||
from app.services import note_service
|
||||
|
||||
with pytest.raises(ApiError) as exc:
|
||||
asyncio.run(
|
||||
note_service.create_note(title="逃逸", markdown="# 逃逸", folder=folder, tags=[])
|
||||
)
|
||||
assert exc.value.status_code == 400
|
||||
assert exc.value.code == "INVALID_PATH"
|
||||
|
||||
|
||||
def test_update_note_rolls_back_file_on_index_error(vault, monkeypatch) -> None:
|
||||
from app.services import note_service
|
||||
|
||||
note = asyncio.run(
|
||||
note_service.create_note(title="回滚", markdown="# 原文\n\n旧内容。", folder="", tags=[])
|
||||
)
|
||||
path = vault / note.file_path
|
||||
before = path.read_text(encoding="utf-8")
|
||||
|
||||
async def _boom(_contents):
|
||||
raise RuntimeError("embedding down")
|
||||
|
||||
monkeypatch.setattr(note_service.embedding, "embed_documents", _boom)
|
||||
with pytest.raises(RuntimeError):
|
||||
asyncio.run(note_service.update_note(note.note_id, markdown="# 新文\n\n新内容。"))
|
||||
|
||||
assert path.read_text(encoding="utf-8") == before # 文件已回滚，无部分提交
|
||||
|
||||
|
||||
def test_update_removes_stale_vectors(vault) -> None:
|
||||
from app.database.db import connect
|
||||
from app.services import note_service
|
||||
|
||||
def vec_count() -> int:
|
||||
conn = connect()
|
||||
try:
|
||||
return conn.execute("SELECT COUNT(*) FROM vec_blocks").fetchone()[0]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
note = asyncio.run(
|
||||
note_service.create_note(
|
||||
title="向量清理", markdown="# 标题\n\n段落一。\n\n段落二。", folder="", tags=[]
|
||||
)
|
||||
)
|
||||
assert vec_count() == 3 # 标题 + 段落一 + 段落二
|
||||
|
||||
asyncio.run(note_service.update_note(note.note_id, markdown="# 标题\n\n段落一。"))
|
||||
assert vec_count() == 2 # 段落二的旧向量被清理，不再残留
|
||||
|
||||
|
||||
def test_search_pagination_total_reflects_all_matches(vault) -> None:
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import index_service
|
||||
|
||||
body = "\n\n".join(f"第{i}段 内容。" for i in range(60))
|
||||
_write_vault(vault, {"多段.md": f"# 大量段落\n\n{body}"})
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
|
||||
page1 = asyncio.run(
|
||||
engine.search(SearchRequest(query="段", mode=SearchMode.fts, limit=10, offset=0))
|
||||
)
|
||||
assert page1.page.total >= 60 # total 反映真实命中数，而非候选池上限 50
|
||||
assert len(page1.items) == 10
|
||||
|
||||
page2 = asyncio.run(
|
||||
engine.search(SearchRequest(query="段", mode=SearchMode.fts, limit=10, offset=55))
|
||||
)
|
||||
assert page2.items # 跨过旧候选池边界仍能取到结果
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 审阅回归：PATCH tags 语义 / 向量-块一致性 / 过滤漏召回 / rebuild 语义与回滚
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_patch_tags_semantics(vault) -> None:
|
||||
"""PATCH 省略 tags 保留、tags=[] 清空、非空替换（审阅 #5）。"""
|
||||
from app.services import note_service
|
||||
|
||||
note = asyncio.run(
|
||||
note_service.create_note(title="标签语义", markdown="# 标题\n\n正文。", folder="", tags=["a"])
|
||||
)
|
||||
assert note.tags == ["a"]
|
||||
|
||||
updated = asyncio.run(note_service.update_note(note.note_id, title="改名")) # tags=None
|
||||
assert updated.tags == ["a"] # 省略 tags 保留原标签
|
||||
|
||||
updated = asyncio.run(note_service.update_note(note.note_id, tags=["b"]))
|
||||
assert updated.tags == ["b"] # 非空列表替换
|
||||
|
||||
updated = asyncio.run(note_service.update_note(note.note_id, tags=[]))
|
||||
assert updated.tags == [] # 空列表清空
|
||||
|
||||
|
||||
def test_patch_partial_content_no_orphan_vectors(vault) -> None:
|
||||
"""修改正文只删部分 block 后，vec_blocks 与 blocks 的 ID 集合一致（审阅 #2/#3）。"""
|
||||
from app.database.db import connect
|
||||
from app.services import note_service
|
||||
|
||||
def ids(table: str) -> set[str]:
|
||||
conn = connect()
|
||||
try:
|
||||
return {row[0] for row in conn.execute(f"SELECT block_id FROM {table}")}
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
note = asyncio.run(
|
||||
note_service.create_note(
|
||||
title="部分修改", markdown="# 标题\n\n段落一。\n\n段落二。", folder="", tags=[]
|
||||
)
|
||||
)
|
||||
assert ids("vec_blocks") == ids("blocks")
|
||||
|
||||
asyncio.run(
|
||||
note_service.update_note(note.note_id, markdown="# 标题\n\n段落一改了。\n\n新增段落。")
|
||||
)
|
||||
# 更新后不变量：向量集合与块集合一一对应，无残留、无缺失
|
||||
assert ids("vec_blocks") == ids("blocks")
|
||||
|
||||
|
||||
def test_fts_metadata_filter_recalls_beyond_candidate_pool(vault) -> None:
|
||||
"""metadata 过滤不能受候选池截断影响：目标块排在 50 名之外也应被召回（审阅 #4）。"""
|
||||
from app.retrieval.engine import engine
|
||||
from app.services import index_service
|
||||
|
||||
files: dict[str, str] = {}
|
||||
# 60 篇短填充笔记：bm25 高，占据 FTS 前 60 位
|
||||
for i in range(60):
|
||||
files[f"批量/填充{i}.md"] = f"---\ntitle: 填充{i}\ntags: 填充\n---\n\n检索\n"
|
||||
# 目标笔记：长正文使 bm25 变低，排在候选池（50）之外
|
||||
long_body = "检索 " + "甲乙丙丁戊己庚辛壬癸子丑寅卯辰巳午未申酉戌亥天地玄黄宇宙洪荒日月盈昃"
|
||||
files["批量/目标.md"] = f"---\ntitle: 目标\ntags: 目标\n---\n\n{long_body}\n"
|
||||
|
||||
_write_vault(vault, files)
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
|
||||
resp = asyncio.run(
|
||||
engine.search(SearchRequest(query="检索", mode=SearchMode.fts, tags=["目标"]))
|
||||
)
|
||||
assert resp.page.total == 1
|
||||
assert resp.items[0].title == "目标"
|
||||
|
||||
|
||||
def test_rebuild_rejects_unsupported_scope_and_note_ids(vault) -> None:
|
||||
"""增量 scope / note_ids 未实现时明确拒绝，而非静默全量重建（审阅 #6）。"""
|
||||
from app.errors import ApiError
|
||||
from app.services import index_service
|
||||
|
||||
with pytest.raises(ApiError) as exc:
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="notes")))
|
||||
assert exc.value.status_code == 400
|
||||
assert exc.value.code == "UNSUPPORTED_SCOPE"
|
||||
|
||||
with pytest.raises(ApiError) as exc:
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all", note_ids=["note_x"])))
|
||||
assert exc.value.code == "UNSUPPORTED_SCOPE"
|
||||
|
||||
|
||||
def test_rebuild_failure_restores_old_index(vault, monkeypatch) -> None:
|
||||
"""重建中途失败应恢复旧索引，不留下半成品（审阅 #6）。"""
|
||||
from app import repository
|
||||
from app.services import index_service
|
||||
from app.services import note_service as ns
|
||||
|
||||
_write_vault(vault, SAMPLE_NOTES)
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
before = repository.stats()
|
||||
|
||||
real_embed = ns.embedding.embed_documents
|
||||
call = {"n": 0}
|
||||
|
||||
async def _flaky(contents):
|
||||
call["n"] += 1
|
||||
if call["n"] > 1:
|
||||
raise RuntimeError("embed down")
|
||||
return await real_embed(contents)
|
||||
|
||||
monkeypatch.setattr(ns.embedding, "embed_documents", _flaky)
|
||||
with pytest.raises(RuntimeError):
|
||||
asyncio.run(index_service.rebuild(IndexRebuildRequest(scope="all")))
|
||||
|
||||
assert repository.stats() == before # 旧索引已恢复，无半成品
|
||||
Generated
+14
@@ -181,6 +181,7 @@ source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "httpx" },
|
||||
{ name = "sqlite-vec" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
|
||||
@@ -193,6 +194,7 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.116,<1.0" },
|
||||
{ name = "httpx", specifier = ">=0.28,<1.0" },
|
||||
{ name = "sqlite-vec", specifier = ">=0.1.9" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.35,<1.0" },
|
||||
]
|
||||
|
||||
@@ -423,6 +425,18 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sqlite-vec"
|
||||
version = "0.1.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/68/85/9fad0045d8e7c8df3e0fa5a56c630e8e15ad6e5ca2e6106fceb666aa6638/sqlite_vec-0.1.9-py3-none-macosx_10_6_x86_64.whl", hash = "sha256:1b62a7f0a060d9475575d4e599bbf94a13d85af896bc1ce86ee80d1b5b48e5fb", size = 131171, upload-time = "2026-03-31T08:02:31.717Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a4/3d/3677e0cd2f92e5ebc43cd29fbf565b75582bff1ccfa0b8327c7508e1084f/sqlite_vec-0.1.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1d52e30513bae4cc9778ddbf6145610434081be4c3afe57cd877893bad9f6b6c", size = 165434, upload-time = "2026-03-31T08:02:32.712Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/d4/f2b936d3bdc38eadcbd2a87875815db36430fab0363182ba5d12cd8e0b51/sqlite_vec-0.1.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e921e592f24a5f9a18f590b6ddd530eb637e2d474e3b1972f9bbeb773aa3cb9", size = 160076, upload-time = "2026-03-31T08:02:33.796Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6f/ad/6afd073b0f817b3e03f9e37ad626ae341805891f23c74b5292818f49ac63/sqlite_vec-0.1.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux1_x86_64.whl", hash = "sha256:1515727990b49e79bcaf75fdee2ffc7d461f8b66905013231251f1c8938e7786", size = 163388, upload-time = "2026-03-31T08:02:34.888Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/42/89/81b2907cda14e566b9bf215e2ad82fc9b349edf07d2010756ffdb902f328/sqlite_vec-0.1.9-py3-none-win_amd64.whl", hash = "sha256:4a28dc12fa4b53d7b1dced22da2488fade444e96b5d16fd2d698cd670675cf32", size = 292804, upload-time = "2026-03-31T08:02:36.035Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "starlette"
|
||||
version = "1.6.0"
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
# Knowledge Core 与 Retrieval Core 开发说明
|
||||
|
||||
> 本文档用于团队开发和模块联调，记录 Knowledge Core / Retrieval Core 已经落地的
|
||||
> 模块边界、数据模型、接口与使用方式，对应分工表中的杨星萱。
|
||||
|
||||
## 当前实现
|
||||
|
||||
当前已经建立第一条可运行的检索链路：
|
||||
|
||||
```text
|
||||
Markdown Vault
|
||||
→ Markdown Parser（Block 切分 / heading_path / offset）
|
||||
→ SQLite（notes / blocks / FTS5）+ sqlite-vec（vec_blocks）
|
||||
→ FTS5(BM25) + Vector(cosine) 双路召回
|
||||
→ RRF 融合 → Reranker 精排 → Metadata Filter → 分页
|
||||
→ Citation + Snippet
|
||||
```
|
||||
|
||||
对应代码：
|
||||
|
||||
```text
|
||||
backend/app/
|
||||
├── constants.py EMBEDDING_DIM = 128
|
||||
├── config.py Settings（data_dir / db_path / vault_path）
|
||||
├── textutils.py 分词、FTS 查询串、摘要片段
|
||||
├── repository.py notes / blocks / blocks_fts 读写（领域记录层）
|
||||
├── database/
|
||||
│ ├── db.py SQLite 连接 + 事务 + 加载 sqlite-vec
|
||||
│ └── migrations.py 轻量迁移（建 notes / blocks / blocks_fts / vec_blocks）
|
||||
├── knowledge/
|
||||
│ └── parser.py Markdown → ParsedNote / NoteBlock
|
||||
├── retrieval/
|
||||
│ ├── embedding.py EmbeddingProvider 接口 + HashEmbeddingProvider
|
||||
│ ├── reranker.py RerankerProvider 接口 + LexicalReranker
|
||||
│ ├── vectorstore.py VectorStore 接口 + SqliteVecStore
|
||||
│ ├── hybrid.py RRF 融合、分数归一化
|
||||
│ └── engine.py RetrievalEngine（编排检索全流程）
|
||||
└── services/
|
||||
├── note_service.py Note CRUD + 索引编排
|
||||
└── index_service.py 全量重建、索引状态、任务查询
|
||||
```
|
||||
|
||||
Router（`backend/app/routes.py`）只负责 HTTP 与错误转换；`/api/notes`、`/api/search`、
|
||||
`/api/index/*` 已接入上述服务，其余端点仍由对应模块负责。
|
||||
|
||||
## 模块边界
|
||||
|
||||
本模块负责（杨星萱）：
|
||||
|
||||
- Markdown Vault、Note / NoteBlock 数据模型与解析；
|
||||
- SQLite 元数据、FTS5 全文检索、sqlite-vec 向量检索；
|
||||
- Embedding、Hybrid RAG、RRF、Reranker、Metadata Filter；
|
||||
- Citation 与笔记定位；
|
||||
- 检索测试数据。
|
||||
|
||||
以下内容保持接口，不在本模块实现：
|
||||
|
||||
- Agent Runtime、Tool Registry、Permission：由 Agent Core 提供；
|
||||
- Provider Adapter、多模型协议：由 Model Core 提供；
|
||||
- Skill / Plugin 生命周期：由 Extension Core 提供；
|
||||
- 文件系统与 API Key 明文读取：由 Rust Host 提供。
|
||||
|
||||
## 数据模型与稳定 ID
|
||||
|
||||
- 笔记元数据存 `notes` 表；正文切成 Block 存 `blocks` 表；`blocks_fts` 是 FTS5 虚拟表；
|
||||
`vec_blocks` 是 sqlite-vec 的 `vec0` 虚拟表。
|
||||
- 稳定 ID（内容/路径不变则 ID 不变）：
|
||||
|
||||
```text
|
||||
note_id = "note_" + sha256(rel_path)[:16]
|
||||
block_id = "blk_" + sha256(note_id | heading_path | content)[:16]
|
||||
citation_id = "cit_" + block_id
|
||||
```
|
||||
|
||||
> 注意：MVP 阶段 `note_id` 由相对路径派生，移动文件会改变 ID；后续 `move` 流程会保留原 ID。
|
||||
|
||||
每个 Block 记录 `heading_path`（章节路径）、`start_offset` / `end_offset`（相对原文的
|
||||
字符偏移，用于前端跳转高亮）、`content_hash`、`token_count`。
|
||||
|
||||
## 分层依赖
|
||||
|
||||
按团队约定，依赖方向为：
|
||||
|
||||
```text
|
||||
Router（HTTP/错误转换）
|
||||
→ Service（note_service / index_service 编排）
|
||||
→ Repository（notes/blocks/FTS5 访问） ← 只在此层访问 SQLite
|
||||
→ Retrieval Infra（embedding/reranker/vectorstore） ← vec0 只在 vectorstore 层访问
|
||||
```
|
||||
|
||||
- 检索 Executor 调用 `RetrievalEngine`，不直接拼接 FTS5 或 sqlite-vec SQL；
|
||||
- 向量实现只在 `retrieval/vectorstore.py`；DB 访问只在 `repository.py`。
|
||||
|
||||
## 分词与中文检索
|
||||
|
||||
FTS5 默认 `unicode61` 不切分中文，因此统一预分词：ASCII 单词 + CJK 单字 + CJK 相邻双字。
|
||||
写入与查询走同一套拆分（`textutils.segment` / `textutils.match_query`），实现中文子串/词级召回。
|
||||
|
||||
## Embedding / Reranker（轻量实现，接口可替换）
|
||||
|
||||
当前是「统一接口 + 轻量实现」，后续接入真实模型时替换实例即可，不改变上层调用：
|
||||
|
||||
- `EmbeddingProvider`（`embed_documents` / `embed_query`）→ `HashEmbeddingProvider`：
|
||||
确定性特征哈希 + L2 归一化，`dim = 128`，`model_id = "hash-v1"`。
|
||||
- `RerankerProvider`（`rerank`）→ `LexicalReranker`：分数归一化 + 词重叠加权，
|
||||
`model_id = "lexical-v1"`。
|
||||
- `VectorStore`（`upsert` / `delete` / `search` / `clear`）→ `SqliteVecStore`：
|
||||
sqlite-vec `vec0`，相似度取余弦 `score = 1 - distance² / 2`。
|
||||
|
||||
## 检索流程
|
||||
|
||||
`RetrievalEngine.search(request)`：
|
||||
|
||||
1. 按 `mode` 收集候选：`fts` / `vector` 各取 Top `CANDIDATE_POOL = 50`；
|
||||
2. `hybrid` 用 RRF（`k = 60`）融合两路排序；
|
||||
3. Metadata Filter：`folders` / `note_ids` / `tags` / 时间范围；
|
||||
4. `hybrid` 再经 Reranker 精排，其余模式按分数排序；
|
||||
5. 分数归一化 → 分页 → 组装 `Citation` 与 `Snippet`。
|
||||
|
||||
模块级单例 `engine = RetrievalEngine(HashEmbeddingProvider(), LexicalReranker(), SqliteVecStore())`，
|
||||
检索入口统一为 `engine.search(request)`。
|
||||
|
||||
## 接口清单
|
||||
|
||||
### Note
|
||||
|
||||
```text
|
||||
GET /api/notes?limit=&offset=&folder=&tag=
|
||||
POST /api/notes
|
||||
GET /api/notes/{note_id}
|
||||
PATCH /api/notes/{note_id}
|
||||
DELETE /api/notes/{note_id}
|
||||
POST /api/notes/{note_id}/move （501，待定语义）
|
||||
```
|
||||
|
||||
创建笔记：
|
||||
|
||||
```json
|
||||
POST /api/notes
|
||||
{"title": "Python 基础", "markdown": "# 变量\n\nPython 是动态类型语言。", "folder": "编程", "tags": ["python"]}
|
||||
```
|
||||
|
||||
### Search
|
||||
|
||||
```text
|
||||
POST /api/search
|
||||
```
|
||||
|
||||
```json
|
||||
{"query": "向量数据库", "mode": "hybrid", "limit": 10}
|
||||
```
|
||||
|
||||
`mode` 取 `fts` / `vector` / `hybrid`；可选 `folders` / `note_ids` / `tags` / 时间范围 /
|
||||
`include_snippet`。结果项含 `score`、`snippet` 与 `citation`（`citation_id`、`file_path`、
|
||||
`heading_path`、`start_offset`、`end_offset`）。
|
||||
|
||||
### Index
|
||||
|
||||
```text
|
||||
GET /api/index/status
|
||||
POST /api/index/rebuild
|
||||
GET /api/index/jobs/{job_id}
|
||||
```
|
||||
|
||||
重建（MVP 同步执行，直接返回 `completed`）：
|
||||
|
||||
```json
|
||||
POST /api/index/rebuild
|
||||
{"scope": "all"}
|
||||
```
|
||||
|
||||
## 检索测试数据
|
||||
|
||||
样例 Vault 位于 `backend/data/vault/`，覆盖中英文、多级标题、frontmatter、子目录与不同 tags：
|
||||
|
||||
```text
|
||||
项目说明.md
|
||||
编程/Python 基础语法.md
|
||||
编程/向量数据库与相似度检索.md
|
||||
产品/RAG 检索增强与引用定位.md
|
||||
日记/2026-08-27 周会.md
|
||||
```
|
||||
|
||||
示例查询：
|
||||
|
||||
```text
|
||||
POST /api/search {"query": "向量数据库", "mode": "hybrid"} → 命中《向量数据库与相似度检索》
|
||||
POST /api/search {"query": "检索", "mode": "fts", "folders": ["产品"]} → 只返回 产品/ 下笔记
|
||||
POST /api/search {"query": "向量", "mode": "hybrid", "tags": ["向量"]} → 按 tag 过滤
|
||||
```
|
||||
|
||||
## 测试
|
||||
|
||||
```powershell
|
||||
cd backend
|
||||
uv run pytest -q
|
||||
```
|
||||
|
||||
当前 26 个用例通过（单元 + 端到端）。测试通过 `tests/conftest.py` 的 autouse fixture 把
|
||||
数据目录/DB/Vault 重定向到临时目录，不读写真实 `backend/data`，任何本机状态下结果确定。
|
||||
|
||||
## 配置
|
||||
|
||||
```text
|
||||
APP_DATA_DIR 默认 backend/data
|
||||
APP_DB_PATH 默认 backend/data/app.db
|
||||
APP_VAULT_PATH 默认 backend/data/vault
|
||||
```
|
||||
|
||||
运行期生成的 `backend/data/*.db*` 已被 `.gitignore` 忽略，vault 下的 Markdown 测试数据会提交。
|
||||
|
||||
## 接入约定（Agent / 其他模块）
|
||||
|
||||
Agent 通过注册 Tool 接入本模块，不让 Agent Runtime 直接依赖具体实现：
|
||||
|
||||
```text
|
||||
notes.search / notes.read / notes.create / notes.update / notes.list / notes.move
|
||||
rag.search
|
||||
```
|
||||
|
||||
- 写操作 Executor 调用 `note_service`，不直接访问 SQLite；
|
||||
- 检索 Executor 调用 `engine.search(request)`，不直接拼接 FTS5 或 sqlite-vec SQL。
|
||||
|
||||
## 当前限制与下一步
|
||||
|
||||
- `move` 接口未实现（需确认移动后 `note_id` 是否保持稳定）。
|
||||
- Embedding / Reranker 为轻量实现，后续替换为真实模型（接口不变）。
|
||||
- 小语料下 hybrid 检索召回偏宽（向量 Top-K 覆盖全部 block），可加相关性阈值收紧。
|
||||
- 重建为同步 + 全量，后续接入增量索引与异步任务队列。
|
||||
- 检索 Benchmark 待建立。
|
||||
Reference in New Issue
Block a user