fix(retrieval): 修复路径逃逸、部分提交、失效向量与分页问题

- 路径逃逸:清洗 folder(拒绝 ..、绝对路径/盘符),_abs_path 增加 Vault 边界校验
- 部分提交:create/update 索引失败时回滚文件
- 失效向量残留:replace_note_metadata 返回旧 id,index_note 清理旧向量
- 分页不完整:fts_count 返回真实命中总数,候选池覆盖 offset+limit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
yxx
2026-08-27 21:38:43 +08:00
co-authored by Claude
parent 39d1aa0fe9
commit 87717450fd
4 changed files with 179 additions and 24 deletions
+65 -18
View File
@@ -32,16 +32,43 @@ def _safe_name(title: str) -> str:
return name or "untitled"
def _rel_path(folder: str | None, title: str) -> str:
folder_part = folder.strip().strip("/") if folder else ""
def _normalize_folder(folder: str | None) -> str:
"""清洗 folder 为安全的相对目录,拒绝 `..`/`.`/绝对路径/盘符/空字节,防路径逃逸。"""
if not folder:
return ""
if "\x00" in folder:
raise ApiError(400, "INVALID_PATH", "folder must not contain NUL bytes", {"folder": folder})
segments: list[str] = []
for part in re.split(r"[\\/]+", folder):
if part == "":
continue
if part in (".", ".."):
raise ApiError(400, "INVALID_PATH", "folder must not contain '.' or '..'", {"folder": folder})
if ":" in part:
raise ApiError(400, "INVALID_PATH", "folder must be a relative path", {"folder": folder})
segments.append(part)
return "/".join(segments)
def _rel_path(folder: str | None, title: str) -> tuple[str, str]:
"""由 folder + title 生成安全的相对路径,返回 (rel_path, 清洗后的 folder)。"""
clean_folder = _normalize_folder(folder)
name = _safe_name(title)
if not name.endswith(".md"):
name += ".md"
return f"{folder_part}/{name}" if folder_part else name
rel = f"{clean_folder}/{name}" if clean_folder else name
return rel, clean_folder
def _abs_path(rel_path: str) -> Path:
return _vault() / rel_path
"""把相对路径解析为 Vault 内的绝对路径;越界即报 400,杜绝路径逃逸。"""
if not rel_path or "\x00" in rel_path:
raise ApiError(400, "INVALID_PATH", "invalid file path", {"file_path": rel_path})
root = _vault().resolve()
candidate = (_vault() / rel_path).resolve()
if not candidate.is_relative_to(root):
raise ApiError(400, "INVALID_PATH", "path escapes vault", {"file_path": rel_path})
return candidate
def _read_markdown(rel_path: str) -> str:
@@ -62,9 +89,13 @@ def _delete_markdown(rel_path: str) -> None:
async def index_note(parsed: ParsedNote) -> None:
"""把解析结果写入元数据 + FTS5 + 向量(三层可重建索引)。"""
"""把解析结果写入元数据 + FTS5 + 向量(三层可重建索引)。
替换元数据时拿到旧 block_id:清理已删除/内容变化的旧向量,只为新增 block 写向量,
避免失效向量残留(内容未变的 block 其向量仍有效,无需重复写入)。
"""
vectors = await embedding.embed_documents([block.content for block in parsed.blocks])
repository.replace_note_metadata(
old_block_ids = repository.replace_note_metadata(
note_id=parsed.note_id,
title=parsed.title,
file_path=parsed.file_path,
@@ -74,24 +105,35 @@ async def index_note(parsed: ParsedNote) -> None:
updated_at=parsed.updated_at,
blocks=parsed.blocks,
)
old_ids = set(old_block_ids)
new_ids = {block.block_id for block in parsed.blocks}
stale_ids = [bid for bid in old_ids if bid not in new_ids]
if stale_ids:
await vector_store.delete(stale_ids)
missing_ids = [bid for bid in new_ids if bid not in old_ids]
records = [
VectorRecord(id=block.block_id, vector=vector)
for block, vector in zip(parsed.blocks, vectors)
if block.block_id in missing_ids
]
await vector_store.upsert(records)
repository.set_index_meta({"embedding_model": embedding.model_id, "embedding_dim": str(embedding.dim)})
async def create_note(*, title: str, markdown: str, folder: str | None, tags: list[str]) -> Note:
rel_path = _rel_path(folder, title)
_write_markdown(rel_path, markdown)
rel_path, clean_folder = _rel_path(folder, title)
now = datetime.now(timezone.utc)
parsed = parse_note(
markdown=markdown, file_path=rel_path, folder=folder or "", tags=tags,
markdown=markdown, file_path=rel_path, folder=clean_folder, tags=tags,
created_at=now, updated_at=now,
)
parsed.title = title # 显式传入的 title 优先于正文推导(与 update_note 保持一致)
await index_note(parsed)
_write_markdown(rel_path, markdown)
try:
await index_note(parsed)
except BaseException:
_delete_markdown(rel_path) # 索引失败时回滚,避免「文件已写、索引缺失」的部分提交
raise
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
parsed.created_at, parsed.updated_at, parsed.blocks, markdown)
@@ -112,18 +154,23 @@ async def update_note(
if record is None:
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
new_md = _read_markdown(record.file_path) if markdown is None else markdown
old_md = _read_markdown(record.file_path)
new_md = old_md if markdown is None else markdown
_write_markdown(record.file_path, new_md)
now = datetime.now(timezone.utc)
parsed = parse_note(
markdown=new_md, file_path=record.file_path, folder=record.folder, tags=tags,
created_at=record.created_at, updated_at=now,
)
if title is not None:
parsed.title = title # 显式传入的 title 覆盖正文推导结果
try:
parsed = parse_note(
markdown=new_md, file_path=record.file_path, folder=record.folder, tags=tags,
created_at=record.created_at, updated_at=now,
)
if title is not None:
parsed.title = title # 显式传入的 title 覆盖正文推导结果
await index_note(parsed)
await index_note(parsed)
except BaseException:
_write_markdown(record.file_path, old_md) # 索引失败时回滚正文,避免部分提交
raise
return _build_note(parsed.note_id, parsed.title, parsed.file_path, parsed.tags,
parsed.created_at, parsed.updated_at, parsed.blocks, new_md)