Kronecker/NotesAgentic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 5 Wiki Activity

feat(workspace): 接入真实Vault数据链路

Browse Source
This commit is contained in:
admin
2026-08-31 21:38:56 +08:00
parent 84077feb18
commit 8da75d4420
23 changed files with 1055 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/note_service.py
+72 -52
View File
@@ -6,13 +6,11 @@ Markdown 文件是笔记正文的持久化载体（Vault），SQLite/FTS5/向量
from __future__ import annotations
import re
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
from app import repository
from app.config import get_settings
from app.contracts import Note, NoteBlock, NoteSummary
from app.database.db import connect, transaction
from app.errors import ApiError
@@ -20,74 +18,40 @@ from app.knowledge.parser import ParsedNote, parse_note
from app.retrieval.embedding import HashEmbeddingProvider
from app.retrieval.vectorstore import SqliteVecStore, VectorRecord
from app.services.coordination import serialized_vault_mutation
from app.services.vault_paths import (
normalize_entry_name,
normalize_folder,
resolve_in_vault,
safe_note_filename,
)
# 轻量实现实例（无状态，可直接复用）；接入真实模型后替换为对应 Provider
embedding = HashEmbeddingProvider()
vector_store = SqliteVecStore()
def _vault() -> Path:
return get_settings().vault_path
def _safe_name(title: str) -> str:
name = re.sub(r'[\\/:*?"<>|]', "_", title).strip()
return name or "untitled"
def _normalize_folder(folder: str | None) -> str:
"""清洗 folder 为安全的相对目录，拒绝 `..`/`.`/绝对路径/盘符/空字节，防路径逃逸。"""
if not folder:
return ""
if "\x00" in folder:
raise ApiError(400, "INVALID_PATH", "folder must not contain NUL bytes", {"folder": folder})
segments: list[str] = []
for part in re.split(r"[\\/]+", folder):
if part == "":
continue
if part in (".", ".."):
raise ApiError(400, "INVALID_PATH", "folder must not contain '.' or '..'", {"folder": folder})
if ":" in part:
raise ApiError(400, "INVALID_PATH", "folder must be a relative path", {"folder": folder})
segments.append(part)
return "/".join(segments)
def _rel_path(folder: str | None, title: str) -> tuple[str, str]:
"""由 folder + title 生成安全的相对路径，返回 (rel_path, 清洗后的 folder)。"""
clean_folder = _normalize_folder(folder)
name = _safe_name(title)
if not name.endswith(".md"):
name += ".md"
clean_folder = normalize_folder(folder)
name = safe_note_filename(title)
rel = f"{clean_folder}/{name}" if clean_folder else name
return rel, clean_folder
def _abs_path(rel_path: str) -> Path:
"""把相对路径解析为 Vault 内的绝对路径；越界即报 400，杜绝路径逃逸。"""
if not rel_path or "\x00" in rel_path:
raise ApiError(400, "INVALID_PATH", "invalid file path", {"file_path": rel_path})
root = _vault().resolve()
candidate = (_vault() / rel_path).resolve()
if not candidate.is_relative_to(root):
raise ApiError(400, "INVALID_PATH", "path escapes vault", {"file_path": rel_path})
return candidate
def _read_markdown(rel_path: str) -> str:
path = _abs_path(rel_path)
path = resolve_in_vault(rel_path)
return path.read_text(encoding="utf-8") if path.exists() else ""
def _write_markdown(rel_path: str, markdown: str) -> None:
path = _abs_path(rel_path)
path = resolve_in_vault(rel_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(markdown, encoding="utf-8")
def _create_markdown(rel_path: str, markdown: str) -> None:
"""排他创建 Markdown；目标已存在时返回资源冲突，不覆盖用户文件。"""
path = _abs_path(rel_path)
path = resolve_in_vault(rel_path)
path.parent.mkdir(parents=True, exist_ok=True)
try:
with path.open("x", encoding="utf-8") as handle:
@@ -102,7 +66,7 @@ def _create_markdown(rel_path: str, markdown: str) -> None:
def _delete_markdown(rel_path: str) -> None:
path = _abs_path(rel_path)
path = resolve_in_vault(rel_path)
if path.exists():
path.unlink()
@@ -222,7 +186,7 @@ async def move_note(note_id: str, *, folder: str) -> Note:
if record is None:
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
clean_folder = _normalize_folder(folder)
clean_folder = normalize_folder(folder)
filename = Path(record.file_path).name
new_rel_path = f"{clean_folder}/{filename}" if clean_folder else filename
if new_rel_path == record.file_path:
@@ -230,8 +194,8 @@ async def move_note(note_id: str, *, folder: str) -> Note:
assert note is not None
return note
source = _abs_path(record.file_path)
target = _abs_path(new_rel_path)
source = resolve_in_vault(record.file_path)
target = resolve_in_vault(new_rel_path)
if not source.is_file():
raise ApiError(
409, "NOTE_FILE_MISSING", "note file is missing from the Vault",
@@ -267,13 +231,69 @@ async def move_note(note_id: str, *, folder: str) -> Note:
)
@serialized_vault_mutation
async def rename_note(note_id: str, *, file_name: str) -> Note:
"""重命名 Markdown 文件并保留 note_id、Block 与向量身份。"""
record = repository.get_note_record(note_id)
if record is None:
raise ApiError(404, "RESOURCE_NOT_FOUND", "note not found", {"note_id": note_id})
normalized = normalize_entry_name(file_name, markdown=True)
source = resolve_in_vault(record.file_path)
folder = normalize_folder(record.folder)
new_file_path = f"{folder}/{normalized}" if folder else normalized
target = resolve_in_vault(new_file_path)
if new_file_path == record.file_path:
note = await get_note(note_id)
assert note is not None
return note
if not source.is_file():
raise ApiError(
409,
"NOTE_FILE_MISSING",
"note file is missing from the Vault",
{"note_id": note_id, "file_path": record.file_path},
)
if target.exists():
raise ApiError(
409,
"RESOURCE_CONFLICT",
"a note already exists with the requested file name",
{"note_id": note_id, "file_path": new_file_path},
)
source.replace(target)
now = datetime.now(timezone.utc)
conn = connect()
try:
with transaction(conn):
repository.update_note_location(
conn=conn,
note_id=note_id,
title=Path(normalized).stem,
file_path=new_file_path,
folder=folder,
updated_at=now,
)
except BaseException:
target.replace(source)
raise
finally:
conn.close()
note = await get_note(note_id)
assert note is not None
return note
@serialized_vault_mutation
async def delete_note(note_id: str) -> bool:
record = repository.get_note_record(note_id)
if record is None:
return False
path = _abs_path(record.file_path)
path = resolve_in_vault(record.file_path)
tombstone = path.with_name(f".{path.name}.{uuid4().hex}.deleting") if path.exists() else None
if tombstone is not None:
path.replace(tombstone)
backend/app/services/vault_paths.py
+78
View File
@@ -0,0 +1,78 @@
"""Vault 相对路径校验；所有文件操作必须先经过本模块。"""
from __future__ import annotations
import re
from pathlib import Path
from app.config import get_settings
from app.errors import ApiError
_INVALID_FILE_CHARS = re.compile(r'[\\/:*?"<>|]')
def normalize_folder(folder: str | None) -> str:
"""返回使用 `/` 的安全相对目录；根目录表示为空字符串。"""
if not folder or folder in {"/", "\\"}:
return ""
if "\x00" in folder:
raise ApiError(400, "INVALID_PATH", "folder must not contain NUL bytes")
segments: list[str] = []
for part in re.split(r"[\\/]+", folder):
if not part:
continue
if part in {".", ".."} or ":" in part:
raise ApiError(
400,
"INVALID_PATH",
"folder must be a relative path without '.' or '..' segments",
{"folder": folder},
)
segments.append(part)
return "/".join(segments)
def normalize_entry_name(name: str, *, markdown: bool = False) -> str:
"""校验单个目录项名称；不静默接受路径分隔符或保留段。"""
value = name.strip()
if not value or value in {".", ".."} or "\x00" in value:
raise ApiError(400, "INVALID_PATH", "entry name is invalid", {"name": name})
if _INVALID_FILE_CHARS.search(value):
raise ApiError(
400,
"INVALID_PATH",
"entry name contains unsupported characters",
{"name": name},
)
if markdown and not value.lower().endswith(".md"):
value += ".md"
return value
def safe_note_filename(title: str) -> str:
"""为创建笔记保留原有的宽松清洗行为。"""
value = _INVALID_FILE_CHARS.sub("_", title).strip() or "untitled"
return value if value.lower().endswith(".md") else f"{value}.md"
def resolve_in_vault(relative_path: str) -> Path:
"""把相对路径解析到当前 Vault，并拒绝符号链接/`..` 导致的越界。"""
if not relative_path or "\x00" in relative_path:
raise ApiError(
400, "INVALID_PATH", "invalid Vault-relative path", {"path": relative_path}
)
root = get_settings().vault_path.resolve()
candidate = (root / relative_path.replace("\\", "/").lstrip("/")).resolve()
if not candidate.is_relative_to(root):
raise ApiError(
400, "INVALID_PATH", "path escapes Vault", {"path": relative_path}
)
return candidate
def relative_to_vault(path: Path) -> str:
return path.resolve().relative_to(get_settings().vault_path.resolve()).as_posix()
backend/app/services/workspace_service.py
+249
View File
@@ -0,0 +1,249 @@
"""Web 联调 Workspace：把单一配置 Vault 映射为前端可用的真实文件树。"""
from __future__ import annotations
import hashlib
import shutil
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
from app import repository
from app.config import get_settings
from app.contracts import (
IndexRebuildRequest,
OperationResponse,
WorkspaceEntry,
WorkspaceInfo,
WorkspaceSnapshot,
)
from app.database.db import connect, transaction
from app.errors import ApiError
from app.retrieval.vectorstore import SqliteVecStore
from app.services import index_service
from app.services.coordination import serialized_vault_mutation
from app.services.vault_paths import normalize_entry_name, normalize_folder, resolve_in_vault
vector_store = SqliteVecStore()
def _entry_id(kind: str, path: str) -> str:
digest = hashlib.sha256(f"{kind}:{path}".encode("utf-8")).hexdigest()[:16]
return f"{kind}_{digest}"
def _disk_markdown_paths() -> set[str]:
root = get_settings().vault_path
if not root.exists():
return set()
resolved_root = root.resolve()
paths: set[str] = set()
for path in root.rglob("*.md"):
if path.is_symlink():
continue
resolved = path.resolve()
if resolved.is_file() and resolved.is_relative_to(resolved_root):
paths.add(resolved.relative_to(resolved_root).as_posix())
return paths
def get_workspace_info() -> WorkspaceInfo:
root = get_settings().vault_path.resolve()
disk_paths = _disk_markdown_paths()
indexed_paths = {item.file_path for item in repository.list_note_locations()}
return WorkspaceInfo(
name=root.name or "Vault",
path=str(root),
file_count=len(disk_paths),
indexed_note_count=len(indexed_paths),
requires_refresh=disk_paths != indexed_paths,
)
def _tree(directory: Path, locations: dict[str, repository.NoteLocation]) -> list[WorkspaceEntry]:
if not directory.exists():
return []
root = get_settings().vault_path.resolve()
entries: list[WorkspaceEntry] = []
children = sorted(
directory.iterdir(), key=lambda item: (not item.is_dir(), item.name.casefold())
)
for child in children:
if child.name.startswith(".") or child.is_symlink():
continue
resolved = child.resolve()
if not resolved.is_relative_to(root):
continue
relative = resolved.relative_to(root).as_posix()
public_path = f"/{relative}"
if resolved.is_dir():
entries.append(
WorkspaceEntry(
entry_id=_entry_id("folder", relative),
name=child.name,
path=public_path,
type="folder",
children=_tree(resolved, locations),
)
)
elif resolved.is_file() and child.suffix.lower() == ".md":
location = locations.get(relative)
entries.append(
WorkspaceEntry(
entry_id=location.note_id if location else _entry_id("file", relative),
note_id=location.note_id if location else None,
name=child.name,
path=public_path,
type="file",
)
)
return entries
def get_workspace_tree() -> list[WorkspaceEntry]:
locations = {item.file_path: item for item in repository.list_note_locations()}
return _tree(get_settings().vault_path.resolve(), locations)
async def open_workspace(requested_path: str | None) -> WorkspaceSnapshot:
"""打开当前配置 Vault；发现未索引文件时先执行一次安全全量刷新。"""
root = get_settings().vault_path.resolve()
if requested_path and Path(requested_path).resolve() != root:
raise ApiError(
409,
"WORKSPACE_PATH_MISMATCH",
"Web development mode can only open the backend configured Vault.",
{"configured_path": str(root)},
)
root.mkdir(parents=True, exist_ok=True)
info = get_workspace_info()
if info.requires_refresh:
await index_service.rebuild(IndexRebuildRequest())
info = get_workspace_info()
return WorkspaceSnapshot(workspace=info, items=get_workspace_tree())
@serialized_vault_mutation
async def create_folder(parent: str, name: str) -> WorkspaceEntry:
clean_parent = normalize_folder(parent)
clean_name = normalize_entry_name(name)
relative = f"{clean_parent}/{clean_name}" if clean_parent else clean_name
target = resolve_in_vault(relative)
if not clean_parent:
get_settings().vault_path.mkdir(parents=True, exist_ok=True)
if target.exists():
raise ApiError(
409, "RESOURCE_CONFLICT", "folder already exists", {"path": relative}
)
if not target.parent.is_dir():
raise ApiError(
404,
"RESOURCE_NOT_FOUND",
"parent folder not found",
{"parent": clean_parent},
)
target.mkdir(parents=False)
return WorkspaceEntry(
entry_id=_entry_id("folder", relative),
name=clean_name,
path=f"/{relative}",
type="folder",
)
@serialized_vault_mutation
async def rename_folder(path: str, new_name: str) -> WorkspaceEntry:
old_folder = normalize_folder(path)
if not old_folder:
raise ApiError(400, "INVALID_PATH", "the Vault root cannot be renamed")
clean_name = normalize_entry_name(new_name)
parent = Path(old_folder).parent.as_posix()
parent = "" if parent == "." else parent
new_folder = f"{parent}/{clean_name}" if parent else clean_name
source = resolve_in_vault(old_folder)
target = resolve_in_vault(new_folder)
if not source.is_dir() or source.is_symlink():
raise ApiError(404, "RESOURCE_NOT_FOUND", "folder not found", {"path": path})
if target.exists():
raise ApiError(
409, "RESOURCE_CONFLICT", "target folder already exists", {"path": new_folder}
)
affected = [
item
for item in repository.list_note_locations()
if item.folder == old_folder or item.folder.startswith(f"{old_folder}/")
]
source.replace(target)
conn = connect()
now = datetime.now(timezone.utc)
try:
with transaction(conn):
for item in affected:
file_suffix = item.file_path[len(old_folder) :].lstrip("/")
folder_suffix = item.folder[len(old_folder) :].lstrip("/")
repository.update_note_location(
conn=conn,
note_id=item.note_id,
title=item.title,
file_path=f"{new_folder}/{file_suffix}",
folder=(
f"{new_folder}/{folder_suffix}" if folder_suffix else new_folder
),
updated_at=now,
)
except BaseException:
target.replace(source)
raise
finally:
conn.close()
return WorkspaceEntry(
entry_id=_entry_id("folder", new_folder),
name=clean_name,
path=f"/{new_folder}",
type="folder",
children=_tree(target, {item.file_path: item for item in repository.list_note_locations()}),
)
@serialized_vault_mutation
async def delete_folder(path: str) -> OperationResponse:
folder = normalize_folder(path)
if not folder:
raise ApiError(400, "INVALID_PATH", "the Vault root cannot be deleted")
source = resolve_in_vault(folder)
if not source.is_dir() or source.is_symlink():
raise ApiError(404, "RESOURCE_NOT_FOUND", "folder not found", {"path": path})
affected = [
item
for item in repository.list_note_locations()
if item.folder == folder or item.folder.startswith(f"{folder}/")
]
tombstone = source.with_name(f".{source.name}.{uuid4().hex}.deleting")
source.replace(tombstone)
conn = connect()
try:
with transaction(conn):
block_ids: list[str] = []
for item in affected:
block_ids.extend(repository.delete_note(item.note_id, conn=conn))
await vector_store.delete(block_ids, conn=conn)
except BaseException:
tombstone.replace(source)
raise
finally:
conn.close()
try:
shutil.rmtree(tombstone)
except OSError:
# 已提交的删除不回滚；隐藏 tombstone 可由后续维护任务清理。
pass
return OperationResponse(
status="completed",
resource_id=_entry_id("folder", folder),
message=f"deleted folder and {len(affected)} indexed notes",
)