diff --git a/backend/app/knowledge/parser.py b/backend/app/knowledge/parser.py index fa2ed89..dabe73b 100644 --- a/backend/app/knowledge/parser.py +++ b/backend/app/knowledge/parser.py @@ -176,11 +176,23 @@ def _split_lines(text: str) -> list[tuple[str, int]]: def _content_start(markdown: str) -> int: """返回正文起始 UTF-16 偏移:有 frontmatter 时跳过 --- 分隔块。""" - if markdown.startswith("---"): - end = markdown.find("\n---", 3) - if end != -1: - return _utf16_len(markdown[: end + 4]) - return 0 + header = _frontmatter(markdown) + return _utf16_len(markdown[:header[1]]) if header else 0 + + +def _frontmatter(markdown: str) -> tuple[str, int] | None: + """Return YAML text and body character offset without changing original text.""" + start = 1 if markdown.startswith("\ufeff") else 0 + opening = re.match(r"---[ \t]*(?:\r\n|\n|\r|\Z)", markdown[start:]) + if opening is None: + return None + content_start = start + opening.end() + offset = content_start + for raw in markdown[content_start:].splitlines(keepends=True): + if re.fullmatch(r"(?:---|\.\.\.)[ \t]*", raw.rstrip("\r\n")): + return markdown[content_start:offset], offset + len(raw) + offset += len(raw) + raise ApiError(422, "INVALID_EMBEDDING_POLICY", "Frontmatter 未闭合,请补全独立一行的结束分隔符后再保存。") def _utf16_len(text: str) -> int: @@ -188,13 +200,13 @@ def _utf16_len(text: str) -> int: def _embedding_policy(markdown: str) -> bool: - end = markdown.find("\n---", 3) if markdown.startswith("---") else -1 - if end == -1: + header = _frontmatter(markdown) + if header is None: return False try: # Compose nodes without constructing objects. This accepts YAML comments, # quoted keys and indentation while retaining duplicate-key information. - node = yaml.compose(markdown[3:end], Loader=yaml.SafeLoader) + node = yaml.compose(header[0], Loader=yaml.SafeLoader) except yaml.YAMLError as exc: raise ApiError(422, "INVALID_EMBEDDING_POLICY", "Frontmatter YAML 无效,无法确认本地索引策略。") from exc if node is None: @@ -218,13 +230,11 @@ def _embedding_policy(markdown: str) -> bool: def _extract_frontmatter(markdown: str) -> dict[str, str]: """极简 frontmatter 解析,只提取 key: value 行。""" - if not markdown.startswith("---"): - return {} - end = markdown.find("\n---", 3) - if end == -1: + header = _frontmatter(markdown) + if header is None: return {} meta: dict[str, str] = {} - for line in markdown[3:end].splitlines(): + for line in header[0].splitlines(): m = _FRONTMATTER_KEY_RE.match(line) if m: meta[m.group(1).lower()] = m.group(2).strip() diff --git a/backend/tests/test_policy_and_migrations.py b/backend/tests/test_policy_and_migrations.py index c65cff2..6fbc7ae 100644 --- a/backend/tests/test_policy_and_migrations.py +++ b/backend/tests/test_policy_and_migrations.py @@ -108,3 +108,60 @@ def test_merge_policy_is_rejected_instead_of_ignored(): parsed('true\n<<: {embedding_local_only: false}') with pytest.raises(ApiError): parsed('!!bool invalid') + + +@pytest.mark.parametrize('bom', ['', '\ufeff']) +@pytest.mark.parametrize('newline', ['\n', '\r\n', '\r']) +@pytest.mark.parametrize('closing', ['---', '...']) +def test_frontmatter_boundaries_preserve_policy_and_utf16_offsets(bom, newline, closing): + markdown = bom + newline.join(['--- ', 'title: Sample', 'embedding_local_only: true # local', closing+' ', '# Heading', '', 'private \U0001f600']) + note = parse_note(markdown=markdown, file_path='note.md', folder='', created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc)) + assert note.embedding_local_only and note.title == 'Sample' + assert all('embedding_local_only' not in block.content for block in note.blocks) + block = next(block for block in note.blocks if block.content == 'private \U0001f600') + original = markdown.encode('utf-16-le')[block.start_offset*2:block.end_offset*2].decode('utf-16-le') + assert original == block.content + + +@pytest.mark.parametrize('ending', ['', '\n---not-a-delimiter', '\n----']) +def test_unclosed_frontmatter_is_rejected_even_with_bom(ending): + for bom in ['', '\ufeff']: + markdown = bom+'---\nembedding_local_only: true'+ending + with pytest.raises(ApiError) as error: + parse_note(markdown=markdown, file_path='note.md', folder='', created_at=datetime.now(timezone.utc), updated_at=datetime.now(timezone.utc)) + assert error.value.code == 'INVALID_EMBEDDING_POLICY' + + +def test_boundary_matching_does_not_truncate_yaml_keys(): + markdown = '---\n---metadata: value\nembedding_local_only: true\n---\nbody' + note = parse_note(markdown=markdown,file_path='note.md',folder='',created_at=datetime.now(timezone.utc),updated_at=datetime.now(timezone.utc)) + assert note.embedding_local_only + + +def test_bom_save_and_invalid_update_never_use_remote(monkeypatch): + import asyncio + from types import SimpleNamespace + from app.local_models.runtime import LocalEmbedding + from app.retrieval import routed_vectors + from app.services import note_service, index_service + from app.contracts import IndexRebuildRequest + from app.config import get_settings + calls=[] + class Routing: + async def embed(self, texts, *, local_only=False): + calls.append(local_only) + assert local_only + return SimpleNamespace(source='local', model_id='local-test', dimensions=2, vectors=[[1.0,0.0] for _ in texts], fallback_reason=None) + monkeypatch.setattr(routed_vectors, 'get_model_routing', lambda: Routing()) + monkeypatch.setattr(note_service, 'embedding', LocalEmbedding()) + async def scenario(): + markdown='\ufeff---\nembedding_local_only: true\n---\nprivate text' + note=await note_service.create_note(title='Private',markdown=markdown,folder=None,tags=[]) + await index_service.rebuild(IndexRebuildRequest()) + count=len(calls) + with pytest.raises(ApiError): + await note_service.update_note(note.note_id,markdown='\ufeff---\nembedding_local_only: true\nprivate text') + assert len(calls)==count + assert (get_settings().vault_path/note.file_path).read_text(encoding='utf-8')==markdown + assert (await note_service.get_note(note.note_id)).markdown==markdown + asyncio.run(scenario()) diff --git a/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md b/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md index 7f7b8cb..46b9dfe 100644 --- a/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md +++ b/docs/retrospectives/阶段F-Embedding与知识库问题与解决方案.md @@ -26,6 +26,7 @@ | F-08 | 将不同处理策略误判为配置漂移 | 普通与仅本地笔记共存时不能重建 | 按策略校验覆盖,独立检索并融合排名 | | F-09 | 简单字符串比较忽略 YAML 语法 | 注释等合法写法可能关闭本地限制 | 解析 YAML 节点,非法策略明确拒绝 | | F-10 | 迁移 DDL 与版本号分开提交 | 中断后重启报重复列 | 原子迁移、并发重检及旧半迁移恢复 | +| F-11 | BOM 与未闭合头部被当作无策略 | 本地限定正文可能进入普通 API 路由 | 统一 frontmatter 边界,异常头部拒绝保存 | ## 3. F-01 / F-03:模型可用不等于索引可用 @@ -136,6 +137,16 @@ embedding_local_only: true ## 9. 工程经验 +### F-11:frontmatter 边界与 BOM + +审阅通过隔离保存链路复现:普通 `---` 头部返回 `local_only=True`,加 UTF-8 BOM 或移除结束分隔线后却返回 `False`。原因是策略、元数据和正文分别使用 `startswith` 与子串查找判断头部;未识别成功时静默按无策略处理。 + +实际方案:三处改用 `_frontmatter` 统一识别。允许一个文件起始 BOM,开头分隔符须为独立的 `---` 行,结束分隔符支持独立的 `---` 或 `...` 行及尾部空白;支持 LF、CRLF、CR。`---metadata`、`----` 等前缀不会被误当成结束分隔符。已识别开头但没有结束行时返回 `INVALID_EMBEDDING_POLICY`,不继续索引。 + +原 Markdown 不做去 BOM 或换行转换,正文偏移仍由原文计算 UTF-16 code unit,保证来源定位。保存和重建使用同一解析路径;更新失败恢复原文件。测试覆盖 BOM 的本地限定保存与重建,未闭合更新不触发模型调用且文件、数据库正文保持原值。 + +F-11 修复后完整后端回归:533 项通过,新增 17 个参数化用例。普通 API 与本地回退、分区检索及迁移测试均通过,未调用真实外部模型。 + ### F-09:本地限制标记的 YAML 解析 再次审阅复现:`embedding_local_only: true # keep local` 被旧字符串比较解析为 `False`。加注释没有改变用户意图,却可能使保存或重建发送正文到远程 Embedding。