Feat/multimodal pipeline #19

Merged
Kronecker merged 9 commits from feat/multimodal-pipeline into main 2026-09-04 20:17:03 +08:00
3 changed files with 83 additions and 1 deletions
Showing only changes of commit cc617ed23e - Show all commits
+34 -1
View File
@@ -190,11 +190,44 @@ def _frontmatter(markdown: str) -> tuple[str, int] | None:
offset = content_start
for raw in markdown[content_start:].splitlines(keepends=True):
if re.fullmatch(r"(?:---|\.\.\.)[ \t]*", raw.rstrip("\r\n")):
return markdown[content_start:offset], offset + len(raw)
candidate = markdown[content_start:offset]
if not candidate.strip() or _metadata_intent(candidate):
return candidate, offset + len(raw)
return None # Ordinary Markdown between thematic breaks.
offset += len(raw)
if not _metadata_intent(markdown[content_start:]):
return None
raise ApiError(422, "INVALID_EMBEDDING_POLICY", "Frontmatter 未闭合,请补全独立一行的结束分隔符后再保存。")
def _metadata_intent(content: str) -> bool:
"""A thematic break alone is not a declaration of YAML metadata."""
# An explicit policy must fail closed even when other header lines are broken.
fence_marker = None
for line in content.splitlines():
fence = _FENCE_RE.match(line)
if fence_marker is not None:
marker = fence.group(1) if fence else ""
if marker.startswith(fence_marker[0]) and len(marker) >= len(fence_marker):
fence_marker = None
continue
if fence:
fence_marker = fence.group(1)
continue
if re.match(r"(?i)^[ \t]*[\"']?embedding_local_only[\"']?[ \t]*:", line):
return True
try:
if isinstance(yaml.compose(content, Loader=yaml.SafeLoader), yaml.MappingNode):
return True
except yaml.YAMLError:
pass
first = next((line.strip() for line in content.splitlines()
if line.strip() and not line.lstrip().startswith("#")), "")
# Preserve errors for incomplete key/value headers, including flow mappings.
return bool(re.match(r"(?:[\w.-]+|[\"'][^\"']+[\"'])\s*:(?:\s|$)", first)
or (first.startswith("{") and ":" in first))
def _utf16_len(text: str) -> int:
return len(text.encode("utf-16-le")) // 2
@@ -165,3 +165,41 @@ def test_bom_save_and_invalid_update_never_use_remote(monkeypatch):
assert (get_settings().vault_path/note.file_path).read_text(encoding='utf-8')==markdown
assert (await note_service.get_note(note.note_id)).markdown==markdown
asyncio.run(scenario())
@pytest.mark.parametrize('markdown', ['---', '---\n\n# Title\n\nNormal body', '---\n\nNormal body\n\n---\n\nLast paragraph', '---\n\n```python\nprint(1)\n```\n---'])
def test_thematic_breaks_are_not_frontmatter(markdown):
note = parse_note(markdown=markdown,file_path='ordinary.md',folder='',created_at=datetime.now(timezone.utc),updated_at=datetime.now(timezone.utc))
assert not note.embedding_local_only
assert note.blocks[0].content == '---'
assert any(block.content == markdown.split('\n\n')[-1] for block in note.blocks) or '```' in markdown
@pytest.mark.parametrize('header', ['title: Sample\nembedding_local_only: true', '"embedding_local_only": true', 'title: [broken\nembedding_local_only: true', '{embedding_local_only: true'])
def test_unclosed_metadata_still_fails_closed(header):
with pytest.raises(ApiError) as error:
parse_note(markdown='---\n'+header,file_path='private.md',folder='',created_at=datetime.now(timezone.utc),updated_at=datetime.now(timezone.utc))
assert error.value.code == 'INVALID_EMBEDDING_POLICY'
def test_thematic_break_note_can_save_and_rebuild():
import asyncio
from app.services import note_service, index_service
from app.contracts import IndexRebuildRequest
async def scenario():
markdown='---\n\n# Title\n\nNormal body'
note=await note_service.create_note(title='Divider',markdown=markdown,folder=None,tags=[])
assert note.blocks[0].content == '---'
assert (await index_service.rebuild(IndexRebuildRequest())).status == 'completed'
loaded=await note_service.get_note(note.note_id)
assert loaded.markdown == markdown
assert [b.content for b in loaded.blocks] == [b.content for b in note.blocks]
asyncio.run(scenario())
def test_thematic_break_with_policy_example_is_ordinary_markdown():
markdown='---\n\n```yaml\nembedding_local_only: true\n```\n\n---\n\nExplanation'
note=parse_note(markdown=markdown,file_path='example.md',folder='',created_at=datetime.now(timezone.utc),updated_at=datetime.now(timezone.utc))
assert not note.embedding_local_only
assert any('embedding_local_only: true' in block.content for block in note.blocks)
assert note.blocks[0].content=='---'
@@ -27,6 +27,7 @@
| F-09 | 简单字符串比较忽略 YAML 语法 | 注释等合法写法可能关闭本地限制 | 解析 YAML 节点,非法策略明确拒绝 |
| F-10 | 迁移 DDL 与版本号分开提交 | 中断后重启报重复列 | 原子迁移、并发重检及旧半迁移恢复 |
| F-11 | BOM 与未闭合头部被当作无策略 | 本地限定正文可能进入普通 API 路由 | 统一 frontmatter 边界,异常头部拒绝保存 |
| F-12 | 普通 Markdown 分割线误判为头部 | 正常笔记保存失败、重建中止 | 按元数据声明识别头部,保留普通正文 |
## 3. F-01 / F-03:模型可用不等于索引可用
@@ -137,6 +138,16 @@ embedding_local_only: true
## 9. 工程经验
### F-12:普通分割线与元数据头部消歧
F-11 修复后,`---``---\n\n# Title\n\n正文` 等合法 Markdown 被误判为未闭合 frontmatter,原先能够保存的笔记被拒绝;库中已有此类文件时全量重建也会失败。
实际方案:开头分隔线仅作为候选,继续判断内容是否声明元数据。YAML 映射、以键值形式开始的头部或显式 `embedding_local_only` 声明按元数据处理,缺少结束行仍报错;普通段落、标题和代码块按正文保留,包括之后再次出现分割线的情况。已有闭合空头部继续兼容。
显式本地策略即使与其他损坏的 YAML 行共存,也不能退成普通正文。无结束分隔符的键值头部仍视为错误;普通文章中有歧义的开头键值形式应避免紧随文件首行 `---`。回归覆盖分割线正文解析、真实保存和重建、BOM 与本地策略原有拒绝规则。
围栏代码块中的策略示例不算真实声明,保留为 Markdown 正文。验证记录:首批修复后全量后端 542 项通过;补充围栏示例识别后,解析、迁移与检索相关 130 项通过。本轮未修改前端,未调用真实外部模型。
### F-11frontmatter 边界与 BOM
审阅通过隔离保存链路复现:普通 `---` 头部返回 `local_only=True`,加 UTF-8 BOM 或移除结束分隔线后却返回 `False`。原因是策略、元数据和正文分别使用 `startswith` 与子串查找判断头部;未识别成功时静默按无策略处理。