fix(export): 修复 PR #17 审阅问题(1 P1 + 5 P2)
- P1 链接/图片 URL 协议白名单校验,危险协议降级为纯文本 + warning - P2 图片 AST 字段映射(src=attrs.url,alt 取 children 文本) - P2 原始 HTML 块转义保留,正文不丢失 + warning - P2 过期/淘汰/重启清理导出产物文件 - P2 解析与渲染移入 asyncio.to_thread,运行中取消生效 - P2 function-plot 围栏别名补全 - 回归测试覆盖全部修复 Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -8,12 +8,28 @@ from __future__ import annotations
|
||||
|
||||
import html
|
||||
from datetime import datetime
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.contracts import ExportOptions
|
||||
from app.export.document import Document, DocumentNode, ExportResult
|
||||
|
||||
_MERMAID_WARNING = "mermaid 需前端渲染,已保留为占位代码块"
|
||||
_FUNCTION_PLOT_WARNING = "函数图像渲染将在后续版本提供,已保留为占位代码块"
|
||||
_RAW_HTML_WARNING = "原始 HTML 已按纯文本转义保留"
|
||||
|
||||
# 链接/图片地址允许的协议;无 scheme 的相对地址视为安全,其余协议一律降级
|
||||
_ALLOWED_URL_SCHEMES = frozenset({"http", "https", "mailto"})
|
||||
|
||||
|
||||
def _safe_url(url: str) -> str | None:
|
||||
"""校验 URL 协议;安全返回原串,不安全返回 None。"""
|
||||
url = url.strip()
|
||||
if not url:
|
||||
return None
|
||||
scheme = urlparse(url).scheme.lower()
|
||||
if scheme and scheme not in _ALLOWED_URL_SCHEMES:
|
||||
return None
|
||||
return url
|
||||
|
||||
_BASE_CSS = """
|
||||
body { margin: 0; background: #f6f7f9; color: #1f2328; font: 15px/1.7 -apple-system, 'Segoe UI', 'Microsoft YaHei', sans-serif; }
|
||||
@@ -46,7 +62,8 @@ hr { border: none; border-top: 1px solid #d0d7de; margin: 1.4em 0; }
|
||||
class HtmlExporter:
|
||||
"""实现 DocumentExporter:递归渲染 Document AST 为完整 HTML5 文档。"""
|
||||
|
||||
async def export(self, document: Document, options: ExportOptions) -> ExportResult:
|
||||
def render(self, document: Document, options: ExportOptions) -> ExportResult:
|
||||
"""同步渲染;CPU 密集,调用方应放入线程执行,避免阻塞事件循环。"""
|
||||
self._options = options
|
||||
warnings: list[str] = []
|
||||
body = self._render_children(document.children, warnings)
|
||||
@@ -55,6 +72,10 @@ class HtmlExporter:
|
||||
content=content.encode("utf-8"), mime_type="text/html", warnings=warnings
|
||||
)
|
||||
|
||||
async def export(self, document: Document, options: ExportOptions) -> ExportResult:
|
||||
"""契约要求的 async 接口;渲染本身同步,直接转发到 render。"""
|
||||
return self.render(document, options)
|
||||
|
||||
def _assemble(
|
||||
self, document: Document, options: ExportOptions, body: str, warnings: list[str]
|
||||
) -> str:
|
||||
@@ -179,6 +200,11 @@ class HtmlExporter:
|
||||
def _render_math_block(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
return f'<div class="math-block">$${html.escape(node.text)}$$</div>'
|
||||
|
||||
def _render_html_block(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
# 原始 HTML 不可信,转义为纯文本展示,保证正文不丢且无注入风险
|
||||
warnings.append(_RAW_HTML_WARNING)
|
||||
return f'<div class="raw-html">{html.escape(node.text)}</div>'
|
||||
|
||||
# --- 行内 ---
|
||||
def _render_text(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
return html.escape(node.text)
|
||||
@@ -190,21 +216,32 @@ class HtmlExporter:
|
||||
return f"<strong>{self._render_children(node.children, warnings)}</strong>"
|
||||
|
||||
def _render_link(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
href = html.escape(str(node.attributes.get("href") or ""))
|
||||
inner = self._render_children(node.children, warnings)
|
||||
href = str(node.attributes.get("href") or "")
|
||||
safe_href = _safe_url(href)
|
||||
if safe_href is None:
|
||||
# 危险协议(如 javascript:)降级为纯文本,不输出可点击链接
|
||||
warnings.append(f"链接协议不安全,已降级为纯文本:{href!r}")
|
||||
return inner
|
||||
title = str(node.attributes.get("title") or "")
|
||||
attrs = [f'href="{href}"']
|
||||
attrs = [f'href="{html.escape(safe_href)}"']
|
||||
if title:
|
||||
attrs.append(f'title="{html.escape(title)}"')
|
||||
return f"<a {' '.join(attrs)}>{self._render_children(node.children, warnings)}</a>"
|
||||
return f"<a {' '.join(attrs)}>{inner}</a>"
|
||||
|
||||
def _render_codespan(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
return f"<code>{html.escape(node.text)}</code>"
|
||||
|
||||
def _render_image(self, node: DocumentNode, warnings: list[str]) -> str:
|
||||
src = html.escape(str(node.attributes.get("src") or ""))
|
||||
alt = html.escape(str(node.attributes.get("alt") or ""))
|
||||
src = str(node.attributes.get("src") or "")
|
||||
alt = str(node.attributes.get("alt") or "")
|
||||
safe_src = _safe_url(src)
|
||||
if safe_src is None:
|
||||
# 危险协议(如 data:/javascript:)跳过图片,仅输出 alt 文本
|
||||
warnings.append(f"图片地址不安全,已跳过:{src!r}")
|
||||
return html.escape(alt) if alt else ""
|
||||
title = str(node.attributes.get("title") or "")
|
||||
attrs = [f'src="{src}"', f'alt="{alt}"']
|
||||
attrs = [f'src="{html.escape(safe_src)}"', f'alt="{html.escape(alt)}"']
|
||||
if title:
|
||||
attrs.append(f'title="{html.escape(title)}"')
|
||||
return f"<img {' '.join(attrs)}>"
|
||||
|
||||
@@ -15,7 +15,7 @@ _PLUGINS = ["table", "math", "url", "task_lists"]
|
||||
|
||||
# fenced code 语言分流:命中则转为专用节点,其余按普通代码块
|
||||
_MERMAID_LANG = "mermaid"
|
||||
_FUNCTION_PLOT_LANGS = {"function_plot", "functionplot"}
|
||||
_FUNCTION_PLOT_LANGS = {"function-plot", "function_plot", "functionplot"}
|
||||
|
||||
|
||||
def parse_document(markdown: str) -> Document:
|
||||
@@ -85,10 +85,19 @@ class _AstMapper:
|
||||
return DocumentNode(type="thematic_break", node_id=self.next_id())
|
||||
if kind == "blank_line":
|
||||
return None
|
||||
# 未知块级 token(如 block_html)保守保留原文,避免静默丢失
|
||||
if kind == "block_html":
|
||||
# 原始 HTML 块降级为纯文本节点,由 HtmlExporter 转义并记 warning,避免静默丢失正文
|
||||
return DocumentNode(
|
||||
type="html_block", node_id=self.next_id(), text=token.get("raw", "")
|
||||
)
|
||||
# 未知块级 token 保守保留原文;映射为带 text 子节点的 paragraph,避免被渲染层丢弃
|
||||
raw = token.get("raw", "")
|
||||
if raw:
|
||||
return DocumentNode(type="paragraph", node_id=self.next_id(), text=raw)
|
||||
return DocumentNode(
|
||||
type="paragraph",
|
||||
node_id=self.next_id(),
|
||||
children=[DocumentNode(type="text", node_id=self.next_id(), text=raw)],
|
||||
)
|
||||
return None
|
||||
|
||||
def map_list_item(self, token: dict) -> DocumentNode:
|
||||
@@ -144,10 +153,16 @@ class _AstMapper:
|
||||
if kind == "codespan":
|
||||
return DocumentNode(type="codespan", node_id=self.next_id(), text=token.get("raw", ""))
|
||||
if kind == "image":
|
||||
# mistune 图片 token:src 在 attrs.url,alt 来自 children 的文本,title 在 attrs.title
|
||||
attrs = token.get("attrs", {})
|
||||
attributes = {"src": attrs.get("src", "")}
|
||||
if attrs.get("alt"):
|
||||
attributes["alt"] = attrs["alt"]
|
||||
alt = "".join(
|
||||
child.get("raw", "")
|
||||
for child in token.get("children", [])
|
||||
if child.get("type") == "text"
|
||||
)
|
||||
attributes = {"src": attrs.get("url", "")}
|
||||
if alt:
|
||||
attributes["alt"] = alt
|
||||
if attrs.get("title"):
|
||||
attributes["title"] = attrs["title"]
|
||||
return DocumentNode(type="image", node_id=self.next_id(), attributes=attributes)
|
||||
|
||||
@@ -28,7 +28,7 @@ from app.contracts import (
|
||||
ExportStatus,
|
||||
)
|
||||
from app.errors import ApiError
|
||||
from app.export.document import Document
|
||||
from app.export.document import Document, ExportResult
|
||||
from app.export.exporters.html import HtmlExporter
|
||||
from app.export.markdown import parse_document
|
||||
from app.services import note_service
|
||||
@@ -61,10 +61,44 @@ def _safe_download_name(title: str) -> str:
|
||||
return name[:80]
|
||||
|
||||
|
||||
def _export_path(job_id: str) -> Path:
|
||||
return get_settings().exports_path / f"{job_id}.html"
|
||||
|
||||
|
||||
def _delete_file(job_id: str) -> None:
|
||||
"""删除导出产物文件;文件不存在时忽略。"""
|
||||
try:
|
||||
_export_path(job_id).unlink(missing_ok=True)
|
||||
except OSError:
|
||||
logger.warning("Failed to delete export file: %s", job_id)
|
||||
|
||||
|
||||
def cleanup_orphan_files() -> int:
|
||||
"""清理 exports 目录下无对应内存任务的孤立产物(服务重启后调用)。"""
|
||||
exports_dir = get_settings().exports_path
|
||||
if not exports_dir.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for path in exports_dir.glob("*.html"):
|
||||
if path.stem not in _jobs:
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
logger.warning("Failed to delete orphan export file: %s", path)
|
||||
return removed
|
||||
|
||||
|
||||
def _render_document(document: Document, options: ExportOptions) -> ExportResult:
|
||||
"""同步渲染辅助,供 asyncio.to_thread 调用;每次新建实例避免跨线程复用。"""
|
||||
return HtmlExporter().render(document, options)
|
||||
|
||||
|
||||
def _forget(job_id: str) -> None:
|
||||
_jobs.pop(job_id, None)
|
||||
_tasks.pop(job_id, None)
|
||||
_cancel_flags.pop(job_id, None)
|
||||
_delete_file(job_id)
|
||||
|
||||
|
||||
def _evict_terminal() -> bool:
|
||||
@@ -166,19 +200,20 @@ async def _execute(
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
|
||||
document = parse_document(markdown)
|
||||
# 解析与渲染都是 CPU 密集的同步工作,放入线程执行避免阻塞事件循环,
|
||||
# 使运行中的取消能在渲染边界生效;写文件前再次检查取消。
|
||||
document = await asyncio.to_thread(parse_document, markdown)
|
||||
document.attributes["title"] = title
|
||||
if metadata:
|
||||
document.attributes["metadata"] = metadata
|
||||
|
||||
exporter = HtmlExporter()
|
||||
result = await exporter.export(document, options)
|
||||
result = await asyncio.to_thread(_render_document, document, options)
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
|
||||
out_dir = get_settings().exports_path
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / f"{job_id}.html"
|
||||
path = _export_path(job_id)
|
||||
path.write_bytes(result.content)
|
||||
|
||||
completed_at = _now()
|
||||
@@ -260,8 +295,9 @@ def get_export_file(job_id: str) -> Path:
|
||||
404, "EXPORT_JOB_NOT_FOUND", "export file not ready", {"job_id": job_id}
|
||||
)
|
||||
if job.file.expires_at <= _now():
|
||||
_forget(job_id) # 过期即清理内存记录与产物文件
|
||||
raise ApiError(410, "EXPORT_FILE_EXPIRED", "export file has expired", {"job_id": job_id})
|
||||
return get_settings().exports_path / f"{job_id}.html"
|
||||
return _export_path(job_id)
|
||||
|
||||
|
||||
async def wait_for_export(job_id: str) -> ExportJob | None:
|
||||
|
||||
@@ -8,6 +8,7 @@ from starlette.exceptions import HTTPException as StarletteHttpException
|
||||
from app.config import get_settings
|
||||
from app.container import container
|
||||
from app.errors import ApiError, api_error_handler, http_error_handler, validation_error_handler
|
||||
from app.export import service as export_service
|
||||
from app.routes import router as api_router
|
||||
from app.schemas import HealthResponse, ServiceStatusResponse
|
||||
|
||||
@@ -16,6 +17,8 @@ settings = get_settings()
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_: FastAPI):
|
||||
# 重启后内存注册表为空,清理上一次运行遗留的导出产物,避免磁盘垃圾堆积。
|
||||
export_service.cleanup_orphan_files()
|
||||
yield
|
||||
# 第三方 MCP Server 必须跟随 AI Core 退出,不能遗留孤儿进程。
|
||||
container.plugins.shutdown()
|
||||
|
||||
@@ -16,6 +16,7 @@ from pydantic import ValidationError
|
||||
from app.config import get_settings
|
||||
from app.contracts import (
|
||||
ExportFormat,
|
||||
ExportJob,
|
||||
ExportOptions,
|
||||
ExportRequest,
|
||||
ExportSource,
|
||||
@@ -133,6 +134,21 @@ def test_parse_document_table_and_math() -> None:
|
||||
assert "math_block" in kinds
|
||||
|
||||
|
||||
def test_parse_document_image_maps_src_alt_title() -> None:
|
||||
doc = parse_document('')
|
||||
img = doc.children[0].children[0]
|
||||
assert img.type == "image"
|
||||
assert img.attributes["src"] == "https://a.b/img.png"
|
||||
assert img.attributes["alt"] == "替代文本"
|
||||
assert img.attributes["title"] == "标题"
|
||||
|
||||
|
||||
def test_parse_document_function_plot_dash_alias() -> None:
|
||||
doc = parse_document("```function-plot\ny = x^2\n```")
|
||||
assert doc.children[0].type == "function_plot"
|
||||
assert doc.children[0].text == "y = x^2"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# HtmlExporter
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -162,6 +178,38 @@ def test_html_exporter_marks_mermaid_and_function_plot() -> None:
|
||||
assert any("mermaid" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_html_exporter_rejects_unsafe_link_protocol() -> None:
|
||||
result = asyncio.run(
|
||||
HtmlExporter().export(parse_document("[点我](javascript:alert(1))"), ExportOptions())
|
||||
)
|
||||
html = result.content.decode("utf-8")
|
||||
assert "javascript:" not in html
|
||||
assert "点我" in html
|
||||
assert any("不安全" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_html_exporter_rejects_unsafe_image_protocol() -> None:
|
||||
result = asyncio.run(
|
||||
HtmlExporter().export(parse_document(""), ExportOptions())
|
||||
)
|
||||
html = result.content.decode("utf-8")
|
||||
assert "data:" not in html
|
||||
assert "<img" not in html
|
||||
assert "alt" in html
|
||||
assert any("不安全" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_html_exporter_preserves_raw_html_block() -> None:
|
||||
result = asyncio.run(
|
||||
HtmlExporter().export(parse_document("<div>重要正文</div>"), ExportOptions())
|
||||
)
|
||||
html = result.content.decode("utf-8")
|
||||
assert "重要正文" in html
|
||||
assert "<div>" not in html
|
||||
assert "<div>重要正文</div>" in html
|
||||
assert any("原始 HTML" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_html_exporter_include_title_and_metadata() -> None:
|
||||
doc = parse_document("正文")
|
||||
doc.attributes["title"] = "操作系统复习"
|
||||
@@ -274,10 +322,73 @@ def test_export_file_expired_410() -> None:
|
||||
return job.job_id
|
||||
|
||||
job_id = asyncio.run(_go())
|
||||
path = get_settings().exports_path / f"{job_id}.html"
|
||||
with pytest.raises(ApiError) as exc:
|
||||
export_service.get_export_file(job_id)
|
||||
assert exc.value.status_code == 410
|
||||
assert exc.value.code == "EXPORT_FILE_EXPIRED"
|
||||
assert not path.exists() # 过期即清理产物文件
|
||||
assert export_service.get_export(job_id) is None # 内存记录一并清理
|
||||
|
||||
|
||||
def test_export_eviction_deletes_file() -> None:
|
||||
finished = _create_and_wait(_markdown_request("# 淘汰"))
|
||||
victim_path = get_settings().exports_path / f"{finished.job_id}.html"
|
||||
assert victim_path.exists()
|
||||
|
||||
# 塞满 MAX_JOBS 个终态任务,下一次 create 会淘汰最旧的终态(finished 最先插入)
|
||||
for i in range(export_service.MAX_JOBS):
|
||||
export_service._jobs[f"export_fake_{i}"] = ExportJob(
|
||||
job_id=f"export_fake_{i}",
|
||||
status=ExportStatus.completed,
|
||||
format=ExportFormat.html,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
)
|
||||
_create_and_wait(_markdown_request("# 触发淘汰"))
|
||||
assert not victim_path.exists()
|
||||
|
||||
|
||||
def test_cleanup_orphan_files() -> None:
|
||||
exports_dir = get_settings().exports_path
|
||||
exports_dir.mkdir(parents=True, exist_ok=True)
|
||||
orphan = exports_dir / "export_orphan.html"
|
||||
orphan.write_text("stale", encoding="utf-8")
|
||||
|
||||
finished = _create_and_wait(_markdown_request("# 保留"))
|
||||
keep_path = exports_dir / f"{finished.job_id}.html"
|
||||
assert keep_path.exists()
|
||||
|
||||
removed = export_service.cleanup_orphan_files()
|
||||
assert removed >= 1
|
||||
assert not orphan.exists()
|
||||
assert keep_path.exists() # 仍在注册表中的任务文件保留
|
||||
|
||||
|
||||
def test_export_cancel_during_running(monkeypatch) -> None:
|
||||
import threading
|
||||
import time
|
||||
|
||||
real_parse = parse_document
|
||||
started = threading.Event()
|
||||
|
||||
def slow_parse(markdown: str):
|
||||
started.set()
|
||||
time.sleep(0.1)
|
||||
return real_parse(markdown)
|
||||
|
||||
monkeypatch.setattr(export_service, "parse_document", slow_parse)
|
||||
|
||||
async def _go():
|
||||
job = await export_service.create_export(_markdown_request("# 运行中取消"))
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
export_service.cancel_export(job.job_id)
|
||||
return await export_service.wait_for_export(job.job_id)
|
||||
|
||||
finished = asyncio.run(_go())
|
||||
assert finished.status == ExportStatus.cancelled
|
||||
assert finished.file is None
|
||||
assert not (get_settings().exports_path / f"{finished.job_id}.html").exists()
|
||||
|
||||
|
||||
def test_export_list_and_get() -> None:
|
||||
|
||||
Reference in New Issue
Block a user