feat(export): 多格式后台导出、主题与警告框渲染 #43

Merged
Kronecker merged 21 commits from feat/export-service into main 2026-09-07 01:38:36 +08:00
5 changed files with 221 additions and 19 deletions
Showing only changes of commit 64af1f5165 - Show all commits
+44 -7
View File
@@ -8,12 +8,28 @@ from __future__ import annotations
import html import html
from datetime import datetime from datetime import datetime
from urllib.parse import urlparse
from app.contracts import ExportOptions from app.contracts import ExportOptions
from app.export.document import Document, DocumentNode, ExportResult from app.export.document import Document, DocumentNode, ExportResult
_MERMAID_WARNING = "mermaid 需前端渲染,已保留为占位代码块" _MERMAID_WARNING = "mermaid 需前端渲染,已保留为占位代码块"
_FUNCTION_PLOT_WARNING = "函数图像渲染将在后续版本提供,已保留为占位代码块" _FUNCTION_PLOT_WARNING = "函数图像渲染将在后续版本提供,已保留为占位代码块"
_RAW_HTML_WARNING = "原始 HTML 已按纯文本转义保留"
# 链接/图片地址允许的协议;无 scheme 的相对地址视为安全,其余协议一律降级
_ALLOWED_URL_SCHEMES = frozenset({"http", "https", "mailto"})
def _safe_url(url: str) -> str | None:
"""校验 URL 协议;安全返回原串,不安全返回 None。"""
url = url.strip()
if not url:
return None
scheme = urlparse(url).scheme.lower()
if scheme and scheme not in _ALLOWED_URL_SCHEMES:
return None
return url
_BASE_CSS = """ _BASE_CSS = """
body { margin: 0; background: #f6f7f9; color: #1f2328; font: 15px/1.7 -apple-system, 'Segoe UI', 'Microsoft YaHei', sans-serif; } body { margin: 0; background: #f6f7f9; color: #1f2328; font: 15px/1.7 -apple-system, 'Segoe UI', 'Microsoft YaHei', sans-serif; }
@@ -46,7 +62,8 @@ hr { border: none; border-top: 1px solid #d0d7de; margin: 1.4em 0; }
class HtmlExporter: class HtmlExporter:
"""实现 DocumentExporter:递归渲染 Document AST 为完整 HTML5 文档。""" """实现 DocumentExporter:递归渲染 Document AST 为完整 HTML5 文档。"""
async def export(self, document: Document, options: ExportOptions) -> ExportResult: def render(self, document: Document, options: ExportOptions) -> ExportResult:
"""同步渲染;CPU 密集,调用方应放入线程执行,避免阻塞事件循环。"""
self._options = options self._options = options
warnings: list[str] = [] warnings: list[str] = []
body = self._render_children(document.children, warnings) body = self._render_children(document.children, warnings)
@@ -55,6 +72,10 @@ class HtmlExporter:
content=content.encode("utf-8"), mime_type="text/html", warnings=warnings content=content.encode("utf-8"), mime_type="text/html", warnings=warnings
) )
async def export(self, document: Document, options: ExportOptions) -> ExportResult:
"""契约要求的 async 接口;渲染本身同步,直接转发到 render。"""
return self.render(document, options)
def _assemble( def _assemble(
self, document: Document, options: ExportOptions, body: str, warnings: list[str] self, document: Document, options: ExportOptions, body: str, warnings: list[str]
) -> str: ) -> str:
@@ -179,6 +200,11 @@ class HtmlExporter:
def _render_math_block(self, node: DocumentNode, warnings: list[str]) -> str: def _render_math_block(self, node: DocumentNode, warnings: list[str]) -> str:
return f'<div class="math-block">$${html.escape(node.text)}$$</div>' return f'<div class="math-block">$${html.escape(node.text)}$$</div>'
def _render_html_block(self, node: DocumentNode, warnings: list[str]) -> str:
# 原始 HTML 不可信,转义为纯文本展示,保证正文不丢且无注入风险
warnings.append(_RAW_HTML_WARNING)
return f'<div class="raw-html">{html.escape(node.text)}</div>'
# --- 行内 --- # --- 行内 ---
def _render_text(self, node: DocumentNode, warnings: list[str]) -> str: def _render_text(self, node: DocumentNode, warnings: list[str]) -> str:
return html.escape(node.text) return html.escape(node.text)
@@ -190,21 +216,32 @@ class HtmlExporter:
return f"<strong>{self._render_children(node.children, warnings)}</strong>" return f"<strong>{self._render_children(node.children, warnings)}</strong>"
def _render_link(self, node: DocumentNode, warnings: list[str]) -> str: def _render_link(self, node: DocumentNode, warnings: list[str]) -> str:
href = html.escape(str(node.attributes.get("href") or "")) inner = self._render_children(node.children, warnings)
href = str(node.attributes.get("href") or "")
safe_href = _safe_url(href)
if safe_href is None:
# 危险协议(如 javascript:)降级为纯文本,不输出可点击链接
warnings.append(f"链接协议不安全,已降级为纯文本:{href!r}")
return inner
title = str(node.attributes.get("title") or "") title = str(node.attributes.get("title") or "")
attrs = [f'href="{href}"'] attrs = [f'href="{html.escape(safe_href)}"']
if title: if title:
attrs.append(f'title="{html.escape(title)}"') attrs.append(f'title="{html.escape(title)}"')
return f"<a {' '.join(attrs)}>{self._render_children(node.children, warnings)}</a>" return f"<a {' '.join(attrs)}>{inner}</a>"
def _render_codespan(self, node: DocumentNode, warnings: list[str]) -> str: def _render_codespan(self, node: DocumentNode, warnings: list[str]) -> str:
return f"<code>{html.escape(node.text)}</code>" return f"<code>{html.escape(node.text)}</code>"
def _render_image(self, node: DocumentNode, warnings: list[str]) -> str: def _render_image(self, node: DocumentNode, warnings: list[str]) -> str:
src = html.escape(str(node.attributes.get("src") or "")) src = str(node.attributes.get("src") or "")
alt = html.escape(str(node.attributes.get("alt") or "")) alt = str(node.attributes.get("alt") or "")
safe_src = _safe_url(src)
if safe_src is None:
# 危险协议(如 data:/javascript:)跳过图片,仅输出 alt 文本
warnings.append(f"图片地址不安全,已跳过:{src!r}")
return html.escape(alt) if alt else ""
title = str(node.attributes.get("title") or "") title = str(node.attributes.get("title") or "")
attrs = [f'src="{src}"', f'alt="{alt}"'] attrs = [f'src="{html.escape(safe_src)}"', f'alt="{html.escape(alt)}"']
if title: if title:
attrs.append(f'title="{html.escape(title)}"') attrs.append(f'title="{html.escape(title)}"')
return f"<img {' '.join(attrs)}>" return f"<img {' '.join(attrs)}>"
+21 -6
View File
@@ -15,7 +15,7 @@ _PLUGINS = ["table", "math", "url", "task_lists"]
# fenced code 语言分流:命中则转为专用节点,其余按普通代码块 # fenced code 语言分流:命中则转为专用节点,其余按普通代码块
_MERMAID_LANG = "mermaid" _MERMAID_LANG = "mermaid"
_FUNCTION_PLOT_LANGS = {"function_plot", "functionplot"} _FUNCTION_PLOT_LANGS = {"function-plot", "function_plot", "functionplot"}
def parse_document(markdown: str) -> Document: def parse_document(markdown: str) -> Document:
@@ -85,10 +85,19 @@ class _AstMapper:
return DocumentNode(type="thematic_break", node_id=self.next_id()) return DocumentNode(type="thematic_break", node_id=self.next_id())
if kind == "blank_line": if kind == "blank_line":
return None return None
# 未知块级 token(如 block_html)保守保留原文,避免静默丢失 if kind == "block_html":
# 原始 HTML 块降级为纯文本节点,由 HtmlExporter 转义并记 warning,避免静默丢失正文
return DocumentNode(
type="html_block", node_id=self.next_id(), text=token.get("raw", "")
)
# 未知块级 token 保守保留原文;映射为带 text 子节点的 paragraph,避免被渲染层丢弃
raw = token.get("raw", "") raw = token.get("raw", "")
if raw: if raw:
return DocumentNode(type="paragraph", node_id=self.next_id(), text=raw) return DocumentNode(
type="paragraph",
node_id=self.next_id(),
children=[DocumentNode(type="text", node_id=self.next_id(), text=raw)],
)
return None return None
def map_list_item(self, token: dict) -> DocumentNode: def map_list_item(self, token: dict) -> DocumentNode:
@@ -144,10 +153,16 @@ class _AstMapper:
if kind == "codespan": if kind == "codespan":
return DocumentNode(type="codespan", node_id=self.next_id(), text=token.get("raw", "")) return DocumentNode(type="codespan", node_id=self.next_id(), text=token.get("raw", ""))
if kind == "image": if kind == "image":
# mistune 图片 tokensrc 在 attrs.urlalt 来自 children 的文本,title 在 attrs.title
attrs = token.get("attrs", {}) attrs = token.get("attrs", {})
attributes = {"src": attrs.get("src", "")} alt = "".join(
if attrs.get("alt"): child.get("raw", "")
attributes["alt"] = attrs["alt"] for child in token.get("children", [])
if child.get("type") == "text"
)
attributes = {"src": attrs.get("url", "")}
if alt:
attributes["alt"] = alt
if attrs.get("title"): if attrs.get("title"):
attributes["title"] = attrs["title"] attributes["title"] = attrs["title"]
return DocumentNode(type="image", node_id=self.next_id(), attributes=attributes) return DocumentNode(type="image", node_id=self.next_id(), attributes=attributes)
+42 -6
View File
@@ -28,7 +28,7 @@ from app.contracts import (
ExportStatus, ExportStatus,
) )
from app.errors import ApiError from app.errors import ApiError
from app.export.document import Document from app.export.document import Document, ExportResult
from app.export.exporters.html import HtmlExporter from app.export.exporters.html import HtmlExporter
from app.export.markdown import parse_document from app.export.markdown import parse_document
from app.services import note_service from app.services import note_service
@@ -61,10 +61,44 @@ def _safe_download_name(title: str) -> str:
return name[:80] return name[:80]
def _export_path(job_id: str) -> Path:
return get_settings().exports_path / f"{job_id}.html"
def _delete_file(job_id: str) -> None:
"""删除导出产物文件;文件不存在时忽略。"""
try:
_export_path(job_id).unlink(missing_ok=True)
except OSError:
logger.warning("Failed to delete export file: %s", job_id)
def cleanup_orphan_files() -> int:
"""清理 exports 目录下无对应内存任务的孤立产物(服务重启后调用)。"""
exports_dir = get_settings().exports_path
if not exports_dir.is_dir():
return 0
removed = 0
for path in exports_dir.glob("*.html"):
if path.stem not in _jobs:
try:
path.unlink()
removed += 1
except OSError:
logger.warning("Failed to delete orphan export file: %s", path)
return removed
def _render_document(document: Document, options: ExportOptions) -> ExportResult:
"""同步渲染辅助,供 asyncio.to_thread 调用;每次新建实例避免跨线程复用。"""
return HtmlExporter().render(document, options)
def _forget(job_id: str) -> None: def _forget(job_id: str) -> None:
_jobs.pop(job_id, None) _jobs.pop(job_id, None)
_tasks.pop(job_id, None) _tasks.pop(job_id, None)
_cancel_flags.pop(job_id, None) _cancel_flags.pop(job_id, None)
_delete_file(job_id)
def _evict_terminal() -> bool: def _evict_terminal() -> bool:
@@ -166,19 +200,20 @@ async def _execute(
if cancel_event.is_set(): if cancel_event.is_set():
raise ExportCancelled() raise ExportCancelled()
document = parse_document(markdown) # 解析与渲染都是 CPU 密集的同步工作,放入线程执行避免阻塞事件循环,
# 使运行中的取消能在渲染边界生效;写文件前再次检查取消。
document = await asyncio.to_thread(parse_document, markdown)
document.attributes["title"] = title document.attributes["title"] = title
if metadata: if metadata:
document.attributes["metadata"] = metadata document.attributes["metadata"] = metadata
exporter = HtmlExporter() result = await asyncio.to_thread(_render_document, document, options)
result = await exporter.export(document, options)
if cancel_event.is_set(): if cancel_event.is_set():
raise ExportCancelled() raise ExportCancelled()
out_dir = get_settings().exports_path out_dir = get_settings().exports_path
out_dir.mkdir(parents=True, exist_ok=True) out_dir.mkdir(parents=True, exist_ok=True)
path = out_dir / f"{job_id}.html" path = _export_path(job_id)
path.write_bytes(result.content) path.write_bytes(result.content)
completed_at = _now() completed_at = _now()
@@ -260,8 +295,9 @@ def get_export_file(job_id: str) -> Path:
404, "EXPORT_JOB_NOT_FOUND", "export file not ready", {"job_id": job_id} 404, "EXPORT_JOB_NOT_FOUND", "export file not ready", {"job_id": job_id}
) )
if job.file.expires_at <= _now(): if job.file.expires_at <= _now():
_forget(job_id) # 过期即清理内存记录与产物文件
raise ApiError(410, "EXPORT_FILE_EXPIRED", "export file has expired", {"job_id": job_id}) raise ApiError(410, "EXPORT_FILE_EXPIRED", "export file has expired", {"job_id": job_id})
return get_settings().exports_path / f"{job_id}.html" return _export_path(job_id)
async def wait_for_export(job_id: str) -> ExportJob | None: async def wait_for_export(job_id: str) -> ExportJob | None:
+3
View File
@@ -8,6 +8,7 @@ from starlette.exceptions import HTTPException as StarletteHttpException
from app.config import get_settings from app.config import get_settings
from app.container import container from app.container import container
from app.errors import ApiError, api_error_handler, http_error_handler, validation_error_handler from app.errors import ApiError, api_error_handler, http_error_handler, validation_error_handler
from app.export import service as export_service
from app.routes import router as api_router from app.routes import router as api_router
from app.schemas import HealthResponse, ServiceStatusResponse from app.schemas import HealthResponse, ServiceStatusResponse
@@ -16,6 +17,8 @@ settings = get_settings()
@asynccontextmanager @asynccontextmanager
async def lifespan(_: FastAPI): async def lifespan(_: FastAPI):
# 重启后内存注册表为空,清理上一次运行遗留的导出产物,避免磁盘垃圾堆积。
export_service.cleanup_orphan_files()
yield yield
# 第三方 MCP Server 必须跟随 AI Core 退出,不能遗留孤儿进程。 # 第三方 MCP Server 必须跟随 AI Core 退出,不能遗留孤儿进程。
container.plugins.shutdown() container.plugins.shutdown()
+111
View File
@@ -16,6 +16,7 @@ from pydantic import ValidationError
from app.config import get_settings from app.config import get_settings
from app.contracts import ( from app.contracts import (
ExportFormat, ExportFormat,
ExportJob,
ExportOptions, ExportOptions,
ExportRequest, ExportRequest,
ExportSource, ExportSource,
@@ -133,6 +134,21 @@ def test_parse_document_table_and_math() -> None:
assert "math_block" in kinds assert "math_block" in kinds
def test_parse_document_image_maps_src_alt_title() -> None:
doc = parse_document('![替代文本](https://a.b/img.png "标题")')
img = doc.children[0].children[0]
assert img.type == "image"
assert img.attributes["src"] == "https://a.b/img.png"
assert img.attributes["alt"] == "替代文本"
assert img.attributes["title"] == "标题"
def test_parse_document_function_plot_dash_alias() -> None:
doc = parse_document("```function-plot\ny = x^2\n```")
assert doc.children[0].type == "function_plot"
assert doc.children[0].text == "y = x^2"
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
# HtmlExporter # HtmlExporter
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
@@ -162,6 +178,38 @@ def test_html_exporter_marks_mermaid_and_function_plot() -> None:
assert any("mermaid" in w for w in result.warnings) assert any("mermaid" in w for w in result.warnings)
def test_html_exporter_rejects_unsafe_link_protocol() -> None:
result = asyncio.run(
HtmlExporter().export(parse_document("[点我](javascript:alert(1))"), ExportOptions())
)
html = result.content.decode("utf-8")
assert "javascript:" not in html
assert "点我" in html
assert any("不安全" in w for w in result.warnings)
def test_html_exporter_rejects_unsafe_image_protocol() -> None:
result = asyncio.run(
HtmlExporter().export(parse_document("![alt](data:text/html,<script>)"), ExportOptions())
)
html = result.content.decode("utf-8")
assert "data:" not in html
assert "<img" not in html
assert "alt" in html
assert any("不安全" in w for w in result.warnings)
def test_html_exporter_preserves_raw_html_block() -> None:
result = asyncio.run(
HtmlExporter().export(parse_document("<div>重要正文</div>"), ExportOptions())
)
html = result.content.decode("utf-8")
assert "重要正文" in html
assert "<div>" not in html
assert "&lt;div&gt;重要正文&lt;/div&gt;" in html
assert any("原始 HTML" in w for w in result.warnings)
def test_html_exporter_include_title_and_metadata() -> None: def test_html_exporter_include_title_and_metadata() -> None:
doc = parse_document("正文") doc = parse_document("正文")
doc.attributes["title"] = "操作系统复习" doc.attributes["title"] = "操作系统复习"
@@ -274,10 +322,73 @@ def test_export_file_expired_410() -> None:
return job.job_id return job.job_id
job_id = asyncio.run(_go()) job_id = asyncio.run(_go())
path = get_settings().exports_path / f"{job_id}.html"
with pytest.raises(ApiError) as exc: with pytest.raises(ApiError) as exc:
export_service.get_export_file(job_id) export_service.get_export_file(job_id)
assert exc.value.status_code == 410 assert exc.value.status_code == 410
assert exc.value.code == "EXPORT_FILE_EXPIRED" assert exc.value.code == "EXPORT_FILE_EXPIRED"
assert not path.exists() # 过期即清理产物文件
assert export_service.get_export(job_id) is None # 内存记录一并清理
def test_export_eviction_deletes_file() -> None:
finished = _create_and_wait(_markdown_request("# 淘汰"))
victim_path = get_settings().exports_path / f"{finished.job_id}.html"
assert victim_path.exists()
# 塞满 MAX_JOBS 个终态任务,下一次 create 会淘汰最旧的终态(finished 最先插入)
for i in range(export_service.MAX_JOBS):
export_service._jobs[f"export_fake_{i}"] = ExportJob(
job_id=f"export_fake_{i}",
status=ExportStatus.completed,
format=ExportFormat.html,
created_at=datetime.now(timezone.utc),
)
_create_and_wait(_markdown_request("# 触发淘汰"))
assert not victim_path.exists()
def test_cleanup_orphan_files() -> None:
exports_dir = get_settings().exports_path
exports_dir.mkdir(parents=True, exist_ok=True)
orphan = exports_dir / "export_orphan.html"
orphan.write_text("stale", encoding="utf-8")
finished = _create_and_wait(_markdown_request("# 保留"))
keep_path = exports_dir / f"{finished.job_id}.html"
assert keep_path.exists()
removed = export_service.cleanup_orphan_files()
assert removed >= 1
assert not orphan.exists()
assert keep_path.exists() # 仍在注册表中的任务文件保留
def test_export_cancel_during_running(monkeypatch) -> None:
import threading
import time
real_parse = parse_document
started = threading.Event()
def slow_parse(markdown: str):
started.set()
time.sleep(0.1)
return real_parse(markdown)
monkeypatch.setattr(export_service, "parse_document", slow_parse)
async def _go():
job = await export_service.create_export(_markdown_request("# 运行中取消"))
while not started.is_set():
await asyncio.sleep(0)
export_service.cancel_export(job.job_id)
return await export_service.wait_for_export(job.job_id)
finished = asyncio.run(_go())
assert finished.status == ExportStatus.cancelled
assert finished.file is None
assert not (get_settings().exports_path / f"{finished.job_id}.html").exists()
def test_export_list_and_get() -> None: def test_export_list_and_get() -> None: