feat(export): 多格式后台导出、主题与警告框渲染 #43
@@ -24,6 +24,9 @@ _ALLOWED_URL_SCHEMES = frozenset({"http", "https", "mailto"})
|
|||||||
|
|
||||||
# 单篇文档允许的函数图像数量上限,超出部分回退占位,防止多图块并发采样耗尽内存/线程
|
# 单篇文档允许的函数图像数量上限,超出部分回退占位,防止多图块并发采样耗尽内存/线程
|
||||||
_MAX_FUNCTION_PLOTS = 16
|
_MAX_FUNCTION_PLOTS = 16
|
||||||
|
# 单篇文档允许的函数图像累计 AST 节点预算,超出部分回退占位,防止组合复杂度(多图块
|
||||||
|
# × 多表达式 × 深表达式)在采样求值时长时间占满 CPU
|
||||||
|
_MAX_TOTAL_PLOT_NODES = 8000
|
||||||
|
|
||||||
|
|
||||||
def _safe_url(url: str) -> str | None:
|
def _safe_url(url: str) -> str | None:
|
||||||
@@ -73,6 +76,7 @@ class HtmlExporter:
|
|||||||
"""同步渲染;CPU 密集,调用方应放入线程执行,避免阻塞事件循环。"""
|
"""同步渲染;CPU 密集,调用方应放入线程执行,避免阻塞事件循环。"""
|
||||||
self._options = options
|
self._options = options
|
||||||
self._plot_count = 0
|
self._plot_count = 0
|
||||||
|
self._plot_nodes = 0
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
body = self._render_children(document.children, warnings)
|
body = self._render_children(document.children, warnings)
|
||||||
content = self._assemble(document, options, body, warnings)
|
content = self._assemble(document, options, body, warnings)
|
||||||
@@ -222,6 +226,13 @@ class HtmlExporter:
|
|||||||
warnings.append(self._format_plot_diagnostic(diag))
|
warnings.append(self._format_plot_diagnostic(diag))
|
||||||
if parsed.plot is None:
|
if parsed.plot is None:
|
||||||
return f'<pre class="function-plot">{html.escape(node.text)}</pre>'
|
return f'<pre class="function-plot">{html.escape(node.text)}</pre>'
|
||||||
|
# 文档级累计复杂度预算:超出后回退占位,不再采样求值
|
||||||
|
if self._plot_nodes + parsed.plot.node_count > _MAX_TOTAL_PLOT_NODES:
|
||||||
|
warnings.append(
|
||||||
|
f"函数图像:文档内函数图像累计复杂度超过上限 {_MAX_TOTAL_PLOT_NODES} 节点,已回退为源码占位"
|
||||||
|
)
|
||||||
|
return f'<pre class="function-plot">{html.escape(node.text)}</pre>'
|
||||||
|
self._plot_nodes += parsed.plot.node_count
|
||||||
rendered = render_svg(parsed.plot)
|
rendered = render_svg(parsed.plot)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
warnings.append(f"函数图像:解析或渲染失败,已回退占位({exc})")
|
warnings.append(f"函数图像:解析或渲染失败,已回退占位({exc})")
|
||||||
|
|||||||
@@ -43,6 +43,10 @@ MAX_JOBS = 100
|
|||||||
MAX_MARKDOWN_CHARS = 200_000
|
MAX_MARKDOWN_CHARS = 200_000
|
||||||
# 最终导出产物大小上限,防止超大 HTML 耗尽内存/磁盘
|
# 最终导出产物大小上限,防止超大 HTML 耗尽内存/磁盘
|
||||||
MAX_EXPORT_BYTES = 20 * 1024 * 1024 # 20 MB
|
MAX_EXPORT_BYTES = 20 * 1024 * 1024 # 20 MB
|
||||||
|
# 并发渲染上限:解析/渲染是 CPU 密集的同步工作,限制同时执行的任务数,
|
||||||
|
# 防止大量任务同时占满工作线程与内存
|
||||||
|
MAX_CONCURRENT_RENDERS = 2
|
||||||
|
_render_slots = asyncio.Semaphore(MAX_CONCURRENT_RENDERS)
|
||||||
# 产物有效期
|
# 产物有效期
|
||||||
FILE_TTL = timedelta(hours=24)
|
FILE_TTL = timedelta(hours=24)
|
||||||
|
|
||||||
@@ -208,6 +212,9 @@ async def _execute(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
|
# 并发渲染限额:解析/渲染是 CPU 密集的同步工作,用信号量限制同时执行的任务数,
|
||||||
|
# 超出限额的任务在此排队等待,避免大量任务同时占满工作线程与内存
|
||||||
|
async with _render_slots:
|
||||||
# 让出一次,使「创建后立即取消」的 queued 任务能及时进入 cancelled
|
# 让出一次,使「创建后立即取消」的 queued 任务能及时进入 cancelled
|
||||||
await asyncio.sleep(0)
|
await asyncio.sleep(0)
|
||||||
if cancel_event.is_set():
|
if cancel_event.is_set():
|
||||||
|
|||||||
@@ -31,6 +31,8 @@ class FunctionPlot(BaseModel):
|
|||||||
domain: tuple[float, float] = (-10.0, 10.0)
|
domain: tuple[float, float] = (-10.0, 10.0)
|
||||||
range: tuple[float, float] | None = None
|
range: tuple[float, float] | None = None
|
||||||
axes: PlotAxes = Field(default_factory=PlotAxes)
|
axes: PlotAxes = Field(default_factory=PlotAxes)
|
||||||
|
# 该块所有表达式 AST 节点数之和,供导出器做文档级累计复杂度预算
|
||||||
|
node_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
class PlotDiagnostic(BaseModel):
|
class PlotDiagnostic(BaseModel):
|
||||||
|
|||||||
@@ -215,6 +215,13 @@ def parse_expression(expr: str) -> ast.Expression:
|
|||||||
return tree
|
return tree
|
||||||
|
|
||||||
|
|
||||||
|
def _count_nodes(node: ast.AST) -> int:
|
||||||
|
"""统计已通过校验的表达式 AST 节点数,供文档级累计复杂度预算使用。"""
|
||||||
|
counter = [0]
|
||||||
|
_check_node(node, counter=counter)
|
||||||
|
return counter[0]
|
||||||
|
|
||||||
|
|
||||||
def evaluate(expr_ast: ast.Expression, x: float) -> float:
|
def evaluate(expr_ast: ast.Expression, x: float) -> float:
|
||||||
"""递归解释已校验 AST 得到数值,全程不编译/执行代码。"""
|
"""递归解释已校验 AST 得到数值,全程不编译/执行代码。"""
|
||||||
return _eval_node(expr_ast.body, x)
|
return _eval_node(expr_ast.body, x)
|
||||||
@@ -282,6 +289,7 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
|||||||
ylabel: str | None = None
|
ylabel: str | None = None
|
||||||
grid: bool = True
|
grid: bool = True
|
||||||
has_error = False
|
has_error = False
|
||||||
|
total_nodes = 0
|
||||||
|
|
||||||
for lineno, raw_line in enumerate(source.splitlines(), start=1):
|
for lineno, raw_line in enumerate(source.splitlines(), start=1):
|
||||||
line = raw_line.strip()
|
line = raw_line.strip()
|
||||||
@@ -363,12 +371,13 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
parse_expression(expr_text)
|
tree = parse_expression(expr_text)
|
||||||
except PlotParseError as exc:
|
except PlotParseError as exc:
|
||||||
exc.diagnostic.line = lineno
|
exc.diagnostic.line = lineno
|
||||||
diagnostics.append(exc.diagnostic)
|
diagnostics.append(exc.diagnostic)
|
||||||
has_error = True
|
has_error = True
|
||||||
continue
|
continue
|
||||||
|
total_nodes += _count_nodes(tree.body)
|
||||||
expressions.append(FunctionPlotExpression(expression=expr_text))
|
expressions.append(FunctionPlotExpression(expression=expr_text))
|
||||||
# 表达式数量超限:整块回退并提前终止,避免对海量表达式做采样求值
|
# 表达式数量超限:整块回退并提前终止,避免对海量表达式做采样求值
|
||||||
if len(expressions) > _MAX_EXPRESSIONS:
|
if len(expressions) > _MAX_EXPRESSIONS:
|
||||||
@@ -398,5 +407,6 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
|||||||
domain=domain,
|
domain=domain,
|
||||||
range=range_,
|
range=range_,
|
||||||
axes=PlotAxes(xlabel=xlabel, ylabel=ylabel, grid=grid),
|
axes=PlotAxes(xlabel=xlabel, ylabel=ylabel, grid=grid),
|
||||||
|
node_count=total_nodes,
|
||||||
)
|
)
|
||||||
return FunctionPlotParseResult(plot=plot, diagnostics=diagnostics)
|
return FunctionPlotParseResult(plot=plot, diagnostics=diagnostics)
|
||||||
|
|||||||
@@ -449,3 +449,37 @@ def test_export_output_too_large(monkeypatch) -> None:
|
|||||||
assert finished.error_code == "EXPORT_OUTPUT_TOO_LARGE"
|
assert finished.error_code == "EXPORT_OUTPUT_TOO_LARGE"
|
||||||
assert finished.file is None
|
assert finished.file is None
|
||||||
assert not (get_settings().exports_path / f"{finished.job_id}.html").exists()
|
assert not (get_settings().exports_path / f"{finished.job_id}.html").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_export_limits_concurrent_rendering(monkeypatch) -> None:
|
||||||
|
# P1:并发渲染受 MAX_CONCURRENT_RENDERS 限制,大量任务不会同时占满工作线程
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
real_render = export_service._render_document
|
||||||
|
active = 0
|
||||||
|
peak = 0
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def slow_render(document, options):
|
||||||
|
nonlocal active, peak
|
||||||
|
with lock:
|
||||||
|
active += 1
|
||||||
|
peak = max(peak, active)
|
||||||
|
time.sleep(0.05)
|
||||||
|
with lock:
|
||||||
|
active -= 1
|
||||||
|
return real_render(document, options)
|
||||||
|
|
||||||
|
monkeypatch.setattr(export_service, "_render_document", slow_render)
|
||||||
|
|
||||||
|
async def _go():
|
||||||
|
jobs = [
|
||||||
|
await export_service.create_export(_markdown_request(f"# t{i}"))
|
||||||
|
for i in range(6)
|
||||||
|
]
|
||||||
|
return [await export_service.wait_for_export(j.job_id) for j in jobs]
|
||||||
|
|
||||||
|
finished = asyncio.run(_go())
|
||||||
|
assert all(j.status == ExportStatus.completed for j in finished)
|
||||||
|
assert peak <= export_service.MAX_CONCURRENT_RENDERS
|
||||||
|
|||||||
@@ -269,3 +269,17 @@ def test_html_exporter_limits_function_plot_count() -> None:
|
|||||||
assert html.count('<figure class="function-plot">') == 16
|
assert html.count('<figure class="function-plot">') == 16
|
||||||
assert html.count('<pre class="function-plot">') == 4
|
assert html.count('<pre class="function-plot">') == 4
|
||||||
assert any("函数图像数量超过上限" in w for w in result.warnings)
|
assert any("函数图像数量超过上限" in w for w in result.warnings)
|
||||||
|
|
||||||
|
|
||||||
|
def test_html_exporter_limits_total_plot_nodes(monkeypatch) -> None:
|
||||||
|
# P1:文档级累计 AST 节点预算超限后,后续图像回退占位,防止组合复杂度耗尽 CPU
|
||||||
|
import app.export.exporters.html as html_mod
|
||||||
|
|
||||||
|
monkeypatch.setattr(html_mod, "_MAX_TOTAL_PLOT_NODES", 5)
|
||||||
|
# 第一个图块 y=x(1 节点)在预算内;第二个图块 y=x+x+x+x(7 节点)累计超限
|
||||||
|
md = "```function-plot\ny = x\n```\n\n```function-plot\ny = x + x + x + x\n```"
|
||||||
|
result = asyncio.run(HtmlExporter().export(parse_document(md), ExportOptions()))
|
||||||
|
html = result.content.decode("utf-8")
|
||||||
|
assert html.count('<figure class="function-plot">') == 1
|
||||||
|
assert html.count('<pre class="function-plot">') == 1
|
||||||
|
assert any("累计复杂度" in w for w in result.warnings)
|
||||||
|
|||||||
@@ -63,20 +63,31 @@ fenced code 按语言分流:`mermaid` → `mermaid` 节点、`function_plot`/`
|
|||||||
- 输入源(`note` 与 `markdown`)统一限制 `MAX_MARKDOWN_CHARS = 200_000` 字符,超限返回 `EXPORT_OPTIONS_INVALID`。
|
- 输入源(`note` 与 `markdown`)统一限制 `MAX_MARKDOWN_CHARS = 200_000` 字符,超限返回 `EXPORT_OPTIONS_INVALID`。
|
||||||
- 单个 `function-plot` 图块最多 16 条表达式,超限整块回退占位并记结构化诊断 `FUNCTION_PLOT_TOO_MANY_EXPRESSIONS`。
|
- 单个 `function-plot` 图块最多 16 条表达式,超限整块回退占位并记结构化诊断 `FUNCTION_PLOT_TOO_MANY_EXPRESSIONS`。
|
||||||
- 单篇文档最多 16 个函数图像,超出部分回退占位并记 warning。
|
- 单篇文档最多 16 个函数图像,超出部分回退占位并记 warning。
|
||||||
|
- 单篇文档累计函数图像 AST 节点预算 `_MAX_TOTAL_PLOT_NODES = 8000`,超出部分回退占位并记 warning,防止多图块 × 多表达式 × 深表达式组合在采样求值时长时间占满 CPU。
|
||||||
|
- 并发渲染上限 `MAX_CONCURRENT_RENDERS = 2`,解析/渲染是 CPU 密集工作,超出限额的任务在内存中排队等待渲染槽位,避免大量任务同时占满工作线程与内存。
|
||||||
- 最终产物大小上限 `MAX_EXPORT_BYTES = 20 MB`,超限任务标记 failed 并返回 `EXPORT_OUTPUT_TOO_LARGE`。
|
- 最终产物大小上限 `MAX_EXPORT_BYTES = 20 MB`,超限任务标记 failed 并返回 `EXPORT_OUTPUT_TOO_LARGE`。
|
||||||
|
|
||||||
## 错误码
|
## 错误码
|
||||||
|
|
||||||
|
错误分两类:**同步错误**在创建/查询请求的 HTTP 响应里直接返回对应状态码;**异步任务错误**在创建时已返回 `202`,后续轮询 `GET /api/exports/{job_id}` 仍返回 `200`,错误通过任务状态与 `error_code` 字段暴露,**不映射 HTTP 状态码**。
|
||||||
|
|
||||||
|
同步错误:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
EXPORT_SOURCE_NOT_FOUND 404
|
EXPORT_SOURCE_NOT_FOUND 404
|
||||||
EXPORT_FORMAT_UNSUPPORTED 400
|
EXPORT_FORMAT_UNSUPPORTED 400
|
||||||
EXPORT_OPTIONS_INVALID 400
|
EXPORT_OPTIONS_INVALID 400
|
||||||
EXPORT_RENDER_FAILED 500
|
EXPORT_UNSUPPORTED_CONTENT 422(预留)
|
||||||
EXPORT_UNSUPPORTED_CONTENT 422
|
|
||||||
EXPORT_JOB_NOT_FOUND 404
|
EXPORT_JOB_NOT_FOUND 404
|
||||||
EXPORT_FILE_EXPIRED 410
|
EXPORT_FILE_EXPIRED 410
|
||||||
EXPORT_CAPACITY_EXCEEDED 429
|
EXPORT_CAPACITY_EXCEEDED 429
|
||||||
EXPORT_OUTPUT_TOO_LARGE 413
|
```
|
||||||
|
|
||||||
|
异步任务错误(轮询返回 `200`,字段形如 `{"status": "failed", "error_code": "..."}`):
|
||||||
|
|
||||||
|
```text
|
||||||
|
EXPORT_RENDER_FAILED
|
||||||
|
EXPORT_OUTPUT_TOO_LARGE
|
||||||
```
|
```
|
||||||
|
|
||||||
## 测试
|
## 测试
|
||||||
|
|||||||
Reference in New Issue
Block a user