fix(export): 增加文档级组合复杂度预算与并发渲染限制
针对 PR 审阅 P1「组合复杂度仍可长时间占满导出线程」与 P3「EXPORT_OUTPUT_TOO_LARGE 误标 HTTP 413」： - plot: FunctionPlot 记录整块 AST 节点数（node_count），parser 累计 - html: 单篇文档累计节点预算 _MAX_TOTAL_PLOT_NODES=8000，超限回退占位 - service: 并发渲染信号量 MAX_CONCURRENT_RENDERS=2，超限额任务排队等待 - docs: 错误码区分同步 HTTP 错误与异步任务错误，EXPORT_OUTPUT_TOO_LARGE 由 error_code 返回而非 HTTP 413 - 补充节点预算与并发限制两条回归测试（全量 627 通过） Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -24,6 +24,9 @@ _ALLOWED_URL_SCHEMES = frozenset({"http", "https", "mailto"})
|
||||
|
||||
# 单篇文档允许的函数图像数量上限，超出部分回退占位，防止多图块并发采样耗尽内存/线程
|
||||
_MAX_FUNCTION_PLOTS = 16
|
||||
# 单篇文档允许的函数图像累计 AST 节点预算，超出部分回退占位，防止组合复杂度（多图块
|
||||
# × 多表达式 × 深表达式）在采样求值时长时间占满 CPU
|
||||
_MAX_TOTAL_PLOT_NODES = 8000
|
||||
|
||||
|
||||
def _safe_url(url: str) -> str | None:
|
||||
@@ -73,6 +76,7 @@ class HtmlExporter:
|
||||
"""同步渲染；CPU 密集，调用方应放入线程执行，避免阻塞事件循环。"""
|
||||
self._options = options
|
||||
self._plot_count = 0
|
||||
self._plot_nodes = 0
|
||||
warnings: list[str] = []
|
||||
body = self._render_children(document.children, warnings)
|
||||
content = self._assemble(document, options, body, warnings)
|
||||
@@ -222,6 +226,13 @@ class HtmlExporter:
|
||||
warnings.append(self._format_plot_diagnostic(diag))
|
||||
if parsed.plot is None:
|
||||
return f'<pre class="function-plot">{html.escape(node.text)}</pre>'
|
||||
# 文档级累计复杂度预算：超出后回退占位，不再采样求值
|
||||
if self._plot_nodes + parsed.plot.node_count > _MAX_TOTAL_PLOT_NODES:
|
||||
warnings.append(
|
||||
f"函数图像：文档内函数图像累计复杂度超过上限 {_MAX_TOTAL_PLOT_NODES} 节点，已回退为源码占位"
|
||||
)
|
||||
return f'<pre class="function-plot">{html.escape(node.text)}</pre>'
|
||||
self._plot_nodes += parsed.plot.node_count
|
||||
rendered = render_svg(parsed.plot)
|
||||
except Exception as exc:
|
||||
warnings.append(f"函数图像：解析或渲染失败，已回退占位（{exc}）")
|
||||
|
||||
@@ -43,6 +43,10 @@ MAX_JOBS = 100
|
||||
MAX_MARKDOWN_CHARS = 200_000
|
||||
# 最终导出产物大小上限，防止超大 HTML 耗尽内存/磁盘
|
||||
MAX_EXPORT_BYTES = 20 * 1024 * 1024 # 20 MB
|
||||
# 并发渲染上限：解析/渲染是 CPU 密集的同步工作，限制同时执行的任务数，
|
||||
# 防止大量任务同时占满工作线程与内存
|
||||
MAX_CONCURRENT_RENDERS = 2
|
||||
_render_slots = asyncio.Semaphore(MAX_CONCURRENT_RENDERS)
|
||||
# 产物有效期
|
||||
FILE_TTL = timedelta(hours=24)
|
||||
|
||||
@@ -208,47 +212,50 @@ async def _execute(
|
||||
}
|
||||
)
|
||||
try:
|
||||
# 让出一次，使「创建后立即取消」的 queued 任务能及时进入 cancelled
|
||||
await asyncio.sleep(0)
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
# 并发渲染限额：解析/渲染是 CPU 密集的同步工作，用信号量限制同时执行的任务数，
|
||||
# 超出限额的任务在此排队等待，避免大量任务同时占满工作线程与内存
|
||||
async with _render_slots:
|
||||
# 让出一次，使「创建后立即取消」的 queued 任务能及时进入 cancelled
|
||||
await asyncio.sleep(0)
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
|
||||
# 解析与渲染都是 CPU 密集的同步工作，放入线程执行避免阻塞事件循环，
|
||||
# 使运行中的取消能在渲染边界生效；写文件前再次检查取消。
|
||||
document = await asyncio.to_thread(parse_document, markdown)
|
||||
document.attributes["title"] = title
|
||||
if metadata:
|
||||
document.attributes["metadata"] = metadata
|
||||
# 解析与渲染都是 CPU 密集的同步工作，放入线程执行避免阻塞事件循环，
|
||||
# 使运行中的取消能在渲染边界生效；写文件前再次检查取消。
|
||||
document = await asyncio.to_thread(parse_document, markdown)
|
||||
document.attributes["title"] = title
|
||||
if metadata:
|
||||
document.attributes["metadata"] = metadata
|
||||
|
||||
result = await asyncio.to_thread(_render_document, document, options)
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
if len(result.content) > MAX_EXPORT_BYTES:
|
||||
raise ExportTooLarge()
|
||||
result = await asyncio.to_thread(_render_document, document, options)
|
||||
if cancel_event.is_set():
|
||||
raise ExportCancelled()
|
||||
if len(result.content) > MAX_EXPORT_BYTES:
|
||||
raise ExportTooLarge()
|
||||
|
||||
out_dir = get_settings().exports_path
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = _export_path(job_id)
|
||||
path.write_bytes(result.content)
|
||||
out_dir = get_settings().exports_path
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = _export_path(job_id)
|
||||
path.write_bytes(result.content)
|
||||
|
||||
completed_at = _now()
|
||||
_jobs[job_id] = _jobs[job_id].model_copy(
|
||||
update={
|
||||
"status": ExportStatus.completed,
|
||||
"progress": ExportProgress(
|
||||
phase="completed", current=1, total=1, percent=1.0
|
||||
),
|
||||
"file": ExportFile(
|
||||
file_name=f"{_safe_download_name(title)}.html",
|
||||
mime_type=result.mime_type,
|
||||
size=len(result.content),
|
||||
sha256=hashlib.sha256(result.content).hexdigest(),
|
||||
expires_at=completed_at + FILE_TTL,
|
||||
),
|
||||
"warnings": result.warnings,
|
||||
"completed_at": completed_at,
|
||||
}
|
||||
)
|
||||
completed_at = _now()
|
||||
_jobs[job_id] = _jobs[job_id].model_copy(
|
||||
update={
|
||||
"status": ExportStatus.completed,
|
||||
"progress": ExportProgress(
|
||||
phase="completed", current=1, total=1, percent=1.0
|
||||
),
|
||||
"file": ExportFile(
|
||||
file_name=f"{_safe_download_name(title)}.html",
|
||||
mime_type=result.mime_type,
|
||||
size=len(result.content),
|
||||
sha256=hashlib.sha256(result.content).hexdigest(),
|
||||
expires_at=completed_at + FILE_TTL,
|
||||
),
|
||||
"warnings": result.warnings,
|
||||
"completed_at": completed_at,
|
||||
}
|
||||
)
|
||||
except ExportCancelled:
|
||||
_jobs[job_id] = _jobs[job_id].model_copy(
|
||||
update={
|
||||
|
||||
@@ -31,6 +31,8 @@ class FunctionPlot(BaseModel):
|
||||
domain: tuple[float, float] = (-10.0, 10.0)
|
||||
range: tuple[float, float] | None = None
|
||||
axes: PlotAxes = Field(default_factory=PlotAxes)
|
||||
# 该块所有表达式 AST 节点数之和，供导出器做文档级累计复杂度预算
|
||||
node_count: int = 0
|
||||
|
||||
|
||||
class PlotDiagnostic(BaseModel):
|
||||
|
||||
@@ -215,6 +215,13 @@ def parse_expression(expr: str) -> ast.Expression:
|
||||
return tree
|
||||
|
||||
|
||||
def _count_nodes(node: ast.AST) -> int:
|
||||
"""统计已通过校验的表达式 AST 节点数，供文档级累计复杂度预算使用。"""
|
||||
counter = [0]
|
||||
_check_node(node, counter=counter)
|
||||
return counter[0]
|
||||
|
||||
|
||||
def evaluate(expr_ast: ast.Expression, x: float) -> float:
|
||||
"""递归解释已校验 AST 得到数值，全程不编译/执行代码。"""
|
||||
return _eval_node(expr_ast.body, x)
|
||||
@@ -282,6 +289,7 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
||||
ylabel: str | None = None
|
||||
grid: bool = True
|
||||
has_error = False
|
||||
total_nodes = 0
|
||||
|
||||
for lineno, raw_line in enumerate(source.splitlines(), start=1):
|
||||
line = raw_line.strip()
|
||||
@@ -363,12 +371,13 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
||||
continue
|
||||
|
||||
try:
|
||||
parse_expression(expr_text)
|
||||
tree = parse_expression(expr_text)
|
||||
except PlotParseError as exc:
|
||||
exc.diagnostic.line = lineno
|
||||
diagnostics.append(exc.diagnostic)
|
||||
has_error = True
|
||||
continue
|
||||
total_nodes += _count_nodes(tree.body)
|
||||
expressions.append(FunctionPlotExpression(expression=expr_text))
|
||||
# 表达式数量超限：整块回退并提前终止，避免对海量表达式做采样求值
|
||||
if len(expressions) > _MAX_EXPRESSIONS:
|
||||
@@ -398,5 +407,6 @@ def parse_source(source: str) -> FunctionPlotParseResult:
|
||||
domain=domain,
|
||||
range=range_,
|
||||
axes=PlotAxes(xlabel=xlabel, ylabel=ylabel, grid=grid),
|
||||
node_count=total_nodes,
|
||||
)
|
||||
return FunctionPlotParseResult(plot=plot, diagnostics=diagnostics)
|
||||
|
||||
@@ -449,3 +449,37 @@ def test_export_output_too_large(monkeypatch) -> None:
|
||||
assert finished.error_code == "EXPORT_OUTPUT_TOO_LARGE"
|
||||
assert finished.file is None
|
||||
assert not (get_settings().exports_path / f"{finished.job_id}.html").exists()
|
||||
|
||||
|
||||
def test_export_limits_concurrent_rendering(monkeypatch) -> None:
|
||||
# P1：并发渲染受 MAX_CONCURRENT_RENDERS 限制，大量任务不会同时占满工作线程
|
||||
import threading
|
||||
import time
|
||||
|
||||
real_render = export_service._render_document
|
||||
active = 0
|
||||
peak = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
def slow_render(document, options):
|
||||
nonlocal active, peak
|
||||
with lock:
|
||||
active += 1
|
||||
peak = max(peak, active)
|
||||
time.sleep(0.05)
|
||||
with lock:
|
||||
active -= 1
|
||||
return real_render(document, options)
|
||||
|
||||
monkeypatch.setattr(export_service, "_render_document", slow_render)
|
||||
|
||||
async def _go():
|
||||
jobs = [
|
||||
await export_service.create_export(_markdown_request(f"# t{i}"))
|
||||
for i in range(6)
|
||||
]
|
||||
return [await export_service.wait_for_export(j.job_id) for j in jobs]
|
||||
|
||||
finished = asyncio.run(_go())
|
||||
assert all(j.status == ExportStatus.completed for j in finished)
|
||||
assert peak <= export_service.MAX_CONCURRENT_RENDERS
|
||||
|
||||
@@ -269,3 +269,17 @@ def test_html_exporter_limits_function_plot_count() -> None:
|
||||
assert html.count('<figure class="function-plot">') == 16
|
||||
assert html.count('<pre class="function-plot">') == 4
|
||||
assert any("函数图像数量超过上限" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_html_exporter_limits_total_plot_nodes(monkeypatch) -> None:
|
||||
# P1：文档级累计 AST 节点预算超限后，后续图像回退占位，防止组合复杂度耗尽 CPU
|
||||
import app.export.exporters.html as html_mod
|
||||
|
||||
monkeypatch.setattr(html_mod, "_MAX_TOTAL_PLOT_NODES", 5)
|
||||
# 第一个图块 y=x（1 节点）在预算内；第二个图块 y=x+x+x+x（7 节点）累计超限
|
||||
md = "```function-plot\ny = x\n```\n\n```function-plot\ny = x + x + x + x\n```"
|
||||
result = asyncio.run(HtmlExporter().export(parse_document(md), ExportOptions()))
|
||||
html = result.content.decode("utf-8")
|
||||
assert html.count('<figure class="function-plot">') == 1
|
||||
assert html.count('<pre class="function-plot">') == 1
|
||||
assert any("累计复杂度" in w for w in result.warnings)
|
||||
|
||||
Reference in New Issue
Block a user