fix: address phase two review and theme benchmark page

This commit is contained in:
2026-09-07 13:26:14 +08:00
parent 0e3f2a7325
commit 47c53b6f38
12 changed files with 451 additions and 35 deletions
+21 -9
View File
@@ -11,15 +11,27 @@ INVALID = {'TOOL_NOT_FOUND', 'TOOL_NOT_ALLOWED', 'TOOL_ARGUMENT_INVALID', 'TOOL_
def score(case, run, events, latency, repeat):
calls = [e.data for e in events if e.event.value == 'ToolCall']
unmatched = list(calls)
selected = accurate = 0
for expected in case.expected_tools:
candidates = [c for c in unmatched if c.get('name') == expected.name]
if not candidates:
continue
exact = next((c for c in candidates if all(k in c.get('arguments', {}) and c['arguments'][k] == v for k,v in expected.arguments.items())), None)
chosen = exact or candidates[0]
unmatched.remove(chosen); selected += 1; accurate += int(exact is not None)
# Maximum bipartite matching: broad parameter subsets must not consume the
# only call satisfying a more specific expectation. Each call is used once.
matched = {}
def assign(expected_index, visited):
expected = case.expected_tools[expected_index]
for call_index, call in enumerate(calls):
if call_index in visited or call.get('name') != expected.name:
continue
arguments = call.get('arguments', {})
if not all(key in arguments and arguments[key] == value for key, value in expected.arguments.items()):
continue
visited.add(call_index)
if call_index not in matched or assign(matched[call_index], visited):
matched[call_index] = expected_index
return True
return False
accurate = sum(assign(index, set()) for index in range(len(case.expected_tools)))
from collections import Counter
actual_names = Counter(call.get('name') for call in calls)
expected_names = Counter(tool.name for tool in case.expected_tools)
selected = sum(min(count, actual_names[name]) for name, count in expected_names.items())
results = run.tool_results
checks = {
'completed': run.status.value == 'completed',
+7 -1
View File
@@ -115,7 +115,13 @@ class DocxExporter:
from PIL import Image
png = node.attributes['static_png']
with Image.open(BytesIO(png)) as image:
width = min(5.8, image.width / (180 if node.type == 'math_block' else 96))
section = self._doc.sections[-1]
available_width = (section.page_width - section.left_margin - section.right_margin) / 914400
# Leave room for Word's containing paragraph line/spacing.
available_height = (section.page_height - section.top_margin - section.bottom_margin) / 914400 - 0.25
width = min(5.8, available_width,
image.width / (180 if node.type == 'math_block' else 96),
available_height * image.width / image.height)
self._doc.add_picture(BytesIO(png), width=Inches(width))
return
handler = getattr(self, f"_block_{node.type}", None)
+33
View File
@@ -164,3 +164,36 @@ def test_repeated_static_assets_share_document_resource_budget():
warnings = enrich_document(document)
assert sum(bool(node.attributes.get('static_png')) for node in document.children) == 64
assert any('预算' in warning for warning in warnings)
@pytest.mark.parametrize('order', [(1, 2), (2, 1)])
def test_agent_parameter_matching_is_independent_of_call_order(order):
from types import SimpleNamespace as NS
from app.benchmarks.agent import score
from app.contracts import AgentDatasetCase
case = AgentDatasetCase(case_id='overlap', prompt='test', allowed_tools=['math.add'],
expected_tools=[{'name':'math.add','arguments':{}}, {'name':'math.add','arguments':{'left':1}}])
events = [NS(event=NS(value='ToolCall'), data={'name':'math.add','arguments':{'left':value}}) for value in order]
run = NS(status=NS(value='completed'),tool_results=[],output='',citations=[],run_id='test',current_step=1,token_usage=0,error_code=None)
result = score(case, run, events, 1, 0)
assert result.success and result.accurate_calls == result.selected_calls == 2
# Two expectations cannot reuse one matching call.
result = score(case, run, events[:1], 1, 0)
assert not result.success and result.accurate_calls == 1
@pytest.mark.parametrize('page_size', ['A4', 'Letter'])
@pytest.mark.parametrize('dimensions', [(200, 2000), (2000, 200)])
def test_docx_static_images_fit_both_page_dimensions(page_size, dimensions):
from app.export.markdown import parse_document
from app.export.exporters.docx import DocxExporter
from app.contracts import ExportOptions
from docx import Document
png = BytesIO(); Image.new('RGB', dimensions, 'white').save(png, 'PNG')
document = parse_document('```mermaid\nflowchart TD\n A-->B\n```')
document.children[0].attributes['static_png'] = png.getvalue()
result = DocxExporter().render(document, ExportOptions(page_size=page_size))
word = Document(BytesIO(result.content)); section = word.sections[0]; shape = word.inline_shapes[0]
assert shape.width <= section.page_width - section.left_margin - section.right_margin
assert shape.height < section.page_height - section.top_margin - section.bottom_margin
assert shape.width / shape.height == pytest.approx(dimensions[0] / dimensions[1], rel=1e-5)