fix: address phase two review and theme benchmark page
This commit is contained in:
@@ -11,15 +11,27 @@ INVALID = {'TOOL_NOT_FOUND', 'TOOL_NOT_ALLOWED', 'TOOL_ARGUMENT_INVALID', 'TOOL_
|
||||
|
||||
def score(case, run, events, latency, repeat):
|
||||
calls = [e.data for e in events if e.event.value == 'ToolCall']
|
||||
unmatched = list(calls)
|
||||
selected = accurate = 0
|
||||
for expected in case.expected_tools:
|
||||
candidates = [c for c in unmatched if c.get('name') == expected.name]
|
||||
if not candidates:
|
||||
continue
|
||||
exact = next((c for c in candidates if all(k in c.get('arguments', {}) and c['arguments'][k] == v for k,v in expected.arguments.items())), None)
|
||||
chosen = exact or candidates[0]
|
||||
unmatched.remove(chosen); selected += 1; accurate += int(exact is not None)
|
||||
# Maximum bipartite matching: broad parameter subsets must not consume the
|
||||
# only call satisfying a more specific expectation. Each call is used once.
|
||||
matched = {}
|
||||
def assign(expected_index, visited):
|
||||
expected = case.expected_tools[expected_index]
|
||||
for call_index, call in enumerate(calls):
|
||||
if call_index in visited or call.get('name') != expected.name:
|
||||
continue
|
||||
arguments = call.get('arguments', {})
|
||||
if not all(key in arguments and arguments[key] == value for key, value in expected.arguments.items()):
|
||||
continue
|
||||
visited.add(call_index)
|
||||
if call_index not in matched or assign(matched[call_index], visited):
|
||||
matched[call_index] = expected_index
|
||||
return True
|
||||
return False
|
||||
accurate = sum(assign(index, set()) for index in range(len(case.expected_tools)))
|
||||
from collections import Counter
|
||||
actual_names = Counter(call.get('name') for call in calls)
|
||||
expected_names = Counter(tool.name for tool in case.expected_tools)
|
||||
selected = sum(min(count, actual_names[name]) for name, count in expected_names.items())
|
||||
results = run.tool_results
|
||||
checks = {
|
||||
'completed': run.status.value == 'completed',
|
||||
|
||||
@@ -115,7 +115,13 @@ class DocxExporter:
|
||||
from PIL import Image
|
||||
png = node.attributes['static_png']
|
||||
with Image.open(BytesIO(png)) as image:
|
||||
width = min(5.8, image.width / (180 if node.type == 'math_block' else 96))
|
||||
section = self._doc.sections[-1]
|
||||
available_width = (section.page_width - section.left_margin - section.right_margin) / 914400
|
||||
# Leave room for Word's containing paragraph line/spacing.
|
||||
available_height = (section.page_height - section.top_margin - section.bottom_margin) / 914400 - 0.25
|
||||
width = min(5.8, available_width,
|
||||
image.width / (180 if node.type == 'math_block' else 96),
|
||||
available_height * image.width / image.height)
|
||||
self._doc.add_picture(BytesIO(png), width=Inches(width))
|
||||
return
|
||||
handler = getattr(self, f"_block_{node.type}", None)
|
||||
|
||||
@@ -164,3 +164,36 @@ def test_repeated_static_assets_share_document_resource_budget():
|
||||
warnings = enrich_document(document)
|
||||
assert sum(bool(node.attributes.get('static_png')) for node in document.children) == 64
|
||||
assert any('预算' in warning for warning in warnings)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('order', [(1, 2), (2, 1)])
|
||||
def test_agent_parameter_matching_is_independent_of_call_order(order):
|
||||
from types import SimpleNamespace as NS
|
||||
from app.benchmarks.agent import score
|
||||
from app.contracts import AgentDatasetCase
|
||||
case = AgentDatasetCase(case_id='overlap', prompt='test', allowed_tools=['math.add'],
|
||||
expected_tools=[{'name':'math.add','arguments':{}}, {'name':'math.add','arguments':{'left':1}}])
|
||||
events = [NS(event=NS(value='ToolCall'), data={'name':'math.add','arguments':{'left':value}}) for value in order]
|
||||
run = NS(status=NS(value='completed'),tool_results=[],output='',citations=[],run_id='test',current_step=1,token_usage=0,error_code=None)
|
||||
result = score(case, run, events, 1, 0)
|
||||
assert result.success and result.accurate_calls == result.selected_calls == 2
|
||||
# Two expectations cannot reuse one matching call.
|
||||
result = score(case, run, events[:1], 1, 0)
|
||||
assert not result.success and result.accurate_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize('page_size', ['A4', 'Letter'])
|
||||
@pytest.mark.parametrize('dimensions', [(200, 2000), (2000, 200)])
|
||||
def test_docx_static_images_fit_both_page_dimensions(page_size, dimensions):
|
||||
from app.export.markdown import parse_document
|
||||
from app.export.exporters.docx import DocxExporter
|
||||
from app.contracts import ExportOptions
|
||||
from docx import Document
|
||||
png = BytesIO(); Image.new('RGB', dimensions, 'white').save(png, 'PNG')
|
||||
document = parse_document('```mermaid\nflowchart TD\n A-->B\n```')
|
||||
document.children[0].attributes['static_png'] = png.getvalue()
|
||||
result = DocxExporter().render(document, ExportOptions(page_size=page_size))
|
||||
word = Document(BytesIO(result.content)); section = word.sections[0]; shape = word.inline_shapes[0]
|
||||
assert shape.width <= section.page_width - section.left_margin - section.right_margin
|
||||
assert shape.height < section.page_height - section.top_margin - section.bottom_margin
|
||||
assert shape.width / shape.height == pytest.approx(dimensions[0] / dimensions[1], rel=1e-5)
|
||||
|
||||
Reference in New Issue
Block a user