107 lines
2.9 KiB
JSON
107 lines
2.9 KiB
JSON
{
|
|
"run_id": "benchmark_f98741e7649a",
|
|
"kind": "agent",
|
|
"dataset_id": "agent-integration-v1",
|
|
"dataset_hash": "sha256:af10b953037a35cad6fae678cbc118b519b209e88906f80721080560d49a508c",
|
|
"status": "completed",
|
|
"config_snapshot": {
|
|
"dataset_id": "agent-integration-v1",
|
|
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
|
"model": "deepseek-v4-flash",
|
|
"max_steps": 10,
|
|
"timeout_seconds": 150,
|
|
"token_budget": 10000,
|
|
"repeat": 1,
|
|
"allow_network": true,
|
|
"offline": false,
|
|
"dataset_hash": "sha256:af10b953037a35cad6fae678cbc118b519b209e88906f80721080560d49a508c",
|
|
"dataset_version": "1.0.0",
|
|
"execution": "real_agent_runtime",
|
|
"provider_type": "openai_compatible",
|
|
"scoring_version": "1.0",
|
|
"permission_policy": "runtime_user_decision",
|
|
"active_agent_run_id": "run_d55e28a22cb0433498ae59d0eabce566"
|
|
},
|
|
"metrics": {
|
|
"total_cases": 2,
|
|
"task_success_rate": 1.0,
|
|
"tool_selection_accuracy": 1.0,
|
|
"tool_argument_accuracy": 1.0,
|
|
"invalid_tool_call_rate": 0.0,
|
|
"average_steps": 2.5,
|
|
"average_latency_ms": 10539.258899982087,
|
|
"token_usage": 16070,
|
|
"tool_calls": 6,
|
|
"expected_calls": 6
|
|
},
|
|
"cases": [
|
|
{
|
|
"case_id": "retrieval-tasks",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_dc5ad9f709c64b59b792bebafad77653",
|
|
"success": true,
|
|
"tool_calls": 5,
|
|
"expected_calls": 5,
|
|
"selected_calls": 5,
|
|
"accurate_calls": 5,
|
|
"invalid_calls": 0,
|
|
"steps": 3,
|
|
"latency_ms": 13835.580999962986,
|
|
"token_usage": 8651,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "mcp-search",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_d55e28a22cb0433498ae59d0eabce566",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 7242.936800001189,
|
|
"token_usage": 7419,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
}
|
|
],
|
|
"error": null,
|
|
"error_code": null,
|
|
"permission_approvals": 3,
|
|
"mcp_present": true,
|
|
"tasks": [
|
|
{
|
|
"task_id": "task_7380b0dd0b2c4161a20f3b6c1a19dffa",
|
|
"title": "验收-锁顺序"
|
|
},
|
|
{
|
|
"task_id": "task_5e0584d287c442b597ca4c22f8998214",
|
|
"title": "验收-循环等待"
|
|
},
|
|
{
|
|
"task_id": "task_22ac49c4e4214c98951b387a6f128a94",
|
|
"title": "验收-互斥条件"
|
|
}
|
|
]
|
|
} |