142 lines
3.8 KiB
JSON
142 lines
3.8 KiB
JSON
{
|
|
"run_id": "benchmark_e3047791a92d",
|
|
"kind": "agent",
|
|
"dataset_id": "agent-core-v1",
|
|
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
|
"status": "completed",
|
|
"config_snapshot": {
|
|
"dataset_id": "agent-core-v1",
|
|
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
|
"model": "deepseek-v4-flash",
|
|
"max_steps": 6,
|
|
"timeout_seconds": 90,
|
|
"token_budget": 6000,
|
|
"repeat": 1,
|
|
"allow_network": false,
|
|
"offline": false,
|
|
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
|
"dataset_version": "1.0.0",
|
|
"execution": "real_agent_runtime",
|
|
"provider_type": "openai_compatible",
|
|
"scoring_version": "1.0",
|
|
"permission_policy": "runtime_user_decision",
|
|
"active_agent_run_id": "run_1230c37a1c2c4559b69cbc4e01b4721b"
|
|
},
|
|
"metrics": {
|
|
"total_cases": 4,
|
|
"evaluated_cases": 4,
|
|
"task_success_rate": 1,
|
|
"tool_selection_accuracy": 1,
|
|
"tool_argument_accuracy": 1,
|
|
"invalid_tool_call_rate": 0,
|
|
"average_steps": 1.75,
|
|
"average_latency_ms": 4646.515175001696,
|
|
"token_usage": 5939,
|
|
"tool_calls": 3,
|
|
"expected_calls": 3
|
|
},
|
|
"cases": [
|
|
{
|
|
"case_id": "arithmetic",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_f4baf45c66c34e3f94b60b90f0846a95",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 4902.329800010193,
|
|
"token_usage": 1585,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "echo",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_5cf8341f73fa431ab7978f9ed4fe84c7",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 5341.693700000178,
|
|
"token_usage": 1821,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "no-tool",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_226a20145fbc4737ae2d3e812ff8f25c",
|
|
"success": true,
|
|
"tool_calls": 0,
|
|
"expected_calls": 0,
|
|
"selected_calls": 0,
|
|
"accurate_calls": 0,
|
|
"invalid_calls": 0,
|
|
"steps": 1,
|
|
"latency_ms": 1806.5612000063993,
|
|
"token_usage": 541,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "markdown-catalog",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_1230c37a1c2c4559b69cbc4e01b4721b",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 6535.4759999900125,
|
|
"token_usage": 1992,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
}
|
|
],
|
|
"error": null,
|
|
"error_code": null
|
|
} |