197 lines
5.3 KiB
JSON
197 lines
5.3 KiB
JSON
{
|
|
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
|
"protocol": "openai_compatible",
|
|
"model": "deepseek-v4-flash",
|
|
"checks": {
|
|
"discovery": {
|
|
"passed": true,
|
|
"count": 3,
|
|
"elapsed_ms": 1904.77
|
|
},
|
|
"normal_chat": {
|
|
"passed": true,
|
|
"input_tokens": 87,
|
|
"output_tokens": 16,
|
|
"elapsed_ms": 2107.22
|
|
},
|
|
"stream_usage_reasoning": {
|
|
"passed": true,
|
|
"events": [
|
|
"Done",
|
|
"TextDelta",
|
|
"ThinkingDelta",
|
|
"Usage"
|
|
],
|
|
"usage_reported": true,
|
|
"reasoning_observed": true,
|
|
"elapsed_ms": 2118.0
|
|
},
|
|
"stream_cancel": {
|
|
"passed": true,
|
|
"first_event": "ThinkingDelta",
|
|
"close_ms": 12.160799989942461,
|
|
"scope": "client stream resource close; provider billing cessation not observable",
|
|
"elapsed_ms": 1842.69
|
|
},
|
|
"error_mapping": {
|
|
"passed": true,
|
|
"error_code": "PROVIDER_INVALID_REQUEST",
|
|
"elapsed_ms": 1590.6
|
|
}
|
|
},
|
|
"unconfigured_protocols": [
|
|
"openai_responses",
|
|
"anthropic_messages",
|
|
"ollama"
|
|
],
|
|
"not_tested": [
|
|
"vendor_maximum_context_boundary"
|
|
],
|
|
"max_direct_requests": 5,
|
|
"agent": {
|
|
"run_id": "benchmark_8ebbf8c6f27f",
|
|
"kind": "agent",
|
|
"dataset_id": "agent-core-v1",
|
|
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
|
"status": "completed",
|
|
"config_snapshot": {
|
|
"dataset_id": "agent-core-v1",
|
|
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
|
"model": "deepseek-v4-flash",
|
|
"max_steps": 6,
|
|
"timeout_seconds": 90,
|
|
"token_budget": 6000,
|
|
"repeat": 1,
|
|
"allow_network": false,
|
|
"offline": false,
|
|
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
|
"dataset_version": "1.0.0",
|
|
"execution": "real_agent_runtime",
|
|
"provider_type": "openai_compatible",
|
|
"scoring_version": "1.0",
|
|
"permission_policy": "runtime_user_decision",
|
|
"active_agent_run_id": "run_c9296aaea4c7429db97528edc72b6180"
|
|
},
|
|
"metrics": {
|
|
"total_cases": 4,
|
|
"task_success_rate": 1.0,
|
|
"tool_selection_accuracy": 1.0,
|
|
"tool_argument_accuracy": 1.0,
|
|
"invalid_tool_call_rate": 0.0,
|
|
"average_steps": 1.75,
|
|
"average_latency_ms": 5682.329325005412,
|
|
"token_usage": 5888,
|
|
"tool_calls": 3,
|
|
"expected_calls": 3
|
|
},
|
|
"cases": [
|
|
{
|
|
"case_id": "arithmetic",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_480c159ffc0341f9883a09914ff003b5",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 4888.349500019103,
|
|
"token_usage": 1545,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "echo",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_4fdbd0508bb84bcda424d97de1b91241",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 4978.496600000653,
|
|
"token_usage": 1601,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "no-tool",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_5dfb4ae0a6c547b79706ad102e577bab",
|
|
"success": true,
|
|
"tool_calls": 0,
|
|
"expected_calls": 0,
|
|
"selected_calls": 0,
|
|
"accurate_calls": 0,
|
|
"invalid_calls": 0,
|
|
"steps": 1,
|
|
"latency_ms": 2549.696500005666,
|
|
"token_usage": 541,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
},
|
|
{
|
|
"case_id": "markdown-catalog",
|
|
"repeat": 0,
|
|
"agent_run_id": "run_c9296aaea4c7429db97528edc72b6180",
|
|
"success": true,
|
|
"tool_calls": 1,
|
|
"expected_calls": 1,
|
|
"selected_calls": 1,
|
|
"accurate_calls": 1,
|
|
"invalid_calls": 0,
|
|
"steps": 2,
|
|
"latency_ms": 10312.774699996226,
|
|
"token_usage": 2201,
|
|
"checks": {
|
|
"completed": true,
|
|
"tools_selected": true,
|
|
"tool_arguments": true,
|
|
"no_extra_calls": true,
|
|
"tool_results": true,
|
|
"output": true,
|
|
"citation": true,
|
|
"tasks_created": true
|
|
},
|
|
"error_code": null
|
|
}
|
|
],
|
|
"error": null,
|
|
"error_code": null
|
|
},
|
|
"followup_evidence": [
|
|
"context.json",
|
|
"integration.json"
|
|
]
|
|
}
|