Record phase two acceptance evidence and synchronize delivery status
This commit is contained in:
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
||||
"protocol": "openai_compatible",
|
||||
"model": "deepseek-v4-flash",
|
||||
"checks": {
|
||||
"discovery": {
|
||||
"passed": true,
|
||||
"count": 3,
|
||||
"elapsed_ms": 1904.77
|
||||
},
|
||||
"normal_chat": {
|
||||
"passed": true,
|
||||
"input_tokens": 87,
|
||||
"output_tokens": 16,
|
||||
"elapsed_ms": 2107.22
|
||||
},
|
||||
"stream_usage_reasoning": {
|
||||
"passed": true,
|
||||
"events": [
|
||||
"Done",
|
||||
"TextDelta",
|
||||
"ThinkingDelta",
|
||||
"Usage"
|
||||
],
|
||||
"usage_reported": true,
|
||||
"reasoning_observed": true,
|
||||
"elapsed_ms": 2118.0
|
||||
},
|
||||
"stream_cancel": {
|
||||
"passed": true,
|
||||
"first_event": "ThinkingDelta",
|
||||
"close_ms": 12.160799989942461,
|
||||
"scope": "client stream resource close; provider billing cessation not observable",
|
||||
"elapsed_ms": 1842.69
|
||||
},
|
||||
"error_mapping": {
|
||||
"passed": true,
|
||||
"error_code": "PROVIDER_INVALID_REQUEST",
|
||||
"elapsed_ms": 1590.6
|
||||
}
|
||||
},
|
||||
"unconfigured_protocols": [
|
||||
"openai_responses",
|
||||
"anthropic_messages",
|
||||
"ollama"
|
||||
],
|
||||
"not_tested": [
|
||||
"vendor_maximum_context_boundary"
|
||||
],
|
||||
"max_direct_requests": 5,
|
||||
"agent": {
|
||||
"run_id": "benchmark_8ebbf8c6f27f",
|
||||
"kind": "agent",
|
||||
"dataset_id": "agent-core-v1",
|
||||
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
||||
"status": "completed",
|
||||
"config_snapshot": {
|
||||
"dataset_id": "agent-core-v1",
|
||||
"provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
|
||||
"model": "deepseek-v4-flash",
|
||||
"max_steps": 6,
|
||||
"timeout_seconds": 90,
|
||||
"token_budget": 6000,
|
||||
"repeat": 1,
|
||||
"allow_network": false,
|
||||
"offline": false,
|
||||
"dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
|
||||
"dataset_version": "1.0.0",
|
||||
"execution": "real_agent_runtime",
|
||||
"provider_type": "openai_compatible",
|
||||
"scoring_version": "1.0",
|
||||
"permission_policy": "runtime_user_decision",
|
||||
"active_agent_run_id": "run_c9296aaea4c7429db97528edc72b6180"
|
||||
},
|
||||
"metrics": {
|
||||
"total_cases": 4,
|
||||
"task_success_rate": 1.0,
|
||||
"tool_selection_accuracy": 1.0,
|
||||
"tool_argument_accuracy": 1.0,
|
||||
"invalid_tool_call_rate": 0.0,
|
||||
"average_steps": 1.75,
|
||||
"average_latency_ms": 5682.329325005412,
|
||||
"token_usage": 5888,
|
||||
"tool_calls": 3,
|
||||
"expected_calls": 3
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "arithmetic",
|
||||
"repeat": 0,
|
||||
"agent_run_id": "run_480c159ffc0341f9883a09914ff003b5",
|
||||
"success": true,
|
||||
"tool_calls": 1,
|
||||
"expected_calls": 1,
|
||||
"selected_calls": 1,
|
||||
"accurate_calls": 1,
|
||||
"invalid_calls": 0,
|
||||
"steps": 2,
|
||||
"latency_ms": 4888.349500019103,
|
||||
"token_usage": 1545,
|
||||
"checks": {
|
||||
"completed": true,
|
||||
"tools_selected": true,
|
||||
"tool_arguments": true,
|
||||
"no_extra_calls": true,
|
||||
"tool_results": true,
|
||||
"output": true,
|
||||
"citation": true,
|
||||
"tasks_created": true
|
||||
},
|
||||
"error_code": null
|
||||
},
|
||||
{
|
||||
"case_id": "echo",
|
||||
"repeat": 0,
|
||||
"agent_run_id": "run_4fdbd0508bb84bcda424d97de1b91241",
|
||||
"success": true,
|
||||
"tool_calls": 1,
|
||||
"expected_calls": 1,
|
||||
"selected_calls": 1,
|
||||
"accurate_calls": 1,
|
||||
"invalid_calls": 0,
|
||||
"steps": 2,
|
||||
"latency_ms": 4978.496600000653,
|
||||
"token_usage": 1601,
|
||||
"checks": {
|
||||
"completed": true,
|
||||
"tools_selected": true,
|
||||
"tool_arguments": true,
|
||||
"no_extra_calls": true,
|
||||
"tool_results": true,
|
||||
"output": true,
|
||||
"citation": true,
|
||||
"tasks_created": true
|
||||
},
|
||||
"error_code": null
|
||||
},
|
||||
{
|
||||
"case_id": "no-tool",
|
||||
"repeat": 0,
|
||||
"agent_run_id": "run_5dfb4ae0a6c547b79706ad102e577bab",
|
||||
"success": true,
|
||||
"tool_calls": 0,
|
||||
"expected_calls": 0,
|
||||
"selected_calls": 0,
|
||||
"accurate_calls": 0,
|
||||
"invalid_calls": 0,
|
||||
"steps": 1,
|
||||
"latency_ms": 2549.696500005666,
|
||||
"token_usage": 541,
|
||||
"checks": {
|
||||
"completed": true,
|
||||
"tools_selected": true,
|
||||
"tool_arguments": true,
|
||||
"no_extra_calls": true,
|
||||
"tool_results": true,
|
||||
"output": true,
|
||||
"citation": true,
|
||||
"tasks_created": true
|
||||
},
|
||||
"error_code": null
|
||||
},
|
||||
{
|
||||
"case_id": "markdown-catalog",
|
||||
"repeat": 0,
|
||||
"agent_run_id": "run_c9296aaea4c7429db97528edc72b6180",
|
||||
"success": true,
|
||||
"tool_calls": 1,
|
||||
"expected_calls": 1,
|
||||
"selected_calls": 1,
|
||||
"accurate_calls": 1,
|
||||
"invalid_calls": 0,
|
||||
"steps": 2,
|
||||
"latency_ms": 10312.774699996226,
|
||||
"token_usage": 2201,
|
||||
"checks": {
|
||||
"completed": true,
|
||||
"tools_selected": true,
|
||||
"tool_arguments": true,
|
||||
"no_extra_calls": true,
|
||||
"tool_results": true,
|
||||
"output": true,
|
||||
"citation": true,
|
||||
"tasks_created": true
|
||||
},
|
||||
"error_code": null
|
||||
}
|
||||
],
|
||||
"error": null,
|
||||
"error_code": null
|
||||
},
|
||||
"followup_evidence": [
|
||||
"context.json",
|
||||
"integration.json"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user