{
  "provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
  "protocol": "openai_compatible",
  "model": "deepseek-v4-flash",
  "checks": {
    "discovery": {
      "passed": true,
      "count": 3,
      "elapsed_ms": 1904.77
    },
    "normal_chat": {
      "passed": true,
      "input_tokens": 87,
      "output_tokens": 16,
      "elapsed_ms": 2107.22
    },
    "stream_usage_reasoning": {
      "passed": true,
      "events": [
        "Done",
        "TextDelta",
        "ThinkingDelta",
        "Usage"
      ],
      "usage_reported": true,
      "reasoning_observed": true,
      "elapsed_ms": 2118.0
    },
    "stream_cancel": {
      "passed": true,
      "first_event": "ThinkingDelta",
      "close_ms": 12.160799989942461,
      "scope": "client stream resource close; provider billing cessation not observable",
      "elapsed_ms": 1842.69
    },
    "error_mapping": {
      "passed": true,
      "error_code": "PROVIDER_INVALID_REQUEST",
      "elapsed_ms": 1590.6
    }
  },
  "unconfigured_protocols": [
    "openai_responses",
    "anthropic_messages",
    "ollama"
  ],
  "not_tested": [
    "vendor_maximum_context_boundary"
  ],
  "max_direct_requests": 5,
  "agent": {
    "run_id": "benchmark_8ebbf8c6f27f",
    "kind": "agent",
    "dataset_id": "agent-core-v1",
    "dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
    "status": "completed",
    "config_snapshot": {
      "dataset_id": "agent-core-v1",
      "provider_id": "provider_65fd326edba646e3bc735c25fbf5ffe8",
      "model": "deepseek-v4-flash",
      "max_steps": 6,
      "timeout_seconds": 90,
      "token_budget": 6000,
      "repeat": 1,
      "allow_network": false,
      "offline": false,
      "dataset_hash": "sha256:82fd1b84cc2dc2feec0fc67357d64b145dcdf6c44ec2660f182d550e85b9d27f",
      "dataset_version": "1.0.0",
      "execution": "real_agent_runtime",
      "provider_type": "openai_compatible",
      "scoring_version": "1.0",
      "permission_policy": "runtime_user_decision",
      "active_agent_run_id": "run_c9296aaea4c7429db97528edc72b6180"
    },
    "metrics": {
      "total_cases": 4,
      "task_success_rate": 1.0,
      "tool_selection_accuracy": 1.0,
      "tool_argument_accuracy": 1.0,
      "invalid_tool_call_rate": 0.0,
      "average_steps": 1.75,
      "average_latency_ms": 5682.329325005412,
      "token_usage": 5888,
      "tool_calls": 3,
      "expected_calls": 3
    },
    "cases": [
      {
        "case_id": "arithmetic",
        "repeat": 0,
        "agent_run_id": "run_480c159ffc0341f9883a09914ff003b5",
        "success": true,
        "tool_calls": 1,
        "expected_calls": 1,
        "selected_calls": 1,
        "accurate_calls": 1,
        "invalid_calls": 0,
        "steps": 2,
        "latency_ms": 4888.349500019103,
        "token_usage": 1545,
        "checks": {
          "completed": true,
          "tools_selected": true,
          "tool_arguments": true,
          "no_extra_calls": true,
          "tool_results": true,
          "output": true,
          "citation": true,
          "tasks_created": true
        },
        "error_code": null
      },
      {
        "case_id": "echo",
        "repeat": 0,
        "agent_run_id": "run_4fdbd0508bb84bcda424d97de1b91241",
        "success": true,
        "tool_calls": 1,
        "expected_calls": 1,
        "selected_calls": 1,
        "accurate_calls": 1,
        "invalid_calls": 0,
        "steps": 2,
        "latency_ms": 4978.496600000653,
        "token_usage": 1601,
        "checks": {
          "completed": true,
          "tools_selected": true,
          "tool_arguments": true,
          "no_extra_calls": true,
          "tool_results": true,
          "output": true,
          "citation": true,
          "tasks_created": true
        },
        "error_code": null
      },
      {
        "case_id": "no-tool",
        "repeat": 0,
        "agent_run_id": "run_5dfb4ae0a6c547b79706ad102e577bab",
        "success": true,
        "tool_calls": 0,
        "expected_calls": 0,
        "selected_calls": 0,
        "accurate_calls": 0,
        "invalid_calls": 0,
        "steps": 1,
        "latency_ms": 2549.696500005666,
        "token_usage": 541,
        "checks": {
          "completed": true,
          "tools_selected": true,
          "tool_arguments": true,
          "no_extra_calls": true,
          "tool_results": true,
          "output": true,
          "citation": true,
          "tasks_created": true
        },
        "error_code": null
      },
      {
        "case_id": "markdown-catalog",
        "repeat": 0,
        "agent_run_id": "run_c9296aaea4c7429db97528edc72b6180",
        "success": true,
        "tool_calls": 1,
        "expected_calls": 1,
        "selected_calls": 1,
        "accurate_calls": 1,
        "invalid_calls": 0,
        "steps": 2,
        "latency_ms": 10312.774699996226,
        "token_usage": 2201,
        "checks": {
          "completed": true,
          "tools_selected": true,
          "tool_arguments": true,
          "no_extra_calls": true,
          "tool_results": true,
          "output": true,
          "citation": true,
          "tasks_created": true
        },
        "error_code": null
      }
    ],
    "error": null,
    "error_code": null
  },
  "followup_evidence": [
    "context.json",
    "integration.json"
  ]
}