intelligence-radar/catalog/anthropic-reported-benchmarks-2026-09-13.json

{
  "schema_version": 1,
  "status": "reviewed source transcription; not deployable-profile ranking evidence",
  "source_url": "https://www-cdn.anthropic.com/283ef97c476cf442c91d9a37d5b214242a55bb92/Claude%20Sonnet%205%20System%20Card.pdf",
  "results": [
    {
      "model": "claude-sonnet-5",
      "benchmark": "swe-bench-pro",
      "version_or_protocol": "Anthropic-card-2026-06-30",
      "score_percent": 63.2,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 116",
      "execution_conditions": "Internal SWE harness; Sonnet 5 adaptive max, five trials; reference model exact settings not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-4.6",
      "benchmark": "swe-bench-pro",
      "version_or_protocol": "Anthropic-card-2026-06-30",
      "score_percent": 58.1,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 116",
      "execution_conditions": "Internal SWE harness; Sonnet 5 adaptive max, five trials; reference model exact settings not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-5",
      "benchmark": "terminal-bench",
      "version_or_protocol": "2.1",
      "score_percent": 80.4,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 116-117",
      "execution_conditions": "mini-SWE-agent; GKE; 1x timeout/3x memory; Sonnet 5 xhigh vs Sonnet 4.6 high; 89 tasks, five trials.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-4.6",
      "benchmark": "terminal-bench",
      "version_or_protocol": "2.1",
      "score_percent": 67.0,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 116-117",
      "execution_conditions": "mini-SWE-agent; GKE; 1x timeout/3x memory; Sonnet 5 xhigh vs Sonnet 4.6 high; 89 tasks, five trials.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-5",
      "benchmark": "browsecomp",
      "version_or_protocol": "Anthropic-blocklist-2026-06-30",
      "score_percent": 84.7,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,123",
      "execution_conditions": "Single-agent search/fetch/programmatic tools/code; Sonnet 5 max, 10M total tokens, compaction at 200k; reference settings not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-4.6",
      "benchmark": "browsecomp",
      "version_or_protocol": "Anthropic-blocklist-2026-06-30",
      "score_percent": 76.2,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,123",
      "execution_conditions": "Single-agent search/fetch/programmatic tools/code; Sonnet 5 max, 10M total tokens, compaction at 200k; reference settings not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-5",
      "benchmark": "hle-with-tools",
      "version_or_protocol": "Anthropic-Opus4.6-grader-2026-06-30",
      "score_percent": 57.4,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,122",
      "execution_conditions": "Search/fetch/code tools; 1M token cap; no compaction; Opus 4.6 grader; reference conditions not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-4.6",
      "benchmark": "hle-with-tools",
      "version_or_protocol": "Anthropic-Opus4.6-grader-2026-06-30",
      "score_percent": 46.8,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,122",
      "execution_conditions": "Search/fetch/code tools; 1M token cap; no compaction; Opus 4.6 grader; reference conditions not fully restated.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-5",
      "benchmark": "osworld-verified",
      "version_or_protocol": "361-tasks-100-steps-zoom-fix-128k",
      "score_percent": 81.2,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,126",
      "execution_conditions": "1080p Ubuntu VM; adaptive max; 100 actions; 128k per-turn limit; updated zoom tool; pass@1 averaged over five runs.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    },
    {
      "model": "claude-sonnet-4.6",
      "benchmark": "osworld-verified",
      "version_or_protocol": "361-tasks-100-steps-zoom-fix-128k",
      "score_percent": 78.5,
      "capture_id": "e2bcc2bcebc960f4b094ffc47a68a88b83edb7e3f6a8579fa479c975913a3483",
      "observed_at": "2026-09-13T21:15:41.720784+00:00",
      "source_locator": "PDF pages 115,126",
      "execution_conditions": "1080p Ubuntu VM; adaptive max; 100 actions; 128k per-turn limit; updated zoom tool; pass@1 averaged over five runs.",
      "evidence_kind": "vendor_reported",
      "comparability": "preserve per-result conditions; no direct-API transfer"
    }
  ]
}