{
  "schema": "cain.e138.external_benchmarks.v1",
  "generated_at": "2026-11-21T21:56:38Z",
  "total_benchmarks_audited": 8,
  "official_completed_count": 0,
  "blocked_count": 8,
  "status": "BLOCKED",
  "benchmarks": [
    {
      "benchmark": "ARC-AGI-2 / ARC-AGI-3",
      "official_source": "https://arcprize.org/ (ARC Prize Foundation)",
      "license_access": "Apache-2.0 public training/eval; private test evaluation requires official Kaggle/ARC Prize submission",
      "version": "ARC-AGI-2 (2025/2026) / ARC-AGI-3 interactive preview",
      "required_infra": "Frontier reasoning compute or official ARC Prize sandbox submission",
      "local_cpu_feasible": false,
      "contamination_risk": "Public train/eval widely memorized by pre-trained LLMs; only sealed private evaluation is valid",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "No official ARC Prize submission credentials or GPU frontier compute; refusing unofficial self-scores."
    },
    {
      "benchmark": "SWE-bench Verified",
      "official_source": "https://www.swebench.com/ (Princeton NLP / OpenAI Verified 500-task subset)",
      "license_access": "MIT harness; requires Docker evaluation images + official leaderboard submission for official rank",
      "version": "SWE-bench Verified (500 instances)",
      "required_infra": "High-throughput frontier coding model + Docker harness (~100GB+ images)",
      "local_cpu_feasible": false,
      "contamination_risk": "Historical GitHub issues/PRs exist in model pretraining corpora",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "Zero paid API budget ($0.00) and CPU-only 0.5B/1.5B models cannot run full official 500-repo Docker evaluation; prior self-run 257/500 is historical interim only."
    },
    {
      "benchmark": "METR Long-Horizon Autonomy Evaluations",
      "official_source": "https://metr.org/ (METR Task Standard / Vivaria)",
      "license_access": "Restricted private task suite; public sample tasks only",
      "version": "METR Autonomy Eval Suite (2025/2026)",
      "required_infra": "Vivaria isolated cloud VMs + frontier models + third-party METR auditor",
      "local_cpu_feasible": false,
      "contamination_risk": "Private task bank specifically prevents contamination",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "Requires third-party METR evaluation access and isolated cloud VMs."
    },
    {
      "benchmark": "GPQA Diamond / ScienceQA / FrontierMath",
      "official_source": "Rein et al. (GPQA) / Epoch AI (FrontierMath)",
      "license_access": "Gated / private test sets (FrontierMath closed; GPQA gated)",
      "version": "GPQA Diamond (198 Qs) / FrontierMath",
      "required_infra": "Frontier reasoning LLM + gated access agreement",
      "local_cpu_feasible": false,
      "contamination_risk": "High on public mirrors; requires gated evaluation",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "Requires gated access and GPU/frontier model budget."
    },
    {
      "benchmark": "LiveCodeBench / Codeforces Algorithmic Suite",
      "official_source": "https://livecodebench.github.io/",
      "license_access": "MIT (time-stamped post-cutoff contest tasks)",
      "version": "LiveCodeBench v5/v6 (2026)",
      "required_infra": "Downloadable dataset + larger local or API coding model",
      "local_cpu_feasible": false,
      "contamination_risk": "Mitigated by strict contest date filtering after model cutoff",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "Offline host / $0.00 paid budget; internal 40-task E138 algorithmic benchmark is NOT claimed as official LiveCodeBench."
    },
    {
      "benchmark": "Avalanche / Continual-Learning Benchmarks",
      "official_source": "https://avalanche.continualai.org/",
      "license_access": "MIT",
      "version": "Avalanche 0.6+",
      "required_infra": "GPU for neural weight continual learning",
      "local_cpu_feasible": false,
      "contamination_risk": "Low for synthetic/split benchmarks",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "CPU-only host; no neural weight continual-training GPU available."
    },
    {
      "benchmark": "CausalBench / CLADDER / Corr2Cause",
      "official_source": "Jin et al. (CLADDER / Corr2Cause); GSK CausalBench",
      "license_access": "CC-BY / MIT",
      "version": "CLADDER v1 / Corr2Cause",
      "required_infra": "Dataset download + standardized evaluation harness",
      "local_cpu_feasible": true,
      "contamination_risk": "Moderate for public Hugging Face releases",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "External dataset download not provisioned in offline E138 run; prioritized for E139."
    },
    {
      "benchmark": "OSWorld / WebArena / NetHack Interactive Suites",
      "official_source": "OSWorld / CMU WebArena / MiniHack",
      "license_access": "Apache-2.0 / MIT",
      "version": "OSWorld 2025 / WebArena",
      "required_infra": "Desktop VM / Docker web-app stack + multimodal agent model",
      "local_cpu_feasible": false,
      "contamination_risk": "Low (live interactive execution)",
      "official_completed": false,
      "status": "BLOCKED",
      "reason": "Requires VM/Docker interactive container fleet and multimodal frontier model."
    }
  ],
  "honest_disclosure": "0 external third-party benchmarks were officially completed during E138. Internal CAIN-42 benchmarks are never presented as official external leaderboard results."
}