{
  "schema": "cain.e140.frontier_benchmarks.v1",
  "generated_at": "2026-11-22T00:34:27Z",
  "duration_s": 693.64,
  "paid_api_spend_usd": 0.0,
  "neural_weights_modified": false,
  "agi_status": "NOT ESTABLISHED",
  "rsi_status": "EXPERIMENTAL",
  "official_public_suites_evaluated": 4,
  "private_restricted_suites_blocked": 4,
  "total_official_frontier_items_evaluated": 11088,
  "cladder_v1_neurips2023": {
    "benchmark": "CLADDER-v1 (Official NeurIPS 2023 Pearl Causal Hierarchy)",
    "source_url": "https://github.com/causalNLP/cladder",
    "license": "MIT / CC-BY",
    "dataset_sha256": {
      "cladder-v1-questions.json": "59ecdc5ac2a6b4583b798b1aaf7bab3744316762769325adea69317ac1b4ae40",
      "cladder-v1-meta-models.json": "f288ed9046a1ef5e39c0dbf6b971b8a3a485887130b7831dc5da4fe2af5d64b5"
    },
    "zero_leakage_contract": "Solver received ONLY (background, given_info, question); zero access to answer, reasoning, or meta labels.",
    "total_questions_evaluated": 10560,
    "total_correct": 9702,
    "cain42_overall_accuracy": 0.9187,
    "cain42_overall_ci_95": [
      0.9134,
      0.9238
    ],
    "by_pearl_rung": {
      "rung_1": {
        "n": 3288,
        "correct": 3174,
        "accuracy": 0.9653,
        "ci_95": [
          0.9585,
          0.9711
        ]
      },
      "rung_2": {
        "n": 3288,
        "correct": 2954,
        "accuracy": 0.8984,
        "ci_95": [
          0.8876,
          0.9083
        ]
      },
      "rung_3": {
        "n": 3984,
        "correct": 3574,
        "accuracy": 0.8971,
        "ci_95": [
          0.8873,
          0.9061
        ]
      }
    },
    "by_query_type": {
      "ate": {
        "n": 1476,
        "accuracy": 0.9851
      },
      "backadj": {
        "n": 1644,
        "accuracy": 0.8102
      },
      "collider_bias": {
        "n": 168,
        "accuracy": 1.0
      },
      "correlation": {
        "n": 1476,
        "accuracy": 0.9566
      },
      "det-counterfactual": {
        "n": 1476,
        "accuracy": 0.8137
      },
      "ett": {
        "n": 1296,
        "accuracy": 0.9321
      },
      "exp_away": {
        "n": 168,
        "accuracy": 0.9762
      },
      "marginal": {
        "n": 1644,
        "accuracy": 0.972
      },
      "nde": {
        "n": 384,
        "accuracy": 0.9401
      },
      "nie": {
        "n": 828,
        "accuracy": 0.971
      }
    },
    "by_causal_graph": {
      "IV": {
        "n": 900,
        "accuracy": 0.7456
      },
      "arrowhead": {
        "n": 1536,
        "accuracy": 0.9564
      },
      "chain": {
        "n": 1092,
        "accuracy": 0.9432
      },
      "collision": {
        "n": 672,
        "accuracy": 0.9911
      },
      "confounding": {
        "n": 1008,
        "accuracy": 0.9673
      },
      "diamond": {
        "n": 1092,
        "accuracy": 0.9176
      },
      "diamondcut": {
        "n": 792,
        "accuracy": 0.9141
      },
      "fork": {
        "n": 1008,
        "accuracy": 0.9692
      },
      "frontdoor": {
        "n": 924,
        "accuracy": 0.7543
      },
      "mediation": {
        "n": 1536,
        "accuracy": 0.9707
      }
    },
    "published_reference_baselines_jin_et_al_neurips2023": {
      "random_chance": 0.5,
      "gpt_3_5_zero_shot": 0.5266,
      "gpt_4_zero_shot": 0.6203,
      "gpt_4_causalcot_sota": 0.704
    },
    "beats_published_gpt4_causalcot_sota": true,
    "live_local_ollama_qwen05b_comparison": {
      "n_stratified": 12,
      "ollama_qwen05b_accuracy": 0.5833,
      "cain42_same_slice_accuracy": 0.8333,
      "ollama_tokens": 1930
    },
    "sample_audit_records": [
      {
        "question_id": 19,
        "rung": 3,
        "query_type": "nie",
        "graph": "mediation",
        "pred": "no",
        "gold": "no",
        "correct": true
      },
      {
        "question_id": 21106,
        "rung": 3,
        "query_type": "nie",
        "graph": "arrowhead",
        "pred": "no",
        "gold": "no",
        "correct": true
      },
      {
        "question_id": 41273,
        "rung": 3,
        "query_type": "nie",
        "graph": "diamond",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      },
      {
        "question_id": 52356,
        "rung": 2,
        "query_type": "ate",
        "graph": "IV",
        "pred": "no",
        "gold": "no",
        "correct": true
      },
      {
        "question_id": 55768,
        "rung": 1,
        "query_type": "correlation",
        "graph": "IV",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      },
      {
        "question_id": 59203,
        "rung": 3,
        "query_type": "nde",
        "graph": "arrowhead",
        "pred": "no",
        "gold": "no",
        "correct": true
      },
      {
        "question_id": 62657,
        "rung": 3,
        "query_type": "nie",
        "graph": "diamond",
        "pred": "no",
        "gold": "no",
        "correct": true
      },
      {
        "question_id": 66055,
        "rung": 1,
        "query_type": "marginal",
        "graph": "diamond",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      },
      {
        "question_id": 69347,
        "rung": 2,
        "query_type": "ate",
        "graph": "arrowhead",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      },
      {
        "question_id": 72801,
        "rung": 1,
        "query_type": "correlation",
        "graph": "diamond",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      },
      {
        "question_id": 76209,
        "rung": 3,
        "query_type": "det-counterfactual",
        "graph": "diamondcut",
        "pred": "yes",
        "gold": "yes",
        "correct": true
      }
    ],
    "duration_s": 99.43,
    "status": "PASS"
  },
  "arc_agi_official_public": {
    "duration_s": 24.75,
    "suites": {
      "ARC-AGI-1 (Official Public Eval Split, 400 Tasks)": {
        "source_url": "https://github.com/fchollet/ARC-AGI",
        "license": "Apache-2.0",
        "archive_sha256": "41d34778e944fe3a6e8192a6b3777d0c78818839b4fb6db98526f129781b2c58",
        "split": "official_public_evaluation",
        "public_vs_private_disclosure": "Official Apache-2.0 public evaluation split only; NOT a private ARC Prize leaderboard submission.",
        "zero_leakage_contract": "Program synthesized strictly from train pairs only; accepted only if 100% exact on all train pairs; evaluated pixel-for-pixel on held-out test pairs.",
        "total_tasks": 400,
        "train_exact_programs_found": 18,
        "test_exact_solved": 17,
        "solve_rate": 0.0425,
        "ci_95": [
          0.0267,
          0.067
        ],
        "identity_and_random_baseline_solved": 0,
        "solved_task_records": [
          {
            "task_id": "0c786b71",
            "synthesized_program": "mirror_quad(rot180)",
            "n_train": 3,
            "n_test": 1
          },
          {
            "task_id": "0c9aba6e",
            "synthesized_program": "colormap(split_v_or)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "195ba7dc",
            "synthesized_program": "colormap(split_h_or)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "31d5ba1a",
            "synthesized_program": "colormap(split_v_xor)",
            "n_train": 5,
            "n_test": 2
          },
          {
            "task_id": "34b99a2b",
            "synthesized_program": "colormap(split_h_xor)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "506d28a5",
            "synthesized_program": "colormap(split_v_or)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "5b6cbef5",
            "synthesized_program": "fractal_mask_tile",
            "n_train": 5,
            "n_test": 1
          },
          {
            "task_id": "5d2a5c43",
            "synthesized_program": "colormap(split_h_or)",
            "n_train": 5,
            "n_test": 2
          },
          {
            "task_id": "60c09cac",
            "synthesized_program": "scale_2x2",
            "n_train": 2,
            "n_test": 1
          },
          {
            "task_id": "66f2d22f",
            "synthesized_program": "colormap(split_h_or)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "833dafe3",
            "synthesized_program": "mirror_quad(rot180)",
            "n_train": 2,
            "n_test": 1
          },
          {
            "task_id": "bf699163",
            "synthesized_program": "colormap(extract_rarest_color_obj)",
            "n_train": 2,
            "n_test": 1
          },
          {
            "task_id": "ce8d95cc",
            "synthesized_program": "dedup_rows_cols",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "d19f7514",
            "synthesized_program": "colormap(split_v_or)",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "e133d23d",
            "synthesized_program": "colormap(split_h_or)",
            "n_train": 5,
            "n_test": 1
          },
          {
            "task_id": "e1baa8a4",
            "synthesized_program": "dedup_rows_cols",
            "n_train": 4,
            "n_test": 1
          },
          {
            "task_id": "e345f17b",
            "synthesized_program": "colormap(split_h_or)",
            "n_train": 4,
            "n_test": 2
          }
        ],
        "status": "PASS_PUBLIC_SPLIT"
      },
      "ARC-AGI-2 (Official Public Eval Split, 120 Tasks)": {
        "source_url": "https://github.com/arcprize/ARC-AGI-2",
        "license": "Apache-2.0",
        "archive_sha256": "270f2ef8adafa0d0df099a9335d6ff27f25838de75af3ce778e4c872228b9958",
        "split": "official_public_evaluation",
        "public_vs_private_disclosure": "Official Apache-2.0 public evaluation split only; NOT a private ARC Prize leaderboard submission.",
        "zero_leakage_contract": "Program synthesized strictly from train pairs only; accepted only if 100% exact on all train pairs; evaluated pixel-for-pixel on held-out test pairs.",
        "total_tasks": 120,
        "train_exact_programs_found": 0,
        "test_exact_solved": 0,
        "solve_rate": 0.0,
        "ci_95": [
          0.0,
          0.031
        ],
        "identity_and_random_baseline_solved": 0,
        "solved_task_records": [],
        "status": "PARTIAL"
      }
    }
  },
  "openai_humaneval_live_rsi": {
    "benchmark": "OpenAI HumanEval (Official Public Python Suite, First 12 Tasks)",
    "source_url": "https://github.com/openai/human-eval",
    "license": "MIT",
    "dataset_sha256": "b796127e635a67f93fb35c04f4cb03cf06f38c8072ee7cee8833d7bee06979ef",
    "zero_leakage_contract": "CAIN-42 self-repair inspects ONLY MCPGate AST safety + public docstring examples; hidden official unit tests are used strictly for final scoring.",
    "model": "ollama:qwen2.5-coder:1.5b (local CPU, $0.00 paid spend)",
    "n_tasks": 8,
    "raw_single_shot_passed": 6,
    "raw_single_shot_pass_at_1": 0.75,
    "cain42_governed_passed": 6,
    "cain42_governed_pass_at_1": 0.75,
    "cain42_ci_95": [
      0.4093,
      0.9285
    ],
    "comparison_cain_vs_raw": {
      "test": "exact_mcnemar_arbitrary_precision",
      "n": 8,
      "rate_a": 0.75,
      "rate_b": 0.75,
      "gain": 0.0,
      "a_only": 1,
      "b_only": 1,
      "p_value": 1.0,
      "log10_p": 0.0,
      "p_display": "1.0",
      "underflowed_float64": false,
      "effect_size": 0.0,
      "effect_size_type": "cohens_h",
      "confidence_interval_diff": [
        -0.3465,
        0.3465
      ]
    },
    "ollama_calls": 10,
    "ollama_tokens": 2390,
    "task_records": [
      {
        "task_id": "HumanEval/0",
        "entry_point": "has_close_elements",
        "raw_pass": false,
        "cain42_governed_pass": true,
        "self_repair_triggered": true
      },
      {
        "task_id": "HumanEval/1",
        "entry_point": "separate_paren_groups",
        "raw_pass": true,
        "cain42_governed_pass": false,
        "self_repair_triggered": true
      },
      {
        "task_id": "HumanEval/2",
        "entry_point": "truncate_number",
        "raw_pass": true,
        "cain42_governed_pass": true,
        "self_repair_triggered": false
      },
      {
        "task_id": "HumanEval/3",
        "entry_point": "below_zero",
        "raw_pass": true,
        "cain42_governed_pass": true,
        "self_repair_triggered": false
      },
      {
        "task_id": "HumanEval/4",
        "entry_point": "mean_absolute_deviation",
        "raw_pass": true,
        "cain42_governed_pass": true,
        "self_repair_triggered": false
      },
      {
        "task_id": "HumanEval/5",
        "entry_point": "intersperse",
        "raw_pass": true,
        "cain42_governed_pass": true,
        "self_repair_triggered": false
      },
      {
        "task_id": "HumanEval/6",
        "entry_point": "parse_nested_parens",
        "raw_pass": false,
        "cain42_governed_pass": false,
        "self_repair_triggered": false
      },
      {
        "task_id": "HumanEval/7",
        "entry_point": "filter_by_substring",
        "raw_pass": true,
        "cain42_governed_pass": true,
        "self_repair_triggered": false
      }
    ],
    "duration_s": 569.4,
    "status": "PASS_PUBLIC_SPLIT"
  },
  "still_blocked_private_or_restricted": [
    {
      "benchmark": "ARC-AGI-2 Private Leaderboard & ARC-AGI-3 Interactive",
      "status": "BLOCKED",
      "reason": "Private ARC Prize submission credentials / interactive harness required; public Apache-2.0 split reported separately above."
    },
    {
      "benchmark": "METR Long-Horizon Autonomy (Vivaria)",
      "status": "BLOCKED",
      "reason": "Restricted private Vivaria task bank & third-party METR auditor required."
    },
    {
      "benchmark": "SWE-bench Pro",
      "status": "BLOCKED",
      "reason": "Requires frontier coding model + multi-node Docker fleet; $0.00 paid budget."
    },
    {
      "benchmark": "OSWorld / WebArena",
      "status": "BLOCKED",
      "reason": "Requires multimodal desktop/web VM infrastructure unavailable on CPU host."
    }
  ],
  "headline_summary": {
    "cladder_v1_n10560": "9,702/10,560 (91.87% vs GPT-4 62.03% & GPT-4+CausalCoT 70.40%)",
    "arc_agi_1_public_eval_n400": "17/400 (4.25%, zero-leakage DSL vs 0/400 baseline)",
    "arc_agi_2_public_eval_n120": "0/120 (0.00%, zero-leakage DSL vs 0/120 baseline)",
    "openai_humaneval_n12_cpu": "CAIN-42 6/8 (75.0%) vs Raw 1.5B 6/8 (75.0%)"
  },
  "digest": "3b5da827319934d1ae47c96a58707c1c4026972c98695b27bd80c628192a69e3"
}