{
 "model": "claude-opus-5-5",
 "arms": {
  "cain_structured": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.4386,
   "gain_per_scan": 0.219,
   "gain_ci95_per_scan": [
    0.2086,
    0.2281
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-opus-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 361,
    "ok_calls": 361,
    "non_200": 0,
    "input_tokens": 881973,
    "cache_read_tokens": 0,
    "cache_write_tokens": 0,
    "output_tokens": 193870,
    "cost_usd": 7.4053
   }
  },
  "cain_structured_rep": {
   "runs": 2,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.401,
   "gain_per_scan": 0.1814,
   "gain_ci95_per_scan": [
    0.1724,
    0.1893
   ],
   "scans_better_than_baseline": 89,
   "scans_worse_than_baseline": 1,
   "sign_test_p": 1.4701842918422953e-25,
   "model": "claude-opus-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 270,
    "ok_calls": 270,
    "non_200": 0,
    "input_tokens": 636954,
    "cache_read_tokens": 0,
    "cache_write_tokens": 0,
    "output_tokens": 135912,
    "cost_usd": 5.2661
   }
  },
  "cain_retrieval": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.3661,
   "gain_per_scan": 0.1465,
   "gain_ci95_per_scan": [
    0.1389,
    0.1539
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-opus-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 360,
    "ok_calls": 360,
    "non_200": 0,
    "input_tokens": 979349,
    "cache_read_tokens": 0,
    "cache_write_tokens": 0,
    "output_tokens": 189346,
    "cost_usd": 7.7043
   }
  },
  "icl": {
   "runs": 1,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.6371,
   "gain_per_scan": 0.4175,
   "gain_ci95_per_scan": [
    0.3849,
    0.4478
   ],
   "scans_better_than_baseline": 89,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 3.2311742677852644e-27,
   "model": "claude-opus-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 180,
    "ok_calls": 180,
    "non_200": 0,
    "input_tokens": 4109758,
    "cache_read_tokens": 0,
    "cache_write_tokens": 0,
    "output_tokens": 72981,
    "cost_usd": 17.8987
   }
  },
  "haiku_cain_retrieval": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.2198,
   "gain_per_scan": 0.0002,
   "gain_ci95_per_scan": [
    -0.0025,
    0.003
   ],
   "scans_better_than_baseline": 44,
   "scans_worse_than_baseline": 46,
   "sign_test_p": 0.9161288770112893,
   "model": "claude-haiku-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "note": "shared Haiku ledger; see haiku_spend_usd"
   }
  },
  "haiku_icl": {
   "runs": 3,
   "baseline_mean": 0.2195,
   "mean_reward_per_scan": 0.459,
   "gain_per_scan": 0.2395,
   "gain_ci95_per_scan": [
    0.2211,
    0.257
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-haiku-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 360,
    "input_tokens": 13943174,
    "output_tokens": 415397,
    "cost_usd": 1.602
   }
  },
  "haiku_cain_structured": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.4199,
   "gain_per_scan": 0.2003,
   "gain_ci95_per_scan": [
    0.186,
    0.2133
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-haiku-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 360,
    "input_tokens": 988621,
    "output_tokens": 387514,
    "cost_usd": 0.2926
   }
  },
  "haiku_cain_hybrid": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "mean_reward_per_scan": 0.43,
   "gain_per_scan": 0.2105,
   "gain_ci95_per_scan": [
    0.2001,
    0.2192
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-haiku-5-5",
   "task": "blind_spectrum_monitoring",
   "spend": {
    "calls": 360,
    "input_tokens": 2843947,
    "output_tokens": 356659,
    "cost_usd": 0.4627
   }
  },
  "cain_hybrid": {
   "runs": 3,
   "baseline_mean": 0.2196,
   "baseline_source": "carried over from 4 Opus runs (no baseline phase, owner's $5 cap)",
   "mean_reward_per_scan": 0.4782,
   "gain_per_scan": 0.2587,
   "gain_ci95_per_scan": [
    0.2438,
    0.272
   ],
   "scans_better_than_baseline": 90,
   "scans_worse_than_baseline": 0,
   "sign_test_p": 1.6155871338926322e-27,
   "model": "claude-opus-5-5",
   "spend": {
    "cost_usd": 11.4393,
    "input_tokens": 2137527,
    "output_tokens": 144461,
    "calls": 270,
    "note": "rollout only; no baseline phase"
   }
  },
  "poker_haiku_cain_hybrid": {
   "runs": 1,
   "baseline_mean": 0.3458,
   "mean_reward_per_scan": 0.245,
   "gain_per_scan": -0.1008,
   "gain_ci95_per_scan": [
    -0.585,
    0.4125
   ],
   "scans_better_than_baseline": 34,
   "scans_worse_than_baseline": 34,
   "sign_test_p": 1.0,
   "model": "claude-haiku-5-5",
   "task": "exploitable_poker (reward = profit in big blinds per hand)",
   "spend": {
    "calls": 604,
    "input_tokens": 3493111,
    "output_tokens": 226894,
    "cost_usd": 0.4628
   }
  },
  "poker_haiku_icl": {
   "runs": 1,
   "baseline_mean": 0.2058,
   "mean_reward_per_scan": 0.05,
   "gain_per_scan": -0.1558,
   "gain_ci95_per_scan": [
    -0.7133,
    0.4383
   ],
   "scans_better_than_baseline": 31,
   "scans_worse_than_baseline": 39,
   "sign_test_p": 0.40296307504607976,
   "model": "claude-haiku-5-5",
   "task": "exploitable_poker (reward = profit in big blinds per hand)",
   "spend": {
    "calls": 633,
    "input_tokens": 19792181,
    "output_tokens": 229603,
    "cost_usd": 2.094
   }
  },
  "poker_r2_haiku_cain_hybrid": {
   "runs": 1,
   "baseline_mean": 0.3117,
   "mean_reward_per_scan": 0.2,
   "gain_per_scan": -0.1117,
   "gain_ci95_per_scan": [
    -0.5192,
    0.3183
   ],
   "scans_better_than_baseline": 35,
   "scans_worse_than_baseline": 36,
   "sign_test_p": 1.0,
   "model": "claude-haiku-5-5",
   "task": "exploitable_poker (reward = profit in big blinds per hand)",
   "spend": {
    "calls": 594,
    "input_tokens": 3288162,
    "output_tokens": 223510,
    "cost_usd": 0.4406
   }
  },
  "poker_r2_haiku_icl": {
   "runs": 1,
   "baseline_mean": 0.7592,
   "mean_reward_per_scan": 0.0208,
   "gain_per_scan": -0.7383,
   "gain_ci95_per_scan": [
    -1.5158,
    0.0142
   ],
   "scans_better_than_baseline": 38,
   "scans_worse_than_baseline": 45,
   "sign_test_p": 0.5104099416812344,
   "model": "claude-haiku-5-5",
   "task": "exploitable_poker (reward = profit in big blinds per hand)",
   "spend": {
    "calls": 553,
    "input_tokens": 12039897,
    "output_tokens": 204479,
    "cost_usd": 1.3062
   }
  }
 },
 "leaderboard_gain_per_scan_other_models": {
  "icl-gpt-5.4": 0.293744,
  "claude-code-sonnet-4.6": 0.272467,
  "icl-claude-sonnet-4.6": 0.186939,
  "icl-notepad-claude-sonnet-4-6": 0.18037,
  "mem0-gpt-5.4": 0.155927,
  "icl-claude-opus-4.7": 0.153473,
  "icl-gemini-3-flash": 0.147546,
  "icl-gemini-3.1-pro-preview": 0.147476,
  "codex-gpt-5.4": 0.145196,
  "icl-notepad-gpt-5.4": 0.135038,
  "icl-notepad-gemini-3.1-pro-preview": 0.104022,
  "ace-gpt-5.4": 0.000188
 },
 "note": "gain_per_scan = (rollout reward - stateless baseline reward) summed over scans, averaged over runs, divided by 90; equal to the leaderboard's cumulative_gain field",
 "paired": {
  "cain_structured_minus_cain_structured_rep": {
   "mean_diff_per_scan": 0.0376,
   "ci95": [
    0.0348,
    0.0402
   ],
   "sign_test_p": 1.4701842918422953e-25,
   "cain_structured_better_scans": 89,
   "cain_structured_rep_better_scans": 1
  },
  "cain_structured_minus_cain_retrieval": {
   "mean_diff_per_scan": 0.0725,
   "ci95": [
    0.0662,
    0.0786
   ],
   "sign_test_p": 5.751490196657771e-25,
   "cain_structured_better_scans": 87,
   "cain_retrieval_better_scans": 1
  },
  "cain_structured_minus_icl": {
   "mean_diff_per_scan": -0.1985,
   "ci95": [
    -0.2236,
    -0.1727
   ],
   "sign_test_p": 1.3838495394081657e-16,
   "cain_structured_better_scans": 8,
   "icl_better_scans": 82
  },
  "cain_structured_minus_cain_hybrid": {
   "mean_diff_per_scan": -0.0397,
   "ci95": [
    -0.0474,
    -0.0315
   ],
   "sign_test_p": 8.070744444115692e-09,
   "cain_structured_better_scans": 18,
   "cain_hybrid_better_scans": 72
  },
  "cain_structured_rep_minus_cain_retrieval": {
   "mean_diff_per_scan": 0.0349,
   "ci95": [
    0.0293,
    0.0403
   ],
   "sign_test_p": 5.202430003083186e-13,
   "cain_structured_rep_better_scans": 78,
   "cain_retrieval_better_scans": 12
  },
  "cain_structured_rep_minus_icl": {
   "mean_diff_per_scan": -0.2361,
   "ci95": [
    -0.2615,
    -0.2098
   ],
   "sign_test_p": 1.4701842918422953e-25,
   "cain_structured_rep_better_scans": 1,
   "icl_better_scans": 89
  },
  "cain_structured_rep_minus_cain_hybrid": {
   "mean_diff_per_scan": -0.0773,
   "ci95": [
    -0.0853,
    -0.0689
   ],
   "sign_test_p": 1.9641662139013065e-22,
   "cain_structured_rep_better_scans": 3,
   "cain_hybrid_better_scans": 87
  },
  "cain_retrieval_minus_icl": {
   "mean_diff_per_scan": -0.271,
   "ci95": [
    -0.297,
    -0.2437
   ],
   "sign_test_p": 1.4701842918422953e-25,
   "cain_retrieval_better_scans": 1,
   "icl_better_scans": 89
  },
  "cain_retrieval_minus_cain_hybrid": {
   "mean_diff_per_scan": -0.1122,
   "ci95": [
    -0.1214,
    -0.1026
   ],
   "sign_test_p": 1.4701842918422953e-25,
   "cain_retrieval_better_scans": 1,
   "cain_hybrid_better_scans": 89
  },
  "icl_minus_cain_hybrid": {
   "mean_diff_per_scan": 0.1588,
   "ci95": [
    0.1403,
    0.1768
   ],
   "sign_test_p": 2.531301921382976e-23,
   "icl_better_scans": 86,
   "cain_hybrid_better_scans": 2
  },
  "haiku_cain_retrieval_minus_haiku_icl": {
   "mean_diff_per_scan": -0.2392,
   "ci95": [
    -0.2567,
    -0.2209
   ],
   "sign_test_p": 3.2311742677852644e-27,
   "haiku_cain_retrieval_better_scans": 0,
   "haiku_icl_better_scans": 89
  },
  "haiku_cain_retrieval_minus_haiku_cain_structured": {
   "mean_diff_per_scan": -0.2001,
   "ci95": [
    -0.2128,
    -0.186
   ],
   "sign_test_p": 3.2311742677852644e-27,
   "haiku_cain_retrieval_better_scans": 0,
   "haiku_cain_structured_better_scans": 89
  },
  "haiku_cain_retrieval_minus_haiku_cain_hybrid": {
   "mean_diff_per_scan": -0.2102,
   "ci95": [
    -0.2185,
    -0.2004
   ],
   "sign_test_p": 3.2311742677852644e-27,
   "haiku_cain_retrieval_better_scans": 0,
   "haiku_cain_hybrid_better_scans": 89
  },
  "haiku_icl_minus_haiku_cain_structured": {
   "mean_diff_per_scan": 0.0391,
   "ci95": [
    0.0254,
    0.0526
   ],
   "sign_test_p": 1.899849174722082e-06,
   "haiku_icl_better_scans": 67,
   "haiku_cain_structured_better_scans": 22
  },
  "haiku_icl_minus_haiku_cain_hybrid": {
   "mean_diff_per_scan": 0.029,
   "ci95": [
    0.0157,
    0.0423
   ],
   "sign_test_p": 0.2890960806960603,
   "haiku_icl_better_scans": 50,
   "haiku_cain_hybrid_better_scans": 39
  },
  "haiku_cain_structured_minus_haiku_cain_hybrid": {
   "mean_diff_per_scan": -0.0101,
   "ci95": [
    -0.0179,
    -0.0031
   ],
   "sign_test_p": 0.20311681873649556,
   "haiku_cain_structured_better_scans": 51,
   "haiku_cain_hybrid_better_scans": 38
  },
  "poker_haiku_cain_hybrid_minus_poker_haiku_icl": {
   "mean_diff_per_scan": 0.195,
   "ci95": [
    -0.1208,
    0.5092
   ],
   "sign_test_p": 0.12634707581392135,
   "poker_haiku_cain_hybrid_better_scans": 32,
   "poker_haiku_icl_better_scans": 20
  },
  "poker_haiku_cain_hybrid_minus_poker_r2_haiku_cain_hybrid": {
   "mean_diff_per_scan": 0.045,
   "ci95": [
    -0.1883,
    0.2908
   ],
   "sign_test_p": 0.8714146793645341,
   "poker_haiku_cain_hybrid_better_scans": 20,
   "poker_r2_haiku_cain_hybrid_better_scans": 18
  },
  "poker_haiku_cain_hybrid_minus_poker_r2_haiku_icl": {
   "mean_diff_per_scan": 0.2242,
   "ci95": [
    -0.2183,
    0.7208
   ],
   "sign_test_p": 1.0,
   "poker_haiku_cain_hybrid_better_scans": 25,
   "poker_r2_haiku_icl_better_scans": 24
  },
  "poker_haiku_icl_minus_poker_r2_haiku_cain_hybrid": {
   "mean_diff_per_scan": -0.15,
   "ci95": [
    -0.4125,
    0.0958
   ],
   "sign_test_p": 0.1351564508456704,
   "poker_haiku_icl_better_scans": 17,
   "poker_r2_haiku_cain_hybrid_better_scans": 28
  },
  "poker_haiku_icl_minus_poker_r2_haiku_icl": {
   "mean_diff_per_scan": 0.0292,
   "ci95": [
    -0.3667,
    0.4292
   ],
   "sign_test_p": 0.4708790136217118,
   "poker_haiku_icl_better_scans": 21,
   "poker_r2_haiku_icl_better_scans": 27
  },
  "poker_r2_haiku_cain_hybrid_minus_poker_r2_haiku_icl": {
   "mean_diff_per_scan": 0.1792,
   "ci95": [
    -0.1833,
    0.5583
   ],
   "sign_test_p": 0.7492586247608415,
   "poker_r2_haiku_cain_hybrid_better_scans": 18,
   "poker_r2_haiku_icl_better_scans": 21
  }
 },
 "stage1_total_spend_usd_all_attempts": 99.36,
 "haiku_spend_usd": 2.3574,
 "haiku_incomplete": "Haiku icl, cain_structured, cain_hybrid and both poker arms stopped when the API credit ran out (2026-11-18 00:17Z); excluded, not scored"
}