{
 "models": {
  "claude-haiku-5-5": {
   "baselines": {
    "A": {
     "accepted": 82,
     "calls_per_accepted": 1.951,
     "ci95": [
      0.4357,
      0.5887
     ],
     "denied_tool_calls": 0,
     "governed_tool_calls": 0,
     "model_calls": 160,
     "n": 160,
     "rate": 0.5125,
     "tokens_in": 54371,
     "tokens_out": 15865,
     "wall_s": 157.2
    },
    "B": {
     "accepted": 85,
     "calls_per_accepted": 1.882,
     "ci95": [
      0.4541,
      0.6069
     ],
     "denied_tool_calls": 0,
     "governed_tool_calls": 0,
     "model_calls": 160,
     "n": 160,
     "rate": 0.5312,
     "tokens_in": 170331,
     "tokens_out": 9214,
     "wall_s": 137.1
    },
    "C": {
     "accepted": 90,
     "calls_per_accepted": 1.778,
     "ci95": [
      0.4851,
      0.637
     ],
     "denied_tool_calls": 0,
     "governed_tool_calls": 0,
     "model_calls": 160,
     "n": 160,
     "rate": 0.5625,
     "tokens_in": 118035,
     "tokens_out": 8554,
     "wall_s": 123.0
    },
    "D": {
     "accepted": 114,
     "calls_per_accepted": 2.667,
     "ci95": [
      0.638,
      0.777
     ],
     "denied_tool_calls": 0,
     "governed_tool_calls": 112,
     "model_calls": 304,
     "n": 160,
     "rate": 0.7125,
     "tokens_in": 128204,
     "tokens_out": 22374,
     "wall_s": 277.1
    }
   },
   "by_vertical": {
    "A": {
     "business": {
      "accepted": 7,
      "calls_per_accepted": 5.714,
      "ci95": [
       0.0875,
       0.3195
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 40,
      "n": 40,
      "rate": 0.175,
      "tokens_in": 12640,
      "tokens_out": 6747,
      "wall_s": 47.9
     },
     "education": {
      "accepted": 19,
      "calls_per_accepted": 1.053,
      "ci95": [
       0.7639,
       0.9911
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.95,
      "tokens_in": 8169,
      "tokens_out": 568,
      "wall_s": 13.3
     },
     "engineering": {
      "accepted": 12,
      "calls_per_accepted": 2.5,
      "ci95": [
       0.2459,
       0.5768
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.4,
      "tokens_in": 10746,
      "tokens_out": 3732,
      "wall_s": 33.8
     },
     "science": {
      "accepted": 16,
      "calls_per_accepted": 1.875,
      "ci95": [
       0.3614,
       0.6977
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.5333,
      "tokens_in": 11120,
      "tokens_out": 1004,
      "wall_s": 29.2
     },
     "security": {
      "accepted": 18,
      "calls_per_accepted": 1.111,
      "ci95": [
       0.699,
       0.9721
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.9,
      "tokens_in": 6682,
      "tokens_out": 2159,
      "wall_s": 16.3
     },
     "software": {
      "accepted": 10,
      "calls_per_accepted": 2.0,
      "ci95": [
       0.2993,
       0.7007
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.5,
      "tokens_in": 5014,
      "tokens_out": 1655,
      "wall_s": 16.7
     }
    },
    "B": {
     "business": {
      "accepted": 4,
      "calls_per_accepted": 10.0,
      "ci95": [
       0.0396,
       0.2305
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 40,
      "n": 40,
      "rate": 0.1,
      "tokens_in": 40240,
      "tokens_out": 2560,
      "wall_s": 34.3
     },
     "education": {
      "accepted": 20,
      "calls_per_accepted": 1.0,
      "ci95": [
       0.8389,
       1.0
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 1.0,
      "tokens_in": 23009,
      "tokens_out": 455,
      "wall_s": 12.9
     },
     "engineering": {
      "accepted": 13,
      "calls_per_accepted": 2.308,
      "ci95": [
       0.2738,
       0.608
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.4333,
      "tokens_in": 32496,
      "tokens_out": 2396,
      "wall_s": 28.5
     },
     "science": {
      "accepted": 19,
      "calls_per_accepted": 1.579,
      "ci95": [
       0.4551,
       0.7813
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.6333,
      "tokens_in": 30290,
      "tokens_out": 1007,
      "wall_s": 28.8
     },
     "security": {
      "accepted": 18,
      "calls_per_accepted": 1.111,
      "ci95": [
       0.699,
       0.9721
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.9,
      "tokens_in": 25122,
      "tokens_out": 1867,
      "wall_s": 17.8
     },
     "software": {
      "accepted": 11,
      "calls_per_accepted": 1.818,
      "ci95": [
       0.3421,
       0.7418
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.55,
      "tokens_in": 19174,
      "tokens_out": 929,
      "wall_s": 14.8
     }
    },
    "C": {
     "business": {
      "accepted": 12,
      "calls_per_accepted": 3.333,
      "ci95": [
       0.1807,
       0.4543
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 40,
      "n": 40,
      "rate": 0.3,
      "tokens_in": 28468,
      "tokens_out": 2394,
      "wall_s": 30.6
     },
     "education": {
      "accepted": 20,
      "calls_per_accepted": 1.0,
      "ci95": [
       0.8389,
       1.0
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 1.0,
      "tokens_in": 15833,
      "tokens_out": 455,
      "wall_s": 12.8
     },
     "engineering": {
      "accepted": 13,
      "calls_per_accepted": 2.308,
      "ci95": [
       0.2738,
       0.608
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.4333,
      "tokens_in": 22725,
      "tokens_out": 1748,
      "wall_s": 26.7
     },
     "science": {
      "accepted": 18,
      "calls_per_accepted": 1.667,
      "ci95": [
       0.4232,
       0.7541
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 30,
      "n": 30,
      "rate": 0.6,
      "tokens_in": 22249,
      "tokens_out": 1006,
      "wall_s": 22.1
     },
     "security": {
      "accepted": 17,
      "calls_per_accepted": 1.176,
      "ci95": [
       0.6396,
       0.9476
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.85,
      "tokens_in": 15100,
      "tokens_out": 1926,
      "wall_s": 16.5
     },
     "software": {
      "accepted": 10,
      "calls_per_accepted": 2.0,
      "ci95": [
       0.2993,
       0.7007
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.5,
      "tokens_in": 13660,
      "tokens_out": 1025,
      "wall_s": 14.2
     }
    },
    "D": {
     "business": {
      "accepted": 22,
      "calls_per_accepted": 3.773,
      "ci95": [
       0.3983,
       0.6929
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 21,
      "model_calls": 83,
      "n": 40,
      "rate": 0.55,
      "tokens_in": 38305,
      "tokens_out": 10297,
      "wall_s": 82.7
     },
     "education": {
      "accepted": 19,
      "calls_per_accepted": 1.053,
      "ci95": [
       0.7639,
       0.9911
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.95,
      "tokens_in": 8169,
      "tokens_out": 507,
      "wall_s": 12.8
     },
     "engineering": {
      "accepted": 19,
      "calls_per_accepted": 3.737,
      "ci95": [
       0.4551,
       0.7813
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 31,
      "model_calls": 71,
      "n": 30,
      "rate": 0.6333,
      "tokens_in": 30951,
      "tokens_out": 4478,
      "wall_s": 61.9
     },
     "science": {
      "accepted": 24,
      "calls_per_accepted": 2.917,
      "ci95": [
       0.6269,
       0.905
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 40,
      "model_calls": 70,
      "n": 30,
      "rate": 0.8,
      "tokens_in": 31032,
      "tokens_out": 2399,
      "wall_s": 70.8
     },
     "security": {
      "accepted": 18,
      "calls_per_accepted": 1.111,
      "ci95": [
       0.699,
       0.9721
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 0,
      "model_calls": 20,
      "n": 20,
      "rate": 0.9,
      "tokens_in": 6772,
      "tokens_out": 2028,
      "wall_s": 17.2
     },
     "software": {
      "accepted": 12,
      "calls_per_accepted": 3.333,
      "ci95": [
       0.3866,
       0.7812
      ],
      "denied_tool_calls": 0,
      "governed_tool_calls": 20,
      "model_calls": 40,
      "n": 20,
      "rate": 0.6,
      "tokens_in": 12975,
      "tokens_out": 2665,
      "wall_s": 31.7
     }
    }
   },
   "note": "Baselines A-D recomputed from raw rows by scripts/evolution91/analyze.py; same 160 held-out tasks as HELD_OUT_TASK_MANIFEST.json.",
   "schema": "cain.e91.baseline_results.v1"
  }
 },
 "schema": "cain.e91.baseline_results.v1"
}