{
 "arms": {
  "fable-5-2-stratified": {
   "accepted": 96,
   "cost_usd": 0.581936,
   "model": "claude-fable-5-2",
   "n": 100,
   "per_domain": {
    "business_decision": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "data_analysis": {
     "accepted": 9,
     "n": 10,
     "rate": 0.9
    },
    "governance": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "planning": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "security_triage": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 7,
     "n": 10,
     "rate": 0.7
    },
    "tool_orchestration": {
     "accepted": 10,
     "n": 10,
     "rate": 1.0
    }
   },
   "rate": 0.96,
   "wilson95": [
    0.9016,
    0.9843
   ]
  },
  "haiku-5-5-full": {
   "accepted": 481,
   "cost_usd": 0.047506,
   "model": "claude-haiku-5-5",
   "n": 500,
   "per_domain": {
    "business_decision": {
     "accepted": 35,
     "n": 50,
     "rate": 0.7
    },
    "data_analysis": {
     "accepted": 48,
     "n": 50,
     "rate": 0.96
    },
    "governance": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "memory_retrieval": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "planning": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "recovery": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "science_calculation": {
     "accepted": 48,
     "n": 50,
     "rate": 0.96
    },
    "security_triage": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "software_repair": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    },
    "tool_orchestration": {
     "accepted": 50,
     "n": 50,
     "rate": 1.0
    }
   },
   "rate": 0.962,
   "wilson95": [
    0.9414,
    0.9755
   ]
  }
 },
 "finding": "Haiku 5.5 scored 481/500 (96.2%) and Fable 5.2 scored 96/100 (96.0%) when given the published policy card. The hand-written library scores 500/500 on the same suite. The 500/500 is therefore a procedure ceiling, not evidence of a capability a frontier model lacks. The 5/100 shift result is the more informative number.",
 "heldout_tasks": 500,
 "published_context": {
  "distribution_shift_library": {
   "accepted": 5,
   "n": 100,
   "per_domain": {
    "business_decision": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "data_analysis": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "governance": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "memory_retrieval": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "planning": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "recovery": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "science_calculation": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "security_triage": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    },
    "software_repair": {
     "accepted": 5,
     "n": 10,
     "rate": 0.5
    },
    "tool_orchestration": {
     "accepted": 0,
     "n": 10,
     "rate": 0.0
    }
   },
   "rate": 0.05
  },
  "fixed": {
   "accepted": 112,
   "n": 500,
   "rate": 0.224
  },
  "governed": {
   "accepted": 500,
   "n": 500,
   "rate": 1.0
  },
  "library": {
   "accepted": 500,
   "domains_met": 10,
   "n": 500,
   "rate": 1.0
  },
  "none_floor": {
   "accepted": 72,
   "n": 500,
   "rate": 0.144
  }
 },
 "schema": "cain42.vagi_suite.model_benchmark.v1",
 "suite_seed": 9402
}