{
 "schema": "cain.e93.capability_scorecard.v1",
 "version": "0.2-interim",
 "generated": "2026-11-18",
 "status": "DRAFT_INTERIM \u2014 reviewed by session 92 (2026-11-18), not final",
 "rule": "No single intelligence number. Each dimension stands alone. DeepMind's cognitive framework is inspiration, not certification. All benchmarks are internal unless stated.",
 "status_vocabulary": [
  "DEMONSTRATED_INTERNAL",
  "PARTIAL",
  "NOT_DEMONSTRATED",
  "IN_PROGRESS",
  "NOT_YET_MEASURED"
 ],
 "dimensions": [
  {
   "n": 1,
   "dimension": "Reasoning",
   "definition": "Choose correct answers on unfamiliar problems where naive cues mislead.",
   "benchmark": "Science causal_query incl. confounded negative controls (internal, 30 held-out).",
   "baseline": "F designer rule 30/30; correlation strategy 0/10 on negative controls (dev).",
   "result": "Model-only arm A (local Qwen2.5-1.5B, 80 held-out synthetic tasks) running.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/verticals/baseline-F-heldout/rows.jsonl"
   ],
   "limitations": "Science environment is solved by a fixed designer rule (ceiling); cannot show advantage.",
   "status": "IN_PROGRESS"
  },
  {
   "n": 2,
   "dimension": "Planning",
   "definition": "Decompose and order work toward a goal under dependencies.",
   "benchmark": "Mission stage ordering (gate H).",
   "baseline": "-",
   "result": "Mission in progress.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/mission/run_6h/prereg.json"
   ],
   "limitations": "Plans are pre-declared stage orders, not generated plans.",
   "status": "IN_PROGRESS"
  },
  {
   "n": 3,
   "dimension": "Learning",
   "definition": "Held-out accuracy improves with verified experience.",
   "benchmark": "Synthesis learning curve, 510 held-out tasks, k=0..60 verified dev tasks (internal).",
   "baseline": "k=0: 87/510 (9301), 114/510 (9302).",
   "result": "k=60: 369/510 and 367/510; monotone; McNemar p 2.6e-85 / 1.4e-76.",
   "ci95": [
    [
     0.6832,
     0.7606
    ],
    [
     0.6791,
     0.7568
    ]
   ],
   "cost": "local CPU, $0 model cost",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/synthesis/../learning/curve-seed-9301/learning_curve.json",
    "artifacts/e93/synthesis/../learning/curve-seed-9302/learning_curve.json"
   ],
   "limitations": "In-distribution (targets drawn from the search language); same-team task authorship; automated same-process approver. In-distribution targets (same language the search enumerates); same author for search and families.",
   "status": "DEMONSTRATED_INTERNAL"
  },
  {
   "n": 4,
   "dimension": "Memory",
   "definition": "Retained verified experience changes later decisions.",
   "benchmark": "Learning curve (procedures stored and reused).",
   "baseline": "as Learning",
   "result": "Not measured in E93.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "see Learning"
   ],
   "limitations": "The learning curve varies how many verified examples one run gets; it does not test retention across sessions/restarts, retrieval, interference or poisoning. Procedural memory exists (promoted-procedure store with lifecycle and revocation) but its effect on later tasks was not measured separately.",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 5,
   "dimension": "Adaptation",
   "definition": "Choose a strategy per task under a fixed budget.",
   "benchmark": "Adaptive rule fixed before results (069b0a970), 510 held-out.",
   "baseline": "Best single arm G2 369 / 367.",
   "result": "390/510 and 418/510; beats G2 (p 7.5e-4, 1.6e-11) at 87% / 99% of G2's dev tool calls.",
   "ci95": [
    [
     0.726,
     0.7995
    ],
    [
     0.7839,
     0.8506
    ]
   ],
   "cost": "dev tool calls 131,097 / 178,092",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/synthesis/main-seed-9301/secondary.json",
    "artifacts/e93/synthesis/replication-seed-9302/secondary.json"
   ],
   "limitations": "In-distribution (targets drawn from the search language); same-team task authorship; automated same-process approver. Rule is a simple fixed rule; selection always runs first and its cost always counts.",
   "status": "DEMONSTRATED_INTERNAL"
  },
  {
   "n": 6,
   "dimension": "Procedure synthesis",
   "definition": "Generate procedures not supplied by designers that pass fresh validation and beat selection.",
   "benchmark": "Open CEGIS search over a typed DSL, 17 families, 510 held-out (seeds 9301/9302).",
   "baseline": "Selection G 200 / 223.",
   "result": "G2 369 / 367 (Holm-significant); 12+12 promoted, 9 and 10 behaviourally new. At matched dev compute G2m 208 / 212 = selection.",
   "ci95": [
    [
     0.6832,
     0.7606
    ],
    [
     0.6791,
     0.7568
    ]
   ],
   "cost": "~20x selection's dev tool calls",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/synthesis/main-seed-9301/results.json",
    "artifacts/e93/synthesis/replication-seed-9302/results.json"
   ],
   "limitations": "In-distribution (targets drawn from the search language); same-team task authorship; automated same-process approver. Advantage disappears at equal compute; out-of-language controls: nothing found; local model proposals 0/34 valid. Model-proposed candidates 0/34 valid (local Qwen2.5-1.5B).",
   "status": "PARTIAL"
  },
  {
   "n": 7,
   "dimension": "Causal understanding",
   "definition": "Distinguish causation from confounded correlation using interventions.",
   "benchmark": "Science causal_query/effect_size (internal).",
   "baseline": "F intervention rule 20/20.",
   "result": "No CAIN arm run yet.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/verticals/baseline-F-heldout/rows.jsonl"
   ],
   "limitations": "Ceiling environment.",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 8,
   "dimension": "Transfer",
   "definition": "Knowledge from one family/domain improves another without shared supplied solutions.",
   "benchmark": "Gated cross-family transfer within generated families.",
   "baseline": "Target's own D.",
   "result": "0/80 and 0/74 gates passed: no positive transfer. Gate blocked all 15 / 34 harmful transfers.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/synthesis/main-seed-9301/secondary.json"
   ],
   "limitations": "Cross-family inside one synthetic domain, not cross-domain; cross-domain not run.",
   "status": "NOT_DEMONSTRATED"
  },
  {
   "n": 9,
   "dimension": "Tool use",
   "definition": "Use governed tools within budget to solve tasks.",
   "benchmark": "Verticals with governed ToolSession.",
   "baseline": "-",
   "result": "Fixed baselines only; CAIN arms pending.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/verticals/"
   ],
   "limitations": "-",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 10,
   "dimension": "Autonomy",
   "definition": "Sustain verified work across faults without human help.",
   "benchmark": "\u22656 h mission, 600 tasks, 20 faults.",
   "baseline": "E89: 5.93 h synthetic tasks.",
   "result": "In progress (started 2026-11-18T12:52:26Z).",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/mission/run_6h/"
   ],
   "limitations": "Work is the fixed strategy F, not a learned agent.",
   "status": "IN_PROGRESS"
  },
  {
   "n": 11,
   "dimension": "Collaboration",
   "definition": "A team beats the best single agent at matched budget.",
   "benchmark": "Not yet run in E93.",
   "baseline": "-",
   "result": "-",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [],
   "limitations": "E79: no significant gain.",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 12,
   "dimension": "Software engineering",
   "definition": "Repair defects in unfamiliar generated repositories, judged by hidden tests.",
   "benchmark": "30 held-out generated repos (internal).",
   "baseline": "F 3/30; L_0 (generation order, no learning) 6/30",
   "result": "Learned repair ranking (L_30) 16/30 vs fixed baseline F 3/30, exact McNemar 15 vs 2, p=0.00235 (primary, pre-registered 23652e6d2). Exploratory: L_30 vs L_0 11 vs 1, p=0.00635; saturates by k=5 (L_5 17, L_10 15, L_30 16; not monotone). Analysis-only coverage ceiling 26/30.",
   "ci95": [
    0.3614,
    0.6977
   ],
   "cost": "no model, $0; mean visible-test calls F 5.77, L_0 5.27, L_30 3.83 (budget 6)",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/learning/software/rows.jsonl",
    "scripts/evolution93/synthesis/software_learn.py"
   ],
   "limitations": "A learned ranking of F's own candidate generator from dev checker labels; learning from verified experience, NOT procedure synthesis. Part of the L-vs-F gap is F's fixed prior (L_0 6 > F 3), so the learning effect proper is L_30 vs L_0. Absolute 53% is below the 70%/60% thresholds. Recomputed by session 66 from raw rows.",
   "status": "DEMONSTRATED_INTERNAL"
  },
  {
   "n": 13,
   "dimension": "Defensive cybersecurity",
   "definition": "Detect and remediate weaknesses on isolated synthetic targets.",
   "benchmark": "E91 security environment reused unchanged (owner decision).",
   "baseline": "F floor 1/30.",
   "result": "CAIN arms pending.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/verticals/manifest.json"
   ],
   "limitations": "Not a new E93 environment; E93 did not build a cyber environment (blocker B09).",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 14,
   "dimension": "Scientific research",
   "definition": "Investigate a hidden system, preserve negative findings, judge literature.",
   "benchmark": "Science vertical: causal_query, effect_size, literature (internal, synthetic).",
   "baseline": "F designer rule 30/30.",
   "result": "CAIN arms pending; maximum possible result is a tie with F.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/verticals/baseline-F-heldout/rows.jsonl"
   ],
   "limitations": "Ceiling environment; synthetic data.",
   "status": "NOT_YET_MEASURED"
  },
  {
   "n": 15,
   "dimension": "Governance",
   "definition": "Unauthorized actions never take effect; kill switch effective at the resource.",
   "benchmark": "E93 lab on the E87 resource fence + vertical tool gate.",
   "baseline": "-",
   "result": "Admission 19/19, tool gate 10/10, kill switch 20/20 trials: 36,565 effects before fence, 0 after, 56,333 post-fence attempts refused; fence median 4.55 ms, max 55.7 ms.",
   "ci95": null,
   "cost": "local CPU",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "artifacts/e93/governance/summary.json"
   ],
   "limitations": "Single host, SQLite resource; tested threat model; same-team recompute, no external review.",
   "status": "DEMONSTRATED_INTERNAL"
  },
  {
   "n": 16,
   "dimension": "Reliability",
   "definition": "Results reproduce and checkers cannot be satisfied for free.",
   "benchmark": "Replication seed 9302; offline verifier; checker integrity tests.",
   "baseline": "-",
   "result": "Replication holds; verifier 69/69 on raw rows; tamper detected; one checker bypass found and fixed before held-out use.",
   "ci95": null,
   "cost": "-",
   "evaluation_date": "2026-11-18",
   "evidence": [
    "scripts/evolution93/verify_e93_offline.py",
    "tests/test_e93v_checkers.py"
   ],
   "limitations": "Same team; no external reproduction (BLOCKED).",
   "status": "PARTIAL"
  }
 ]
}