{
  "base_commit": "2d444e70746447f3b8c70ff089e9ce0eb5f22109",
  "claims": {
    "arms": {
      "A": {
        "accepted": 26,
        "n": 120,
        "rate": 0.216667
      },
      "B": {
        "accepted": 26,
        "n": 120,
        "rate": 0.216667
      },
      "C": {
        "accepted": 22,
        "n": 120,
        "rate": 0.183333
      },
      "D": {
        "accepted": 96,
        "n": 120,
        "rate": 0.8
      },
      "E": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      },
      "F": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      },
      "G": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      },
      "G0": {
        "accepted": 26,
        "n": 120,
        "rate": 0.216667
      },
      "H": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      },
      "I": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      },
      "J": {
        "accepted": 120,
        "n": 120,
        "rate": 1.0
      }
    },
    "collaboration": {
      "F": {
        "accepted": 120,
        "ci95": [
          0.968981,
          1
        ],
        "n": 120,
        "rate": 1.0
      },
      "I": {
        "accepted": 120,
        "ci95": [
          0.968981,
          1
        ],
        "n": 120,
        "rate": 1.0
      },
      "abstentions_I": 0,
      "false_accepts_I": 0
    },
    "cost_usd": 0.0,
    "heldout_tasks": 120,
    "independent_validation": false,
    "learning": {
      "E_memory": {
        "accepted": 120,
        "ci95": [
          0.968981,
          1
        ],
        "n": 120,
        "rate": 1.0
      },
      "F_planning": {
        "accepted": 120,
        "ci95": [
          0.968981,
          1
        ],
        "n": 120,
        "rate": 1.0
      },
      "G_after": {
        "accepted": 120,
        "ci95": [
          0.968981,
          1
        ],
        "n": 120,
        "rate": 1.0
      },
      "G_before": {
        "accepted": 26,
        "ci95": [
          0.152367,
          0.298544
        ],
        "n": 120,
        "rate": 0.216667
      }
    },
    "model_calls": 0,
    "negative_controls": {
      "logged_ab": {
        "accepted": 5,
        "ci95": [
          0.565518,
          1
        ],
        "n": 5,
        "rate": 1.0
      },
      "skewed_prior_diagnosis": {
        "accepted": 7,
        "ci95": [
          0.64567,
          1
        ],
        "n": 7,
        "rate": 1.0
      }
    },
    "overall_status": "PARTIAL",
    "paired": {
      "A_vs_D": {
        "b": 3,
        "c": 73,
        "control": "A",
        "control_accepted": 26,
        "difference_pp": 58.333,
        "p_value": 0.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "D",
        "treatment_accepted": 96
      },
      "A_vs_E": {
        "b": 0,
        "c": 94,
        "control": "A",
        "control_accepted": 26,
        "difference_pp": 78.333,
        "p_value": 0.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "E",
        "treatment_accepted": 120
      },
      "D_vs_F": {
        "b": 0,
        "c": 24,
        "control": "D",
        "control_accepted": 96,
        "difference_pp": 20.0,
        "p_value": 1.2e-07,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "F",
        "treatment_accepted": 120
      },
      "E_vs_G": {
        "b": 0,
        "c": 0,
        "control": "E",
        "control_accepted": 120,
        "difference_pp": 0.0,
        "p_value": 1.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "G",
        "treatment_accepted": 120
      },
      "F_vs_I": {
        "b": 0,
        "c": 0,
        "control": "F",
        "control_accepted": 120,
        "difference_pp": 0.0,
        "p_value": 1.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "I",
        "treatment_accepted": 120
      },
      "F_vs_J": {
        "b": 0,
        "c": 0,
        "control": "F",
        "control_accepted": 120,
        "difference_pp": 0.0,
        "p_value": 1.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "J",
        "treatment_accepted": 120
      },
      "G0_vs_G": {
        "b": 0,
        "c": 94,
        "control": "G0",
        "control_accepted": 26,
        "difference_pp": 78.333,
        "p_value": 0.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "G",
        "treatment_accepted": 120
      },
      "H_vs_J": {
        "b": 0,
        "c": 0,
        "control": "H",
        "control_accepted": 120,
        "difference_pp": 0.0,
        "p_value": 1.0,
        "pairs": 120,
        "test": "mcnemar_exact",
        "treatment": "J",
        "treatment_accepted": 120
      },
      "transfer_A_vs_F": {
        "b": 0,
        "c": 36,
        "control": "A",
        "control_accepted": 4,
        "difference_pp": 90.0,
        "p_value": 0.0,
        "pairs": 40,
        "test": "mcnemar_exact",
        "treatment": "F",
        "treatment_accepted": 40
      },
      "transfer_A_vs_G_source": {
        "b": 0,
        "c": 36,
        "control": "A",
        "control_accepted": 4,
        "difference_pp": 90.0,
        "p_value": 0.0,
        "pairs": 40,
        "test": "mcnemar_exact",
        "treatment": "G_source",
        "treatment_accepted": 40
      }
    },
    "production": false,
    "tasks_per_vertical": {
      "business": 20,
      "education": 20,
      "engineering": 20,
      "science": 20,
      "security": 20,
      "software": 20
    },
    "transfer": {
      "A": {
        "accepted": 4,
        "ci95": [
          0.03958,
          0.230518
        ],
        "n": 40,
        "rate": 0.1
      },
      "C": {
        "accepted": 0,
        "ci95": [
          0,
          0.087622
        ],
        "n": 40,
        "rate": 0.0
      },
      "D": {
        "accepted": 34,
        "ci95": [
          0.709277,
          0.929388
        ],
        "n": 40,
        "rate": 0.85
      },
      "E_source": {
        "accepted": 4,
        "ci95": [
          0.03958,
          0.230518
        ],
        "n": 40,
        "rate": 0.1
      },
      "F": {
        "accepted": 40,
        "ci95": [
          0.912378,
          1
        ],
        "n": 40,
        "rate": 1.0
      },
      "G_source": {
        "accepted": 40,
        "ci95": [
          0.912378,
          1
        ],
        "n": 40,
        "rate": 1.0
      }
    }
  },
  "created_at": "2026-11-18T04:58:23Z",
  "evolution": "E91",
  "files": [
    {
      "path": "artifacts/e91/eval/results.json",
      "sha256": "490beb422528f2f11fbc150a6755b36968599e2ccc5230313b3c574560b21739",
      "size": 18043
    },
    {
      "path": "artifacts/e91/eval/rows.jsonl",
      "sha256": "17238ed756e10988793ad4d1aa37ca3ae08334ce5bba9ac14a6f513dcfcabedb",
      "size": 609377
    },
    {
      "path": "benchmarks/e91/eval/BASELINE_RESULTS.json",
      "sha256": "9f1840c8945d4ab92d6d8fb292f2752f94c7604a693a66880099f92124dc6069",
      "size": 6306
    },
    {
      "path": "benchmarks/e91/eval/EXPERIMENT_PROTOCOL.md",
      "sha256": "d39572923bfbbe5c82db5bd6f4a83352dea402d8f2892ade9ceb4f2315af4d0b",
      "size": 3807
    },
    {
      "path": "benchmarks/e91/eval/HELD_OUT_TASK_MANIFEST.json",
      "sha256": "12525e1311b523c563aa673cb7a30db578dcf488e5977adbff50e05bb6455979",
      "size": 316817
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/__init__.py",
      "sha256": "67197f5365b44c7b953b29c15b4a478d6a8ef0d19f0e0702f69155c155b5d480",
      "size": 371
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/schema.py",
      "sha256": "a8135f289a49f15c3710cbd66f90dfb6155b345221ec9359fe62e7453612b71b",
      "size": 3421
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/__init__.py",
      "sha256": "996f81e289df777fae8c0a4d06d0451539c4700e4ae58d771f63a60fd2853f35",
      "size": 3829
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/business.py",
      "sha256": "4e0021a6a8116df9caccb99e82894324ebb9ec7a130983a1e75c931e5df9c2ad",
      "size": 10598
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/education.py",
      "sha256": "4596cd0b5e59029bbcb285f2f42b9e1abfbe8056236acb4276cd23d42277be99",
      "size": 6601
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/engineering.py",
      "sha256": "b1c6a922a96dc9064161c32f38b0b822926307aa68eb71ab173b85c3549af45c",
      "size": 6987
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/science.py",
      "sha256": "2e17afde858d792d5c8fc35dbd9c7630dc5a17f9322da46fca888b06dff6daf0",
      "size": 8429
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/security.py",
      "sha256": "91e41946336b6840622fc3a21909d7fb50859df3d79fdc0c5e98488c1a84aed8",
      "size": 7194
    },
    {
      "path": "benchmarks/e91/eval/_vertical_snapshot/verticals/software.py",
      "sha256": "c6d4fabff83b3af8d788051ee85681a367eef02e4c6844fcae37d491ffe3edc7",
      "size": 11395
    },
    {
      "path": "benchmarks/e91/eval/common.py",
      "sha256": "5aa76611862a30ef4d9e116a5fa27cfcd5f0744e13ae2eea265a76c251b9cccf",
      "size": 364
    },
    {
      "path": "benchmarks/e91/eval/harness.py",
      "sha256": "417d0dee799014c57a562da2063c5b6098cd5f14d48a864897e453cdd42eb75c",
      "size": 18117
    },
    {
      "path": "benchmarks/e91/eval/solvers.py",
      "sha256": "946e43556da972e584e8a32402b6f30d98210596836c990e566d7d32f41a0a4e",
      "size": 13972
    },
    {
      "path": "docs/evolution-91-eval/ACCEPTANCE.json",
      "sha256": "2167e1c43b003c86cd165cf9ebecf0ba033659f12aa6e1e1e9219217aa63386f",
      "size": 2211
    },
    {
      "path": "docs/evolution-91-eval/E91_EVAL_REPORT.md",
      "sha256": "6f5f56fe9218c4b86e2cf490f880bc54f993a0831cf3e8ba3293354e24889f66",
      "size": 7989
    },
    {
      "path": "scripts/evolution91/eval/verify_e91_eval_offline.py",
      "sha256": "6e79bcbbe91b85371650b4d9c9a16aa2443c46c23e1fa8e6a369daf60fbf37fb",
      "size": 4216
    },
    {
      "path": "tests/test_e91_eval.py",
      "sha256": "f32daf6f51e7a9c9b44ad339fd5dcf58cc9363d06aaf4f29165cec50abf9c8a1",
      "size": 2604
    }
  ],
  "independent_witness": false,
  "integration_head": "8df7eb289 (evolution-91-integration)",
  "limits": [
    "deterministic hand-written expert policies, not an LLM; this measures the memory/planning/governance architecture",
    "120 constructed held-out tasks across six verticals, not real customer codebases or real science",
    "the vertical generators and checkers are written by the project, not an external benchmark",
    "no remote model, no paid service; cost is $0",
    "some verticals are simulated (science response surface, engineering digital simulation)",
    "negative-transfer controls are included but a null result is reported honestly",
    "the six-vertical suite is saturated by a hand-written expert",
    "this is a local deterministic evaluation, not an external benchmark or customer deployment"
  ],
  "manifest_task_count": 120,
  "overall_status": "PARTIAL",
  "schema": "cain.e91.eval.public.bundle.v1",
  "signature": "91yTTxC6AwjsV6q/ovuGTU/kqjL5f2WJsiLcnjsfKbKRYr8YNIRpg/nJNIDe8UGHiQshJ3wYGfJ+KylsBFadDA==",
  "signature_class": "self_generated_key",
  "slice": "evaluation-harness"
}