{
  "baselines": {
    "A": {
      "abstentions": 0,
      "accepted": 26,
      "ci95": [
        0.152367,
        0.298544
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.008,
      "n": 120,
      "rate": 0.216667,
      "tool_calls": 0
    },
    "B": {
      "abstentions": 0,
      "accepted": 26,
      "ci95": [
        0.152367,
        0.298544
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.007,
      "n": 120,
      "rate": 0.216667,
      "tool_calls": 0
    },
    "C": {
      "abstentions": 0,
      "accepted": 22,
      "ci95": [
        0.124303,
        0.262009
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.695,
      "n": 120,
      "rate": 0.183333,
      "tool_calls": 0
    },
    "D": {
      "abstentions": 0,
      "accepted": 96,
      "ci95": [
        0.719633,
        0.861755
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.476,
      "n": 120,
      "rate": 0.8,
      "tool_calls": 78
    }
  },
  "full_arms": {
    "E": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.507,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 155
    },
    "F": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.488,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 155
    },
    "G": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.464,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 155
    },
    "H": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.535,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 155
    },
    "I": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 1.0,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 233
    },
    "J": {
      "abstentions": 0,
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "false_accepts": 0,
      "mean_latency_ms": 0.487,
      "n": 120,
      "rate": 1.0,
      "tool_calls": 155
    }
  },
  "heldout_tasks": 120,
  "learning": {
    "E_memory": {
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "n": 120,
      "rate": 1.0
    },
    "F_planning": {
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "n": 120,
      "rate": 1.0
    },
    "G_after": {
      "accepted": 120,
      "ci95": [
        0.968981,
        1
      ],
      "n": 120,
      "rate": 1.0
    },
    "G_before": {
      "accepted": 26,
      "ci95": [
        0.152367,
        0.298544
      ],
      "n": 120,
      "rate": 0.216667
    }
  },
  "limits": [
    "deterministic hand-written expert policies, not an LLM; this measures the memory/planning/governance architecture",
    "120 constructed held-out tasks across six verticals, not real customer codebases or real science",
    "the vertical generators and checkers are written by the project, not an external benchmark",
    "no remote model, no paid service; cost is $0",
    "some verticals are simulated (science response surface, engineering digital simulation)",
    "negative-transfer controls are included but a null result is reported honestly"
  ],
  "paired": {
    "A_vs_D": {
      "b": 3,
      "c": 73,
      "control": "A",
      "control_accepted": 26,
      "difference_pp": 58.333,
      "p_value": 0.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "D",
      "treatment_accepted": 96
    },
    "A_vs_E": {
      "b": 0,
      "c": 94,
      "control": "A",
      "control_accepted": 26,
      "difference_pp": 78.333,
      "p_value": 0.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "E",
      "treatment_accepted": 120
    },
    "D_vs_F": {
      "b": 0,
      "c": 24,
      "control": "D",
      "control_accepted": 96,
      "difference_pp": 20.0,
      "p_value": 1.2e-07,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "F",
      "treatment_accepted": 120
    },
    "E_vs_G": {
      "b": 0,
      "c": 0,
      "control": "E",
      "control_accepted": 120,
      "difference_pp": 0.0,
      "p_value": 1.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "G",
      "treatment_accepted": 120
    },
    "F_vs_I": {
      "b": 0,
      "c": 0,
      "control": "F",
      "control_accepted": 120,
      "difference_pp": 0.0,
      "p_value": 1.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "I",
      "treatment_accepted": 120
    },
    "F_vs_J": {
      "b": 0,
      "c": 0,
      "control": "F",
      "control_accepted": 120,
      "difference_pp": 0.0,
      "p_value": 1.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "J",
      "treatment_accepted": 120
    },
    "G0_vs_G": {
      "b": 0,
      "c": 94,
      "control": "G0",
      "control_accepted": 26,
      "difference_pp": 78.333,
      "p_value": 0.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "G",
      "treatment_accepted": 120
    },
    "H_vs_J": {
      "b": 0,
      "c": 0,
      "control": "H",
      "control_accepted": 120,
      "difference_pp": 0.0,
      "p_value": 1.0,
      "pairs": 120,
      "test": "mcnemar_exact",
      "treatment": "J",
      "treatment_accepted": 120
    },
    "transfer_A_vs_F": {
      "b": 0,
      "c": 36,
      "control": "A",
      "control_accepted": 4,
      "difference_pp": 90.0,
      "p_value": 0.0,
      "pairs": 40,
      "test": "mcnemar_exact",
      "treatment": "F",
      "treatment_accepted": 40
    },
    "transfer_A_vs_G_source": {
      "b": 0,
      "c": 36,
      "control": "A",
      "control_accepted": 4,
      "difference_pp": 90.0,
      "p_value": 0.0,
      "pairs": 40,
      "test": "mcnemar_exact",
      "treatment": "G_source",
      "treatment_accepted": 40
    }
  },
  "schema": "cain.e91.eval.baseline.v1"
}