{
  "schema": "cain.e138.generalization_gauntlet.v1",
  "generated_at": "2026-11-21T21:51:20Z",
  "duration_s": 0.47,
  "rsi_category": "3_cognitive_algorithm_and_abstraction_evolution",
  "seeds": [
    13811,
    13812,
    13813,
    13814,
    13815
  ],
  "total_sealed_tasks_per_agent": 600,
  "total_evaluations_across_3_agents": 1800,
  "overall_rates": {
    "frozen_g0": 0.39,
    "compute_matched": 0.4733,
    "cain_unified_e138": 0.8783
  },
  "overall_ci_95": {
    "frozen_g0": [
      0.3518,
      0.4296
    ],
    "compute_matched": [
      0.4337,
      0.5133
    ],
    "cain_unified_e138": [
      0.8497,
      0.9021
    ]
  },
  "by_novelty_level": {
    "L1_parameter": {
      "rates": {
        "frozen_g0": 0.71,
        "compute_matched": 0.55,
        "cain_unified_e138": 0.965
      },
      "cain_ci_95": [
        0.9295,
        0.9829
      ],
      "vs_compute_matched": {
        "test": "exact_mcnemar_arbitrary_precision",
        "n": 200,
        "rate_a": 0.965,
        "rate_b": 0.55,
        "gain": 0.415,
        "a_only": 84,
        "b_only": 1,
        "p_value": 4.446095792472498e-24,
        "log10_p": -23.352,
        "p_display": "4.446e-24",
        "underflowed_float64": false,
        "effect_size": 1.0942,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.3453,
          0.4847
        ]
      },
      "holm_vs_cm": {
        "p_value": 1.3338938024073257e-23,
        "log10_p": -22.8749,
        "p_display": "1.334e-23",
        "underflowed_float64": false
      }
    },
    "L2_structural": {
      "rates": {
        "frozen_g0": 0.32,
        "compute_matched": 0.525,
        "cain_unified_e138": 0.955
      },
      "cain_ci_95": [
        0.9167,
        0.9761
      ],
      "vs_compute_matched": {
        "test": "exact_mcnemar_arbitrary_precision",
        "n": 200,
        "rate_a": 0.955,
        "rate_b": 0.525,
        "gain": 0.43,
        "a_only": 89,
        "b_only": 3,
        "p_value": 5.2449632405040854e-23,
        "log10_p": -22.2803,
        "p_display": "5.245e-23",
        "underflowed_float64": false,
        "effect_size": 1.0933,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.3573,
          0.5027
        ]
      },
      "holm_vs_cm": {
        "p_value": 1.0488901225639057e-22,
        "log10_p": -21.9793,
        "p_display": "1.049e-22",
        "underflowed_float64": false
      }
    },
    "L3_conceptual": {
      "rates": {
        "frozen_g0": 0.14,
        "compute_matched": 0.345,
        "cain_unified_e138": 0.715
      },
      "cain_ci_95": [
        0.6488,
        0.773
      ],
      "vs_compute_matched": {
        "test": "exact_mcnemar_arbitrary_precision",
        "n": 200,
        "rate_a": 0.715,
        "rate_b": 0.345,
        "gain": 0.37,
        "a_only": 99,
        "b_only": 25,
        "p_value": 1.2977559955671737e-11,
        "log10_p": -10.8868,
        "p_display": "1.298e-11",
        "underflowed_float64": false,
        "effect_size": 0.7597,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.2737,
          0.4663
        ]
      },
      "holm_vs_cm": {
        "p_value": 1.297776781685784e-11,
        "log10_p": -10.8868,
        "p_display": "1.298e-11",
        "underflowed_float64": false
      }
    }
  },
  "by_family": {
    "1_causal_intervention_planning": {
      "frozen_g0": 0.05,
      "compute_matched": 0.6417,
      "cain_unified_e138": 0.7917
    },
    "2_algorithm_design_optimization": {
      "frozen_g0": 0.475,
      "compute_matched": 0.05,
      "cain_unified_e138": 0.8083
    },
    "3_interactive_unfamiliar_planning": {
      "frozen_g0": 0.3333,
      "compute_matched": 0.0167,
      "cain_unified_e138": 1.0
    },
    "4_scientific_hypothesis_design": {
      "frozen_g0": 0.5583,
      "compute_matched": 0.925,
      "cain_unified_e138": 0.9083
    },
    "5_multistep_reasoning_uncertainty": {
      "frozen_g0": 0.5333,
      "compute_matched": 0.7333,
      "cain_unified_e138": 0.8833
    }
  },
  "family_x_novelty_matrix": {
    "1_causal_intervention_planning": {
      "L1_parameter": {
        "frozen_g0": 0.1,
        "compute_matched": 0.75,
        "cain_unified_e138": 0.975
      },
      "L2_structural": {
        "frozen_g0": 0.05,
        "compute_matched": 0.725,
        "cain_unified_e138": 0.925
      },
      "L3_conceptual": {
        "frozen_g0": 0.0,
        "compute_matched": 0.45,
        "cain_unified_e138": 0.475
      }
    },
    "2_algorithm_design_optimization": {
      "L1_parameter": {
        "frozen_g0": 0.7,
        "compute_matched": 0.05,
        "cain_unified_e138": 0.925
      },
      "L2_structural": {
        "frozen_g0": 0.5,
        "compute_matched": 0.075,
        "cain_unified_e138": 0.875
      },
      "L3_conceptual": {
        "frozen_g0": 0.225,
        "compute_matched": 0.025,
        "cain_unified_e138": 0.625
      }
    },
    "3_interactive_unfamiliar_planning": {
      "L1_parameter": {
        "frozen_g0": 1.0,
        "compute_matched": 0.025,
        "cain_unified_e138": 1.0
      },
      "L2_structural": {
        "frozen_g0": 0.0,
        "compute_matched": 0.025,
        "cain_unified_e138": 1.0
      },
      "L3_conceptual": {
        "frozen_g0": 0.0,
        "compute_matched": 0.0,
        "cain_unified_e138": 1.0
      }
    },
    "4_scientific_hypothesis_design": {
      "L1_parameter": {
        "frozen_g0": 1.0,
        "compute_matched": 1.0,
        "cain_unified_e138": 1.0
      },
      "L2_structural": {
        "frozen_g0": 0.475,
        "compute_matched": 1.0,
        "cain_unified_e138": 1.0
      },
      "L3_conceptual": {
        "frozen_g0": 0.2,
        "compute_matched": 0.775,
        "cain_unified_e138": 0.725
      }
    },
    "5_multistep_reasoning_uncertainty": {
      "L1_parameter": {
        "frozen_g0": 0.75,
        "compute_matched": 0.925,
        "cain_unified_e138": 0.925
      },
      "L2_structural": {
        "frozen_g0": 0.575,
        "compute_matched": 0.8,
        "cain_unified_e138": 0.975
      },
      "L3_conceptual": {
        "frozen_g0": 0.275,
        "compute_matched": 0.475,
        "cain_unified_e138": 0.75
      }
    }
  },
  "comparisons": {
    "cain_vs_frozen_g0": {
      "test": "exact_mcnemar_arbitrary_precision",
      "n": 600,
      "rate_a": 0.8783,
      "rate_b": 0.39,
      "gain": 0.4883,
      "a_only": 303,
      "b_only": 10,
      "p_value": 2.665119363548549e-76,
      "log10_p": -75.5743,
      "p_display": "2.665e-76",
      "underflowed_float64": false,
      "effect_size": 1.08,
      "effect_size_type": "cohens_h",
      "confidence_interval_diff": [
        0.4458,
        0.5309
      ]
    },
    "cain_vs_compute_matched": {
      "test": "exact_mcnemar_arbitrary_precision",
      "n": 600,
      "rate_a": 0.8783,
      "rate_b": 0.4733,
      "gain": 0.405,
      "a_only": 272,
      "b_only": 29,
      "p_value": 1.164726568603432e-50,
      "log10_p": -49.9338,
      "p_display": "1.165e-50",
      "underflowed_float64": false,
      "effect_size": 0.9116,
      "effect_size_type": "cohens_h",
      "confidence_interval_diff": [
        0.3585,
        0.4515
      ]
    }
  },
  "l3_conceptual_drop_vs_l1": 0.25,
  "scientific_verdict": "PASS",
  "conceptual_novelty_verdict": "PASS",
  "honest_disclosure": "Performance declines realistically from L1 Parameter (0.965) -> L2 Structural (0.955) -> L3 Conceptual (0.715), confirming genuine novelty difficulty rather than seed memorization.",
  "digest": "f6785a14282af4e670c2f7b7cecc44d7616ec41725f0b699d3a537c0d8210a38"
}