{
  "schema": "cain.e138.replication_audit.v1",
  "generated_at": "2026-11-21T21:51:19Z",
  "duration_s": 38.75,
  "rsi_category": "2_workflow_and_research_process_evolution",
  "source_commit_audited": "15bc7d25f",
  "total_replication_evaluations": 540,
  "zero_p_value_bugs_found_and_corrected": [
    {
      "file": "causal_apex.json",
      "field": "holm_adjusted_p.vs_random_naive",
      "original_reported": 0.0,
      "recomputed_exact_display": "4.086e-25",
      "recomputed_log10_p": -24.3887
    },
    {
      "file": "causal_apex.json",
      "field": "holm_adjusted_p.vs_e128_eig_div_cost",
      "original_reported": 0.0,
      "recomputed_exact_display": "2.175e-26",
      "recomputed_log10_p": -25.6626
    },
    {
      "file": "unification_3domain.json",
      "field": "holm_adjusted_p.D_vs_A",
      "original_reported": 0.0,
      "recomputed_exact_display": "2.928e-55",
      "recomputed_log10_p": -54.5335
    },
    {
      "file": "unification_3domain.json",
      "field": "holm_adjusted_p.D_vs_B",
      "original_reported": 0.0,
      "recomputed_exact_display": "4.950e-28",
      "recomputed_log10_p": -27.3054
    },
    {
      "file": "unification_3domain.json",
      "field": "holm_adjusted_p.D_vs_C",
      "original_reported": 0.0,
      "recomputed_exact_display": "3.415e-35",
      "recomputed_log10_p": -34.4667
    }
  ],
  "preregistration_deviations": [
    {
      "breakthrough": "2_code_rsi",
      "planned": "AST-verified sandboxed Python Code-RSI using local qwen2.5-coder:3b matches or improves sealed edge-case pass rate over zero-shot & blind best-of-2 while blocking 100% of AST escape payloads.",
      "executed": "Fell back to qwen2.5-coder:1.5b (CPU timeout on 3b); scored 0.500 (4/8) < 0.625 target -> honestly PARTIAL.",
      "severity": "DISCLOSED_IN_ORIGINAL_REPORT"
    },
    {
      "breakthrough": "5_neuro_symbolic",
      "planned": "CAIN Neuro-Symbolic Hybrid (3B neural + symbolic verification + calibrated abstention) outperforms or matches raw 3B zero-shot with 100% OOD abstention and Brier <= 0.15.",
      "executed": "BudgetTracker wall-time cap (900.0s) triggered right after Code-RSI (921.08s), so ALL 8 raw LLM calls returned 0 tokens / raw_3b_acc=0.0. Raw-vs-hybrid LLM contrast in original APEX is NOT valid as a neural comparison; only symbolic accuracy (1.0) and OOD abstention (1.0) executed.",
      "severity": "CRITICAL_METHODOLOGICAL_DEVIATION_FLAGGED_BY_E138"
    },
    {
      "breakthrough": "1_causal_and_4_unification",
      "planned": "Exact Holm-Bonferroni & BH FDR p-values",
      "executed": "stats.py applied round(p, 12), rounding extreme tail probabilities down to exact 0.0. E138 recomputes exact arbitrary-precision log10(p) and never emits 0.0.",
      "severity": "STATISTICAL_REPORTING_DEFECT_CORRECTED_BY_E138"
    },
    {
      "breakthrough": "3_multigen_g0_g5",
      "planned": "Monotonic generalization improvement G0->G5",
      "executed": "G3/G4 reached 0.9167 unseen while G5 dropped slightly to 0.875 unseen (while sealed rose 0.8333->0.9167), showing mild late-generation overfitting.",
      "severity": "MINOR_OVERFITTING_SIGNAL_INVESTIGATED_IN_E138_G0_G10"
    }
  ],
  "breakthroughs": {
    "1_causal_bayesian_mixture": {
      "original_reported": {
        "passive": 0.8787,
        "random_naive": 0.7709,
        "e128_eig_div_cost": 0.7832,
        "random_mixture": 0.4632,
        "apex_bayesian_mixture_a_optimal": 0.4119
      },
      "independently_recomputed": {
        "passive": 0.8787,
        "random_naive": 0.7709,
        "e128_eig_div_cost": 0.7832,
        "random_mixture": 0.4632,
        "apex_bayesian_mixture_a_optimal": 0.4119
      },
      "fresh_seed_replication": {
        "passive": 0.8108,
        "random_naive": 0.7804,
        "e128_eig_div_cost": 0.78,
        "random_mixture": 0.4678,
        "apex_bayesian_mixture_a_optimal": 0.3834
      },
      "exact_reconstruction_match": true,
      "fresh_seed_vs_matched_random_holm": {
        "p_value": 0.0023163276959874465,
        "log10_p": -2.6352,
        "p_display": "0.002316",
        "underflowed_float64": false
      },
      "fresh_seed_effect_size_d": 0.2927,
      "cluster_seed_neff": {
        "n_observations": 200,
        "n_clusters": 5,
        "mean_cluster_size": 40.0,
        "icc": 0.0,
        "design_effect": 1.0,
        "n_effective": 200.0
      },
      "cluster_seed_paired_t_vs_mix": {
        "test": "paired_cluster_t_test",
        "n_clusters": 5,
        "mean_diff": 0.0513,
        "t_stat": 3.5879,
        "df": 4,
        "effect_size_cohens_d": 1.6046,
        "ci_95": [
          0.0116,
          0.091
        ],
        "p_value": 0.0230040758045941,
        "log10_p": -1.6382,
        "p_display": "0.023004",
        "underflowed_float64": false
      },
      "differences": "Original reported exact p=0.0 for naive/E128 due to round(p,12); exact log10(p) recomputed. Fresh-seed 100-world replication confirms advantage over random_mixture.",
      "limitations": "Assumes binary hidden confounder U in {0,1} and discrete W_GRID; stress-tested under misspecification in Section 6.",
      "verdict": "PASS"
    },
    "2_sandboxed_python_code_rsi": {
      "original_reported": {
        "g0": 0.375,
        "cm": 0.375,
        "unified": 0.5,
        "ast_blocked": "4/4"
      },
      "independently_recomputed": {
        "g0": 0.375,
        "cm": 0.375,
        "unified": 0.5,
        "p_vs_g0_display": "1.0",
        "ast_blocked": "4/4"
      },
      "replication_status": "Exact raw-record reconstruction verified; extended to 40-task ceiling benchmark in Section 5.",
      "differences": "Model fell back from preregistered 3b to 1.5b; small n=8 yields McNemar p=1.0 (not statistically significant).",
      "limitations": "n=8 tasks underpowered; below 0.625 preregistered threshold.",
      "verdict": "PARTIAL"
    },
    "3_multigen_g0_to_g5": {
      "original_reported": {
        "g0": 0.25,
        "cm2x": 0.25,
        "g5": 0.875
      },
      "independently_recomputed": {
        "g0": 0.25,
        "cm2x": 0.25,
        "g5": 0.875,
        "p_display": "6.104e-05",
        "cohens_h": 1.3717
      },
      "fresh_seed_replication": {
        "seed": 13809,
        "g0": 0.25,
        "g5": 0.8333,
        "p_display": "1.221e-04",
        "cohens_h": 1.2533
      },
      "differences": "Exact reconstruction matches 0.25 -> 0.875 on seed 13012; G5 unseen (0.875) slightly below G3/G4 (0.9167).",
      "limitations": "Bounded DSL macro synthesis; extended to G0-G10 branching DAG in Section 7.",
      "verdict": "PASS"
    },
    "4_unification_3domain_full_compute": {
      "original_reported": {
        "A": 0.14,
        "B": 0.3667,
        "C": 0.3433,
        "D": 0.8133
      },
      "independently_recomputed_from_1200_raw_rows": {
        "A": 0.14,
        "B": 0.3667,
        "C": 0.3433,
        "D": 0.8133
      },
      "exact_mcnemar_rigor": {
        "D_vs_A": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 300,
          "rate_a": 0.8133,
          "rate_b": 0.14,
          "gain": 0.6733,
          "a_only": 206,
          "b_only": 4,
          "p_value": 9.757777411530882e-56,
          "log10_p": -55.0106,
          "p_display": "9.758e-56",
          "underflowed_float64": false,
          "effect_size": 1.4811,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.6171,
            0.7295
          ]
        },
        "D_vs_B": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 300,
          "rate_a": 0.8133,
          "rate_b": 0.3667,
          "gain": 0.4467,
          "a_only": 151,
          "b_only": 17,
          "p_value": 4.949592375030885e-28,
          "log10_p": -27.3054,
          "p_display": "4.950e-28",
          "underflowed_float64": false,
          "effect_size": 0.9472,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.3787,
            0.5146
          ]
        },
        "D_vs_C": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 300,
          "rate_a": 0.8133,
          "rate_b": 0.3433,
          "gain": 0.47,
          "a_only": 148,
          "b_only": 7,
          "p_value": 1.70740140229607e-35,
          "log10_p": -34.7677,
          "p_display": "1.707e-35",
          "underflowed_float64": false,
          "effect_size": 0.996,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.4085,
            0.5315
          ]
        }
      },
      "holm_rigor": {
        "D_vs_A": {
          "p_value": 2.9276641500908906e-55,
          "log10_p": -54.5335,
          "p_display": "2.928e-55",
          "underflowed_float64": false
        },
        "D_vs_C": {
          "p_value": 3.414522632212301e-35,
          "log10_p": -34.4667,
          "p_display": "3.415e-35",
          "underflowed_float64": false
        },
        "D_vs_B": {
          "p_value": 4.949940743969364e-28,
          "log10_p": -27.3054,
          "p_display": "4.950e-28",
          "underflowed_float64": false
        }
      },
      "clustering_audit": {
        "headline_evaluations": 1200,
        "paired_tasks_per_arm": 300,
        "independent_seed_clusters": 5,
        "domain_clusters": 3,
        "seed_cluster_neff_arm_d": {
          "n_observations": 300,
          "n_clusters": 5,
          "mean_cluster_size": 60.0,
          "icc": 0.0,
          "design_effect": 1.0,
          "n_effective": 300.0
        },
        "domain_cluster_neff_arm_d": {
          "n_observations": 300,
          "n_clusters": 3,
          "mean_cluster_size": 100.0,
          "icc": 0.0437,
          "design_effect": 5.3267,
          "n_effective": 56.32
        },
        "seed_level_paired_t_tests": {
          "D_vs_A": {
            "test": "paired_cluster_t_test",
            "n_clusters": 5,
            "mean_diff": 0.6733,
            "t_stat": 49.729,
            "df": 4,
            "effect_size_cohens_d": 22.2395,
            "ci_95": [
              0.6357,
              0.7109
            ],
            "p_value": 9.784616088988554e-07,
            "log10_p": -6.0095,
            "p_display": "9.785e-07",
            "underflowed_float64": false
          },
          "D_vs_B": {
            "test": "paired_cluster_t_test",
            "n_clusters": 5,
            "mean_diff": 0.4467,
            "t_stat": 36.4702,
            "df": 4,
            "effect_size_cohens_d": 16.31,
            "ci_95": [
              0.4127,
              0.4807
            ],
            "p_value": 3.374628197988746e-06,
            "log10_p": -5.4718,
            "p_display": "3.375e-06",
            "underflowed_float64": false
          },
          "D_vs_C": {
            "test": "paired_cluster_t_test",
            "n_clusters": 5,
            "mean_diff": 0.47,
            "t_stat": 17.6943,
            "df": 4,
            "effect_size_cohens_d": 7.9131,
            "ci_95": [
              0.3963,
              0.5437
            ],
            "p_value": 5.992814077806199e-05,
            "log10_p": -4.2224,
            "p_display": "5.993e-05",
            "underflowed_float64": false
          }
        }
      },
      "fresh_seed_120eval_replication": {
        "n_evals": 120,
        "rates": {
          "A": 0.2,
          "B": 0.3,
          "C": 0.3333,
          "D": 0.8
        },
        "D_vs_A": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 30,
          "rate_a": 0.8,
          "rate_b": 0.2,
          "gain": 0.6,
          "a_only": 20,
          "b_only": 2,
          "p_value": 0.00012111663818359366,
          "log10_p": -3.9168,
          "p_display": "1.211e-04",
          "underflowed_float64": false,
          "effect_size": 1.287,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.3814,
            0.8186
          ]
        },
        "D_vs_B": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 30,
          "rate_a": 0.8,
          "rate_b": 0.3,
          "gain": 0.5,
          "a_only": 17,
          "b_only": 2,
          "p_value": 0.0007286071777343749,
          "log10_p": -3.1375,
          "p_display": "7.286e-04",
          "underflowed_float64": false,
          "effect_size": 1.055,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.2784,
            0.7216
          ]
        },
        "D_vs_C": {
          "test": "exact_mcnemar_arbitrary_precision",
          "n": 30,
          "rate_a": 0.8,
          "rate_b": 0.3333,
          "gain": 0.4667,
          "a_only": 14,
          "b_only": 0,
          "p_value": 0.00012207031250000003,
          "log10_p": -3.9134,
          "p_display": "1.221e-04",
          "underflowed_float64": false,
          "effect_size": 0.9833,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.2881,
            0.6452
          ]
        }
      },
      "differences": "Original reported exact Holm p=0.0; exact log10(p) recomputed. Effective N under domain clustering is much smaller than headline 1,200, yet seed-level paired t-tests (df=4) still confirm D > A, B, C (p < 0.005).",
      "limitations": "Same evolved G5 library reused across all 5 seeds; constructed 3-domain suite.",
      "verdict": "PASS"
    },
    "5_neuro_symbolic_and_immune": {
      "original_reported": {
        "raw_3b_acc": 0.0,
        "hybrid_acc": 1.0,
        "ood_abstain": 1.0,
        "brier": 0.0144,
        "immune": "10/10"
      },
      "independently_recomputed": {
        "raw_llm_calls_completed": 0,
        "budget_halted_raw_arm": true,
        "symbolic_rows_verified": 8
      },
      "differences": "E138 downgrades original PASS to PARTIAL because BudgetTracker halted raw LLM calls at 900s wall-time (0 tokens), invalidating the raw-vs-hybrid LLM contrast! Re-run cleanly inside budget in Section 9.",
      "limitations": "Only 12 symbolic tasks in APEX; budget cap prevented valid raw LLM comparison.",
      "verdict": "PARTIAL"
    }
  },
  "summary_verdicts": {
    "1_causal_bayesian_mixture": "PASS",
    "2_sandboxed_python_code_rsi": "PARTIAL",
    "3_multigen_g0_to_g5": "PASS",
    "4_unification_3domain_full_compute": "PASS",
    "5_neuro_symbolic_and_immune": "PARTIAL"
  },
  "overall_replication_verdict": "PASS",
  "statistical_integrity_audit_verdict": "PASS",
  "digest": "9c75326c31437e44b531c64bb713e47ae7197425d089f599b1ce519dfaa91ff9"
}