{"schema":"cain.e138.replication_audit.v1","generated_at":"2026-11-21T21:51:19Z","duration_s":38.75,"rsi_category":"2_workflow_and_research_process_evolution","source_commit_audited":"15bc7d25f","total_replication_evaluations":540,"zero_p_value_bugs_found_and_corrected":[{"file":"causal_apex.json","field":"holm_adjusted_p.vs_random_naive","original_reported":0.0,"recomputed_exact_display":"4.086e-25","recomputed_log10_p":-24.3887},{"file":"causal_apex.json","field":"holm_adjusted_p.vs_e128_eig_div_cost","original_reported":0.0,"recomputed_exact_display":"2.175e-26","recomputed_log10_p":-25.6626},{"file":"unification_3domain.json","field":"holm_adjusted_p.D_vs_A","original_reported":0.0,"recomputed_exact_display":"2.928e-55","recomputed_log10_p":-54.5335},{"file":"unification_3domain.json","field":"holm_adjusted_p.D_vs_B","original_reported":0.0,"recomputed_exact_display":"4.950e-28","recomputed_log10_p":-27.3054},{"file":"unification_3domain.json","field":"holm_adjusted_p.D_vs_C","original_reported":0.0,"recomputed_exact_display":"3.415e-35","recomputed_log10_p":-34.4667}],"preregistration_deviations":[{"breakthrough":"2_code_rsi","planned":"AST-verified sandboxed Python Code-RSI using local qwen2.5-coder:3b matches or improves sealed edge-case pass rate over zero-shot & blind best-of-2 while blocking 100% of AST escape payloads.","executed":"Fell back to qwen2.5-coder:1.5b (CPU timeout on 3b); scored 0.500 (4/8) < 0.625 target -> honestly PARTIAL.","severity":"DISCLOSED_IN_ORIGINAL_REPORT"},{"breakthrough":"5_neuro_symbolic","planned":"CAIN Neuro-Symbolic Hybrid (3B neural + symbolic verification + calibrated abstention) outperforms or matches raw 3B zero-shot with 100% OOD abstention and Brier <= 0.15.","executed":"BudgetTracker wall-time cap (900.0s) triggered right after Code-RSI (921.08s), so ALL 8 raw LLM calls returned 0 tokens / raw_3b_acc=0.0. Raw-vs-hybrid LLM contrast in original APEX is NOT valid as a neural comparison; only symbolic accuracy (1.0) and OOD abstention (1.0) executed.","severity":"CRITICAL_METHODOLOGICAL_DEVIATION_FLAGGED_BY_E138"},{"breakthrough":"1_causal_and_4_unification","planned":"Exact Holm-Bonferroni & BH FDR p-values","executed":"stats.py applied round(p, 12), rounding extreme tail probabilities down to exact 0.0. E138 recomputes exact arbitrary-precision log10(p) and never emits 0.0.","severity":"STATISTICAL_REPORTING_DEFECT_CORRECTED_BY_E138"},{"breakthrough":"3_multigen_g0_g5","planned":"Monotonic generalization improvement G0->G5","executed":"G3/G4 reached 0.9167 unseen while G5 dropped slightly to 0.875 unseen (while sealed rose 0.8333->0.9167), showing mild late-generation overfitting.","severity":"MINOR_OVERFITTING_SIGNAL_INVESTIGATED_IN_E138_G0_G10"}],"breakthroughs":{"1_causal_bayesian_mixture":{"original_reported":{"passive":0.8787,"random_naive":0.7709,"e128_eig_div_cost":0.7832,"random_mixture":0.4632,"apex_bayesian_mixture_a_optimal":0.4119},"independently_recomputed":{"passive":0.8787,"random_naive":0.7709,"e128_eig_div_cost":0.7832,"random_mixture":0.4632,"apex_bayesian_mixture_a_optimal":0.4119},"fresh_seed_replication":{"passive":0.8108,"random_naive":0.7804,"e128_eig_div_cost":0.78,"random_mixture":0.4678,"apex_bayesian_mixture_a_optimal":0.3834},"exact_reconstruction_match":true,"fresh_seed_vs_matched_random_holm":{"p_value":0.0023163276959874465,"log10_p":-2.6352,"p_display":"0.002316","underflowed_float64":false},"fresh_seed_effect_size_d":0.2927,"cluster_seed_neff":{"n_observations":200,"n_clusters":5,"mean_cluster_size":40.0,"icc":0.0,"design_effect":1.0,"n_effective":200.0},"cluster_seed_paired_t_vs_mix":{"test":"paired_cluster_t_test","n_clusters":5,"mean_diff":0.0513,"t_stat":3.5879,"df":4,"effect_size_cohens_d":1.6046,"ci_95":[0.0116,0.091],"p_value":0.0230040758045941,"log10_p":-1.6382,"p_display":"0.023004","underflowed_float64":false},"differences":"Original reported exact p=0.0 for naive/E128 due to round(p,12); exact log10(p) recomputed. Fresh-seed 100-world replication confirms advantage over random_mixture.","limitations":"Assumes binary hidden confounder U in {0,1} and discrete W_GRID; stress-tested under misspecification in Section 6.","verdict":"PASS"},"2_sandboxed_python_code_rsi":{"original_reported":{"g0":0.375,"cm":0.375,"unified":0.5,"ast_blocked":"4/4"},"independently_recomputed":{"g0":0.375,"cm":0.375,"unified":0.5,"p_vs_g0_display":"1.0","ast_blocked":"4/4"},"replication_status":"Exact raw-record reconstruction verified; extended to 40-task ceiling benchmark in Section 5.","differences":"Model fell back from preregistered 3b to 1.5b; small n=8 yields McNemar p=1.0 (not statistically significant).","limitations":"n=8 tasks underpowered; below 0.625 preregistered threshold.","verdict":"PARTIAL"},"3_multigen_g0_to_g5":{"original_reported":{"g0":0.25,"cm2x":0.25,"g5":0.875},"independently_recomputed":{"g0":0.25,"cm2x":0.25,"g5":0.875,"p_display":"6.104e-05","cohens_h":1.3717},"fresh_seed_replication":{"seed":13809,"g0":0.25,"g5":0.8333,"p_display":"1.221e-04","cohens_h":1.2533},"differences":"Exact reconstruction matches 0.25 -> 0.875 on seed 13012; G5 unseen (0.875) slightly below G3/G4 (0.9167).","limitations":"Bounded DSL macro synthesis; extended to G0-G10 branching DAG in Section 7.","verdict":"PASS"},"4_unification_3domain_full_compute":{"original_reported":{"A":0.14,"B":0.3667,"C":0.3433,"D":0.8133},"independently_recomputed_from_1200_raw_rows":{"A":0.14,"B":0.3667,"C":0.3433,"D":0.8133},"exact_mcnemar_rigor":{"D_vs_A":{"test":"exact_mcnemar_arbitrary_precision","n":300,"rate_a":0.8133,"rate_b":0.14,"gain":0.6733,"a_only":206,"b_only":4,"p_value":9.757777411530882e-56,"log10_p":-55.0106,"p_display":"9.758e-56","underflowed_float64":false,"effect_size":1.4811,"effect_size_type":"cohens_h","confidence_interval_diff":[0.6171,0.7295]},"D_vs_B":{"test":"exact_mcnemar_arbitrary_precision","n":300,"rate_a":0.8133,"rate_b":0.3667,"gain":0.4467,"a_only":151,"b_only":17,"p_value":4.949592375030885e-28,"log10_p":-27.3054,"p_display":"4.950e-28","underflowed_float64":false,"effect_size":0.9472,"effect_size_type":"cohens_h","confidence_interval_diff":[0.3787,0.5146]},"D_vs_C":{"test":"exact_mcnemar_arbitrary_precision","n":300,"rate_a":0.8133,"rate_b":0.3433,"gain":0.47,"a_only":148,"b_only":7,"p_value":1.70740140229607e-35,"log10_p":-34.7677,"p_display":"1.707e-35","underflowed_float64":false,"effect_size":0.996,"effect_size_type":"cohens_h","confidence_interval_diff":[0.4085,0.5315]}},"holm_rigor":{"D_vs_A":{"p_value":2.9276641500908906e-55,"log10_p":-54.5335,"p_display":"2.928e-55","underflowed_float64":false},"D_vs_C":{"p_value":3.414522632212301e-35,"log10_p":-34.4667,"p_display":"3.415e-35","underflowed_float64":false},"D_vs_B":{"p_value":4.949940743969364e-28,"log10_p":-27.3054,"p_display":"4.950e-28","underflowed_float64":false}},"clustering_audit":{"headline_evaluations":1200,"paired_tasks_per_arm":300,"independent_seed_clusters":5,"domain_clusters":3,"seed_cluster_neff_arm_d":{"n_observations":300,"n_clusters":5,"mean_cluster_size":60.0,"icc":0.0,"design_effect":1.0,"n_effective":300.0},"domain_cluster_neff_arm_d":{"n_observations":300,"n_clusters":3,"mean_cluster_size":100.0,"icc":0.0437,"design_effect":5.3267,"n_effective":56.32},"seed_level_paired_t_tests":{"D_vs_A":{"test":"paired_cluster_t_test","n_clusters":5,"mean_diff":0.6733,"t_stat":49.729,"df":4,"effect_size_cohens_d":22.2395,"ci_95":[0.6357,0.7109],"p_value":9.784616088988554e-7,"log10_p":-6.0095,"p_display":"9.785e-07","underflowed_float64":false},"D_vs_B":{"test":"paired_cluster_t_test","n_clusters":5,"mean_diff":0.4467,"t_stat":36.4702,"df":4,"effect_size_cohens_d":16.31,"ci_95":[0.4127,0.4807],"p_value":3.374628197988746e-6,"log10_p":-5.4718,"p_display":"3.375e-06","underflowed_float64":false},"D_vs_C":{"test":"paired_cluster_t_test","n_clusters":5,"mean_diff":0.47,"t_stat":17.6943,"df":4,"effect_size_cohens_d":7.9131,"ci_95":[0.3963,0.5437],"p_value":0.00005992814077806199,"log10_p":-4.2224,"p_display":"5.993e-05","underflowed_float64":false}}},"fresh_seed_120eval_replication":{"n_evals":120,"rates":{"A":0.2,"B":0.3,"C":0.3333,"D":0.8},"D_vs_A":{"test":"exact_mcnemar_arbitrary_precision","n":30,"rate_a":0.8,"rate_b":0.2,"gain":0.6,"a_only":20,"b_only":2,"p_value":0.00012111663818359366,"log10_p":-3.9168,"p_display":"1.211e-04","underflowed_float64":false,"effect_size":1.287,"effect_size_type":"cohens_h","confidence_interval_diff":[0.3814,0.8186]},"D_vs_B":{"test":"exact_mcnemar_arbitrary_precision","n":30,"rate_a":0.8,"rate_b":0.3,"gain":0.5,"a_only":17,"b_only":2,"p_value":0.0007286071777343749,"log10_p":-3.1375,"p_display":"7.286e-04","underflowed_float64":false,"effect_size":1.055,"effect_size_type":"cohens_h","confidence_interval_diff":[0.2784,0.7216]},"D_vs_C":{"test":"exact_mcnemar_arbitrary_precision","n":30,"rate_a":0.8,"rate_b":0.3333,"gain":0.4667,"a_only":14,"b_only":0,"p_value":0.00012207031250000003,"log10_p":-3.9134,"p_display":"1.221e-04","underflowed_float64":false,"effect_size":0.9833,"effect_size_type":"cohens_h","confidence_interval_diff":[0.2881,0.6452]}},"differences":"Original reported exact Holm p=0.0; exact log10(p) recomputed. Effective N under domain clustering is much smaller than headline 1,200, yet seed-level paired t-tests (df=4) still confirm D > A, B, C (p < 0.005).","limitations":"Same evolved G5 library reused across all 5 seeds; constructed 3-domain suite.","verdict":"PASS"},"5_neuro_symbolic_and_immune":{"original_reported":{"raw_3b_acc":0.0,"hybrid_acc":1.0,"ood_abstain":1.0,"brier":0.0144,"immune":"10/10"},"independently_recomputed":{"raw_llm_calls_completed":0,"budget_halted_raw_arm":true,"symbolic_rows_verified":8},"differences":"E138 downgrades original PASS to PARTIAL because BudgetTracker halted raw LLM calls at 900s wall-time (0 tokens), invalidating the raw-vs-hybrid LLM contrast! Re-run cleanly inside budget in Section 9.","limitations":"Only 12 symbolic tasks in APEX; budget cap prevented valid raw LLM comparison.","verdict":"PARTIAL"}},"summary_verdicts":{"1_causal_bayesian_mixture":"PASS","2_sandboxed_python_code_rsi":"PARTIAL","3_multigen_g0_to_g5":"PASS","4_unification_3domain_full_compute":"PASS","5_neuro_symbolic_and_immune":"PARTIAL"},"overall_replication_verdict":"PASS","statistical_integrity_audit_verdict":"PASS","digest":"9c75326c31437e44b531c64bb713e47ae7197425d089f599b1ce519dfaa91ff9"}