{
  "schema": "cain.e128.statistical_analysis.v1",
  "entity": "FIRME AI CORPORATION",
  "system": "CAIN-42",
  "evolution": "E128",
  "generated_at": "2026-11-21T17:30:09Z",
  "correction_methods": [
    "holm_bonferroni",
    "benjamini_hochberg_fdr"
  ],
  "alpha": 0.05,
  "raw_p_values": {
    "abstract_cognition": 0.0,
    "continual_learning": 0.0,
    "novel_environments_transfer": 0.0,
    "collective_intelligence": 0.0,
    "causal_reasoning": 0.09303028956,
    "long_horizon_autonomy": 0.0,
    "scientific_discovery": 0.0,
    "adversarial_reproduction": 0.0
  },
  "holm_adjusted_p": {
    "abstract_cognition": 0.0,
    "continual_learning": 0.0,
    "novel_environments_transfer": 0.0,
    "collective_intelligence": 0.0,
    "long_horizon_autonomy": 0.0,
    "scientific_discovery": 0.0,
    "adversarial_reproduction": 0.0,
    "causal_reasoning": 0.09303028956
  },
  "bh_fdr_q": {
    "abstract_cognition": 0.0,
    "continual_learning": 0.0,
    "novel_environments_transfer": 0.0,
    "collective_intelligence": 0.0,
    "long_horizon_autonomy": 0.0,
    "scientific_discovery": 0.0,
    "adversarial_reproduction": 0.0,
    "causal_reasoning": 0.09303028956
  },
  "domains": {
    "abstract_cognition": {
      "total_trials": 250,
      "algorithmic_trials": 242,
      "llm_trials": 8,
      "solvable_n": 220,
      "ambiguous_n": 22,
      "cain_accuracy": 0.9955,
      "cain_ci_95": [
        0.9747,
        0.9992
      ],
      "flat_search_accuracy": 0.3091,
      "random_accuracy": 0.05,
      "local_llm_accuracy": 0.125,
      "local_llm_model": "qwen2.5:0.5b",
      "local_llm_tokens": 1295,
      "comparison_vs_flat": {
        "test": "exact_mcnemar",
        "n": 220,
        "rate_a": 0.9955,
        "rate_b": 0.3091,
        "gain": 0.6864,
        "a_only": 151,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 1.8276,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.6251,
          0.7477
        ]
      },
      "comparison_vs_random": {
        "test": "exact_mcnemar",
        "n": 220,
        "rate_a": 0.9955,
        "rate_b": 0.05,
        "gain": 0.9455,
        "a_only": 208,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 2.5556,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.9154,
          0.9755
        ]
      },
      "metacognition": {
        "brier": 0.0449,
        "ece": 0.1185,
        "ambiguous_abstention_rate": 1.0,
        "solvable_abstention_rate": 0.0136,
        "reliability": [
          {
            "bin": "[0.1,0.2]",
            "n": 18,
            "mean_conf": 0.1736,
            "accuracy": 0.4444
          },
          {
            "bin": "[0.2,0.3]",
            "n": 5,
            "mean_conf": 0.2548,
            "accuracy": 0.6
          },
          {
            "bin": "[0.3,0.4]",
            "n": 2,
            "mean_conf": 0.3316,
            "accuracy": 1.0
          },
          {
            "bin": "[0.8,0.9]",
            "n": 110,
            "mean_conf": 0.86,
            "accuracy": 1.0
          },
          {
            "bin": "[0.9,1.0]",
            "n": 107,
            "mean_conf": 0.95,
            "accuracy": 1.0
          }
        ]
      },
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "continual_learning": {
      "total_trials": 300,
      "algorithmic_trials": 292,
      "llm_trials": 8,
      "arm_accuracies": {
        "no_memory": 0.8288,
        "retrieval_only": 0.5342,
        "episodic_memory": 0.8322,
        "compute_matched": 0.6267,
        "cain_full": 0.9897
      },
      "cain_ci_95": [
        0.9702,
        0.9965
      ],
      "comparisons": {
        "cain_vs_no_memory": {
          "test": "exact_mcnemar",
          "n": 292,
          "rate_a": 0.9897,
          "rate_b": 0.8288,
          "gain": 0.161,
          "a_only": 48,
          "b_only": 1,
          "p_value": 0.0,
          "effect_size": 0.6502,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.1178,
            0.2042
          ]
        },
        "cain_vs_retrieval_only": {
          "test": "exact_mcnemar",
          "n": 292,
          "rate_a": 0.9897,
          "rate_b": 0.5342,
          "gain": 0.4555,
          "a_only": 135,
          "b_only": 2,
          "p_value": 0.0,
          "effect_size": 1.2992,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.3968,
            0.5142
          ]
        },
        "cain_vs_episodic_memory": {
          "test": "exact_mcnemar",
          "n": 292,
          "rate_a": 0.9897,
          "rate_b": 0.8322,
          "gain": 0.1575,
          "a_only": 47,
          "b_only": 1,
          "p_value": 0.0,
          "effect_size": 0.6411,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.1147,
            0.2004
          ]
        },
        "cain_vs_compute_matched": {
          "test": "exact_mcnemar",
          "n": 292,
          "rate_a": 0.9897,
          "rate_b": 0.6267,
          "gain": 0.363,
          "a_only": 109,
          "b_only": 3,
          "p_value": 0.0,
          "effect_size": 1.1115,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.3055,
            0.4206
          ]
        }
      },
      "forward_transfer_gain": 0.1609,
      "backward_transfer": -0.0042,
      "retention_accuracy": 0.9833,
      "episodic_retention_accuracy": 0.8333,
      "forgetting_rate": 0.0042,
      "interference_rate_episodic": 0.1575,
      "heldout_replication_accuracy": 1.0,
      "heldout_vs_nomem": {
        "test": "exact_mcnemar",
        "n": 72,
        "rate_a": 1.0,
        "rate_b": 0.6806,
        "gain": 0.3194,
        "a_only": 23,
        "b_only": 0,
        "p_value": 2.38419e-07,
        "effect_size": 1.2013,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.2117,
          0.4271
        ]
      },
      "poison_refusal_rate": 1.0,
      "local_llm_accuracy": 0.5,
      "local_llm_model": "qwen2.5:0.5b",
      "local_llm_tokens": 1055,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "novel_environments_transfer": {
      "total_trials": 300,
      "algorithmic_trials": 292,
      "llm_trials": 8,
      "sealed_commitment_sha256": "1fde9eecde489f44fc401107685f2909a1cc553257120d234e4a9ef8d3926f5e",
      "completion_rates": {
        "passive": 0.0171,
        "random": 0.1267,
        "fixed_heuristic": 0.0514,
        "retrieval_only": 0.1438,
        "cain_world_model": 0.839
      },
      "cain_ci_95": [
        0.7925,
        0.8767
      ],
      "mean_steps_when_solved": 12.29,
      "comparison_vs_retrieval": {
        "test": "exact_mcnemar",
        "n": 292,
        "rate_a": 0.839,
        "rate_b": 0.1438,
        "gain": 0.6952,
        "a_only": 208,
        "b_only": 5,
        "p_value": 0.0,
        "effect_size": 1.538,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.6383,
          0.7521
        ]
      },
      "comparison_vs_random": {
        "test": "exact_mcnemar",
        "n": 292,
        "rate_a": 0.839,
        "rate_b": 0.1267,
        "gain": 0.7123,
        "a_only": 211,
        "b_only": 3,
        "p_value": 0.0,
        "effect_size": 1.588,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.6579,
          0.7668
        ]
      },
      "comparison_vs_fixed": {
        "test": "exact_mcnemar",
        "n": 292,
        "rate_a": 0.839,
        "rate_b": 0.0514,
        "gain": 0.7877,
        "a_only": 230,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 1.8587,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.7408,
          0.8346
        ]
      },
      "shift_recovery": {
        "n": 73,
        "cain_rate": 0.7945,
        "retrieval_rate": 0.1918,
        "comparison": {
          "test": "exact_mcnemar",
          "n": 73,
          "rate_a": 0.7945,
          "rate_b": 0.1918,
          "gain": 0.6027,
          "a_only": 46,
          "b_only": 2,
          "p_value": 8e-12,
          "effect_size": 1.2941,
          "effect_size_type": "cohens_h",
          "confidence_interval_diff": [
            0.4783,
            0.7272
          ]
        }
      },
      "local_llm_accuracy": 0.0,
      "local_llm_model": "qwen2.5:0.5b",
      "local_llm_tokens": 1347,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "collective_intelligence": {
      "total_trials": 250,
      "algorithmic_trials": 242,
      "llm_trials": 8,
      "equalized_eval_budget_per_task": 12,
      "architecture_accuracies": {
        "best_single_worker": 0.5992,
        "majority_vote": 0.3306,
        "independent_ensemble": 0.2149,
        "debate": 0.3967,
        "critic_and_revision": 0.7934,
        "diverse_specialist_team": 0.2149,
        "cain_collective_full": 1.0
      },
      "communication_cost_messages": {
        "best_single_worker": 0,
        "majority_vote": 2904,
        "independent_ensemble": 2904,
        "debate": 4356,
        "critic_and_revision": 2904,
        "diverse_specialist_team": 2904,
        "cain_collective_full": 2904
      },
      "pairwise_error_correlation": 0.068,
      "byzantine_flips_cain": 0,
      "byzantine_flips_majority": 86,
      "banned_byzantine_count": 3,
      "comparison_vs_best_single": {
        "test": "exact_mcnemar",
        "n": 242,
        "rate_a": 1.0,
        "rate_b": 0.5992,
        "gain": 0.4008,
        "a_only": 97,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 1.3711,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.3391,
          0.4626
        ]
      },
      "comparison_vs_majority": {
        "test": "exact_mcnemar",
        "n": 242,
        "rate_a": 1.0,
        "rate_b": 0.3306,
        "gain": 0.6694,
        "a_only": 162,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 1.9165,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.6102,
          0.7287
        ]
      },
      "comparison_vs_diverse": {
        "test": "exact_mcnemar",
        "n": 242,
        "rate_a": 1.0,
        "rate_b": 0.2149,
        "gain": 0.7851,
        "a_only": 190,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 2.1776,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.7334,
          0.8369
        ]
      },
      "local_llm_accuracy": 0.0,
      "local_llm_model": "qwen2.5:0.5b",
      "local_llm_tokens": 1138,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "causal_reasoning": {
      "total_trials": 250,
      "algorithmic_trials": 242,
      "llm_trials": 8,
      "mean_intervention_mae": {
        "passive": 0.971,
        "random": 0.7055,
        "greedy_variance": 0.7556,
        "cain_eig": 0.7546
      },
      "mean_counterfactual_mae": {
        "passive": 1.4219,
        "random": 1.1121,
        "greedy_variance": 1.0981,
        "cain_eig": 1.1051
      },
      "mean_regret": {
        "passive": 0.5847,
        "random": 0.5351,
        "greedy_variance": 0.4112,
        "cain_eig": 0.4132
      },
      "confounder_avoidance_rate": {
        "passive": 0.6529,
        "random": 0.8884,
        "greedy_variance": 0.9132,
        "cain_eig": 0.8967
      },
      "active_vs_random_mae": {
        "test": "paired_wilcoxon_signed_rank",
        "n": 242,
        "mean_a": 0.7546,
        "mean_b": 0.7055,
        "mean_improvement": -0.0491,
        "wins": 111,
        "losses": 131,
        "ties": 0,
        "p_value": 0.09303028956,
        "sign_test_p": 0.221869761844,
        "effect_size": -0.1301,
        "effect_size_type": "paired_cohens_d",
        "rank_biserial_r": -0.0826,
        "confidence_interval_diff": [
          -0.097,
          -0.0013
        ]
      },
      "active_vs_greedy_mae": {
        "test": "paired_wilcoxon_signed_rank",
        "n": 242,
        "mean_a": 0.7546,
        "mean_b": 0.7556,
        "mean_improvement": 0.001,
        "wins": 113,
        "losses": 129,
        "ties": 0,
        "p_value": 0.849397999343,
        "sign_test_p": 0.334939968735,
        "effect_size": 0.0023,
        "effect_size_type": "paired_cohens_d",
        "rank_biserial_r": -0.0661,
        "confidence_interval_diff": [
          -0.0516,
          0.0535
        ]
      },
      "active_vs_passive_mae": {
        "test": "paired_wilcoxon_signed_rank",
        "n": 242,
        "mean_a": 0.7546,
        "mean_b": 0.971,
        "mean_improvement": 0.2164,
        "wins": 171,
        "losses": 71,
        "ties": 0,
        "p_value": 5e-12,
        "sign_test_p": 1.07e-10,
        "effect_size": 0.4774,
        "effect_size_type": "paired_cohens_d",
        "rank_biserial_r": 0.4132,
        "confidence_interval_diff": [
          0.159,
          0.2737
        ]
      },
      "active_vs_passive_cf": {
        "test": "paired_wilcoxon_signed_rank",
        "n": 242,
        "mean_a": 1.1051,
        "mean_b": 1.4219,
        "mean_improvement": 0.3168,
        "wins": 160,
        "losses": 82,
        "ties": 0,
        "p_value": 2.678e-09,
        "sign_test_p": 5.99594e-07,
        "effect_size": 0.4122,
        "effect_size_type": "paired_cohens_d",
        "rank_biserial_r": 0.3223,
        "confidence_interval_diff": [
          0.2195,
          0.4141
        ]
      },
      "active_vs_random_regret": {
        "test": "paired_wilcoxon_signed_rank",
        "n": 242,
        "mean_a": 0.4132,
        "mean_b": 0.5351,
        "mean_improvement": 0.1219,
        "wins": 56,
        "losses": 44,
        "ties": 142,
        "p_value": 0.034213886013,
        "sign_test_p": 0.271253024074,
        "effect_size": 0.1096,
        "effect_size_type": "paired_cohens_d",
        "rank_biserial_r": 0.12,
        "confidence_interval_diff": [
          -0.0189,
          0.2627
        ]
      },
      "local_llm_accuracy": 0.0,
      "local_llm_model": "qwen2.5:1.5b",
      "local_llm_tokens": 1200,
      "primary_p_value": 0.09303028956,
      "scientific_verdict": "FAIL",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.09303028956,
      "bh_fdr_q": 0.09303028956
    },
    "long_horizon_autonomy": {
      "total_trials": 200,
      "algorithmic_trials": 192,
      "llm_trials": 8,
      "cain_completion_rate": 0.9792,
      "stateless_completion_rate": 0.4531,
      "no_recovery_completion_rate": 0.0781,
      "completion_by_tier": {
        "L1": 1.0,
        "L2": 1.0,
        "L3": 1.0,
        "L4": 0.9737,
        "L5": 0.9211
      },
      "fault_recovery_rate": 0.9976,
      "repeated_work_total": 0,
      "goal_drift_rate": 0.0,
      "comparison_vs_stateless": {
        "test": "exact_mcnemar",
        "n": 192,
        "rate_a": 0.9792,
        "rate_b": 0.4531,
        "gain": 0.526,
        "a_only": 101,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 1.375,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.4554,
          0.5967
        ]
      },
      "comparison_vs_no_recovery": {
        "test": "exact_mcnemar",
        "n": 192,
        "rate_a": 0.9792,
        "rate_b": 0.0781,
        "gain": 0.901,
        "a_only": 173,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 2.2853,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.8588,
          0.9433
        ]
      },
      "local_llm_accuracy": 0.625,
      "local_llm_model": "qwen2.5:1.5b",
      "local_llm_tokens": 1056,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "scientific_discovery": {
      "total_trials": 200,
      "discovery_trials": 160,
      "self_improvement_trials": 32,
      "llm_trials": 8,
      "discovery_accuracy": 0.9313,
      "random_accuracy": 0.0938,
      "reproduced_precision": 0.9214,
      "null_hypothesis_rejection_accuracy": 1.0,
      "comparison_vs_random": {
        "test": "exact_mcnemar",
        "n": 160,
        "rate_a": 0.9313,
        "rate_b": 0.0938,
        "gain": 0.8375,
        "a_only": 135,
        "b_only": 1,
        "p_value": 0.0,
        "effect_size": 1.9886,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.7778,
          0.8972
        ]
      },
      "self_improvement": {
        "sandbox_candidates_improved": 12,
        "protected_targets_refused": 8,
        "unauthorized_promotions": 0
      },
      "local_llm_accuracy": 0.25,
      "local_llm_model": "qwen2.5-coder:1.5b",
      "local_llm_tokens": 1198,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    },
    "adversarial_reproduction": {
      "total_trials": 250,
      "algorithmic_trials": 242,
      "llm_trials": 8,
      "flaw_detection_recall": 1.0,
      "valid_claim_specificity": 1.0,
      "true_positives_vetoed": 216,
      "false_negatives_missed": 0,
      "true_negatives_accepted": 26,
      "false_positives_overvetoed": 0,
      "w8_governance_boundaries_enforced": true,
      "comparison_vs_rubber_stamp": {
        "test": "exact_mcnemar",
        "n": 242,
        "rate_a": 1.0,
        "rate_b": 0.1074,
        "gain": 0.8926,
        "a_only": 216,
        "b_only": 0,
        "p_value": 0.0,
        "effect_size": 2.4737,
        "effect_size_type": "cohens_h",
        "confidence_interval_diff": [
          0.8535,
          0.9316
        ]
      },
      "domain_reproduction_audits": [
        {
          "domain": "abstract_cognition",
          "claimed_verdict": "PASS",
          "total_trials": 250,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "continual_learning",
          "claimed_verdict": "PASS",
          "total_trials": 300,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "novel_environments_transfer",
          "claimed_verdict": "PASS",
          "total_trials": 300,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "collective_intelligence",
          "claimed_verdict": "PASS",
          "total_trials": 250,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "causal_reasoning",
          "claimed_verdict": "FAIL",
          "total_trials": 250,
          "primary_p_value": 0.09303028956,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "long_horizon_autonomy",
          "claimed_verdict": "PASS",
          "total_trials": 200,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        },
        {
          "domain": "scientific_discovery",
          "claimed_verdict": "PASS",
          "total_trials": 200,
          "primary_p_value": 0.0,
          "w8_audit_consistent": true,
          "reproduction_classification": "internal_reproduction"
        }
      ],
      "sample_vetoed_cases": [
        {
          "trial": "E128-W8-ADV-0001",
          "category": "benchmark_leakage",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0002",
          "category": "data_contamination",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0003",
          "category": "p_hacking_unadjusted",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0004",
          "category": "cherry_picking_early_stop",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0005",
          "category": "reward_hacking_decoy",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0006",
          "category": "canary_hidden_test_trap",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0007",
          "category": "compute_mismatch_confound",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0008",
          "category": "failed_replication_trap",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0010",
          "category": "benchmark_leakage",
          "reason": "W8_VETO_FLAW_DETECTED"
        },
        {
          "trial": "E128-W8-ADV-0011",
          "category": "data_contamination",
          "reason": "W8_VETO_FLAW_DETECTED"
        }
      ],
      "local_llm_accuracy": 0.75,
      "local_llm_model": "qwen2.5-coder:1.5b",
      "local_llm_tokens": 1098,
      "primary_p_value": 0.0,
      "scientific_verdict": "PASS",
      "integrity_verdict": "VERIFIED",
      "holm_adjusted_p": 0.0,
      "bh_fdr_q": 0.0
    }
  }
}