{
  "schema": "cain.e138.neuro_symbolic_metacognition.v1",
  "generated_at": "2026-11-21T21:52:59Z",
  "duration_s": 19.9,
  "rsi_category": "4_model_assisted_candidate_generation + 3_cognitive_algorithm_and_abstraction_evolution",
  "live_llm_comparison": {
    "model_id": "qwen2.5:0.5b",
    "calls": 8,
    "tokens": 575,
    "paid_cost_usd": 0.0,
    "budget_halted_calls": 0,
    "raw_llm_accuracy": 0.125,
    "hybrid_accuracy": 1.0,
    "comparison_hybrid_vs_raw": {
      "test": "exact_mcnemar_arbitrary_precision",
      "n": 8,
      "rate_a": 1.0,
      "rate_b": 0.125,
      "gain": 0.875,
      "a_only": 7,
      "b_only": 0,
      "p_value": 0.015625000000000003,
      "log10_p": -1.8062,
      "p_display": "0.015625",
      "underflowed_float64": false,
      "effect_size": 2.4189,
      "effect_size_type": "cohens_h",
      "confidence_interval_diff": [
        0.6458,
        1.1042
      ]
    },
    "rows": [
      {
        "id": "NS138-01",
        "gold": 29,
        "raw_pred": 73,
        "raw_ok": 0,
        "hybrid_pred": 29,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: linear(3x+2)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-02",
        "gold": 50,
        "raw_pred": 26,
        "raw_ok": 0,
        "hybrid_pred": 50,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: quad(1x^2+1)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-03",
        "gold": 5,
        "raw_pred": 9,
        "raw_ok": 0,
        "hybrid_pred": 5,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: parity(4,0)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-04",
        "gold": 37,
        "raw_pred": 59,
        "raw_ok": 0,
        "hybrid_pred": 37,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: linear(3x+1)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-05",
        "gold": 73,
        "raw_pred": 75,
        "raw_ok": 0,
        "hybrid_pred": 73,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: quad(2x^2+1)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-06",
        "gold": 7,
        "raw_pred": 45,
        "raw_ok": 0,
        "hybrid_pred": 7,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: parity(1,5)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-07",
        "gold": 121,
        "raw_pred": 121,
        "raw_ok": 1,
        "hybrid_pred": 121,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: quad(1x^2+0)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      },
      {
        "id": "NS138-08",
        "gold": 52,
        "raw_pred": 82,
        "raw_ok": 0,
        "hybrid_pred": 52,
        "hybrid_ok": 1,
        "evidence": "Unique symbolic fit: linear(5x+2)",
        "unverified_assumptions": [
          "Assumes rule stays stationary outside training range"
        ],
        "budget_halted": false
      }
    ]
  },
  "sealed_metacognition_n200": {
    "n": 200,
    "coverage": 0.425,
    "selective_accuracy": 0.8353,
    "overall_useful_decision_rate": 0.93,
    "brier": 0.0616,
    "ece": 0.1866,
    "abstention_precision": 1.0,
    "abstention_recall": 0.92,
    "abstention_f1": 0.9583,
    "false_confidence_rate": 0.0,
    "confusion": {
      "tp_abstain": 115,
      "fp_abstain": 0,
      "fn_abstain": 10,
      "tn_acted_solvable": 75
    }
  },
  "contradiction_detection_rate": 0.9,
  "recovery_after_correction_rate": 1.0,
  "per_category": {
    "solvable_linear_quad": {
      "n": 25,
      "coverage": 1.0,
      "selective_accuracy": 1.0,
      "overall_useful_decision_rate": 1.0,
      "brier": 0.0036,
      "ece": 0.06,
      "abstention_precision": 0.0,
      "abstention_recall": 0.0,
      "abstention_f1": 0.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 0,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 25
      }
    },
    "solvable_multi_solution_valid": {
      "n": 25,
      "coverage": 1.0,
      "selective_accuracy": 1.0,
      "overall_useful_decision_rate": 1.0,
      "brier": 0.01,
      "ece": 0.1,
      "abstention_precision": 0.0,
      "abstention_recall": 0.0,
      "abstention_f1": 0.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 0,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 25
      }
    },
    "solvable_misleading_correlation": {
      "n": 25,
      "coverage": 1.0,
      "selective_accuracy": 0.84,
      "overall_useful_decision_rate": 0.84,
      "brier": 0.0994,
      "ece": 0.2328,
      "abstention_precision": 0.0,
      "abstention_recall": 0.0,
      "abstention_f1": 0.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 0,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 25
      }
    },
    "ambiguous_underdetermined": {
      "n": 25,
      "coverage": 0.0,
      "selective_accuracy": 0.0,
      "overall_useful_decision_rate": 1.0,
      "brier": 0.0484,
      "ece": 0.22,
      "abstention_precision": 1.0,
      "abstention_recall": 1.0,
      "abstention_f1": 1.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 25,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 0
      }
    },
    "missing_critical_info": {
      "n": 25,
      "coverage": 0.0,
      "selective_accuracy": 0.0,
      "overall_useful_decision_rate": 1.0,
      "brier": 0.0324,
      "ece": 0.18,
      "abstention_precision": 1.0,
      "abstention_recall": 1.0,
      "abstention_f1": 1.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 25,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 0
      }
    },
    "conflicting_evidence": {
      "n": 25,
      "coverage": 0.0,
      "selective_accuracy": 0.0,
      "overall_useful_decision_rate": 1.0,
      "brier": 0.0144,
      "ece": 0.12,
      "abstention_precision": 1.0,
      "abstention_recall": 1.0,
      "abstention_f1": 1.0,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 25,
        "fp_abstain": 0,
        "fn_abstain": 0,
        "tn_acted_solvable": 0
      }
    },
    "ood_unseen_regime": {
      "n": 25,
      "coverage": 0.2,
      "selective_accuracy": 0.0,
      "overall_useful_decision_rate": 0.8,
      "brier": 0.1537,
      "ece": 0.316,
      "abstention_precision": 1.0,
      "abstention_recall": 0.8,
      "abstention_f1": 0.8889,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 20,
        "fp_abstain": 0,
        "fn_abstain": 5,
        "tn_acted_solvable": 0
      }
    },
    "false_premise_contradiction": {
      "n": 25,
      "coverage": 0.2,
      "selective_accuracy": 0.0,
      "overall_useful_decision_rate": 0.8,
      "brier": 0.1312,
      "ece": 0.264,
      "abstention_precision": 1.0,
      "abstention_recall": 0.8,
      "abstention_f1": 0.8889,
      "false_confidence_rate": 0.0,
      "confusion": {
        "tp_abstain": 20,
        "fp_abstain": 0,
        "fn_abstain": 5,
        "tn_acted_solvable": 0
      }
    }
  },
  "scientific_verdict": "PARTIAL",
  "digest": "13a7eddf0f84327ef9f51d4ee23645b91b5202bc68cf67885285b1acc9ab3b82"
}