{"schema":"cain.e138.neuro_symbolic_metacognition.v1","generated_at":"2026-11-21T21:52:59Z","duration_s":19.9,"rsi_category":"4_model_assisted_candidate_generation + 3_cognitive_algorithm_and_abstraction_evolution","live_llm_comparison":{"model_id":"qwen2.5:0.5b","calls":8,"tokens":575,"paid_cost_usd":0.0,"budget_halted_calls":0,"raw_llm_accuracy":0.125,"hybrid_accuracy":1.0,"comparison_hybrid_vs_raw":{"test":"exact_mcnemar_arbitrary_precision","n":8,"rate_a":1.0,"rate_b":0.125,"gain":0.875,"a_only":7,"b_only":0,"p_value":0.015625000000000003,"log10_p":-1.8062,"p_display":"0.015625","underflowed_float64":false,"effect_size":2.4189,"effect_size_type":"cohens_h","confidence_interval_diff":[0.6458,1.1042]},"rows":[{"id":"NS138-01","gold":29,"raw_pred":73,"raw_ok":0,"hybrid_pred":29,"hybrid_ok":1,"evidence":"Unique symbolic fit: linear(3x+2)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-02","gold":50,"raw_pred":26,"raw_ok":0,"hybrid_pred":50,"hybrid_ok":1,"evidence":"Unique symbolic fit: quad(1x^2+1)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-03","gold":5,"raw_pred":9,"raw_ok":0,"hybrid_pred":5,"hybrid_ok":1,"evidence":"Unique symbolic fit: parity(4,0)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-04","gold":37,"raw_pred":59,"raw_ok":0,"hybrid_pred":37,"hybrid_ok":1,"evidence":"Unique symbolic fit: linear(3x+1)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-05","gold":73,"raw_pred":75,"raw_ok":0,"hybrid_pred":73,"hybrid_ok":1,"evidence":"Unique symbolic fit: quad(2x^2+1)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-06","gold":7,"raw_pred":45,"raw_ok":0,"hybrid_pred":7,"hybrid_ok":1,"evidence":"Unique symbolic fit: parity(1,5)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-07","gold":121,"raw_pred":121,"raw_ok":1,"hybrid_pred":121,"hybrid_ok":1,"evidence":"Unique symbolic fit: quad(1x^2+0)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false},{"id":"NS138-08","gold":52,"raw_pred":82,"raw_ok":0,"hybrid_pred":52,"hybrid_ok":1,"evidence":"Unique symbolic fit: linear(5x+2)","unverified_assumptions":["Assumes rule stays stationary outside training range"],"budget_halted":false}]},"sealed_metacognition_n200":{"n":200,"coverage":0.425,"selective_accuracy":0.8353,"overall_useful_decision_rate":0.93,"brier":0.0616,"ece":0.1866,"abstention_precision":1.0,"abstention_recall":0.92,"abstention_f1":0.9583,"false_confidence_rate":0.0,"confusion":{"tp_abstain":115,"fp_abstain":0,"fn_abstain":10,"tn_acted_solvable":75}},"contradiction_detection_rate":0.9,"recovery_after_correction_rate":1.0,"per_category":{"solvable_linear_quad":{"n":25,"coverage":1.0,"selective_accuracy":1.0,"overall_useful_decision_rate":1.0,"brier":0.0036,"ece":0.06,"abstention_precision":0.0,"abstention_recall":0.0,"abstention_f1":0.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":0,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":25}},"solvable_multi_solution_valid":{"n":25,"coverage":1.0,"selective_accuracy":1.0,"overall_useful_decision_rate":1.0,"brier":0.01,"ece":0.1,"abstention_precision":0.0,"abstention_recall":0.0,"abstention_f1":0.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":0,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":25}},"solvable_misleading_correlation":{"n":25,"coverage":1.0,"selective_accuracy":0.84,"overall_useful_decision_rate":0.84,"brier":0.0994,"ece":0.2328,"abstention_precision":0.0,"abstention_recall":0.0,"abstention_f1":0.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":0,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":25}},"ambiguous_underdetermined":{"n":25,"coverage":0.0,"selective_accuracy":0.0,"overall_useful_decision_rate":1.0,"brier":0.0484,"ece":0.22,"abstention_precision":1.0,"abstention_recall":1.0,"abstention_f1":1.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":25,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":0}},"missing_critical_info":{"n":25,"coverage":0.0,"selective_accuracy":0.0,"overall_useful_decision_rate":1.0,"brier":0.0324,"ece":0.18,"abstention_precision":1.0,"abstention_recall":1.0,"abstention_f1":1.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":25,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":0}},"conflicting_evidence":{"n":25,"coverage":0.0,"selective_accuracy":0.0,"overall_useful_decision_rate":1.0,"brier":0.0144,"ece":0.12,"abstention_precision":1.0,"abstention_recall":1.0,"abstention_f1":1.0,"false_confidence_rate":0.0,"confusion":{"tp_abstain":25,"fp_abstain":0,"fn_abstain":0,"tn_acted_solvable":0}},"ood_unseen_regime":{"n":25,"coverage":0.2,"selective_accuracy":0.0,"overall_useful_decision_rate":0.8,"brier":0.1537,"ece":0.316,"abstention_precision":1.0,"abstention_recall":0.8,"abstention_f1":0.8889,"false_confidence_rate":0.0,"confusion":{"tp_abstain":20,"fp_abstain":0,"fn_abstain":5,"tn_acted_solvable":0}},"false_premise_contradiction":{"n":25,"coverage":0.2,"selective_accuracy":0.0,"overall_useful_decision_rate":0.8,"brier":0.1312,"ece":0.264,"abstention_precision":1.0,"abstention_recall":0.8,"abstention_f1":0.8889,"false_confidence_rate":0.0,"confusion":{"tp_abstain":20,"fp_abstain":0,"fn_abstain":5,"tn_acted_solvable":0}}},"scientific_verdict":"PARTIAL","digest":"13a7eddf0f84327ef9f51d4ee23645b91b5202bc68cf67885285b1acc9ab3b82"}