{
 "budget": {
  "agents": {
   "bench-a": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-a",
    "remaining": 176,
    "spent": 24,
    "tokens": 1304,
    "units": 200
   },
   "bench-b": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-b",
    "remaining": 176,
    "spent": 24,
    "tokens": 1319,
    "units": 200
   },
   "bench-conf": {
    "cost_usd": 0.0,
    "entries": 41,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-conf",
    "remaining": 159,
    "spent": 41,
    "tokens": 2666,
    "units": 200
   },
   "bench-critic": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-critic",
    "remaining": 176,
    "spent": 24,
    "tokens": 1914,
    "units": 200
   },
   "bench-producer": {
    "cost_usd": 0.0,
    "entries": 48,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-producer",
    "remaining": 152,
    "spent": 48,
    "tokens": 2859,
    "units": 200
   },
   "bench-reader": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-reader",
    "remaining": 176,
    "spent": 24,
    "tokens": 1794,
    "units": 200
   },
   "bench-reasoner": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-reasoner",
    "remaining": 176,
    "spent": 24,
    "tokens": 1304,
    "units": 200
   },
   "bench-researcher": {
    "cost_usd": 0.0,
    "entries": 72,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-researcher",
    "remaining": 128,
    "spent": 72,
    "tokens": 3947,
    "units": 200
   },
   "bench-router": {
    "cost_usd": 0.0,
    "entries": 72,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-router",
    "remaining": 128,
    "spent": 72,
    "tokens": 3912,
    "units": 200
   },
   "bench-statistician": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-statistician",
    "remaining": 176,
    "spent": 24,
    "tokens": 1468,
    "units": 200
   },
   "bench-synth": {
    "cost_usd": 0.0,
    "entries": 24,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-synth",
    "remaining": 176,
    "spent": 24,
    "tokens": 2007,
    "units": 200
   },
   "bench-tie": {
    "cost_usd": 0.0,
    "entries": 11,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "agent:bench-tie",
    "remaining": 189,
    "spent": 11,
    "tokens": 819,
    "units": 200
   }
  },
  "closed": false,
  "created_utc": 1794828568.842441,
  "mission": {
   "cost_usd": 0.0,
   "entries": 412,
   "max_cost_usd": 0.0,
   "max_retries": 3,
   "max_tokens": 2000000,
   "name": "mission:M-bench",
   "remaining": 4588,
   "spent": 412,
   "tokens": 25313,
   "units": 5000
  },
  "mission_id": "M-bench",
  "models": {
   "qwen2.5-0.5b-instruct": {
    "cost_usd": 0.0,
    "entries": 412,
    "max_cost_usd": null,
    "max_retries": 3,
    "max_tokens": null,
    "name": "model:qwen2.5-0.5b-instruct",
    "remaining": 4974687,
    "spent": 25313,
    "tokens": 25313,
    "units": 5000000
   }
  },
  "schema": "cain2027.e81.mission_budget.v1"
 },
 "comparisons_vs_baseline": {
  "B_hierarchical": {
   "bootstrap95": [
    -0.2083,
    0.2083
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 3,
    "b_only": 3,
    "discordant": 6,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "C_parallel_research": {
   "bootstrap95": [
    0.0,
    0.0
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 0,
    "b_only": 0,
    "discordant": 0,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "D_evidence_first": {
   "bootstrap95": [
    -0.4167,
    0.1667
   ],
   "delta_vs_baseline": -0.125,
   "mcnemar": {
    "a_only": 8,
    "b_only": 5,
    "discordant": 13,
    "p_two_sided": 0.581055,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "E_adversarial_review": {
   "bootstrap95": [
    -0.25,
    0.0833
   ],
   "delta_vs_baseline": -0.083333,
   "mcnemar": {
    "a_only": 3,
    "b_only": 1,
    "discordant": 4,
    "p_two_sided": 0.625,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "F_debate_synthesis": {
   "bootstrap95": [
    -0.2083,
    0.0
   ],
   "delta_vs_baseline": -0.083333,
   "mcnemar": {
    "a_only": 2,
    "b_only": 0,
    "discordant": 2,
    "p_two_sided": 0.5,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "G_shared_workspace": {
   "bootstrap95": [
    -0.25,
    0.0833
   ],
   "delta_vs_baseline": -0.083333,
   "mcnemar": {
    "a_only": 3,
    "b_only": 1,
    "discordant": 4,
    "p_two_sided": 0.625,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "H_dynamic_routing": {
   "bootstrap95": [
    0.0,
    0.0
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 0,
    "b_only": 0,
    "discordant": 0,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "I_fixed_routing": {
   "bootstrap95": [
    0.0,
    0.0
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 0,
    "b_only": 0,
    "discordant": 0,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "J_confidence_aware": {
   "bootstrap95": [
    -0.625,
    -0.2083
   ],
   "delta_vs_baseline": -0.416667,
   "mcnemar": {
    "a_only": 10,
    "b_only": 0,
    "discordant": 10,
    "p_two_sided": 0.001953,
    "significant_at_0.05": true
   },
   "n": 24
  },
  "K_model_specialist": {
   "bootstrap95": [
    0.0,
    0.0
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 0,
    "b_only": 0,
    "discordant": 0,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "L_adaptive_comm": {
   "bootstrap95": [
    -0.125,
    0.0
   ],
   "delta_vs_baseline": -0.041667,
   "mcnemar": {
    "a_only": 1,
    "b_only": 0,
    "discordant": 1,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "M_independent_vote": {
   "bootstrap95": [
    -0.125,
    0.125
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 1,
    "b_only": 1,
    "discordant": 2,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "N_multistage_verify": {
   "bootstrap95": [
    -0.25,
    0.0833
   ],
   "delta_vs_baseline": -0.083333,
   "mcnemar": {
    "a_only": 3,
    "b_only": 1,
    "discordant": 4,
    "p_two_sided": 0.625,
    "significant_at_0.05": false
   },
   "n": 24
  },
  "O_cross_domain": {
   "bootstrap95": [
    -0.2083,
    0.2083
   ],
   "delta_vs_baseline": 0.0,
   "mcnemar": {
    "a_only": 3,
    "b_only": 3,
    "discordant": 6,
    "p_two_sided": 1.0,
    "significant_at_0.05": false
   },
   "n": 24
  }
 },
 "decision": {
  "baseline_accuracy": 0.416667,
  "best_collaborative_accuracy": 0.416667,
  "best_collaborative_arm": "B_hierarchical",
  "best_delta": 0.0,
  "best_mcnemar_p": 1.0,
  "collaboration_superior": false,
  "recommendation": "use the single agent; collaboration cost more without a measured benefit",
  "verdict": "COLLABORATION_NOT_SUPERIOR_HONESTLY_REJECTED",
  "worst_accuracy": 0.0,
  "worst_arm": "J_confidence_aware"
 },
 "elapsed_s": 1544.5,
 "execution_class": "REAL_LOCAL_MODEL",
 "grading_note": "correctness is recomputed from the stored answers with the final normaliser; no model call was repeated",
 "limits": [
  "one 0.5B local CPU model",
  "24 constructed tasks with code-computed ground truth",
  "no cloud model was called",
  "small sample; Wilson intervals reported"
 ],
 "model_key": "qwen2.5-0.5b-instruct",
 "preregistration": {
  "arms": {
   "A_single": "one agent, one answer",
   "B_hierarchical": "planner then executor",
   "C_parallel_research": "three independent samples, majority",
   "D_evidence_first": "answer with an explicit rule/evidence step",
   "E_adversarial_review": "producer then independent critic",
   "F_debate_synthesis": "two debaters then synthesis",
   "G_shared_workspace": "producer writes intermediate, second agent reads it",
   "H_dynamic_routing": "route by task family to the best measured model",
   "I_fixed_routing": "route every task to one fixed model",
   "J_confidence_aware": "second sample only when confidence is low",
   "K_model_specialist": "match model family to discipline",
   "L_adaptive_comm": "communicate only on disagreement",
   "M_independent_vote": "three independent answers, majority",
   "N_multistage_verify": "answer then independent verification",
   "O_cross_domain": "two disciplines then synthesis"
  },
  "created_utc": 1794828568.809831,
  "execution_class": "REAL_LOCAL_MODEL",
  "frozen": true,
  "model": "qwen2.5-0.5b-instruct",
  "note": "The task set is computed deterministically in code; ground truth is not model-generated.",
  "null_hypothesis": "H0: no collaborative arm achieves that; report collaboration as not superior.",
  "primary_hypothesis": "H1: the best collaborative arm beats A_single by >= 0.10 absolute exact-match accuracy with McNemar exact p < 0.05 on the same frozen tasks.",
  "primary_metric": "exact-match accuracy (normalised final answer)",
  "schema": "cain2027.e81.preregistration.v1",
  "secondary_metrics": [
   "latency_ms",
   "tokens",
   "cost_usd",
   "communication_messages",
   "communication_chars",
   "calls"
  ],
  "statistics": [
   "Wilson 95% interval",
   "McNemar exact test vs A_single",
   "paired bootstrap 95% interval of the difference"
  ],
  "task_count": 24,
  "task_set_sha256": "4246f22a9c0c602578f7c9a3847fb69ae35174a43db1025ab96e21028962d61b",
  "task_set_version": "e81.collaboration.v1"
 },
 "real_local_calls": 412,
 "real_remote_calls": 0,
 "recomputed_utc": 1794830739.819615,
 "rows_file": "artifacts/e81/collaboration_rows.jsonl",
 "schema": "cain2027.e81.collaboration_benchmark.v1",
 "summary": {
  "A_single": {
   "accuracy": 0.416667,
   "calls": 24,
   "communication_chars": 0,
   "communication_messages": 0,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "one agent, one answer",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 4495.126,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 10104.547,
   "tokens": 1304,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "B_hierarchical": {
   "accuracy": 0.416667,
   "calls": 48,
   "communication_chars": 2270,
   "communication_messages": 48,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "planner then executor",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 14666.723,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 26586.205,
   "tokens": 3838,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "C_parallel_research": {
   "accuracy": 0.416667,
   "calls": 72,
   "communication_chars": 1034,
   "communication_messages": 72,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "three independent samples, majority",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 14567.127,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 36286.778,
   "tokens": 3931,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "D_evidence_first": {
   "accuracy": 0.291667,
   "calls": 24,
   "communication_chars": 0,
   "communication_messages": 0,
   "correct": 7,
   "cost_usd": 0.0,
   "description": "answer with an explicit rule/evidence step",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 8165.147,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 11578.143,
   "tokens": 2162,
   "wilson95": [
    0.1491,
    0.4917
   ]
  },
  "E_adversarial_review": {
   "accuracy": 0.333333,
   "calls": 48,
   "communication_chars": 753,
   "communication_messages": 48,
   "correct": 8,
   "cost_usd": 0.0,
   "description": "producer then independent critic",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 5335.487,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 12760.452,
   "tokens": 3218,
   "wilson95": [
    0.1797,
    0.5329
   ]
  },
  "F_debate_synthesis": {
   "accuracy": 0.333333,
   "calls": 72,
   "communication_chars": 1097,
   "communication_messages": 72,
   "correct": 8,
   "cost_usd": 0.0,
   "description": "two debaters then synthesis",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 9464.121,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 21213.256,
   "tokens": 4702,
   "wilson95": [
    0.1797,
    0.5329
   ]
  },
  "G_shared_workspace": {
   "accuracy": 0.333333,
   "calls": 48,
   "communication_chars": 1105,
   "communication_messages": 48,
   "correct": 8,
   "cost_usd": 0.0,
   "description": "producer writes intermediate, second agent reads it",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 8803.209,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 17077.638,
   "tokens": 3349,
   "wilson95": [
    0.1797,
    0.5329
   ]
  },
  "H_dynamic_routing": {
   "accuracy": 0.416667,
   "calls": 24,
   "communication_chars": 0,
   "communication_messages": 0,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "route by task family to the best measured model",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 2366.824,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 4593.782,
   "tokens": 1304,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "I_fixed_routing": {
   "accuracy": 0.416667,
   "calls": 24,
   "communication_chars": 0,
   "communication_messages": 0,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "route every task to one fixed model",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 3954.864,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 9357.919,
   "tokens": 1304,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "J_confidence_aware": {
   "accuracy": 0.0,
   "calls": 41,
   "communication_chars": 271,
   "communication_messages": 17,
   "correct": 0,
   "cost_usd": 0.0,
   "description": "second sample only when confidence is low",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 7357.129,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 12893.521,
   "tokens": 2666,
   "wilson95": [
    0.0,
    0.138
   ]
  },
  "K_model_specialist": {
   "accuracy": 0.416667,
   "calls": 24,
   "communication_chars": 0,
   "communication_messages": 0,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "match model family to discipline",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 3133.542,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 7226.2,
   "tokens": 1304,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "L_adaptive_comm": {
   "accuracy": 0.375,
   "calls": 59,
   "communication_chars": 895,
   "communication_messages": 59,
   "correct": 9,
   "cost_usd": 0.0,
   "description": "communicate only on disagreement",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 9363.196,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 16799.71,
   "tokens": 3442,
   "wilson95": [
    0.2116,
    0.5729
   ]
  },
  "M_independent_vote": {
   "accuracy": 0.416667,
   "calls": 72,
   "communication_chars": 1160,
   "communication_messages": 72,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "three independent answers, majority",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 10996.818,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 23973.46,
   "tokens": 3947,
   "wilson95": [
    0.2447,
    0.6117
   ]
  },
  "N_multistage_verify": {
   "accuracy": 0.333333,
   "calls": 48,
   "communication_chars": 753,
   "communication_messages": 48,
   "correct": 8,
   "cost_usd": 0.0,
   "description": "answer then independent verification",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 7334.646,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 18901.497,
   "tokens": 3218,
   "wilson95": [
    0.1797,
    0.5329
   ]
  },
  "O_cross_domain": {
   "accuracy": 0.416667,
   "calls": 72,
   "communication_chars": 1373,
   "communication_messages": 72,
   "correct": 10,
   "cost_usd": 0.0,
   "description": "two disciplines then synthesis",
   "execution_classes": [
    "REAL_LOCAL_MODEL"
   ],
   "mean_latency_ms": 11020.162,
   "models": [
    "qwen2.5-0.5b-instruct"
   ],
   "n": 24,
   "p95_latency_ms": 19081.062,
   "tokens": 4779,
   "wilson95": [
    0.2447,
    0.6117
   ]
  }
 },
 "task_count": 24,
 "task_set_sha256": "4246f22a9c0c602578f7c9a3847fb69ae35174a43db1025ab96e21028962d61b",
 "utc": 1794830113.326772
}