{
 "design": {
  "arms_compared": [
   "B_single",
   "T_tool_single",
   "D_four",
   "SC2_self_consistency"
  ],
  "blinding": "the grader is a pure function of (answer, task); arm identity is attached after grading",
  "grader": "exact match against computed ground truth",
  "main_tasks": 32,
  "replication_seed": 791,
  "replication_tasks": 32,
  "type": "paired within-task comparison"
 },
 "epistemic_ledger": {
  "DERIVED": {
   "paired_tests": {
    "D_vs_B": null,
    "T_vs_B": {
     "both": 1,
     "discordant_pairs": 12,
     "neither": 19,
     "only_B": 10,
     "only_T": 2,
     "only_first": 2,
     "only_second": 10,
     "p_value_two_sided": 0.0386,
     "tasks": 32
    }
   },
   "wilson_95": {
    "B_single": [
     0.2041,
     0.5169
    ],
    "D_four": [
     0.1325,
     0.4211
    ],
    "SC2_self_consistency": [
     0.2041,
     0.5169
    ],
    "T_tool_single": [
     0.0686,
     0.3175
    ]
   }
  },
  "HYPOTHESIS": {
   "H2_four_vs_single": {
    "direction": "FOUR_LOWER",
    "main_delta": -0.0938,
    "main_paired": null,
    "reason_not_replicated": "The replication run covers arms A/B/T (cheap, single-call) only; D was not re-run.",
    "replicated": false
   },
   "H3_four_vs_self_consistency": {
    "direction": "FOUR_LOWER",
    "main_delta": -0.0938,
    "reason_not_replicated": "SC2 was not re-run in the replication process.",
    "replicated": false
   }
  },
  "MODEL_INFERENCE": {
   "promoted_to_fact": false,
   "text": "The benchmark report indicates that:\n\n- The single specialist's accuracy is 0.3438, which supports the idea that individual specialists can perform tasks with high precision but may still have room for improvement in certain areas.\n  \n- The tool-augmented single specialist's accuracy is 0.1562, suggesting that combining tools can enhance performance but also implies limitations on how well specialized skills can be leveraged.\n\n- The four-specialist's accuracy is 0.25, indicating that even with multiple specialists working together, there might still be room for significant improvements in specific areas"
  },
  "OBSERVED": {
   "main_arm_accuracies": {
    "B_single": 0.3438,
    "D_four": 0.25,
    "SC2_self_consistency": 0.3438,
    "T_tool_single": 0.1562
   },
   "tasks": 32
  },
  "REPRODUCED_RESULT": {
   "direction_main": "TOOL_LOWER",
   "direction_replication": "TOOL_LOWER",
   "finding": "The tool effect direction (tool_lower) reproduced in a separate run (tool_lower).",
   "replicated": true
  },
  "UNRESOLVED": [
   "Whether collaboration helps at a larger model scale is untested here.",
   "Whether the tool effect holds on tasks whose arithmetic cannot be expressed in one line is untested.",
   "The verifier shares the producer's model family, so verifier agreement is weak evidence of correctness."
  ]
 },
 "findings": {
  "hypotheses": [
   "H2_four_vs_single",
   "H3_four_vs_self_consistency"
  ],
  "new_observations": [
   {
    "observation": "Main-run tool delta",
    "value": -0.1876
   }
  ],
  "reproduced": [
   {
    "direction_main": "TOOL_LOWER",
    "direction_replication": "TOOL_LOWER",
    "finding": "The tool effect direction (tool_lower) reproduced in a separate run (tool_lower).",
    "replicated": true
   }
  ],
  "unresolved": [
   "Whether collaboration helps at a larger model scale is untested here.",
   "Whether the tool effect holds on tasks whose arithmetic cannot be expressed in one line is untested.",
   "The verifier shares the producer's model family, so verifier agreement is weak evidence of correctness."
  ]
 },
 "follow_up_experiments": [
  "Repeat the 4-arm comparison with Qwen2.5-1.5B under the same task set.",
  "Give the verifier a different model or a deterministic check so agreement is independent.",
  "Test whether a model-authored expression that disagrees with its own answer can be resolved by a second tool call."
 ],
 "generated_utc": 1794757496.694,
 "governed_execution": {
  "cloud_calls": 0,
  "cost_usd": 0.0,
  "execution_class": "LOCAL_MODEL_INFERENCE",
  "model": "Qwen/Qwen2.5-0.5B-Instruct",
  "tool_execution_class": "DETERMINISTIC_TOOL_EXECUTION"
 },
 "grants_authority": false,
 "literature": {
  "corpus": [
   {
    "claim": "Paired designs control for task difficulty by evaluating every arm on identical inputs.",
    "kind": "METHOD_NOTE",
    "ref": "local:METHOD-01"
   },
   {
    "claim": "Exact-binomial McNemar tests are appropriate for paired binary outcomes.",
    "kind": "METHOD_NOTE",
    "ref": "local:METHOD-02"
   },
   {
    "claim": "Wilson score intervals are preferable to Wald intervals for small samples.",
    "kind": "METHOD_NOTE",
    "ref": "local:METHOD-03"
   },
   {
    "claim": "Tool use can raise arithmetic reliability when the tool, not the model, performs the exact operation.",
    "kind": "PRIOR_HYPOTHESIS_NOT_TESTED_HERE",
    "ref": "local:METHOD-04"
   },
   {
    "claim": "Majority voting helps only when errors are not correlated across voters.",
    "kind": "PRIOR_HYPOTHESIS_NOT_TESTED_HERE",
    "ref": "local:METHOD-05"
   },
   {
    "claim": "A verifier that shares the producer's model and prompt family cannot be assumed independent.",
    "kind": "METHOD_NOTE",
    "ref": "local:METHOD-06"
   }
  ],
  "external_search": "NOT_PERFORMED",
  "retrieval": "LOCAL_FIXTURE_CORPUS"
 },
 "memory_record": {
  "promoted": true,
  "record_id": "mem-31c653d6f851",
  "state": "QUARANTINED_PROPOSAL",
  "verified_by": "agent-verification"
 },
 "model_interpretation": {
  "execution_class": "LOCAL_MODEL_INFERENCE",
  "input_sha256": "716fd48a4e15d41f30cca9f5e9e357f1c0e93e1612b3c912c1f1e778080dcfd8",
  "label": "MODEL_INFERENCE",
  "output_sha256": "f541677670507c61e88c9a14d64a8580c71c1a6c7cf3b4412b07d85b59ada708",
  "promoted_to_fact": false,
  "request_id": "req-ab2fa51b3b0d",
  "status": "PASS",
  "text": "The benchmark report indicates that:\n\n- The single specialist's accuracy is 0.3438, which supports the idea that individual specialists can perform tasks with high precision but may still have room for improvement in certain areas.\n  \n- The tool-augmented single specialist's accuracy is 0.1562, suggesting that combining tools can enhance performance but also implies limitations on how well specialized skills can be leveraged.\n\n- The four-specialist's accuracy is 0.25, indicating that even with multiple specialists working together, there might still be room for significant improvements in specific areas",
  "tokens": 274
 },
 "question": "Does an authorized calculator tool, and does multi-specialist collaboration, change exact-match accuracy on a paired closed-form benchmark compared with a single specialist?",
 "raw_data": {
  "main": "artifacts/e79/bench/*.jsonl",
  "replication": "artifacts/e79/bench_replication/*.jsonl"
 },
 "schema": "cain2027.e79.research_lab.v2",
 "scientific_discovery": "NOT_CLAIMED",
 "statistics": {
  "H1_tool_beats_single": {
   "direction_main": "TOOL_LOWER",
   "direction_replication": "TOOL_LOWER",
   "main_delta": -0.1876,
   "main_paired": {
    "both": 1,
    "discordant_pairs": 12,
    "neither": 19,
    "only_B": 10,
    "only_T": 2,
    "only_first": 2,
    "only_second": 10,
    "p_value_two_sided": 0.0386,
    "tasks": 32
   },
   "replication_delta": -0.25,
   "replication_paired": {
    "both": 1,
    "discordant_pairs": 12,
    "neither": 19,
    "only_B": 10,
    "only_T": 2,
    "only_first": 2,
    "only_second": 10,
    "p_value_two_sided": 0.0386,
    "tasks": 32
   }
  },
  "H2_four_beats_single": {
   "direction": "FOUR_LOWER",
   "main_delta": -0.0938,
   "main_paired": null,
   "reason_not_replicated": "The replication run covers arms A/B/T (cheap, single-call) only; D was not re-run.",
   "replicated": false
  },
  "H3_four_beats_budget_matched_control": {
   "direction": "FOUR_LOWER",
   "main_delta": -0.0938,
   "reason_not_replicated": "SC2 was not re-run in the replication process.",
   "replicated": false
  },
  "main": {
   "B_single": 0.3438,
   "D_four": 0.25,
   "SC2_self_consistency": 0.3438,
   "T_tool_single": 0.1562
  },
  "main_wilson": {
   "B_single": [
    0.2041,
    0.5169
   ],
   "D_four": [
    0.1325,
    0.4211
   ],
   "SC2_self_consistency": [
    0.2041,
    0.5169
   ],
   "T_tool_single": [
    0.0686,
    0.3175
   ]
  },
  "replication": {
   "B_single": 0.3438,
   "T_tool_single": 0.0938,
   "seed": 791,
   "tasks": 32
  }
 }
}