{
 "at": 1794630240.7483666,
 "at_end": 1794631580.716222,
 "calibration_n": 16,
 "families": [
  "arithmetic",
  "code_symbol",
  "data_stats",
  "extraction",
  "logic",
  "memory_recall",
  "planning",
  "unit_conversion"
 ],
 "models": {
  "qwen2.5-0.5b": {
   "calibration": {
    "accuracy": 0.125,
    "completion_rate": 1.0,
    "correct": 2,
    "latency_s": 98.079,
    "n": 16,
    "output_tokens": 320,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 1.0,
      "correct": 2,
      "n": 2,
      "wilson95": [
       0.3424,
       1.0
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.035,
     0.3602
    ]
   },
   "elapsed_s": 510.36,
   "model_id": "Qwen/Qwen2.5-0.5B-Instruct",
   "test": {
    "accuracy": 0.125,
    "completion_rate": 1.0,
    "correct": 4,
    "latency_s": 411.19,
    "n": 32,
    "output_tokens": 640,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 1.0,
      "correct": 4,
      "n": 4,
      "wilson95": [
       0.5101,
       1.0
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0497,
     0.2807
    ]
   }
  },
  "smollm2-135m": {
   "calibration": {
    "accuracy": 0.0,
    "completion_rate": 0.5625,
    "correct": 0,
    "latency_s": 243.522,
    "n": 16,
    "output_tokens": 176,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1936
    ]
   },
   "elapsed_s": 336.26,
   "model_id": "HuggingFaceTB/SmolLM2-135M-Instruct",
   "test": {
    "accuracy": 0.0,
    "completion_rate": 0.5312,
    "correct": 0,
    "latency_s": 58.174,
    "n": 32,
    "output_tokens": 282,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1072
    ]
   }
  },
  "smollm2-360m": {
   "calibration": {
    "accuracy": 0.0,
    "completion_rate": 0.375,
    "correct": 0,
    "latency_s": 69.592,
    "n": 16,
    "output_tokens": 116,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1936
    ]
   },
   "elapsed_s": 209.14,
   "model_id": "HuggingFaceTB/SmolLM2-360M-Instruct",
   "test": {
    "accuracy": 0.0,
    "completion_rate": 0.375,
    "correct": 0,
    "latency_s": 139.01,
    "n": 32,
    "output_tokens": 246,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1072
    ]
   }
  }
 },
 "note": "constructed held-out benchmark with deterministic checkers; not general intelligence",
 "routing": {
  "arithmetic": "smollm2-360m",
  "code_symbol": "smollm2-360m",
  "data_stats": "smollm2-360m",
  "extraction": "smollm2-360m",
  "logic": "smollm2-360m",
  "memory_recall": "qwen2.5-0.5b",
  "planning": "smollm2-360m",
  "unit_conversion": "smollm2-360m"
 },
 "schema": "cain72.benchmark.v1",
 "test_n": 32,
 "tool_assisted": {
  "accuracy": 1.0,
  "completion_rate": 1.0,
  "correct": 32,
  "latency_s": 0.0,
  "n": 32,
  "output_tokens": 0,
  "per_family": {
   "arithmetic": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "code_symbol": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "data_stats": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "extraction": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "logic": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "memory_recall": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "planning": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   },
   "unit_conversion": {
    "accuracy": 1.0,
    "correct": 4,
    "n": 4,
    "wilson95": [
     0.5101,
     1.0
    ]
   }
  },
  "wilson95": [
   0.8928,
   1.0
  ]
 },
 "tool_assisted_model": "none (all test families have a deterministic tool)",
 "tool_assisted_note": "deterministic tools only; no model call for these families"
}