{
 "at": 1794630240.7483666,
 "calibration_n": 16,
 "families": [
  "arithmetic",
  "code_symbol",
  "data_stats",
  "extraction",
  "logic",
  "memory_recall",
  "planning",
  "unit_conversion"
 ],
 "models": {
  "qwen2.5-0.5b": {
   "calibration": {
    "accuracy": 0.125,
    "completion_rate": 1.0,
    "correct": 2,
    "latency_s": 98.079,
    "n": 16,
    "output_tokens": 320,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 1.0,
      "correct": 2,
      "n": 2,
      "wilson95": [
       0.3424,
       1.0
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.035,
     0.3602
    ]
   },
   "elapsed_s": 510.36,
   "model_id": "Qwen/Qwen2.5-0.5B-Instruct",
   "test": {
    "accuracy": 0.125,
    "completion_rate": 1.0,
    "correct": 4,
    "latency_s": 411.19,
    "n": 32,
    "output_tokens": 640,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 1.0,
      "correct": 4,
      "n": 4,
      "wilson95": [
       0.5101,
       1.0
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0497,
     0.2807
    ]
   }
  },
  "smollm2-135m": {
   "calibration": {
    "accuracy": 0.0,
    "completion_rate": 0.5625,
    "correct": 0,
    "latency_s": 243.522,
    "n": 16,
    "output_tokens": 176,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1936
    ]
   },
   "elapsed_s": 336.26,
   "model_id": "HuggingFaceTB/SmolLM2-135M-Instruct",
   "test": {
    "accuracy": 0.0,
    "completion_rate": 0.5312,
    "correct": 0,
    "latency_s": 58.174,
    "n": 32,
    "output_tokens": 282,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1072
    ]
   }
  },
  "smollm2-360m": {
   "calibration": {
    "accuracy": 0.0,
    "completion_rate": 0.375,
    "correct": 0,
    "latency_s": 69.592,
    "n": 16,
    "output_tokens": 116,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 2,
      "wilson95": [
       0.0,
       0.6576
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1936
    ]
   },
   "elapsed_s": 209.14,
   "model_id": "HuggingFaceTB/SmolLM2-360M-Instruct",
   "test": {
    "accuracy": 0.0,
    "completion_rate": 0.375,
    "correct": 0,
    "latency_s": 139.01,
    "n": 32,
    "output_tokens": 246,
    "per_family": {
     "arithmetic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "code_symbol": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "data_stats": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "extraction": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "logic": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "memory_recall": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "planning": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     },
     "unit_conversion": {
      "accuracy": 0.0,
      "correct": 0,
      "n": 4,
      "wilson95": [
       0.0,
       0.4899
      ]
     }
    },
    "wilson95": [
     0.0,
     0.1072
    ]
   }
  }
 },
 "note": "constructed held-out benchmark with deterministic checkers; not general intelligence",
 "schema": "cain72.benchmark.v1",
 "test_n": 32,
 "tool_assisted": {}
}