{
 "calls": [
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "14.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 14.694,
   "task_id": "gsm0",
   "text": "14",
   "tokens_out": 3
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "540.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 1.054,
   "task_id": "gsm1",
   "text": "1800",
   "tokens_out": 5
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "50.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 5.582,
   "task_id": "gsm2",
   "text": "To determine the percentage chance that a baby wearing a bow is wearing purple, we need to follow these steps:\n\n1.",
   "tokens_out": 24
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "220.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 1.369,
   "task_id": "gsm3",
   "text": "2240",
   "tokens_out": 5
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "64800.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 9.503,
   "task_id": "gsm4",
   "text": "To determine how much more money Fern will make by choosing the WidgetMaster 9000 over the Widgeteer",
   "tokens_out": 24
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 2.389,
   "task_id": "gsm5",
   "text": "16",
   "tokens_out": 3
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "410.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 2.464,
   "task_id": "gsm6",
   "text": "80 feet",
   "tokens_out": 4
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "5.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 7.395,
   "task_id": "gsm7",
   "text": "To determine how many candy bars George bought, we need to follow these steps:\n\n1. Calculate the total amount of money",
   "tokens_out": 24
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "77.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 2.792,
   "task_id": "gsm8",
   "text": "Elvira will have \u20ac865 left for her clothing.",
   "tokens_out": 15
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "112.0",
   "family": "arithmetic",
   "keyword_score": 0.0,
   "latency_s": 4.331,
   "task_id": "gsm9",
   "text": "To determine how much money Paul spent on driving his car over the two-week period, we need to calculate the total cost",
   "tokens_out": 24
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2611.15207",
   "family": "citation_extract",
   "keyword_score": 1.0,
   "latency_s": 5.098,
   "task_id": "cite0",
   "text": "http://arxiv.org/abs/2611.15207v1",
   "tokens_out": 21
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2610.34634",
   "family": "citation_extract",
   "keyword_score": 1.0,
   "latency_s": 8.721,
   "task_id": "cite1",
   "text": "http://arxiv.org/abs/2610.34634v1",
   "tokens_out": 21
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2610.33633",
   "family": "citation_extract",
   "keyword_score": 1.0,
   "latency_s": 6.583,
   "task_id": "cite2",
   "text": "http://arxiv.org/abs/2610.33633v1",
   "tokens_out": 21
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2610.31130",
   "family": "citation_extract",
   "keyword_score": 0.5,
   "latency_s": 3.9,
   "task_id": "cite3",
   "text": "http://arxiv.org/abs/2610.29722v1",
   "tokens_out": 21
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "2610.29722",
   "family": "citation_extract",
   "keyword_score": 1.0,
   "latency_s": 3.295,
   "task_id": "cite4",
   "text": "http://arxiv.org/abs/2610.29722v1",
   "tokens_out": 21
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "design",
   "family": "planning",
   "keyword_score": 1.0,
   "latency_s": 0.61,
   "task_id": "plan0",
   "text": "Design",
   "tokens_out": 2
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "test",
   "family": "planning",
   "keyword_score": 1.0,
   "latency_s": 0.553,
   "task_id": "plan1",
   "text": "Test",
   "tokens_out": 2
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "deploy",
   "family": "planning",
   "keyword_score": 1.0,
   "latency_s": 0.869,
   "task_id": "plan2",
   "text": "Deploy",
   "tokens_out": 2
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "test",
   "family": "planning",
   "keyword_score": 1.0,
   "latency_s": 0.918,
   "task_id": "plan3",
   "text": "Test",
   "tokens_out": 2
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "design",
   "family": "planning",
   "keyword_score": 1.0,
   "latency_s": 1.589,
   "task_id": "plan4",
   "text": "Design",
   "tokens_out": 2
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "150.0",
   "family": "data_analysis_model",
   "keyword_score": 0.0,
   "latency_s": 2.018,
   "task_id": "data0",
   "text": "100000",
   "tokens_out": 7
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "3.0",
   "family": "data_analysis_model",
   "keyword_score": 0.0,
   "latency_s": 0.925,
   "task_id": "data1",
   "text": "10",
   "tokens_out": 3
  },
  {
   "correct": true,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "5.84",
   "family": "data_analysis_model",
   "keyword_score": 0.0,
   "latency_s": 1.12,
   "task_id": "data2",
   "text": "5.84",
   "tokens_out": 5
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "4.4",
   "family": "data_analysis_model",
   "keyword_score": 0.0,
   "latency_s": 2.632,
   "task_id": "data3",
   "text": "5.1",
   "tokens_out": 4
  },
  {
   "correct": false,
   "error": null,
   "execution_class": "REAL_LOCAL_MODEL",
   "expected": "1.0",
   "family": "data_analysis_model",
   "keyword_score": 0.0,
   "latency_s": 0.848,
   "task_id": "data4",
   "text": "3",
   "tokens_out": 2
  }
 ],
 "comparison": {
  "deterministic_tool": "5/5 repo defects, 5/5 generated regression tests, 5/5 data questions, 20/20 memory",
  "e71_keyword_metric": "would report 0.380 on the same outputs; exact completion is 0.440",
  "local_model_exact": "11/25",
  "not_comparable": "model families and deterministic families use different task types; no aggregate score"
 },
 "elapsed_s": 96.44,
 "families": {
  "arithmetic": {
   "accuracy": 0.1,
   "correct": 1,
   "n": 10,
   "rows": [
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "14.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 14.694,
     "task_id": "gsm0",
     "text": "14",
     "tokens_out": 3
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "540.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 1.054,
     "task_id": "gsm1",
     "text": "1800",
     "tokens_out": 5
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "50.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 5.582,
     "task_id": "gsm2",
     "text": "To determine the percentage chance that a baby wearing a bow is wearing purple, we need to follow these steps:\n\n1.",
     "tokens_out": 24
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "220.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 1.369,
     "task_id": "gsm3",
     "text": "2240",
     "tokens_out": 5
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "64800.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 9.503,
     "task_id": "gsm4",
     "text": "To determine how much more money Fern will make by choosing the WidgetMaster 9000 over the Widgeteer",
     "tokens_out": 24
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 2.389,
     "task_id": "gsm5",
     "text": "16",
     "tokens_out": 3
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "410.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 2.464,
     "task_id": "gsm6",
     "text": "80 feet",
     "tokens_out": 4
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "5.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 7.395,
     "task_id": "gsm7",
     "text": "To determine how many candy bars George bought, we need to follow these steps:\n\n1. Calculate the total amount of money",
     "tokens_out": 24
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "77.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 2.792,
     "task_id": "gsm8",
     "text": "Elvira will have \u20ac865 left for her clothing.",
     "tokens_out": 15
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "112.0",
     "family": "arithmetic",
     "keyword_score": 0.0,
     "latency_s": 4.331,
     "task_id": "gsm9",
     "text": "To determine how much money Paul spent on driving his car over the two-week period, we need to calculate the total cost",
     "tokens_out": 24
    }
   ],
   "wilson95": [
    0.01787574951572113,
    0.4041563854975721
   ]
  },
  "citation_extract": {
   "accuracy": 0.8,
   "correct": 4,
   "n": 5,
   "rows": [
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2611.15207",
     "family": "citation_extract",
     "keyword_score": 1.0,
     "latency_s": 5.098,
     "task_id": "cite0",
     "text": "http://arxiv.org/abs/2611.15207v1",
     "tokens_out": 21
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2610.34634",
     "family": "citation_extract",
     "keyword_score": 1.0,
     "latency_s": 8.721,
     "task_id": "cite1",
     "text": "http://arxiv.org/abs/2610.34634v1",
     "tokens_out": 21
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2610.33633",
     "family": "citation_extract",
     "keyword_score": 1.0,
     "latency_s": 6.583,
     "task_id": "cite2",
     "text": "http://arxiv.org/abs/2610.33633v1",
     "tokens_out": 21
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2610.31130",
     "family": "citation_extract",
     "keyword_score": 0.5,
     "latency_s": 3.9,
     "task_id": "cite3",
     "text": "http://arxiv.org/abs/2610.29722v1",
     "tokens_out": 21
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "2610.29722",
     "family": "citation_extract",
     "keyword_score": 1.0,
     "latency_s": 3.295,
     "task_id": "cite4",
     "text": "http://arxiv.org/abs/2610.29722v1",
     "tokens_out": 21
    }
   ],
   "wilson95": [
    0.3755282641185388,
    0.9637768390302125
   ]
  },
  "data_analysis_deterministic": {
   "accuracy": 1.0,
   "correct": 5,
   "method": "exact recomputation over the public CSV",
   "n": 5,
   "wilson95": [
    0.5655085052479191,
    1.0
   ]
  },
  "data_analysis_model": {
   "accuracy": 0.2,
   "correct": 1,
   "n": 5,
   "rows": [
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "150.0",
     "family": "data_analysis_model",
     "keyword_score": 0.0,
     "latency_s": 2.018,
     "task_id": "data0",
     "text": "100000",
     "tokens_out": 7
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "3.0",
     "family": "data_analysis_model",
     "keyword_score": 0.0,
     "latency_s": 0.925,
     "task_id": "data1",
     "text": "10",
     "tokens_out": 3
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "5.84",
     "family": "data_analysis_model",
     "keyword_score": 0.0,
     "latency_s": 1.12,
     "task_id": "data2",
     "text": "5.84",
     "tokens_out": 5
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "4.4",
     "family": "data_analysis_model",
     "keyword_score": 0.0,
     "latency_s": 2.632,
     "task_id": "data3",
     "text": "5.1",
     "tokens_out": 4
    },
    {
     "correct": false,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "1.0",
     "family": "data_analysis_model",
     "keyword_score": 0.0,
     "latency_s": 0.848,
     "task_id": "data4",
     "text": "3",
     "tokens_out": 2
    }
   ],
   "wilson95": [
    0.03622316096978745,
    0.6244717358814613
   ]
  },
  "memory_retrieval": {
   "accuracy": 1.0,
   "correct": 20,
   "method": "deterministic retrieval over verified memory",
   "n": 20,
   "wilson95": [
    0.8389,
    1.0
   ]
  },
  "planning": {
   "accuracy": 1.0,
   "correct": 5,
   "n": 5,
   "rows": [
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "design",
     "family": "planning",
     "keyword_score": 1.0,
     "latency_s": 0.61,
     "task_id": "plan0",
     "text": "Design",
     "tokens_out": 2
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "test",
     "family": "planning",
     "keyword_score": 1.0,
     "latency_s": 0.553,
     "task_id": "plan1",
     "text": "Test",
     "tokens_out": 2
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "deploy",
     "family": "planning",
     "keyword_score": 1.0,
     "latency_s": 0.869,
     "task_id": "plan2",
     "text": "Deploy",
     "tokens_out": 2
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "test",
     "family": "planning",
     "keyword_score": 1.0,
     "latency_s": 0.918,
     "task_id": "plan3",
     "text": "Test",
     "tokens_out": 2
    },
    {
     "correct": true,
     "error": null,
     "execution_class": "REAL_LOCAL_MODEL",
     "expected": "design",
     "family": "planning",
     "keyword_score": 1.0,
     "latency_s": 1.589,
     "task_id": "plan4",
     "text": "Design",
     "tokens_out": 2
    }
   ],
   "wilson95": [
    0.5655085052479191,
    1.0
   ]
  },
  "repository_defect": {
   "accuracy": 1.0,
   "clean_false_positives": 0,
   "correct": 5,
   "method": "deterministic AST+mutation detector",
   "n": 5,
   "rows": [
    {
     "detected": true,
     "findings": [
      "TRUNCATED_DIGEST"
     ],
     "mutant": 0,
     "test_fails_before": true,
     "test_passes_after": true
    },
    {
     "detected": true,
     "findings": [
      "WEAK_HASH"
     ],
     "mutant": 1,
     "test_fails_before": true,
     "test_passes_after": true
    },
    {
     "detected": true,
     "findings": [
      "UNREACHABLE_RETURN"
     ],
     "mutant": 2,
     "test_fails_before": true,
     "test_passes_after": true
    },
    {
     "detected": true,
     "findings": [
      "TYPE_ERROR"
     ],
     "mutant": 3,
     "test_fails_before": true,
     "test_passes_after": true
    },
    {
     "detected": true,
     "findings": [
      "BYTES_NOT_HEX"
     ],
     "mutant": 4,
     "test_fails_before": true,
     "test_passes_after": true
    }
   ],
   "wilson95": [
    0.5655085052479191,
    1.0
   ]
  },
  "test_generation": {
   "accuracy": 1.0,
   "correct": 5,
   "method": "detector-authored regression test + before/after execution",
   "n": 5,
   "wilson95": [
    0.5655085052479191,
    1.0
   ]
  }
 },
 "limits": "10 GSM8K problems, 5 citation questions, 5 planning tasks, 5 data questions; small samples with Wilson intervals. One 494M-parameter local model, zero remote calls. Held-out tasks are public or system-derived; this is not a general-intelligence benchmark.",
 "model": {
  "calls": 25,
  "exact_accuracy": 0.44,
  "execution_class": "REAL_LOCAL_MODEL",
  "keyword_overstatement": -0.06,
  "latency_p50_s": 2.464,
  "mean_keyword_score": 0.38,
  "name": "qwen2.5-0.5b",
  "params_m": 494.0,
  "tokens_out": 267,
  "wilson95": [
   0.2666537494700256,
   0.6293298298043557
  ]
 },
 "schema": "cain72.benchmark.v1",
 "sources": {
  "arithmetic": "openai/grade-school-math test.jsonl (MIT), 10 sampled problems",
  "citation_extract": "arXiv Atom API response (real primary source)"
 }
}