{ "at": 1794625114.3936083, "calibration": { "qwen2.5-0.5b": { "accuracy": 0.75, "correct": 6, "mean_latency_s": 50.3891, "model": "qwen2.5-0.5b", "tasks": 8, "tokens_out": 64 }, "smollm2-135m": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 5.0288, "model": "smollm2-135m", "tasks": 8, "tokens_out": 183 }, "smollm2-360m": { "accuracy": 0.25, "correct": 2, "mean_latency_s": 4.4662, "model": "smollm2-360m", "tasks": 8, "tokens_out": 53 } }, "call_count": 56, "calls": [ { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "88", "family": "arithmetic", "latency_s": 19.9885, "model": "smollm2-135m", "ok": true, "prompt_sha256": "dd3f5f9c43fe2ffe7ac43044b9f55b0050deb5974e483679993169df33726212", "task_id": "c1", "text": "12 * 7 + 4 = 156.", "tokens_in": 43, "tokens_out": 15 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "43", "family": "arithmetic", "latency_s": 4.1708, "model": "smollm2-135m", "ok": true, "prompt_sha256": "5b62b73df7e339680cfc2a0cfabc83dac6b6daedd24496250d240c6509b9ffe4", "task_id": "c2", "text": "There are 9 rows of 6 bottles.\n11 are removed.\nTherefore, there are 9 - 11 = 3 rows of 6 bottles left.\nThere are 3 rows of 6 bottles.", "tokens_in": 54, "tokens_out": 48 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 0.758, "model": "smollm2-135m", "ok": true, "prompt_sha256": "5f96ccaba2185d21fd6a6d43099ecd7c280dcf7b5d5f88586fb06642d9df188e", "task_id": "c3", "text": "a stolen credential was used to log in", "tokens_in": 50, "tokens_out": 9 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.7837, "model": "smollm2-135m", "ok": true, "prompt_sha256": "b66badf1eb0a546943ff672522628c926df384e48c3ecbe69aa7e2b04629b421", "task_id": "c4", "text": "an invoice did not reconcile with the ledger", "tokens_in": 50, "tokens_out": 9 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"grep\",\"sed\"]}", "family": "extraction", "latency_s": 8.0374, "model": "smollm2-135m", "ok": true, "prompt_sha256": "a0ab676f0ebb5d2f5ea171f169056a6db1a3381dfb6c43791e175ce6d088bdad", "task_id": "c5", "text": "Here is the JSON response:\n\n```json\n{\n \"tools\": [\n {\n \"name\": \"grep\",\n \"description\": \"Search for files in a directory and print the files that match a pattern\",", "tokens_in": 48, "tokens_out": 48 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 2.4102, "model": "smollm2-135m", "ok": true, "prompt_sha256": "f62ff39b5d522ea6253e2be99757a6b4342b331648b7cbb26b8e66883f040a48", "task_id": "c6", "text": "The sentence 'there is no evidence of failure' means failure occurred: true or false?", "tokens_in": 49, "tokens_out": 19 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "READY", "family": "instruction", "latency_s": 1.3438, "model": "smollm2-135m", "ok": true, "prompt_sha256": "c71c5c5cb7962ff961f6104d270992d5a0f307053365c1088c9097ec9665d150", "task_id": "c7", "text": "Reply with exactly the word READY and nothing else.", "tokens_in": 38, "tokens_out": 13 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "3", "family": "code", "latency_s": 2.7377, "model": "smollm2-135m", "ok": true, "prompt_sha256": "4605f9c5de59c20b00d385f18d91c5316b6024decaf187936cf97236c933d555", "task_id": "c8", "text": "The value of the list comprehension `[4, 5, 6]` is `6`.", "tokens_in": 48, "tokens_out": 22 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "88", "family": "arithmetic", "latency_s": 25.7025, "model": "smollm2-360m", "ok": true, "prompt_sha256": "dd3f5f9c43fe2ffe7ac43044b9f55b0050deb5974e483679993169df33726212", "task_id": "c1", "text": "12 * 7 + 4 = 88", "tokens_in": 43, "tokens_out": 13 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "43", "family": "arithmetic", "latency_s": 0.848, "model": "smollm2-360m", "ok": true, "prompt_sha256": "5b62b73df7e339680cfc2a0cfabc83dac6b6daedd24496250d240c6509b9ffe4", "task_id": "c2", "text": "11", "tokens_in": 54, "tokens_out": 3 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 0.7172, "model": "smollm2-360m", "ok": true, "prompt_sha256": "5f96ccaba2185d21fd6a6d43099ecd7c280dcf7b5d5f88586fb06642d9df188e", "task_id": "c3", "text": "security", "tokens_in": 50, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.6636, "model": "smollm2-360m", "ok": true, "prompt_sha256": "b66badf1eb0a546943ff672522628c926df384e48c3ecbe69aa7e2b04629b421", "task_id": "c4", "text": "security", "tokens_in": 50, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"grep\",\"sed\"]}", "family": "extraction", "latency_s": 1.9544, "model": "smollm2-360m", "ok": true, "prompt_sha256": "a0ab676f0ebb5d2f5ea171f169056a6db1a3381dfb6c43791e175ce6d088bdad", "task_id": "c5", "text": "{\n \"tools\": []\n}", "tokens_in": 48, "tokens_out": 9 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 0.6581, "model": "smollm2-360m", "ok": true, "prompt_sha256": "f62ff39b5d522ea6253e2be99757a6b4342b331648b7cbb26b8e66883f040a48", "task_id": "c6", "text": "False", "tokens_in": 49, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "READY", "family": "instruction", "latency_s": 1.921, "model": "smollm2-360m", "ok": true, "prompt_sha256": "c71c5c5cb7962ff961f6104d270992d5a0f307053365c1088c9097ec9665d150", "task_id": "c7", "text": "I am ready to assist you.", "tokens_in": 38, "tokens_out": 8 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "3", "family": "code", "latency_s": 3.2647, "model": "smollm2-360m", "ok": true, "prompt_sha256": "4605f9c5de59c20b00d385f18d91c5316b6024decaf187936cf97236c933d555", "task_id": "c8", "text": "len([4, 5, 6]) = 3", "tokens_in": 48, "tokens_out": 14 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "88", "family": "arithmetic", "latency_s": 377.4168, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "dd3f5f9c43fe2ffe7ac43044b9f55b0050deb5974e483679993169df33726212", "task_id": "c1", "text": "88", "tokens_in": 42, "tokens_out": 3 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "43", "family": "arithmetic", "latency_s": 1.1056, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "5b62b73df7e339680cfc2a0cfabc83dac6b6daedd24496250d240c6509b9ffe4", "task_id": "c2", "text": "54", "tokens_in": 52, "tokens_out": 3 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 0.6819, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "5f96ccaba2185d21fd6a6d43099ecd7c280dcf7b5d5f88586fb06642d9df188e", "task_id": "c3", "text": "Security", "tokens_in": 49, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.7679, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "b66badf1eb0a546943ff672522628c926df384e48c3ecbe69aa7e2b04629b421", "task_id": "c4", "text": "Finance", "tokens_in": 49, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"grep\",\"sed\"]}", "family": "extraction", "latency_s": 18.0057, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "a0ab676f0ebb5d2f5ea171f169056a6db1a3381dfb6c43791e175ce6d088bdad", "task_id": "c5", "text": "```json\n{\n \"tools\": [\n {\n \"name\": \"grep\",\n \"description\": \"Search for patterns in files or strings.\",\n \"usage\": \"grep [options] pattern file1 file2 ...\"", "tokens_in": 46, "tokens_out": 48 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 1.7786, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "f62ff39b5d522ea6253e2be99757a6b4342b331648b7cbb26b8e66883f040a48", "task_id": "c6", "text": "False", "tokens_in": 48, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "READY", "family": "instruction", "latency_s": 1.5294, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "c71c5c5cb7962ff961f6104d270992d5a0f307053365c1088c9097ec9665d150", "task_id": "c7", "text": "READY", "tokens_in": 35, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "3", "family": "code", "latency_s": 1.8266, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "4605f9c5de59c20b00d385f18d91c5316b6024decaf187936cf97236c933d555", "task_id": "c8", "text": "3", "tokens_in": 47, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "97", "family": "arithmetic", "latency_s": 2.4089, "model": "smollm2-135m", "ok": true, "prompt_sha256": "c639761edaa1510e7e806444ee0b31a689622792ffc37075d80bd490aabae664", "task_id": "h1", "text": "15 * 6 + 7 = 120 + 7 = 127.", "tokens_in": 43, "tokens_out": 23 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "31", "family": "arithmetic", "latency_s": 4.1726, "model": "smollm2-135m", "ok": true, "prompt_sha256": "e2e78580f75912ea75d93e8c9db875fe289e36ca5b44e3875167be480a7610fa", "task_id": "h2", "text": "There are 8 rows of 5 books.\n9 are removed.\nThere are 8 - 9 = 1 row left.\nTherefore, there are 1 - 9 = 1 book left.\n####", "tokens_in": 52, "tokens_out": 48 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 0.7382, "model": "smollm2-135m", "ok": true, "prompt_sha256": "08bc1144cb9350e55a5eca23cb25576c3fcc44a1a71ddf91b351b6726de30065", "task_id": "h3", "text": "an anomalous login from a new country", "tokens_in": 49, "tokens_out": 8 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.7263, "model": "smollm2-135m", "ok": true, "prompt_sha256": "d66ac641dc3a50a457d6ea75643ed6ec554384dd2df876f1155200be97b8b8cc", "task_id": "h4", "text": "a duplicate payment was posted twice.", "tokens_in": 48, "tokens_out": 8 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"curl\",\"awk\"]}", "family": "extraction", "latency_s": 6.0788, "model": "smollm2-135m", "ok": true, "prompt_sha256": "58816c9af5fddf2b1909fd08a6d05e26e443064886adce4544878e0bbe3aa707", "task_id": "h5", "text": "Here is the JSON response:\n\n```json\n{\n \"tools\": [\n {\n \"name\": \"curl\",\n \"description\": \"curl\",\n \"version\": \"1.0.0\",\n \"", "tokens_in": 48, "tokens_out": 48 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 1.8054, "model": "smollm2-135m", "ok": true, "prompt_sha256": "0b8018184122384872a5c2e324e1a416c0d0dbfa3d1a4ca8eed9cb2b17d68ea5", "task_id": "h6", "text": "The sentence 'the test did not fail' means the test failed: true or false?", "tokens_in": 49, "tokens_out": 19 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "DONE", "family": "instruction", "latency_s": 5.334, "model": "smollm2-135m", "ok": true, "prompt_sha256": "100a0dd1b700de909e2926338737c9f62db66b0c956f99a4902e32037607636b", "task_id": "h7", "text": "I will respond with the requested information exactly as it is.", "tokens_in": 38, "tokens_out": 13 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "4", "family": "code", "latency_s": 9.948, "model": "smollm2-135m", "ok": true, "prompt_sha256": "abb05eb10db84561dc3cac0540097b9640bf25028fe00cc4617ee85c3fe8ed80", "task_id": "h8", "text": "The value of the list comprehension `[9, 8, 7, 6]` is `10`.", "tokens_in": 51, "tokens_out": 26 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "97", "family": "arithmetic", "latency_s": 3.7759, "model": "smollm2-360m", "ok": true, "prompt_sha256": "c639761edaa1510e7e806444ee0b31a689622792ffc37075d80bd490aabae664", "task_id": "h1", "text": "15 * 6 + 7 = 93", "tokens_in": 43, "tokens_out": 13 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "31", "family": "arithmetic", "latency_s": 0.7869, "model": "smollm2-360m", "ok": true, "prompt_sha256": "e2e78580f75912ea75d93e8c9db875fe289e36ca5b44e3875167be480a7610fa", "task_id": "h2", "text": "9", "tokens_in": 52, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 0.7295, "model": "smollm2-360m", "ok": true, "prompt_sha256": "08bc1144cb9350e55a5eca23cb25576c3fcc44a1a71ddf91b351b6726de30065", "task_id": "h3", "text": "Security", "tokens_in": 49, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.6346, "model": "smollm2-360m", "ok": true, "prompt_sha256": "d66ac641dc3a50a457d6ea75643ed6ec554384dd2df876f1155200be97b8b8cc", "task_id": "h4", "text": "security", "tokens_in": 48, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"curl\",\"awk\"]}", "family": "extraction", "latency_s": 1.4691, "model": "smollm2-360m", "ok": true, "prompt_sha256": "58816c9af5fddf2b1909fd08a6d05e26e443064886adce4544878e0bbe3aa707", "task_id": "h5", "text": "{\n \"tools\": []\n}", "tokens_in": 48, "tokens_out": 9 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 0.4778, "model": "smollm2-360m", "ok": true, "prompt_sha256": "0b8018184122384872a5c2e324e1a416c0d0dbfa3d1a4ca8eed9cb2b17d68ea5", "task_id": "h6", "text": "True", "tokens_in": 49, "tokens_out": 2 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "DONE", "family": "instruction", "latency_s": 1.2673, "model": "smollm2-360m", "ok": true, "prompt_sha256": "100a0dd1b700de909e2926338737c9f62db66b0c956f99a4902e32037607636b", "task_id": "h7", "text": "I am ready to assist you.", "tokens_in": 38, "tokens_out": 8 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "4", "family": "code", "latency_s": 4.3154, "model": "smollm2-360m", "ok": true, "prompt_sha256": "abb05eb10db84561dc3cac0540097b9640bf25028fe00cc4617ee85c3fe8ed80", "task_id": "h8", "text": "len([9, 8, 7, 6]) = 6", "tokens_in": 51, "tokens_out": 17 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "97", "family": "arithmetic", "latency_s": 1.8773, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "c639761edaa1510e7e806444ee0b31a689622792ffc37075d80bd490aabae664", "task_id": "h1", "text": "92", "tokens_in": 42, "tokens_out": 3 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "31", "family": "arithmetic", "latency_s": 0.6464, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "e2e78580f75912ea75d93e8c9db875fe289e36ca5b44e3875167be480a7610fa", "task_id": "h2", "text": "42", "tokens_in": 51, "tokens_out": 3 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "security", "family": "classification", "latency_s": 1.288, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "08bc1144cb9350e55a5eca23cb25576c3fcc44a1a71ddf91b351b6726de30065", "task_id": "h3", "text": "Anomalous Login", "tokens_in": 49, "tokens_out": 5 }, { "correct": false, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "finance", "family": "classification", "latency_s": 0.7862, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "d66ac641dc3a50a457d6ea75643ed6ec554384dd2df876f1155200be97b8b8cc", "task_id": "h4", "text": "Security", "tokens_in": 47, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "{\"tools\":[\"curl\",\"awk\"]}", "family": "extraction", "latency_s": 8.7146, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "58816c9af5fddf2b1909fd08a6d05e26e443064886adce4544878e0bbe3aa707", "task_id": "h5", "text": "```json\n{\n \"tools\": [\n \"curl\",\n \"awk\"\n ]\n}\n```", "tokens_in": 46, "tokens_out": 22 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "false", "family": "reasoning", "latency_s": 3.2075, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "0b8018184122384872a5c2e324e1a416c0d0dbfa3d1a4ca8eed9cb2b17d68ea5", "task_id": "h6", "text": "False.", "tokens_in": 48, "tokens_out": 3 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "DONE", "family": "instruction", "latency_s": 1.0354, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "100a0dd1b700de909e2926338737c9f62db66b0c956f99a4902e32037607636b", "task_id": "h7", "text": "DONE", "tokens_in": 35, "tokens_out": 2 }, { "correct": true, "cost_usd": 0.0, "error": null, "execution_class": "REAL_LOCAL_MODEL", "expected": "4", "family": "code", "latency_s": 1.3061, "model": "qwen2.5-0.5b", "ok": true, "prompt_sha256": "abb05eb10db84561dc3cac0540097b9640bf25028fe00cc4617ee85c3fe8ed80", "task_id": "h8", "text": "4", "tokens_in": 50, "tokens_out": 2 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "arithmetic", "heldout": true, "latency_s": 0.68, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h1", "tokens_in": 42, "tokens_out": 3 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "arithmetic", "heldout": true, "latency_s": 0.9446, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h2", "tokens_in": 51, "tokens_out": 3 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "classification", "heldout": true, "latency_s": 1.2694, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h3", "tokens_in": 49, "tokens_out": 5 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "classification", "heldout": true, "latency_s": 0.4644, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h4", "tokens_in": 47, "tokens_out": 2 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "extraction", "heldout": true, "latency_s": 1.4687, "model": "smollm2-360m", "ok": true, "routed": true, "task_id": "h5", "tokens_in": 48, "tokens_out": 9 }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "reasoning", "heldout": true, "latency_s": 3.4715, "model": "smollm2-360m", "ok": true, "routed": true, "task_id": "h6", "tokens_in": 49, "tokens_out": 2 }, { "correct": true, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "instruction", "heldout": true, "latency_s": 0.5085, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h7", "tokens_in": 35, "tokens_out": 2 }, { "correct": true, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "code", "heldout": true, "latency_s": 1.7614, "model": "qwen2.5-0.5b", "ok": true, "routed": true, "task_id": "h8", "tokens_in": 50, "tokens_out": 2 } ], "cost_usd": 0.0, "discovery": { "at": 1794624551.5178175, "hf_cache": "/tmp/opencode/hf-cache", "hf_cached_models": [], "local_transformers": { "available": true, "class": "REAL_LOCAL_MODEL", "device": "cpu" }, "note": "presence of a credential is not an inference; only a returned result is a model call", "ollama": { "models": [], "reachable": false, "reason": "URLError: ", "url": "http://127.0.0.1:11434" }, "remote": { "anthropic": { "base_url_present": false, "credential_env": "ANTHROPIC_API_KEY", "credentials_present": false, "default_model": "claude-3-5-haiku-latest", "execution_class_if_called": "BLOCKED", "reason": "ANTHROPIC_API_KEY not set" }, "deepseek": { "base_url_present": false, "credential_env": "DEEPSEEK_API_KEY", "credentials_present": false, "default_model": "deepseek-chat", "execution_class_if_called": "BLOCKED", "reason": "DEEPSEEK_API_KEY not set" }, "gemini": { "base_url_present": false, "credential_env": "GEMINI_API_KEY", "credentials_present": false, "default_model": "gemini-2.0-flash", "execution_class_if_called": "BLOCKED", "reason": "GEMINI_API_KEY not set" }, "groq": { "base_url_present": false, "credential_env": "GROQ_API_KEY", "credentials_present": false, "default_model": "llama-3.3-70b-versatile", "execution_class_if_called": "BLOCKED", "reason": "GROQ_API_KEY not set" }, "huggingface": { "base_url_present": false, "credential_env": "HF_TOKEN", "credentials_present": false, "default_model": "HuggingFaceH4/zephyr-7b-beta", "execution_class_if_called": "BLOCKED", "reason": "HF_TOKEN not set" }, "mistral": { "base_url_present": false, "credential_env": "MISTRAL_API_KEY", "credentials_present": false, "default_model": "mistral-small-latest", "execution_class_if_called": "BLOCKED", "reason": "MISTRAL_API_KEY not set" }, "openai": { "base_url_present": false, "credential_env": "OPENAI_API_KEY", "credentials_present": false, "default_model": "gpt-4o-mini", "execution_class_if_called": "BLOCKED", "reason": "OPENAI_API_KEY not set" } }, "schema": "cain72.provider_discovery.v1" }, "elapsed_s": 562.9, "execution_classes": { "REAL_LOCAL_MODEL": 56 }, "fallback": { "attempts": [ { "error": "ProviderError: cain/does-not-exist-e72 is not a local folder and is not a valid model identifier listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a token ha", "execution_class": "BLOCKED", "model": "broken-model", "ok": false }, { "error": null, "execution_class": "REAL_LOCAL_MODEL", "model": "smollm2-360m", "ok": true, "text": "15 * 6 + 7 = 93" } ], "fallback": "smollm2-360m", "note": "the failed primary was a nonexistent model id, not a simulated remote provider", "primary": "broken-model", "recovered": true, "schema": "cain72.fallback.v1" }, "heldout_per_model": { "qwen2.5-0.5b": { "accuracy": 0.5, "correct": 4, "mean_latency_s": 2.3577, "model": "qwen2.5-0.5b", "tasks": 8, "tokens_out": 42, "wilson95": [ 0.2152, 0.7848 ] }, "smollm2-135m": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 3.9015, "model": "smollm2-135m", "tasks": 8, "tokens_out": 193, "wilson95": [ 0.0, 0.3244 ] }, "smollm2-360m": { "accuracy": 0.125, "correct": 1, "mean_latency_s": 1.6821, "model": "smollm2-360m", "tasks": 8, "tokens_out": 55, "wilson95": [ 0.0224, 0.4709 ] } }, "load": { "qwen2.5-0.5b": { "loaded": true }, "smollm2-135m": { "loaded": true }, "smollm2-360m": { "loaded": true } }, "note": "Accuracy is deterministic task completion on 8 held-out prompts, not keyword coverage and not general intelligence. Local models are 135M/362M/494M parameters on CPU; remote providers had no credential and are classified BLOCKED, not executed. Cost is 0 because nothing was billed.", "real_local_calls": 56, "real_remote_calls": 0, "routing": { "baseline_accuracy": 0.5, "baseline_model": "qwen2.5-0.5b", "baseline_wilson95": [ 0.2152, 0.7848 ], "policy": "highest measured family accuracy, latency tie-break", "routed_accuracy": 0.25, "routed_rows": [ { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "arithmetic", "latency_s": 0.68, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h1" }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "arithmetic", "latency_s": 0.9446, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h2" }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "classification", "latency_s": 1.2694, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h3" }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "classification", "latency_s": 0.4644, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h4" }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "extraction", "latency_s": 1.4687, "model": "smollm2-360m", "routed_by": "measured accuracy, latency tie-break", "task_id": "h5" }, { "correct": false, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "reasoning", "latency_s": 3.4715, "model": "smollm2-360m", "routed_by": "measured accuracy, latency tie-break", "task_id": "h6" }, { "correct": true, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "instruction", "latency_s": 0.5085, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h7" }, { "correct": true, "error": null, "execution_class": "REAL_LOCAL_MODEL", "family": "code", "latency_s": 1.7614, "model": "qwen2.5-0.5b", "routed_by": "measured accuracy, latency tie-break", "task_id": "h8" } ], "routed_wilson95": [ 0.0715, 0.5907 ], "router_table": { "qwen2.5-0.5b": { "arithmetic": { "accuracy": 0.5, "correct": 1, "mean_latency_s": 189.2612, "tokens_out": 6, "total": 2 }, "classification": { "accuracy": 1.0, "correct": 2, "mean_latency_s": 0.7249, "tokens_out": 4, "total": 2 }, "code": { "accuracy": 1.0, "correct": 1, "mean_latency_s": 1.8266, "tokens_out": 2, "total": 1 }, "extraction": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 18.0057, "tokens_out": 48, "total": 1 }, "instruction": { "accuracy": 1.0, "correct": 1, "mean_latency_s": 1.5294, "tokens_out": 2, "total": 1 }, "reasoning": { "accuracy": 1.0, "correct": 1, "mean_latency_s": 1.7786, "tokens_out": 2, "total": 1 } }, "smollm2-135m": { "arithmetic": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 12.0796, "tokens_out": 63, "total": 2 }, "classification": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 0.7708, "tokens_out": 18, "total": 2 }, "code": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 2.7377, "tokens_out": 22, "total": 1 }, "extraction": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 8.0374, "tokens_out": 48, "total": 1 }, "instruction": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 1.3438, "tokens_out": 13, "total": 1 }, "reasoning": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 2.4102, "tokens_out": 19, "total": 1 } }, "smollm2-360m": { "arithmetic": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 13.2752, "tokens_out": 16, "total": 2 }, "classification": { "accuracy": 0.5, "correct": 1, "mean_latency_s": 0.6904, "tokens_out": 4, "total": 2 }, "code": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 3.2647, "tokens_out": 14, "total": 1 }, "extraction": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 1.9544, "tokens_out": 9, "total": 1 }, "instruction": { "accuracy": 0.0, "correct": 0, "mean_latency_s": 1.921, "tokens_out": 8, "total": 1 }, "reasoning": { "accuracy": 1.0, "correct": 1, "mean_latency_s": 0.6581, "tokens_out": 2, "total": 1 } } }, "selection_basis": "measured on identical calibration prompts; held-out prompts are disjoint" }, "schema": "cain72.models.v1", "tokens": { "in": 2623, "out": 618 } }