{"answer": "350", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$900", "input_tokens": 95, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "900", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0f986cdd7d11"], "split": "validation", "task_id": "arith_finance:validation:2000"}
{"answer": "260", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$108", "input_tokens": 93, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1b20c05f23d4"], "split": "validation", "task_id": "arith_finance:validation:2001"}
{"answer": "238", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$160", "input_tokens": 93, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "160", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2e71793a8c10"], "split": "validation", "task_id": "arith_finance:validation:2002"}
{"answer": "260", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1083", "input_tokens": 95, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "1083", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-01f1c492e8d5"], "split": "validation", "task_id": "arith_finance:validation:2003"}
{"answer": "571", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1579", "input_tokens": 95, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "1579", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-30d7c0d1f9ef"], "split": "validation", "task_id": "arith_finance:validation:2004"}
{"answer": "440", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "140", "input_tokens": 94, "latency_ms": 4531.891, "output_tokens": 5, "parsed": "140", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b563b60b1773"], "split": "validation", "task_id": "arith_finance:validation:2005"}
{"answer": "111", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "108", "input_tokens": 92, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-876d9ae5cf4c"], "split": "validation", "task_id": "arith_finance:validation:2006"}
{"answer": "215", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$180", "input_tokens": 94, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "180", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b8bf4337d272"], "split": "validation", "task_id": "arith_finance:validation:2007"}
{"answer": "12", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 135, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-31c6ef9220e2"], "split": "validation", "task_id": "constraint_schedule:validation:2000"}
{"answer": "12", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 136, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3b6cafa6da67"], "split": "validation", "task_id": "constraint_schedule:validation:2001"}
{"answer": "9", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 133, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-189113fc2478"], "split": "validation", "task_id": "constraint_schedule:validation:2002"}
{"answer": "17", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 126, "latency_ms": 1972.405, "output_tokens": 5, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1a72fe5bd71a"], "split": "validation", "task_id": "constraint_schedule:validation:2003"}
{"answer": "11", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 131, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-da71c708a45d"], "split": "validation", "task_id": "constraint_schedule:validation:2004"}
{"answer": "12", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 138, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f943a0933ed0"], "split": "validation", "task_id": "constraint_schedule:validation:2005"}
{"answer": "15", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 131, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ff124f4695fc"], "split": "validation", "task_id": "constraint_schedule:validation:2006"}
{"answer": "11", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 127, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fa28d0e473c9"], "split": "validation", "task_id": "constraint_schedule:validation:2007"}
{"answer": "4", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7ef34493a89f"], "split": "validation", "task_id": "dependency_impact:validation:2000"}
{"answer": "2", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1745.574, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3a57f4b71585"], "split": "validation", "task_id": "dependency_impact:validation:2001"}
{"answer": "4", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 95, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7217749fa67c"], "split": "validation", "task_id": "dependency_impact:validation:2002"}
{"answer": "3", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 95, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-70c83419ae50"], "split": "validation", "task_id": "dependency_impact:validation:2003"}
{"answer": "5", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d05705d1c1dd"], "split": "validation", "task_id": "dependency_impact:validation:2004"}
{"answer": "5", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a7a1b12487d3"], "split": "validation", "task_id": "dependency_impact:validation:2005"}
{"answer": "3", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ef74f8bb0a56"], "split": "validation", "task_id": "dependency_impact:validation:2006"}
{"answer": "5", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "0", "input_tokens": 95, "latency_ms": 1314.175, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-03b8c4dccd53"], "split": "validation", "task_id": "dependency_impact:validation:2007"}
{"answer": "COMPLIANT", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0b6d62ad7252"], "split": "validation", "task_id": "contradiction:validation:2000"}
{"answer": "COMPLIANT", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5268e330eb41"], "split": "validation", "task_id": "contradiction:validation:2001"}
{"answer": "VIOLATION", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-08eacefeae3c"], "split": "validation", "task_id": "contradiction:validation:2002"}
{"answer": "VIOLATION", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-73b8288eea94"], "split": "validation", "task_id": "contradiction:validation:2003"}
{"answer": "COMPLIANT", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1ddb589bcdea"], "split": "validation", "task_id": "contradiction:validation:2004"}
{"answer": "COMPLIANT", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1651.486, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e555ab6ea987"], "split": "validation", "task_id": "contradiction:validation:2005"}
{"answer": "COMPLIANT", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7de0424392ac"], "split": "validation", "task_id": "contradiction:validation:2006"}
{"answer": "VIOLATION", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f455beb28809"], "split": "validation", "task_id": "contradiction:validation:2007"}
{"answer": "DENY", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-38a4aafdcf7e"], "split": "validation", "task_id": "access_decision:validation:2000"}
{"answer": "DENY", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e60e09f689bd"], "split": "validation", "task_id": "access_decision:validation:2001"}
{"answer": "DENY", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 88, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c3773fdc941c"], "split": "validation", "task_id": "access_decision:validation:2002"}
{"answer": "ALLOW", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 90, "latency_ms": 1216.319, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-284cb9dd2fb4"], "split": "validation", "task_id": "access_decision:validation:2003"}
{"answer": "ALLOW", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 92, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-043f7c3dbc2b"], "split": "validation", "task_id": "access_decision:validation:2004"}
{"answer": "DENY", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 90, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c7fe3d7898af"], "split": "validation", "task_id": "access_decision:validation:2005"}
{"answer": "DENY", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 90, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8dc9f21abd93"], "split": "validation", "task_id": "access_decision:validation:2006"}
{"answer": "ALLOW", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 98, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3cac5585988d"], "split": "validation", "task_id": "access_decision:validation:2007"}
{"answer": "TEST", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8c465666520d"], "split": "validation", "task_id": "plan_order:validation:2000"}
{"answer": "BUILD", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design", "input_tokens": 82, "latency_ms": 1134.107, "output_tokens": 3, "parsed": "DESIGN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-17068beba49b"], "split": "validation", "task_id": "plan_order:validation:2001"}
{"answer": "DESIGN", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-09517f783504"], "split": "validation", "task_id": "plan_order:validation:2002"}
{"answer": "DEPLOY", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-91f64f2c9647"], "split": "validation", "task_id": "plan_order:validation:2003"}
{"answer": "REVIEW", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c9aa0f30114d"], "split": "validation", "task_id": "plan_order:validation:2004"}
{"answer": "TEST", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fa06a38e0a66"], "split": "validation", "task_id": "plan_order:validation:2005"}
{"answer": "DEPLOY", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "test", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "TEST", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5955a2a8d197"], "split": "validation", "task_id": "plan_order:validation:2006"}
{"answer": "DEPLOY", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1168.7, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dbfc0a93aec2"], "split": "validation", "task_id": "plan_order:validation:2007"}
{"answer": "UNKNOWN", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "58", "input_tokens": 82, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "58", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-43d54fb23111"], "split": "validation", "task_id": "insufficient:validation:2000"}
{"answer": "81", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 83, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9cfa3637d86f"], "split": "validation", "task_id": "insufficient:validation:2001"}
{"answer": "71", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 83, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1e65d76908e5"], "split": "validation", "task_id": "insufficient:validation:2002"}
{"answer": "UNKNOWN", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "143", "input_tokens": 82, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "143", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a0bb3e76739c"], "split": "validation", "task_id": "insufficient:validation:2003"}
{"answer": "UNKNOWN", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "148", "input_tokens": 82, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "148", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-68247cf2069e"], "split": "validation", "task_id": "insufficient:validation:2004"}
{"answer": "30", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "40", "input_tokens": 84, "latency_ms": 1285.477, "output_tokens": 4, "parsed": "40", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-703669f1231d"], "split": "validation", "task_id": "insufficient:validation:2005"}
{"answer": "62", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 84, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bbf6c2680fdb"], "split": "validation", "task_id": "insufficient:validation:2006"}
{"answer": "12", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "30", "input_tokens": 84, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "30", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cb8dbac241b7"], "split": "validation", "task_id": "insufficient:validation:2007"}
{"answer": "1", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e269cb2f0f60"], "split": "validation", "task_id": "error_detect:validation:2000"}
{"answer": "0", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ed327cf53c41"], "split": "validation", "task_id": "error_detect:validation:2001"}
{"answer": "2", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-002a3b2dd3e2"], "split": "validation", "task_id": "error_detect:validation:2002"}
{"answer": "2", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1803.432, "output_tokens": 3, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c47651fa7d0"], "split": "validation", "task_id": "error_detect:validation:2003"}
{"answer": "0", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-481ee2bff36b"], "split": "validation", "task_id": "error_detect:validation:2004"}
{"answer": "3", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bc6582f57f87"], "split": "validation", "task_id": "error_detect:validation:2005"}
{"answer": "0", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e8640562c2d4"], "split": "validation", "task_id": "error_detect:validation:2006"}
{"answer": "2", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 117, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e9a458dfad53"], "split": "validation", "task_id": "error_detect:validation:2007"}
{"answer": "NO", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 86, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3ef3bdc60bdd"], "split": "validation", "task_id": "causal:validation:2000"}
{"answer": "NO", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 1581.125, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3499023fca5c"], "split": "validation", "task_id": "causal:validation:2001"}
{"answer": "YES", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 82, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-eaa1717c10bc"], "split": "validation", "task_id": "causal:validation:2002"}
{"answer": "YES", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2d58dc0ef62f"], "split": "validation", "task_id": "causal:validation:2003"}
{"answer": "YES", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 83, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-be483b8d1e0d"], "split": "validation", "task_id": "causal:validation:2004"}
{"answer": "YES", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 83, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8bc34d442ccb"], "split": "validation", "task_id": "causal:validation:2005"}
{"answer": "YES", "arm": "val:baseline", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 82, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ad69b5cadac3"], "split": "validation", "task_id": "causal:validation:2006"}
{"answer": "NO", "arm": "val:baseline", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1024.122, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3d6cdae3ca47"], "split": "validation", "task_id": "causal:validation:2007"}
{"answer": "350", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "120", "input_tokens": 341, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "120", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b4e25983df6b"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2000"}
{"answer": "260", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "105", "input_tokens": 339, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "105", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-407f7ce88f74"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2001"}
{"answer": "238", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "100", "input_tokens": 339, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "100", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-09ced65abacf"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2002"}
{"answer": "260", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "100", "input_tokens": 341, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "100", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2f60debb023f"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2003"}
{"answer": "571", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "100", "input_tokens": 341, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "100", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-36fdd44d86ce"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2004"}
{"answer": "440", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "142", "input_tokens": 340, "latency_ms": 4595.075, "output_tokens": 4, "parsed": "142", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-676ddcf020aa"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2005"}
{"answer": "111", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "102", "input_tokens": 338, "latency_ms": 4730.596, "output_tokens": 4, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-930481241727"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2006"}
{"answer": "215", "arm": "val:cand:arith_finance:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "125", "input_tokens": 340, "latency_ms": 4730.596, "output_tokens": 4, "parsed": "125", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66eb93b8a4d9"], "split": "validation", "strategy": "arith_finance:v0:fewshot", "task_id": "arith_finance:validation:2007"}
{"answer": "350", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "5*38=190 + 7*16=112 + 2*31=155 = 167; minus 14 = 153\nA: 153", "input_tokens": 470, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "153", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ea6bdefe96b8"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2000"}
{"answer": "260", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "7*15=105 + 2*4=28 + 9*17=151 = 163; minus 6 = 157\nA: 157", "input_tokens": 468, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "157", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-61674f466eb7"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2001"}
{"answer": "238", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "2*4+2*39+8*20=16+78+160=254; minus 8 = 246", "input_tokens": 468, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "246", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3f3b7b7ea378"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2002"}
{"answer": "260", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "2*25=50 + 4*26=104 + 5*27=127 = 271; minus 29 = 242\nA: 242", "input_tokens": 470, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "242", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7bed7cf7221a"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2003"}
{"answer": "571", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "2*40=80 + 8*26=208 + 8*39=208+264=472; minus 29 = 443", "input_tokens": 470, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "443", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2885a7df69be"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2004"}
{"answer": "440", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "2*26=52 + 9*38=342 + 2*27=342 - 8 = 334", "input_tokens": 469, "latency_ms": 11380.684, "output_tokens": 51, "parsed": "334", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8f31a025104f"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2005"}
{"answer": "111", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "2*26=52 + 7*7=52 + 4*4=52 + 4*4=52 + 16=68 - 6 = 62", "input_tokens": 467, "latency_ms": 17826.279, "output_tokens": 46, "parsed": "62", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d91400e76ea8"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2006"}
{"answer": "215", "arm": "val:cand:arith_finance:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "3*34+9*6+7*12=126+54+84=264; minus 25 = 239", "input_tokens": 469, "latency_ms": 17826.279, "output_tokens": 46, "parsed": "239", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-873fa8abb5a9"], "split": "validation", "strategy": "arith_finance:v0:worked", "task_id": "arith_finance:validation:2007"}
{"answer": "12", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 485, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e048d32dbd70"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2000"}
{"answer": "12", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 486, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-57ff722642cd"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2001"}
{"answer": "9", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 483, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-da3460decc9b"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2002"}
{"answer": "17", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "16", "input_tokens": 476, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "16", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-50116dcce74d"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2003"}
{"answer": "11", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 481, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e71420560271"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2004"}
{"answer": "12", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 488, "latency_ms": 6612.469, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bd778d62be70"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2005"}
{"answer": "15", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 481, "latency_ms": 5566.725, "output_tokens": 3, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e0ca7b17e4a"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2006"}
{"answer": "11", "arm": "val:cand:constraint_schedule:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 477, "latency_ms": 5566.725, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1ef6d219d22d"], "split": "validation", "strategy": "constraint_schedule:v0:fewshot", "task_id": "constraint_schedule:validation:2007"}
{"answer": "12", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 551, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-447e14cfdc11"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2000"}
{"answer": "12", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 552, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-96448473d102"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2001"}
{"answer": "9", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "17", "input_tokens": 549, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "17", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-00222bf58a20"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2002"}
{"answer": "17", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "16", "input_tokens": 542, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "16", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-679095fcadd1"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2003"}
{"answer": "11", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "16", "input_tokens": 547, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "16", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-96fdc32bfe80"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2004"}
{"answer": "12", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "17", "input_tokens": 554, "latency_ms": 6877.602, "output_tokens": 3, "parsed": "17", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4608cc1fa134"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2005"}
{"answer": "15", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 547, "latency_ms": 7738.947, "output_tokens": 3, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-096cfeb648b9"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2006"}
{"answer": "11", "arm": "val:cand:constraint_schedule:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "16", "input_tokens": 543, "latency_ms": 7738.947, "output_tokens": 3, "parsed": "16", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fb4b6866e66b"], "split": "validation", "strategy": "constraint_schedule:v0:worked", "task_id": "constraint_schedule:validation:2007"}
{"answer": "4", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 351, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-eb149bee1525"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2000"}
{"answer": "2", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 349, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fba5b324da05"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2001"}
{"answer": "4", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 347, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7737b5ae65f7"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2002"}
{"answer": "3", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 347, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6ed61dff0e87"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2003"}
{"answer": "5", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 349, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8c6665446601"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2004"}
{"answer": "5", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 349, "latency_ms": 4913.104, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-43a2b2b4797e"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2005"}
{"answer": "3", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 351, "latency_ms": 5710.867, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-16623b5a3de7"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2006"}
{"answer": "5", "arm": "val:cand:dependency_impact:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 347, "latency_ms": 5710.867, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d4b9e8ad093b"], "split": "validation", "strategy": "dependency_impact:v0:fewshot", "task_id": "dependency_impact:validation:2007"}
{"answer": "4", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 426, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-795f75022833"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2000"}
{"answer": "2", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 424, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c8407ad6160"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2001"}
{"answer": "4", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 422, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cbf7619b82d1"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2002"}
{"answer": "3", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 422, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4c372040ee5b"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2003"}
{"answer": "5", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 424, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a1e752255f53"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2004"}
{"answer": "5", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 424, "latency_ms": 5938.535, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-13d57bd828d9"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2005"}
{"answer": "3", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 426, "latency_ms": 5551.059, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d6548f3ffcb8"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2006"}
{"answer": "5", "arm": "val:cand:dependency_impact:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 422, "latency_ms": 5551.059, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-63891c5da1cb"], "split": "validation", "strategy": "dependency_impact:v0:worked", "task_id": "dependency_impact:validation:2007"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 308, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-63b01ebaf5b8"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2000"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 309, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cd3ebaa9feb3"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2001"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 309, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5dc6cb12c4b8"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2002"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 308, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "VIOLATION", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d4b008c8b647"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2003"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 308, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2dd7e5039faf"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2004"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 309, "latency_ms": 3837.443, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b0f89ee0eac0"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2005"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 308, "latency_ms": 4769.215, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dbe0041cc9d0"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2006"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 309, "latency_ms": 4769.215, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-22ecb200c673"], "split": "validation", "strategy": "contradiction:v0:fewshot", "task_id": "contradiction:validation:2007"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 369, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6d8c406da0f2"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2000"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 370, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9f168d6465a1"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2001"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 370, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1eafaa9bcc90"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2002"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 369, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4e34e20adc5f"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2003"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 369, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-002835fefcd7"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2004"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 370, "latency_ms": 4670.796, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b9ce71b9ebef"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2005"}
{"answer": "COMPLIANT", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 369, "latency_ms": 4917.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-21964335a87f"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2006"}
{"answer": "VIOLATION", "arm": "val:cand:contradiction:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 370, "latency_ms": 4917.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f54ed13cef47"], "split": "validation", "strategy": "contradiction:v0:worked", "task_id": "contradiction:validation:2007"}
{"answer": "DENY", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 323, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a537bd355008"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2000"}
{"answer": "DENY", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 325, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3c10ab9daf61"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2001"}
{"answer": "DENY", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 317, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-31b0a8a75d64"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2002"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 319, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bd276f9224ee"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2003"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 321, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e5ebc5ca8f47"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2004"}
{"answer": "DENY", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 319, "latency_ms": 3963.047, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-35435b47abd8"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2005"}
{"answer": "DENY", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 319, "latency_ms": 3397.525, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66b1e690a2bc"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2006"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 327, "latency_ms": 3397.525, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b9309e907564"], "split": "validation", "strategy": "access_decision:v0:fewshot", "task_id": "access_decision:validation:2007"}
{"answer": "DENY", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 381, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4c4c09e0b82f"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2000"}
{"answer": "DENY", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 383, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e2baaa45d0b"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2001"}
{"answer": "DENY", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 375, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f91731c179b2"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2002"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 377, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b0bc9dd23d24"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2003"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 379, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-13ba2e01b2ea"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2004"}
{"answer": "DENY", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 377, "latency_ms": 5739.627, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6c2d546f1c26"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2005"}
{"answer": "DENY", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 377, "latency_ms": 7629.393, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c4dc6aa69134"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2006"}
{"answer": "ALLOW", "arm": "val:cand:access_decision:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 385, "latency_ms": 7629.393, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-11427765b911"], "split": "validation", "strategy": "access_decision:v0:worked", "task_id": "access_decision:validation:2007"}
{"answer": "TEST", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-989c6715ef22"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2000"}
{"answer": "BUILD", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dcd5ae9da0e0"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2001"}
{"answer": "DESIGN", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-30e4fa520978"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2002"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0ab347d2777f"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2003"}
{"answer": "REVIEW", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-895ddebe63fb"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2004"}
{"answer": "TEST", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3828.258, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c4c9a8e39734"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2005"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 4653.112, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dc8ef43be9fd"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2006"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 4653.112, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8edae98f1f0a"], "split": "validation", "strategy": "plan_order:v0:fewshot", "task_id": "plan_order:validation:2007"}
{"answer": "TEST", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work: document", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-da7bb637e06f"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2000"}
{"answer": "BUILD", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-094c880d65ab"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2001"}
{"answer": "DESIGN", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c7e0b02204ce"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2002"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work: document", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-74f28e72c6ad"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2003"}
{"answer": "REVIEW", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-44824ae6d412"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2004"}
{"answer": "TEST", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work", "input_tokens": 324, "latency_ms": 6148.213, "output_tokens": 4, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b3a37e87e282"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2005"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work", "input_tokens": 324, "latency_ms": 5913.211, "output_tokens": 4, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cb5a8ac98e03"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2006"}
{"answer": "DEPLOY", "arm": "val:cand:plan_order:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "work: design", "input_tokens": 324, "latency_ms": 5913.211, "output_tokens": 4, "parsed": "DESIGN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e975089dbef"], "split": "validation", "strategy": "plan_order:v0:worked", "task_id": "plan_order:validation:2007"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "42", "input_tokens": 287, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "42", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c39b31dbcd2a"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2000"}
{"answer": "81", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "71", "input_tokens": 288, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "71", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f34c98349328"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2001"}
{"answer": "71", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 288, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "UNKNOWN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-26b92732ed24"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2002"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "74", "input_tokens": 287, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "74", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3bc9392f2eda"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2003"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "40", "input_tokens": 287, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "40", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-194c11975ee5"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2004"}
{"answer": "30", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "22", "input_tokens": 289, "latency_ms": 4785.651, "output_tokens": 3, "parsed": "22", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e6ca2547a007"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2005"}
{"answer": "62", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 289, "latency_ms": 3880.382, "output_tokens": 3, "parsed": "UNKNOWN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-69fadd2c6ccc"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2006"}
{"answer": "12", "arm": "val:cand:insufficient:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "30", "input_tokens": 289, "latency_ms": 3880.382, "output_tokens": 3, "parsed": "30", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c058982b1d36"], "split": "validation", "strategy": "insufficient:v0:fewshot", "task_id": "insufficient:validation:2007"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "34 - 2 = 32", "input_tokens": 329, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "32", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b31a27791991"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2000"}
{"answer": "81", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The number of crates shipped on Tuesday is unknown, so we cannot determine the total number of crates in the warehouse after Tuesday. Therefore, the answer is UNKNOWN.", "input_tokens": 330, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "UNKNOWN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1c6004400acb"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2001"}
{"answer": "71", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The number of crates shipped on Monday is 9, so the total number of crates in the warehouse after Tuesday would be 71 (initial) + 9 (Monday) + 9 (Tuesday) = 89. Therefore, the final answer is 89.", "input_tokens": 330, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "89", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a3d204586ba7"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2002"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The number of crates shipped on Monday is unknown, so we cannot determine the total number of crates in the warehouse after Tuesday. Therefore, the answer is UNKNOWN.", "input_tokens": 329, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "UNKNOWN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f68cd19902d6"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2003"}
{"answer": "UNKNOWN", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The number of crates in the warehouse after Tuesday cannot be determined from the given information.", "input_tokens": 329, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-01b24d3feb41"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2004"}
{"answer": "30", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "36 - 11 + 5 = 28", "input_tokens": 331, "latency_ms": 10933.944, "output_tokens": 58, "parsed": "28", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0386347c6b39"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2005"}
{"answer": "62", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The number of crates shipped on Monday is 19, so the total number of crates after Monday is 74 + 19 = 93. Therefore, the number of crates remaining in the warehouse after Tuesday is 93 - 7 = 86.\n\nFinal Answer: 86", "input_tokens": 331, "latency_ms": 15292.04, "output_tokens": 64, "parsed": "86", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6fa2db962a70"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2006"}
{"answer": "12", "arm": "val:cand:insufficient:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "30", "input_tokens": 331, "latency_ms": 15292.04, "output_tokens": 64, "parsed": "30", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f4b8f680bed2"], "split": "validation", "strategy": "insufficient:v0:worked", "task_id": "insufficient:validation:2007"}
{"answer": "1", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "2", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7222d6797806"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2000"}
{"answer": "0", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-19c142812cfe"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2001"}
{"answer": "2", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "2", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6f4a05c95498"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2002"}
{"answer": "2", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c62b5f0219c6"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2003"}
{"answer": "0", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "2", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-047df9878e1a"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2004"}
{"answer": "3", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 411, "latency_ms": 5334.132, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c9f1abe8dbb2"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2005"}
{"answer": "0", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "2", "input_tokens": 412, "latency_ms": 5192.955, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-757ccabba6f8"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2006"}
{"answer": "2", "arm": "val:cand:error_detect:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "2", "input_tokens": 415, "latency_ms": 5192.955, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f8d68e695710"], "split": "validation", "strategy": "error_detect:v0:fewshot", "task_id": "error_detect:validation:2007"}
{"answer": "1", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "true values: 9x3=27; recompute each shown step from its own left side -> first wrong step = 2", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-46080f091be6"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2000"}
{"answer": "0", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-62a1e1df1767"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2001"}
{"answer": "2", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cbcc1c4a8beb"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2002"}
{"answer": "2", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "true values: 8x6=48; recompute each shown step from its own left side -> first wrong step = 2\nA: 2", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-add3646dbf21"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2003"}
{"answer": "0", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "true values: 6x7=42; recompute each shown step from its own left side -> first wrong step = 2", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4cc4e9a96a75"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2004"}
{"answer": "3", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 505, "latency_ms": 11492.3, "output_tokens": 34, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7f696042ce1a"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2005"}
{"answer": "0", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 506, "latency_ms": 14020.551, "output_tokens": 35, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2432282ebceb"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2006"}
{"answer": "2", "arm": "val:cand:error_detect:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "true values: 8x11=88; recompute each shown step from its own left side -> first wrong step = 2\nA: 2", "input_tokens": 509, "latency_ms": 14020.551, "output_tokens": 35, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ef841bc85ae2"], "split": "validation", "strategy": "error_detect:v0:worked", "task_id": "error_detect:validation:2007"}
{"answer": "NO", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 290, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-661da79b2307"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2000"}
{"answer": "NO", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 288, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ffe2fa0cb82b"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2001"}
{"answer": "YES", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 286, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-59e599564b7e"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2002"}
{"answer": "YES", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 285, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-25dc5542889c"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2003"}
{"answer": "YES", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 287, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-40bb1e913af6"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2004"}
{"answer": "YES", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 287, "latency_ms": 3913.347, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cfedd2ed2d13"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2005"}
{"answer": "YES", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 286, "latency_ms": 4334.763, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-15194c91ae06"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2006"}
{"answer": "NO", "arm": "val:cand:causal:fewshot", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 288, "latency_ms": 4334.763, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f1da090aae36"], "split": "validation", "strategy": "causal:v0:fewshot", "task_id": "causal:validation:2007"}
{"answer": "NO", "arm": "val:cand:causal:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 341, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4414b804ec9c"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2000"}
{"answer": "NO", "arm": "val:cand:causal:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 339, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-679eed90f6bd"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2001"}
{"answer": "YES", "arm": "val:cand:causal:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 337, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9b7c0b5e155b"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2002"}
{"answer": "YES", "arm": "val:cand:causal:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 336, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-728fc9ea7fc6"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2003"}
{"answer": "YES", "arm": "val:cand:causal:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 338, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-840c5700bf2a"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2004"}
{"answer": "YES", "arm": "val:cand:causal:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 338, "latency_ms": 4533.103, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-497a2ba11958"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2005"}
{"answer": "YES", "arm": "val:cand:causal:worked", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 337, "latency_ms": 4411.994, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-aa9418691961"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2006"}
{"answer": "NO", "arm": "val:cand:causal:worked", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 339, "latency_ms": 4411.994, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6957e2bd6d75"], "split": "validation", "strategy": "causal:v0:worked", "task_id": "causal:validation:2007"}
{"answer": "214", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "arith_finance", "final_text": "145", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "145", "providers": [], "request_ids": [], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "10", "providers": [], "request_ids": [], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "COMPLIANT", "providers": [], "request_ids": [], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "ALLOW", "providers": [], "request_ids": [], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "DEPLOY", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "DEPLOY", "providers": [], "request_ids": [], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "UNKNOWN", "providers": [], "request_ids": [], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "2", "providers": [], "request_ids": [], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "causal", "final_text": "YES", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "YES", "providers": [], "request_ids": [], "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "F_floor", "calls": 0, "correct": true, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "F_floor", "calls": 0, "correct": false, "execution_classes": [], "family": "transfer_supply", "final_text": "5", "input_tokens": 0, "latency_ms": 0, "note": "most common training answer", "output_tokens": 0, "parsed": "5", "providers": [], "request_ids": [], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$254", "input_tokens": 94, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "254", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3435aa7b98a3"], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$165", "input_tokens": 93, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "165", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3710e37012e4"], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$804", "input_tokens": 95, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "804", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6ded1393abda"], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "564", "input_tokens": 94, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "564", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e34eb448081c"], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$904", "input_tokens": 94, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "904", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-23a2d242fa67"], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$508", "input_tokens": 95, "latency_ms": 1548.987, "output_tokens": 5, "parsed": "508", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b41174a1e1f7"], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$468", "input_tokens": 94, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "468", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-95ea66dcf727"], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$408", "input_tokens": 94, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "408", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8a79590549e8"], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "546", "input_tokens": 95, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "546", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ca95b1307370"], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$509", "input_tokens": 95, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "509", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ed0634630c55"], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 130, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e56d10169757"], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 130, "latency_ms": 2066.868, "output_tokens": 5, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4d49e2316ed0"], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 139, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ba8acd0117e3"], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 139, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d2b1abd1b3a8"], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 134, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9f5351dc3475"], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 129, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fd940c459ad5"], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 129, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0d4d0d3da2c3"], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 126, "latency_ms": 2011.913, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-df8038822c1a"], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 127, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b75f2cc16b46"], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 123, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1c61c04fe84f"], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 101, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8a7ddd61aed6"], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-38d8f94c9c6e"], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1545626fc88c"], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 95, "latency_ms": 1737.655, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d0ea2452e57e"], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-965bdc5e6361"], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ae7be972604f"], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-edc42e08c4a6"], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-efbbcb4843a4"], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4c70ceeb4abf"], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1428.26, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1ea2e5e0af51"], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8a78d515c7d6"], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1543c9d3eb9c"], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9258b76210b2"], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-adf9e7d4d823"], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ef0ff93d38da"], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1948.843, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-662a4b5815c5"], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-82926dc83d60"], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1aac92d3127b"], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 85, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0405f64c1d78"], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-77d300cdc5c7"], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ccddcfdea6e6"], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 1389.57, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d8ee97261f9a"], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d28494e2a37"], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a1814e5735d2"], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 88, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f4bfee46ebeb"], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0f1ed6f56578"], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 98, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-120dcfb66183"], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 92, "latency_ms": 1374.441, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-857fa2ad55aa"], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0bd42ffea5c3"], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 86, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a0a46d7fcea7"], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "DEPLOY", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d65637a5be23"], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-71b2d853fb7a"], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 82, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e13266151375"], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1269.391, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-91b25431e930"], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-311d55320ddc"], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9cf77af44fe9"], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-96f170dda9b6"], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6b2b3a3e4d3a"], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Deploy", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-41bf9abeb60f"], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 82, "latency_ms": 1038.337, "output_tokens": 2, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e486387f47bf"], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 82, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-47e513cff6ac"], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 84, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-53fafb62bfe5"], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25 crates", "input_tokens": 84, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "25", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dbf42017e833"], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "52", "input_tokens": 84, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "52", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c004b4fe758c"], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 83, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "104", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0d6916a37337"], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 82, "latency_ms": 1245.111, "output_tokens": 4, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2421046e9f44"], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "149", "input_tokens": 82, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "149", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-da449d7dd22c"], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 83, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ff80f19a088d"], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 82, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c4c70bef228e"], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "58", "input_tokens": 82, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "58", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c570134fbcf4"], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 116, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1d8964b7f3f1"], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1439.757, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ab017b4961eb"], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 116, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-542f2429733d"], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3dfb538e02ab"], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fad8c13352ff"], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05c331f3dddf"], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d063f2d1dbd"], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1287.385, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8fb41841baf2"], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 120, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5c7974b2e6dd"], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6245dcab93bd"], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4932e61fe6a5"], "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 85, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-59c3886b86e9"], "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f842cba1ff4a"], "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1482.516, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-49a19da81b43"], "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-afa42d907330"], "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0f15f1dcde66"], "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9382f5fad3a6"], "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9b23bd437db0"], "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2de00218961e"], "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1092.977, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f9699b5d180f"], "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 121, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2295b560ea23"], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 126, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f30bae6c01ff"], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 128, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-37d0296c0165"], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 134, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-821a2bfd9349"], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 122, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e44c37169801"], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "A_single", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 123, "latency_ms": 1567.557, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b14c5324a09f"], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 128, "latency_ms": 1505.409, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-58d4075681d2"], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 129, "latency_ms": 1505.409, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5d8ff83597ab"], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 125, "latency_ms": 1505.409, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-70dcd5dea113"], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "A_single", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 124, "latency_ms": 1505.409, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-304932937d22"], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "528", "input_tokens": 94, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "528", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-e529c899f64b"], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "161", "input_tokens": 93, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "161", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-cbacd995b5f7"], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "480", "input_tokens": 95, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "480", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-6ea3756add44"], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "456", "input_tokens": 94, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "456", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1e59e960a5d4"], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "400", "input_tokens": 94, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "400", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1480d1c58f54"], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "700", "input_tokens": 95, "latency_ms": 13526.66, "output_tokens": 4, "parsed": "700", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-5b00264e59f6"], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "476", "input_tokens": 94, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "476", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-d4eb3334315d"], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "408", "input_tokens": 94, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "408", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-d8c60c3143e2"], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "456", "input_tokens": 95, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "456", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-d0406ea62b03"], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "509", "input_tokens": 95, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "509", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-b3cb3ddcbb8b"], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 130, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "13", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-54cb8fd83d84"], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 130, "latency_ms": 6340.909, "output_tokens": 4, "parsed": "15", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-52d48468299e"], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 139, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-5b48d5c511f0"], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 139, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-01b42724f25f"], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 134, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-4e8fc7a86f70"], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 129, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-0292efea9837"], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 129, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-46e104c24d84"], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 126, "latency_ms": 6690.567, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-06c7b4a8d640"], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 127, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "11", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-aa1f388fc41b"], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 123, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-dc5461c44839"], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 101, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-f1723c8ae500"], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 97, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-6b48fb2c9300"], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 97, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-712356d10cac"], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 95, "latency_ms": 6196.873, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-de86d1140944"], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 99, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-c488ec52c43d"], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-316f83e6804d"], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 97, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-9c3ccdbca5e4"], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-6b1cccd8a276"], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-a23eab4773bc"], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 97, "latency_ms": 4429.283, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-43b68153277d"], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-cfa52071896b"], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 86, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-49180ddcb668"], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 86, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-70a0e50538c0"], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-159a877ff58c"], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-165bf10bd746"], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4000.471, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-80fecb25351e"], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-9c369b54c095"], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 86, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-64a3b39bfcbc"], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 85, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-d151336a4248"], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 87, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "VIOLATION", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-4da46816e0b8"], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-f7554f80abc1"], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 4675.608, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-956a9aa1a906"], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-0cd71b702f91"], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-c1ae68fe5400"], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 88, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1e463a9b0579"], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 96, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-61173481f019"], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 98, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-c1d048a208a1"], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 92, "latency_ms": 4659.604, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-465c0f1e68ee"], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-3c1dd38c355b"], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 86, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-27733cb0424e"], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "DEPLOY", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "document", "input_tokens": 82, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DOCUMENT", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-86e2e8107c27"], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-72a0f93eb090"], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "document", "input_tokens": 82, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DOCUMENT", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-8b496b1e9655"], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 4178.58, "output_tokens": 3, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-6640e24475b1"], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-dcbe672c45bd"], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-5b8d84f2e2d0"], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-61c5e1cd3652"], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-c70104f4fdf6"], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "design", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DESIGN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-496c864032cd"], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "document", "input_tokens": 82, "latency_ms": 3491.678, "output_tokens": 2, "parsed": "DOCUMENT", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-208d491c6537"], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 82, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-be7a4f7ec449"], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 84, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-aed73301fe8a"], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 84, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-cbe5f374106f"], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 84, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-9a08bb0075b7"], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 83, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-77ee2c1527a5"], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 82, "latency_ms": 3719.179, "output_tokens": 2, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1ddb2f94eef0"], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 82, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-bd6b03ba7ce4"], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 83, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-8fb7c2129de9"], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 82, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-fe532fbe01a8"], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "UNKNOWN", "input_tokens": 82, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "UNKNOWN", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-b26ac608d9b2"], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 116, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-3a6ff7212bda"], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 114, "latency_ms": 5718.947, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-812248976c1c"], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 116, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-ba63efce1190"], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 113, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-becd1ec61e5b"], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 114, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-9c2988a83cdf"], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 114, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-39505957a808"], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 113, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-93972fcc1784"], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 114, "latency_ms": 5381.762, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-18d61487dcfe"], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 120, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-937f315bf70f"], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 3", "input_tokens": 113, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-4d79c4ff1e99"], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "NO", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-6be02818e0a7"], "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 85, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "NO", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-75a725ebf11d"], "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1f36e1887550"], "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 5782.204, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-3d9bdcfcc993"], "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 83, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-3fdaaf127390"], "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-9136fd3e8b35"], "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-bd313dff8ace"], "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-1e92810535e5"], "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-cc6f342af549"], "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 4033.476, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-c4a2542a179f"], "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 121, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-bb6baa4217a8"], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "A_single_1p5b", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 126, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-8fa851d3fc52"], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 128, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-7d33c897457c"], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 134, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-031389389742"], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 122, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-5c5ce883c30c"], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 123, "latency_ms": 7191.175, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-49c73e4f0bce"], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 128, "latency_ms": 5668.959, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-45d44d9969b2"], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 129, "latency_ms": 5668.959, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-7d9d4d32fa18"], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 125, "latency_ms": 5668.959, "output_tokens": 2, "parsed": "4", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-38d263e7b81b"], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "A_single_1p5b", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 124, "latency_ms": 5668.959, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-1.5b-instruct"], "request_ids": ["req-4b1f5817707b"], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $16 + $18 - $30 =", "input_tokens": 96, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "30", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-fb24f4b149da"], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $40.", "input_tokens": 95, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "40", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-4c504ae0ed56"], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $19 + $32 - $26 =", "input_tokens": 97, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "26", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3f92d3f3d2b0"], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $7 + $11 + $38 = $", "input_tokens": 96, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "38", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-5bc989665246"], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "An invoice lists: 6 units at $17 each; 6 units", "input_tokens": 96, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "6", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-7a1d88be96ae"], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "An invoice lists: 8 units at $38 each; 2 units", "input_tokens": 97, "latency_ms": 1666.318, "output_tokens": 16, "parsed": "2", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3aef7709c921"], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $10.", "input_tokens": 96, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "10", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-a80a5f316e3f"], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "An invoice lists: 9 units at $6 each; 7 units at", "input_tokens": 96, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "7", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-59938896fa92"], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "The invoice total is $12 + $20 - $20 =", "input_tokens": 97, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "20", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-f2e253dd18fe"], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "An invoice lists: 8 units at $36 each; 6 units", "input_tokens": 97, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "6", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-01f73a8491ff"], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 12, 13", "input_tokens": 133, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "13", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-e54fb76b1aa3"], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 13", "input_tokens": 133, "latency_ms": 1207.611, "output_tokens": 16, "parsed": "13", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-6c0bd8961033"], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 10, 12, 1", "input_tokens": 142, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "1", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-89ee7318fd7e"], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 11, 12", "input_tokens": 142, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "12", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-8212f98eaee6"], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 11", "input_tokens": 137, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "11", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-542bf5cdb997"], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 11, 13", "input_tokens": 132, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "13", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-0754038b5688"], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 11", "input_tokens": 132, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "11", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-c4429f306a14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 11", "input_tokens": 129, "latency_ms": 1128.07, "output_tokens": 16, "parsed": "11", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-c34859b6daf9"], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 11", "input_tokens": 130, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "11", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-1e5af5c0535c"], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "Ana is free at hours 9, 10, 13", "input_tokens": 126, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "13", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-6f9972c05a02"], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: mail depends on nothing. search depends on mail. cache depends on", "input_tokens": 104, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-8c856573cea4"], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: db depends on nothing. mail depends on db. billing depends on", "input_tokens": 100, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-05ff7395f0b0"], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: search, auth, db, billing, cache, query, query", "input_tokens": 100, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-34ec5476ebe7"], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: gateway, cache, auth, search, billing, mail, gateway", "input_tokens": 98, "latency_ms": 1241.675, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-bf7295acd030"], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: billing, gateway, cache, db, auth, mail, Other", "input_tokens": 102, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-f923057a00b0"], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: cache, search, gateway, billing, mail, auth, gateway", "input_tokens": 100, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-04d1a4734e8d"], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services:\n\n- Queue:\n- Gateway:\n- Billing", "input_tokens": 100, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-721fa43a567c"], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: search, mail, db, auth, queue, billing\n\n", "input_tokens": 102, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-749e715e6842"], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: mail, cache, gateway, search, queue, other services\n", "input_tokens": 100, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-9743267d622c"], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "Services: db, auth, mail, cache, gateway, search\n\n", "input_tokens": 100, "latency_ms": 1007.227, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-7cbc7a231932"], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-d2796c1ebcc0"], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 87, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-30e205ef562b"], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 87, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-4d0cec2971e6"], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-a6e1ccf060f5"], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-35da9fd4d87c"], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 492.874, "output_tokens": 4, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-16be7be4e911"], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-111ede80b07a"], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 87, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-373dce876fb6"], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 86, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-f5a6179bc5de"], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "Answer: compliant", "input_tokens": 88, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "COMPLIANT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-b545ed1b9e9a"], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An analyst wants to read. Answer DENY.", "input_tokens": 99, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "DENY", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3def22af31f5"], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An intern wants to delete.\n\nThe answer is:\n\nALLOW", "input_tokens": 97, "latency_ms": 1965.035, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-0e1c9a794241"], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An analyst wants to delete.\n\nThe answer is:\n\nALLOW", "input_tokens": 101, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3838009c6d54"], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An intern wants to delete.\n\nThe answer is:\n\nALLOW", "input_tokens": 97, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-e73b8e01e6c1"], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An auditor wants to export. Answer: DENY.", "input_tokens": 93, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "DENY", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-0e1da7296464"], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An engineer wants to read.\n\nThe answer is:\n\nALLOW", "input_tokens": 101, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-09900dad683f"], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An auditor wants to export. Answer: DENY.", "input_tokens": 103, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "DENY", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-7d99ef1a59f5"], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An engineer wants to read.\n\nThe answer is:\n\nALLOW", "input_tokens": 97, "latency_ms": 2264.332, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-0d266da70f2c"], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An auditor wants to delete.\n\nThe answer is:\n\nALLOW", "input_tokens": 99, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-b98e00394882"], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "An intern wants to read. Answer: DENY.", "input_tokens": 91, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "DENY", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-af5a25e033a8"], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "DEPLOY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "The step to document first is: document requires test first.", "input_tokens": 84, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "DOCUMENT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-9970e1c3e66b"], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design first.", "input_tokens": 84, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "DESIGN", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-cb70e7997bf2"], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design requires review first.", "input_tokens": 84, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "DESIGN", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-8fe39e38cb45"], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "The step to document first is: document requires design first.", "input_tokens": 84, "latency_ms": 1218.57, "output_tokens": 16, "parsed": "DOCUMENT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-641237513d10"], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design requires document first.", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DESIGN", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-2b44e31a47da"], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design requires document first.", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DESIGN", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-0c1954d5cb7d"], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "The step that must be done first is document first.", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DOCUMENT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-9816983e59d6"], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "The step to deploy requires review first.", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DEPLOY", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-838b4172ad5d"], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design first.", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DESIGN", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-7ab11b94933f"], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "The step to document first is:\n\nDocument first.\n\nThe step", "input_tokens": 84, "latency_ms": 2403.196, "output_tokens": 16, "parsed": "DOCUMENT", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-4d8bb5ac3928"], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 73 crates. Some crates are shipped on Monday", "input_tokens": 89, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "73", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-04500cb37e2e"], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 71 crates. 10 crates are shipped", "input_tokens": 91, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "10", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-519727b35adf"], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 50 crates. 10 crates are shipped", "input_tokens": 91, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "10", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3c0d8d8d8558"], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 44 crates. 11 crates are shipped", "input_tokens": 91, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "11", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-b0474b0ab475"], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 80 crates.\n8 crates are shipped on", "input_tokens": 90, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "80", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-c065a193c315"], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 80 crates. Some crates are shipped on Monday", "input_tokens": 89, "latency_ms": 1227.273, "output_tokens": 16, "parsed": "80", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-ad19f776988b"], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 57 crates. Some crates are shipped on Monday", "input_tokens": 89, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "57", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-9fbd1d49b3ca"], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 89 crates. 7 crates are shipped on", "input_tokens": 90, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "7", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-73f86215f086"], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 35 crates. Some crates are shipped on Monday", "input_tokens": 89, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "35", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-7e3681b8aa4f"], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The warehouse holds 46 crates. Some crates are shipped on Monday", "input_tokens": 89, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "46", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-51db3e2cc075"], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 8 x 10 = 80. Step", "input_tokens": 118, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "80", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-69be645025ab"], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 11 x 5 = 55. Step", "input_tokens": 116, "latency_ms": 1160.021, "output_tokens": 16, "parsed": "55", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-a4c7ec563d3b"], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 11 x 5 = 65. Step", "input_tokens": 118, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "65", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-089b81923818"], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 6 x 7 = 45. Step ", "input_tokens": 115, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "45", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-1bd3b4a4a13e"], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 10 x 5 = 50. Step", "input_tokens": 116, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "50", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-9166566284de"], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 6 x 11 = 66. Step", "input_tokens": 116, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "66", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-f54b9924407a"], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 3 x 7 = 21. Step ", "input_tokens": 115, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "21", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-5b89e4897109"], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 12 x 5 = 60. Step", "input_tokens": 116, "latency_ms": 1618.121, "output_tokens": 16, "parsed": "60", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-fba42982527b"], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 10 x 12 = 120", "input_tokens": 122, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "120", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-946fe828d69a"], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "Step 1: 5 x 3 = 15. Step ", "input_tokens": 115, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "15", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-a31ee76c0935"], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 87, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-c079498373bd"], "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "M happens when Q occurs or W occurs.\n\nAnswer: YES", "input_tokens": 88, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-5bcd69b9523b"], "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "M happens.", "input_tokens": 86, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-1d4d4204c787"], "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 86, "latency_ms": 1694.664, "output_tokens": 16, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-2ccb9c053f71"], "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "Y always happens.", "input_tokens": 86, "latency_ms": 868.841, "output_tokens": 5, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-31856e9950f0"], "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "G happens.", "input_tokens": 84, "latency_ms": 868.841, "output_tokens": 5, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-5ce07ccf068d"], "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 87, "latency_ms": 868.841, "output_tokens": 5, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-1dba6a3c6e3d"], "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 87, "latency_ms": 868.841, "output_tokens": 5, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-611598ea4f7a"], "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 84, "latency_ms": 868.841, "output_tokens": 5, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-16a98721ebc0"], "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "A_single_135m", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES.", "input_tokens": 86, "latency_ms": 868.841, "output_tokens": 5, "parsed": "YES", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-6bfc3d2fb7e2"], "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The press has no supplier. The warehouse receives material from the press. The found", "input_tokens": 125, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-71ea79f9ddad"], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The assembly has no supplier. The dock receives material from the assembly. The smel", "input_tokens": 130, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-64fb9f049f67"], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The foundry has no supplier. The mine receives material from the foundry.", "input_tokens": 132, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-3362f7928ab0"], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The paint shop receives material from the smelter. The press receives material from the", "input_tokens": 138, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-30eb9a812614"], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The press shuts down, and the assembly receives material from the press. The", "input_tokens": 126, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-93391b8cf39d"], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The warehouse has no supplier, so the warehouse receives material from the mine and the", "input_tokens": 127, "latency_ms": 1182.12, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-e60103660825"], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The dock has no supplier. The smelter receives material from the dock. The", "input_tokens": 132, "latency_ms": 2220.269, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-229cc0cd32cd"], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The foundry has no supplier. The dock receives material from the foundry.", "input_tokens": 133, "latency_ms": 2220.269, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-aa0a75a1ea98"], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The mine shuts down, and the assembly receives material from the mine. The", "input_tokens": 129, "latency_ms": 2220.269, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-fe1f57036778"], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "A_single_135m", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "The paint shop has no supplier. The dock receives material from the paint shop.", "input_tokens": 128, "latency_ms": 2220.269, "output_tokens": 16, "parsed": "", "providers": ["smollm2-135m-instruct"], "request_ids": ["req-338c82793a1e"], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$254", "input_tokens": 94, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "254", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-856fa2f999dc"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$165", "input_tokens": 93, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "165", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-161af0278658"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$804", "input_tokens": 95, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "804", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-123ff9d37346"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "564", "input_tokens": 94, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "564", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1f64fae94cfb"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$904", "input_tokens": 94, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "904", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cb341497314d"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$508", "input_tokens": 95, "latency_ms": 2906.428, "output_tokens": 5, "parsed": "508", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0fa3cb539f45"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$468", "input_tokens": 94, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "468", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ea6fe2dc92f5"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$408", "input_tokens": 94, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "408", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e6276c0240bc"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "546", "input_tokens": 95, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "546", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c05e3d461089"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$509", "input_tokens": 95, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "509", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-85062659ec1e"], "split": "heldout", "strategy": "none", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 130, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-43c50f9f2928"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 130, "latency_ms": 2075.942, "output_tokens": 5, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5c22a535e6d3"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 139, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c7f4930d6ba8"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 139, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7108bc8e288e"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 134, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6978a9de7bd6"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 129, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f0a382694e06"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 129, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cf42f043154a"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 126, "latency_ms": 2111.815, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-31cc883ec1d8"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 127, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8583c68600a4"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 123, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-17db279e60c8"], "split": "heldout", "strategy": "none", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 101, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2cd816b55c85"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1cf48b2824e6"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5c14f28d6764"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 95, "latency_ms": 1671.337, "output_tokens": 3, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-71e74e253474"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d41d433e50fc"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e50db8b0b0ac"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5e88b7fa5dd6"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 99, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7fb5586cbbbe"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cf4799df9a19"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 97, "latency_ms": 1420.119, "output_tokens": 2, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66fade043ba8"], "split": "heldout", "strategy": "none", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1b68b67c6736"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dc0613d84b00"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d01538fe33f3"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9f00797bc34d"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9e87fb4376fb"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1437.531, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b28ecc090777"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d46472c86181"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7f09c3476b4e"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 85, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c84c66dd830f"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e97f7cf3fa1e"], "split": "heldout", "strategy": "none", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2802ad93d996"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 1491.828, "output_tokens": 5, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-22ec2e2301bc"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-478d9cfb67c8"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 92, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ea7f84560520"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 88, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ed0c953d8745"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 96, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ad50f833b281"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 98, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0ebb0af95e15"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 92, "latency_ms": 1325.919, "output_tokens": 3, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-708a3c3bf5fa"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 94, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-890dd63184e9"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 86, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-99f0277a08f5"], "split": "heldout", "strategy": "none", "task_id": "access_decision:heldout:9009"}
{"answer": "UNKNOWN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 82, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d26d9bae067e"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 84, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dfda11f69113"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25 crates", "input_tokens": 84, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "25", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-beb659c5198a"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "52", "input_tokens": 84, "latency_ms": 1380.78, "output_tokens": 4, "parsed": "52", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a7515f0f2eee"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 83, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "104", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-de156e6e1982"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 82, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-296bd889e8cd"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "149", "input_tokens": 82, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "149", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-39abda8f20fa"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 83, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c625a7bea39e"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 82, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fa6c1901cc23"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "58", "input_tokens": 82, "latency_ms": 1445.193, "output_tokens": 4, "parsed": "58", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c8beadfa30d6"], "split": "heldout", "strategy": "none", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 116, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c92a563e04c"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-28ee0a026e5c"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 116, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d5dc147ddde0"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f05cf51c48e7"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e1293b89d3b5"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1665.489, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-907ee3399b27"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-951905f584ab"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 114, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1e0daaaeb8ed"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 120, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8473564de197"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 113, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a47b2abdfec0"], "split": "heldout", "strategy": "none", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b7ae55bff60e"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 85, "latency_ms": 1628.64, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-515b0a27b534"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0bfdaa664473"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e8e32df22dd2"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a40171005ee2"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3220ccf69488"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c034bc890963"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1020.737, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a85459e762b2"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8e0dfacfa1fb"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1dde696de969"], "split": "heldout", "strategy": "none", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 121, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4e810292a59a"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 126, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f63b2017d4e5"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 128, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c3e2a38c6c8d"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 134, "latency_ms": 1631.802, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-01ef7ed30b60"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 122, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1347fd45d99e"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 123, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4abbd342f5ce"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 128, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a4770110a44b"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 129, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d5e0333501b7"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 125, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-09a50fd66e65"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 124, "latency_ms": 1769.008, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f6b9c4601335"], "split": "heldout", "strategy": "none", "task_id": "transfer_supply:heldout:9009"}
{"answer": "DEPLOY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6937f0621309"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f9fbdc8198a2"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ccba045c0684"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-036b7f890ca7"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "B_learned", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-77c12b3f7b95"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3450.747, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-929bc918c989"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3362.076, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f3e7af3b26c0"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3362.076, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6438b00c3b90"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3362.076, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a04e11f58e65"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "B_learned", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 285, "latency_ms": 3362.076, "output_tokens": 3, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-496f9ae8b54b"], "split": "heldout", "strategy": "plan_order:v1:fewshot", "task_id": "plan_order:heldout:9009"}
{"agreed": true, "answer": "214", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$254", "input_tokens": 208, "latency_ms": 5445.432, "output_tokens": 10, "parsed": "254", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-54033c28c0c5", "req-4432182c8195"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "125", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$165", "input_tokens": 206, "latency_ms": 5445.432, "output_tokens": 10, "parsed": "165", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b7200aa42c33", "req-fb67aae9e60d"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "348", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$804", "input_tokens": 210, "latency_ms": 5445.432, "output_tokens": 10, "parsed": "804", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c6c7dad07f9d", "req-e8a177c25009"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "470", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$654", "input_tokens": 208, "latency_ms": 5445.432, "output_tokens": 10, "parsed": "654", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7265c24f5952", "req-1d83072f9e9a"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "448", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$954", "input_tokens": 338, "latency_ms": 11361.99, "output_tokens": 15, "parsed": "954", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fccd654488eb", "req-c499b7bffd39", "req-5888521c2763"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "474", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$508", "input_tokens": 210, "latency_ms": 5445.432, "output_tokens": 10, "parsed": "508", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-768c72d7036e", "req-e9c6bcf509a4"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9005", "verifier_correct": false}
{"agreed": false, "answer": "261", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$167", "input_tokens": 338, "latency_ms": 13181.681, "output_tokens": 15, "parsed": "167", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dcf9fe1018e6", "req-aa72273a528f", "req-df93450667f2"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "302", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$408", "input_tokens": 208, "latency_ms": 7265.123, "output_tokens": 10, "parsed": "408", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dd0f3b12a8aa", "req-5cdd664332de"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "259", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$465", "input_tokens": 210, "latency_ms": 7265.123, "output_tokens": 10, "parsed": "465", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-387236ecf776", "req-c8792b0e81a2"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "577", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$509", "input_tokens": 210, "latency_ms": 7265.123, "output_tokens": 10, "parsed": "509", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-158eb4d359c5", "req-ef2677fda10b"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "15", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 279, "latency_ms": 7265.123, "output_tokens": 10, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a2b336fe5f6c", "req-3a73c8a8b42b"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "15", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 279, "latency_ms": 7265.123, "output_tokens": 10, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2564eb4e0633", "req-6f9207b420fa"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "10", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 297, "latency_ms": 7760.43, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d408b62ef9a2", "req-c8a433dbb03a"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9002", "verifier_correct": true}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 470, "latency_ms": 13676.988, "output_tokens": 11, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b699f5ac0f4f", "req-d0d8fd087df4", "req-968bf90fa1d2"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "10", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 455, "latency_ms": 13676.988, "output_tokens": 11, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-269e7f6674c0", "req-40603a5876a1", "req-5c0dd07b8ad6"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9004", "verifier_correct": false}
{"agreed": false, "answer": "9", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 440, "latency_ms": 13676.988, "output_tokens": 11, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-23d4045996d3", "req-cc09e6aacc90", "req-9652ea381a6b"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "9", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 277, "latency_ms": 7760.43, "output_tokens": 6, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-146c1b27c73e", "req-11a961639237"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9006", "verifier_correct": false}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 431, "latency_ms": 13676.988, "output_tokens": 11, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3429da7f2ae3", "req-dc2e97fcad8e", "req-4389f654fc20"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "12", "input_tokens": 434, "latency_ms": 12221.483, "output_tokens": 9, "parsed": "12", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fc0a1f1d8e43", "req-69cc3f6bf8eb", "req-11b30120e46e"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "10", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 265, "latency_ms": 7765.293, "output_tokens": 6, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a90cf701b286", "req-f9c85bb894f0"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 220, "latency_ms": 7765.293, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65cb147d1928", "req-f71b2c013630"], "specialist_correct": true, "split": "heldout", "task_id": "dependency_impact:heldout:9000", "verifier_correct": true}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 341, "latency_ms": 12221.483, "output_tokens": 9, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-511e5734884f", "req-00f5b5a4f050", "req-960f44492b6a"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 7765.293, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b31054ad3d9f", "req-be098eaea022"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 208, "latency_ms": 7765.293, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6334d20f48ac", "req-6315019dee00"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "5", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 216, "latency_ms": 5792.587, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6c17a6ceec6e", "req-d5e4d0183c84"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 5792.587, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a308cc4143a3", "req-b3b8a8abe252"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 5792.587, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e9a5960dd95e", "req-36fa37b9a4dc"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 216, "latency_ms": 5792.587, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e3106ea59e19", "req-43fd6f8980ca"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 5792.587, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-763fc8dbf08f", "req-43e66bfda1cd"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9008", "verifier_correct": false}
{"agreed": false, "answer": "3", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 341, "latency_ms": 10248.777, "output_tokens": 7, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-513ba7f037f9", "req-8eeab1cf88e1", "req-4dd9aebc408e"], "specialist_correct": true, "split": "heldout", "task_id": "dependency_impact:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-80394f977810", "req-4d7822b0965b"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9665312ddb38", "req-60f800f550c5"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-635794dfa407", "req-ab12a7120b94"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9002", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0d69d3c7b3d0", "req-0f415d1774e6"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8224d6345140", "req-8e39ccbc8ac8"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 6408.161, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dc10ee6b4d90", "req-0852903f31bd"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9005", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 7137.196, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1e2b8a0ea252", "req-ffb121e3f9e8"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 7137.196, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5cf921a08233", "req-d9be0ee4b2e8"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 190, "latency_ms": 7137.196, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5e671c00e5aa", "req-4a55a45aac1f"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9008", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 7137.196, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-10f519658a29", "req-c8a3ce1319c9"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 206, "latency_ms": 7137.196, "output_tokens": 10, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-42d255bbf62f", "req-16dc6e64e18f"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9000", "verifier_correct": false}
{"agreed": false, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 325, "latency_ms": 11593.386, "output_tokens": 13, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-22963bbf7e0d", "req-ce71074cb95e", "req-80efeed5bee6"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 210, "latency_ms": 6621.744, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f5449ce327c3", "req-917b7e98aae4"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 202, "latency_ms": 6621.744, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7473b55af538", "req-7627f36bab0f"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9003", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 313, "latency_ms": 11077.934, "output_tokens": 9, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ba588799e253", "req-eef7e3a0cba1", "req-588ca96ae4dc"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9004", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 337, "latency_ms": 11077.934, "output_tokens": 9, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9ff313ce089b", "req-b67f5460a174", "req-20edc1879868"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9005", "verifier_correct": true}
{"agreed": false, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 343, "latency_ms": 10705.135, "output_tokens": 10, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f61374346aa4", "req-8c5cbeb2fc04", "req-3959ea857199"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 202, "latency_ms": 6621.744, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bafa76c2d5f1", "req-3d605d9ddee1"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9007", "verifier_correct": true}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 206, "latency_ms": 6241.018, "output_tokens": 7, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-939f608bedb7", "req-d13c4ba2d5ad"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9008", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 307, "latency_ms": 10324.409, "output_tokens": 11, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-059a043a201b", "req-8eaa097a0794", "req-06cc3332c476"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9009", "verifier_correct": true}
{"agreed": true, "answer": "DEPLOY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 183, "latency_ms": 6241.018, "output_tokens": 7, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2cd93ac9063b", "req-f0fbb69d41cc"], "specialist_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9000", "verifier_correct": true}
{"agreed": true, "answer": "DOCUMENT", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 183, "latency_ms": 6241.018, "output_tokens": 7, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f89db5a0b8b6", "req-cd372e5f8221"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Test", "input_tokens": 296, "latency_ms": 10324.409, "output_tokens": 11, "parsed": "TEST", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-198056608380", "req-b11f2c7d06de", "req-9ef8db170606"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "REVIEW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 183, "latency_ms": 6241.018, "output_tokens": 7, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-34efd24e4b0d", "req-7ec94b651faf"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "DEPLOY", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 295, "latency_ms": 10469.127, "output_tokens": 12, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3a0085003051", "req-efc1a1117c36", "req-0ebbcd64b18f"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9004", "verifier_correct": true}
{"agreed": false, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 295, "latency_ms": 10469.127, "output_tokens": 12, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c010b779872", "req-1eb1851232aa", "req-e9d754570bfa"], "specialist_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9005", "verifier_correct": false}
{"agreed": false, "answer": "TEST", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 296, "latency_ms": 10469.127, "output_tokens": 12, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6be47e8ac89f", "req-d266e5e1a150", "req-5540a15b02a3"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "DESIGN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 183, "latency_ms": 6385.736, "output_tokens": 8, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-99e9d73ec2a3", "req-1132dd3689b9"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "REVIEW", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 298, "latency_ms": 12880.011, "output_tokens": 19, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-495d1793222e", "req-83948d80a729", "req-f8848e2ffaa7"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 183, "latency_ms": 6385.736, "output_tokens": 8, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2dbff316baed", "req-f54a4381a00d"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9009", "verifier_correct": false}
{"agreed": false, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The correct final answer is 108.", "input_tokens": 300, "latency_ms": 13033.588, "output_tokens": 20, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9db694aa6293", "req-fd4350583f87", "req-1f9a72cbe5e5"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9000", "verifier_correct": false}
{"agreed": false, "answer": "70", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "54", "input_tokens": 304, "latency_ms": 13033.588, "output_tokens": 20, "parsed": "54", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9edfa0926d8a", "req-e02a6391a0e9", "req-9c66905d432f"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "45", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "40", "input_tokens": 306, "latency_ms": 13033.588, "output_tokens": 20, "parsed": "40", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-62a56cbc34e0", "req-d76d7ad234bd", "req-6871fa172eec"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9002", "verifier_correct": false}
{"agreed": false, "answer": "42", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "55", "input_tokens": 304, "latency_ms": 13033.588, "output_tokens": 20, "parsed": "55", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c46bf962037b", "req-48e7b9d14e9b", "req-544fc3ac0910"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "74", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "396 crates", "input_tokens": 304, "latency_ms": 13033.588, "output_tokens": 20, "parsed": "396", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5fd8c9173eed", "req-03eca8af6ae5", "req-901468363ad3"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 182, "latency_ms": 6539.313, "output_tokens": 9, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-015d14441149", "req-8b856cbbb7d0"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "29", "input_tokens": 182, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "29", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fc1342a48ac3", "req-6b736d6d8154"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "84", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 184, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5f56825815bd", "req-fae7ed3b9aae"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 182, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-42174777fb41", "req-8bda7ae8980f"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "28", "input_tokens": 182, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "28", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9aa1b5fcd3d7", "req-acce0953e12c"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 250, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1c1b6f9cea9d", "req-8390efe3557c"], "specialist_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9000", "verifier_correct": true}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 7786.526, "output_tokens": 8, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9743ed352032", "req-d60401e69c03"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "1", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 250, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8673d8d6ddc6", "req-60598aefec63"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "1", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bdad4b9d8a2b", "req-df5539e2af2f"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c2d85ec43fa0", "req-b3617ddf9f1e"], "specialist_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9004", "verifier_correct": true}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0ace7e52d077", "req-263164aadfdb"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e3dc790bc21b", "req-49eddca92001"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 6431.243, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4f8c51012eaa", "req-89a1b718010d"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 258, "latency_ms": 6953.665, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b78206d0de9f", "req-c7a33470da25"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 6953.665, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d0c9f01ec59", "req-b1a536847c7f"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9009", "verifier_correct": false}
{"agreed": false, "answer": "NO", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 301, "latency_ms": 10339.062, "output_tokens": 6, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ce850b765aeb", "req-581ab8cdb24f", "req-f3819638095e"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 189, "latency_ms": 6953.665, "output_tokens": 4, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2f7537cb2667", "req-6845a06ee98c"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9001", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6953.665, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3ee0298e0f86", "req-b64ad3391795"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9002", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6953.665, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-28bd03e3b450", "req-9272599039dd"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9003", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e712b6c4712", "req-52dc9842e6c7"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9004", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 181, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-04ce2525e6ff", "req-f01a8a5c93f7"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9005", "verifier_correct": true}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 187, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65e17f020a53", "req-a92b39aac104"], "specialist_correct": false, "split": "heldout", "task_id": "causal:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 187, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d0bd9224543e", "req-8341b87d7ef8"], "specialist_correct": false, "split": "heldout", "task_id": "causal:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 181, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ac342e000634", "req-c9aabe57eca6"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9008", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6122.554, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-336f01e37cc7", "req-c8d53cece396"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9009", "verifier_correct": true}
{"agreed": false, "answer": "3", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 418, "latency_ms": 10666.716, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-76717ff24bbc", "req-5f94b06ed4b6", "req-339d855f0a3a"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 274, "latency_ms": 7281.319, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ecb3a00ba5bd", "req-3460db4a7681"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "2", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 439, "latency_ms": 10666.716, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ad08203509ba", "req-454618f8dfe9", "req-401549952b00"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 290, "latency_ms": 7281.319, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-26b39bffe883", "req-7da2f1936639"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 421, "latency_ms": 10666.716, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e2d8605d0d93", "req-26f61ce4c251", "req-a39ef60a62a7"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 268, "latency_ms": 7281.319, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e69f756eb7b8", "req-f9efe2df7def"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 278, "latency_ms": 8331.025, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-64343f96e8ba", "req-a1bb592f471d"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 280, "latency_ms": 8331.025, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fce104ebbcd6", "req-498a1c5efd20"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 430, "latency_ms": 11716.422, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e9ddb12e9217", "req-694ac0c92a27", "req-350ed285c79c"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 270, "latency_ms": 8331.025, "output_tokens": 6, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-419acea18b22", "req-dc5dc9e81a68"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "214", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$254", "input_tokens": 208, "latency_ms": 12934.863, "output_tokens": 10, "parsed": "254", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-14fe7f45e15c", "req-acbd14aee39b"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "125", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$165", "input_tokens": 206, "latency_ms": 12934.863, "output_tokens": 10, "parsed": "165", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7f2a0f93207c", "req-43d9999531e6"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "348", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$804", "input_tokens": 210, "latency_ms": 12934.863, "output_tokens": 10, "parsed": "804", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-179162135315", "req-fae58425aad3"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "470", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$654", "input_tokens": 208, "latency_ms": 12934.863, "output_tokens": 10, "parsed": "654", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bd14c49ad5d9", "req-392873094ad4"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "448", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$954", "input_tokens": 338, "latency_ms": 18690.708, "output_tokens": 15, "parsed": "954", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-055ddffde046", "req-964e281f9ccd", "req-049a1a1a2cc6"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "474", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$508", "input_tokens": 210, "latency_ms": 12934.863, "output_tokens": 10, "parsed": "508", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f0f0888f9aed", "req-20219dc2db3e"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9005", "verifier_correct": false}
{"agreed": false, "answer": "261", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$167", "input_tokens": 338, "latency_ms": 14881.146, "output_tokens": 15, "parsed": "167", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-666400143bc7", "req-d7a2e8204dd1", "req-38b67a68269b"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "302", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$408", "input_tokens": 208, "latency_ms": 9125.301, "output_tokens": 10, "parsed": "408", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3ded37245c34", "req-1809bdf05d9d"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "259", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$465", "input_tokens": 210, "latency_ms": 9125.301, "output_tokens": 10, "parsed": "465", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-012ec83aee1b", "req-14086e51510a"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "577", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "$509", "input_tokens": 210, "latency_ms": 9125.301, "output_tokens": 10, "parsed": "509", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fa7a64ff19b1", "req-91de972846c9"], "specialist_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "15", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 279, "latency_ms": 9125.301, "output_tokens": 10, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-37b747817f22", "req-e08dcb51ebff"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "15", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 279, "latency_ms": 9125.301, "output_tokens": 10, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c48387465f1", "req-b8c1c6193023"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "10", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 297, "latency_ms": 8996.256, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5db0297fcdc8", "req-33f9a24e83f3"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9002", "verifier_correct": true}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 470, "latency_ms": 14752.101, "output_tokens": 11, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2fafb31485b3", "req-8d9784883301", "req-4c2a0bd47691"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "10", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 455, "latency_ms": 14752.101, "output_tokens": 11, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05fb3f9d7dc5", "req-44990e1fe733", "req-24749dd4c6ba"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9004", "verifier_correct": false}
{"agreed": false, "answer": "9", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 440, "latency_ms": 14752.101, "output_tokens": 11, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d632a0b664f3", "req-8b49297e1409", "req-0c407fd3e72c"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "9", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 277, "latency_ms": 8996.256, "output_tokens": 6, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c0f26e8c6be8", "req-2ddf43d5342b"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9006", "verifier_correct": false}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 431, "latency_ms": 14752.101, "output_tokens": 11, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ffadd21d4f55", "req-e679a7d9c81b", "req-2af364553e98"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "11", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "12", "input_tokens": 434, "latency_ms": 13616.474, "output_tokens": 9, "parsed": "12", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3b8f0b841bf6", "req-e06870076a23", "req-fcd69bc38fb5"], "specialist_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "10", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 265, "latency_ms": 9315.635, "output_tokens": 6, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8b050cadf169", "req-5951e079e083"], "specialist_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 220, "latency_ms": 9315.635, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0390b0fbd97a", "req-2d623ff22c97"], "specialist_correct": true, "split": "heldout", "task_id": "dependency_impact:heldout:9000", "verifier_correct": true}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 341, "latency_ms": 13616.474, "output_tokens": 9, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cc3cc91445b8", "req-b0f4a31ab44c", "req-51f6b3ed7eb3"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 9315.635, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-13f983e8fe7a", "req-6fc1596d4c29"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 208, "latency_ms": 9315.635, "output_tokens": 6, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ef61985938b5", "req-027236a5dc8e"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "5", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 216, "latency_ms": 6428.001, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-04d3c67abda4", "req-a852bbe3dd70"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 6428.001, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dcae4dbbbe5d", "req-ddb01e0b7bad"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 6428.001, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2a6cec131e86", "req-78ded2d700da"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 216, "latency_ms": 6428.001, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bc3846ba7e04", "req-ce1da5ff7872"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 212, "latency_ms": 6428.001, "output_tokens": 4, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-336ca3201429", "req-5ba3e4ef05fd"], "specialist_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9008", "verifier_correct": false}
{"agreed": false, "answer": "3", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 341, "latency_ms": 10728.84, "output_tokens": 7, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e8915530968a", "req-706e8071d422", "req-5b6ca7557138"], "specialist_correct": true, "split": "heldout", "task_id": "dependency_impact:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-beb8bd228a76", "req-82f81d4dbbd1"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7cb438ff1e63", "req-c8510b942f6e"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-67f8fee7efb7", "req-797629f9317d"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9002", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e7bac9ed9fc3", "req-f0096112fb38"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f189836f4969", "req-4c1d22880936"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8537.137, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-38eef4429934", "req-00aded33477d"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9005", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8990.814, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4ce6beb88a84", "req-25b58d3ac723"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 192, "latency_ms": 8990.814, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9d2b80a7faaa", "req-47a657f5aca8"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "COMPLIANT", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 190, "latency_ms": 8990.814, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3ed75e10be8c", "req-b23884f2c3af"], "specialist_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9008", "verifier_correct": true}
{"agreed": true, "answer": "VIOLATION", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 194, "latency_ms": 8990.814, "output_tokens": 10, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-151209750d1b", "req-13c4741def07"], "specialist_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 206, "latency_ms": 8990.814, "output_tokens": 10, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-53d89955ef2c", "req-947f7632128b"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9000", "verifier_correct": false}
{"agreed": false, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 325, "latency_ms": 13291.653, "output_tokens": 13, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0f4412d55ffe", "req-7c169646cea0", "req-ff88fabcf133"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 210, "latency_ms": 9067.14, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-db89b946715b", "req-6e2d5eafeff8"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 202, "latency_ms": 9067.14, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6010b2a07353", "req-2308a529773b"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9003", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 313, "latency_ms": 13367.979, "output_tokens": 9, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-db983322a3ce", "req-fa6651180975", "req-4ffc256b3320"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9004", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 337, "latency_ms": 13367.979, "output_tokens": 9, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a411f72878bc", "req-c8d398a4ec7e", "req-6ead329f883f"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9005", "verifier_correct": true}
{"agreed": false, "answer": "ALLOW", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 343, "latency_ms": 12749.788, "output_tokens": 10, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8435c756cfa4", "req-1042b68d033b", "req-300f47d30fa1"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 202, "latency_ms": 9067.14, "output_tokens": 6, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2c51fea3f270", "req-417bd9d75c92"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9007", "verifier_correct": true}
{"agreed": true, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 206, "latency_ms": 6577.2, "output_tokens": 7, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d5b435969864", "req-241d7b61849c"], "specialist_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9008", "verifier_correct": true}
{"agreed": false, "answer": "DENY", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 307, "latency_ms": 10259.848, "output_tokens": 11, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-44e3b55f3feb", "req-244c30f2ded4", "req-13c9aa99b6b7"], "specialist_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9009", "verifier_correct": true}
{"agreed": true, "answer": "DEPLOY", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 183, "latency_ms": 6577.2, "output_tokens": 7, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4e487a4307c2", "req-a9280144d5d7"], "specialist_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9000", "verifier_correct": true}
{"agreed": true, "answer": "DOCUMENT", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 183, "latency_ms": 6577.2, "output_tokens": 7, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05c053324106", "req-82ff7d2eb4bf"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Test", "input_tokens": 296, "latency_ms": 10259.848, "output_tokens": 11, "parsed": "TEST", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b4816a5fd1fa", "req-a936487bbaa4", "req-a81e908c37e1"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "REVIEW", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 183, "latency_ms": 6577.2, "output_tokens": 7, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ba5185cbb7b6", "req-97e90d05a792"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "DEPLOY", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 295, "latency_ms": 11372.219, "output_tokens": 12, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65dc80c04945", "req-73820448ba0e", "req-575b2ac265f8"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9004", "verifier_correct": true}
{"agreed": false, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 295, "latency_ms": 11372.219, "output_tokens": 12, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a711f3419079", "req-198c83eefe8c", "req-cc9863e5fb11"], "specialist_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9005", "verifier_correct": false}
{"agreed": false, "answer": "TEST", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 296, "latency_ms": 11372.219, "output_tokens": 12, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6541c0d718df", "req-111ab84308f9", "req-d99d82bf2ddb"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "DESIGN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 183, "latency_ms": 7689.571, "output_tokens": 8, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c418d64d503f", "req-6841bf876037"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "REVIEW", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 298, "latency_ms": 14189.901, "output_tokens": 19, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d5f329c06563", "req-6cd87c2d9c2a", "req-7e795ed3ff64"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "BUILD", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 183, "latency_ms": 7689.571, "output_tokens": 8, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-39db4c3436f8", "req-a75f5b7ae1d9"], "specialist_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9009", "verifier_correct": false}
{"agreed": false, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "The correct final answer is 108.", "input_tokens": 300, "latency_ms": 15420.505, "output_tokens": 20, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-91ef50ea0592", "req-b56364f326c3", "req-e7f8aef573b3"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9000", "verifier_correct": false}
{"agreed": false, "answer": "70", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "54", "input_tokens": 304, "latency_ms": 15420.505, "output_tokens": 20, "parsed": "54", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1b8ce9d79deb", "req-d1b99f3a3cce", "req-555fac28c4f2"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "45", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "40", "input_tokens": 306, "latency_ms": 15420.505, "output_tokens": 20, "parsed": "40", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a1924a0de148", "req-ee20b3101cc6", "req-6aa2bcf98664"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9002", "verifier_correct": false}
{"agreed": false, "answer": "42", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "55", "input_tokens": 304, "latency_ms": 15420.505, "output_tokens": 20, "parsed": "55", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66d1a0b1cf66", "req-c9c63143e209", "req-dfc3ba7ab2f7"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "74", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "396 crates", "input_tokens": 304, "latency_ms": 15420.505, "output_tokens": 20, "parsed": "396", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b587baea9d43", "req-de5c59161078", "req-c8b512da42cb"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 182, "latency_ms": 8920.175, "output_tokens": 9, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-110f41242b9c", "req-75b93704db39"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "29", "input_tokens": 182, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "29", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f49132a6f290", "req-05e6ceaa6188"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "84", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 184, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fbd73cee9b9f", "req-8f44778c12a1"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 182, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-947512bdb32d", "req-e20fa1a63b06"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "UNKNOWN", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "28", "input_tokens": 182, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "28", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1e9c2e23507b", "req-89d755e6ea34"], "specialist_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9009", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 250, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ad45de512e0c", "req-154c22733ae9"], "specialist_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9000", "verifier_correct": true}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 10474.201, "output_tokens": 8, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7c2f449fc526", "req-39c38c685f6b"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9001", "verifier_correct": false}
{"agreed": true, "answer": "1", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 250, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-102b93a01c09", "req-eba8d540921c"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "1", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0bbcb9c60238", "req-59f96b4a589f"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9003", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-88699cd048ee", "req-5debf1b9eaac"], "specialist_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9004", "verifier_correct": true}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fe60a2caad78", "req-9d5767866aa7"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-947155212640", "req-9ea92261a396"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 246, "latency_ms": 8664.908, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c9a545f3f9a2", "req-ae21c8a32bd7"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 258, "latency_ms": 7224.931, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6235760deb8a", "req-1c7f497f58d6"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 244, "latency_ms": 7224.931, "output_tokens": 4, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a38343ef5248", "req-14f88b1cdb10"], "specialist_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9009", "verifier_correct": false}
{"agreed": false, "answer": "NO", "arm": "C_specialist_verifier", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 301, "latency_ms": 12176.131, "output_tokens": 6, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-39ebd0c81037", "req-e0dd0bb643a8", "req-bb28da50c375"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 189, "latency_ms": 7224.931, "output_tokens": 4, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c231d4ed06f3", "req-6379a875a4c1"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9001", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 7224.931, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-12a75f7b20e2", "req-6376f5e12ea0"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9002", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 7224.931, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d715f1b02805", "req-b22c9f349edc"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9003", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0a4c8fed5fd4", "req-9acaaa33ce1a"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9004", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 181, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f97871c3b05f", "req-900c8703567a"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9005", "verifier_correct": true}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 187, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e2009d5d8edc", "req-bed45c6d0942"], "specialist_correct": false, "split": "heldout", "task_id": "causal:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "NO", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 187, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6dc00955ac33", "req-af4034e1565d"], "specialist_correct": false, "split": "heldout", "task_id": "causal:heldout:9007", "verifier_correct": false}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 181, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e17aaa0f7eb", "req-e0639a04d02a"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9008", "verifier_correct": true}
{"agreed": true, "answer": "YES", "arm": "C_specialist_verifier", "calls": 2, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 185, "latency_ms": 6017.395, "output_tokens": 4, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bea61014641d", "req-2ee67e43d032"], "specialist_correct": true, "split": "heldout", "task_id": "causal:heldout:9009", "verifier_correct": true}
{"agreed": false, "answer": "3", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 418, "latency_ms": 14888.493, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0cea7c3a689e", "req-fa1957b6c848", "req-5807bf29f9ea"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9000", "verifier_correct": false}
{"agreed": true, "answer": "4", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 274, "latency_ms": 9937.293, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f9f277560c2e", "req-77f98decc1ae"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9001", "verifier_correct": false}
{"agreed": false, "answer": "2", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 439, "latency_ms": 14888.493, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-22cda1182464", "req-3283ccac3e78", "req-f8277f195d3e"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9002", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 290, "latency_ms": 9937.293, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4b35b96477f8", "req-c44bbf2b7a7a"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9003", "verifier_correct": false}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 421, "latency_ms": 14888.493, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3e0545bcee0b", "req-922656f55741", "req-cccdc8831cd3"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9004", "verifier_correct": false}
{"agreed": true, "answer": "2", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 268, "latency_ms": 9937.293, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a4c0ca5ecc5f", "req-f37486ea4f06"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9005", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 278, "latency_ms": 9361.528, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f21d93596cf1", "req-7ea5cbbf01f7"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9006", "verifier_correct": false}
{"agreed": true, "answer": "3", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 280, "latency_ms": 9361.528, "output_tokens": 6, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9cdea4dea89e", "req-9df39061b8d9"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9007", "verifier_correct": false}
{"agreed": false, "answer": "5", "arm": "C_specialist_verifier", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 430, "latency_ms": 14312.728, "output_tokens": 8, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-79a147281aad", "req-d6124ee3d8b8", "req-33acfb23c04a"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9008", "verifier_correct": false}
{"agreed": true, "answer": "0", "arm": "C_specialist_verifier", "calls": 2, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 270, "latency_ms": 9361.528, "output_tokens": 6, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-aa0ee80f9f62", "req-eb970dba141d"], "specialist_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9009", "verifier_correct": false}
{"answer": "214", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "524", "input_tokens": 396, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "524", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0c9fddb67476", "req-d110d018650e", "req-9f26caa95bb7"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "169", "input_tokens": 395, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "169", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6cccb3c2bb4e", "req-f172031814a2", "req-d08b087769f9"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "374", "input_tokens": 400, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "374", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6d9ba36eadac", "req-368bbaf10d9a", "req-c80b49feaa32"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "460", "input_tokens": 398, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "460", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2301cf419f84", "req-e1f9077555de", "req-02ebdf65a493"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "454", "input_tokens": 398, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "454", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9b866c363c03", "req-560a8e96d65c", "req-7581f38f1eb3"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "508", "input_tokens": 400, "latency_ms": 21795.862, "output_tokens": 64, "parsed": "508", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6dafc11acb81", "req-ec2e56688e23", "req-a8f786701c5c"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "\\$268", "input_tokens": 397, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "268", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5ce451608054", "req-38b579058c2c", "req-080bd7dc3855"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "\\$208", "input_tokens": 398, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "208", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ed48ad6c7f09", "req-720f4fe7947e", "req-6ddf03f558ef"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "\\$264", "input_tokens": 401, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "264", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c156bb21575c", "req-c2eba6acb3d1", "req-b9a6814ad2c5"], "solver_correct": true, "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "594", "input_tokens": 401, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "594", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-87c12888e47d", "req-eb3d0ecb9a55", "req-38227d0d99d8"], "solver_correct": false, "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 462, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "11", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-245e5a93e667", "req-4bce78f54142", "req-ea90de91a608"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 461, "latency_ms": 19846.489, "output_tokens": 66, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-55fa1260df15", "req-f84b1035c947", "req-5c4e543affe5"], "solver_correct": true, "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "16", "input_tokens": 489, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "16", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-470a378c2577", "req-03106348da05", "req-8570885b712e"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 489, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c66d1811a2a3", "req-9fe736ad7569", "req-7864eb20bd2b"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 474, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e8705f26570a", "req-425739385b47", "req-ae70f7f22877"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 458, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "13", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-64d92fc5f923", "req-765a61946adb", "req-67a0c57f4854"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 458, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dbfe110c98b9", "req-f9c18d96680d", "req-0e427b3fe5d4"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 450, "latency_ms": 11353.502, "output_tokens": 21, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0c8dc6dbc707", "req-8b34eef3d34f", "req-0ab9a7e65ffe"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 453, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "15", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-32d29cad0580", "req-bc9f828d0755", "req-55e004daf4a6"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 441, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "14", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4153324dcfec", "req-71dfbfce1d74", "req-dba040c7f835"], "solver_correct": false, "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 417, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5b5438500017", "req-46afacec9cb0", "req-7e0788f565c2"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "45", "input_tokens": 406, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "45", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-58a835496ea7", "req-4076598c62c5", "req-add8a89b59ee"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 405, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-088d1dc93747", "req-4b218a10021a", "req-923d089664d5"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 399, "latency_ms": 22942.405, "output_tokens": 62, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66f1e68fbe8d", "req-c411acb47992", "req-7e95eb9abb5d"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 378, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-33af705ab48a", "req-ffb912b411e4", "req-496d8837f716"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 405, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a4743cccb95b", "req-885639a0d692", "req-82589751b97d"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 405, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-de8128764114", "req-d92715b9342f", "req-4259b37cc729"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 411, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fb697f9ea19c", "req-bc61f300ef14", "req-84808c0c142a"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 405, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9d5e0306e263", "req-13080525034a", "req-669fd1c1d355"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 405, "latency_ms": 21423.29, "output_tokens": 60, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05a25d6788ab", "req-56bdc3849620", "req-5edb67281eb2"], "solver_correct": false, "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 325, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9b729d518257", "req-a9cf7a9ddbf5", "req-cf931dff4f7a"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 322, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fb101f4991f1", "req-80197ea9b916", "req-99cb13af2bee"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 322, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6e9ed8dfddff", "req-c937cc728f9e", "req-3d68c3960b28"], "solver_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 325, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ffd7c379337d", "req-f9480e849287", "req-012ee58e4037"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 325, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9c1dedb30ebd", "req-a05cb1b9d333", "req-82c252d93dd1"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "VIOLATION", "input_tokens": 324, "latency_ms": 6567.054, "output_tokens": 15, "parsed": "VIOLATION", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ab003e443de4", "req-a2d0c3d6a796", "req-b9a2467aff32"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 325, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f35037f9a3a7", "req-388fb27dd7b6", "req-6c3d62684694"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 322, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-80f3f504d2ff", "req-74dc320e969a", "req-6c8c485480e1"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 319, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-31908669accd", "req-7e7e0faaa4c6", "req-fb31d71f9d77"], "solver_correct": true, "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 325, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c5fa7d8fe396", "req-0862491b22e6", "req-d638070ce812"], "solver_correct": false, "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "ALLOW", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 341, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bc5ac2cb7705", "req-0e71445b9949", "req-260c85728b37"], "solver_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 335, "latency_ms": 7336.789, "output_tokens": 15, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5e138818eaa8", "req-fbdd6cb90fef", "req-1bbdaa10c9d6"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 347, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1eaae7b99cea", "req-28900952ba7f", "req-6fdd54a35237"], "solver_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 335, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d88d8323ea17", "req-9abe67cfb58a", "req-19280264c38e"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 323, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6041e7f712ef", "req-5d0780473977", "req-a764cfa0b5f8"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 347, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3f0949b63358", "req-0edce0f6273d", "req-c6dd1e90a1ea"], "solver_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 353, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-be8ecce5147b", "req-ca516058adb7", "req-f919adc819a3"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 335, "latency_ms": 5594.183, "output_tokens": 9, "parsed": "ALLOW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-29a9ab15c575", "req-b79531835af0", "req-9b491a73f809"], "solver_correct": false, "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 342, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d787c46b95da", "req-1ddac08d6142", "req-f5d856f6c914"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 317, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "DENY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-92017ef5f6b9", "req-8e980cccde92", "req-f14867fad62d"], "solver_correct": true, "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "DEPLOY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Deploy", "input_tokens": 320, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9630fd018158", "req-b78ecaa4d2e9", "req-5ae61ce61907"], "solver_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "build", "input_tokens": 304, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6e396bb5c938", "req-31f7e1e7bdee", "req-e01130bcbdc2"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "test", "input_tokens": 308, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "TEST", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-53b4c75367a6", "req-6005f765406e", "req-38d3b51f3b51"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "build", "input_tokens": 304, "latency_ms": 11014.087, "output_tokens": 25, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f8b0df57b132", "req-5d24ec839a57", "req-cb7bc63b722a"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 308, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9798cb1377d3", "req-3adfaea5d53b", "req-dff967ae103b"], "solver_correct": true, "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 304, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ec6b5f302193", "req-d8995fb7f188", "req-d37139f03d18"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Design requires review first.", "input_tokens": 311, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "DESIGN", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d5060bc1e7cc", "req-e9c34fef509f", "req-fa6af013d918"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 304, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-021b051e4b28", "req-a200c3caca3a", "req-75084082c636"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 322, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cb28b866f7a4", "req-040e96f60d39", "req-6a2ced8a7ceb"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 320, "latency_ms": 10816.298, "output_tokens": 33, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2595dbf2a403", "req-10e3b2aadfec", "req-3b4c22cf0fb4"], "solver_correct": false, "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 348, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9ee61598e6e8", "req-3e5ec39f578a", "req-316e00ff50ae"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "70", "input_tokens": 365, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "70", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-52daafb95d75", "req-4bf042eabdb4", "req-bf6136f1a861"], "solver_correct": true, "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "45", "input_tokens": 359, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "45", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-008182a76717", "req-8954152b08b3", "req-d996ef3d20b8"], "solver_correct": true, "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "42", "input_tokens": 365, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "42", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65bc47bf30fc", "req-ee6428d13d81", "req-4e0b199f6de7"], "solver_correct": true, "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 365, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4bb352ca500a", "req-9c308a64fb1b", "req-5bd08dd189d7"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "76", "input_tokens": 341, "latency_ms": 20901.051, "output_tokens": 63, "parsed": "76", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-39d25dd43c7e", "req-011340f2e392", "req-2eb13fd24021"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "65", "input_tokens": 342, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "65", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f7db86d10136", "req-73e9f4fed223", "req-15f092807a67"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "84", "input_tokens": 360, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "84", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-13518658b003", "req-8f9642045fb7", "req-feba0935d7fa"], "solver_correct": true, "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "45", "input_tokens": 355, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "45", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2aae35e0dc1c", "req-6e4d4fca6a8a", "req-d118e0c22e05"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "84", "input_tokens": 345, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "84", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f4ce68730986", "req-899f802b9ba6", "req-0254c80a8467"], "solver_correct": false, "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "0", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 462, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e8129a8e39e4", "req-6b3336e94069", "req-dde33b4a08d1"], "solver_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 456, "latency_ms": 22157.873, "output_tokens": 62, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4546e8c075d2", "req-6242eb515335", "req-ea06615332a9"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 462, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d1fdacd04424", "req-bbc2ad5ef3ce", "req-f30636f5f715"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 452, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b02e93da8703", "req-3fe4bd0beb6d", "req-00fb274b7590"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 403, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-999849c38b11", "req-e46e9d74677c", "req-3e1e5c6217a6"], "solver_correct": true, "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 403, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65f1b7bdac17", "req-a1deb419d307", "req-2a8ddd7de19f"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 451, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f41d42cfd160", "req-e6966d8676a5", "req-775048b403b9"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 456, "latency_ms": 19284.005, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bd7599370c3f", "req-edcacb12a4b0", "req-d61eee00cef9"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 421, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-54fa2b02362a", "req-f510907076bd", "req-0fb9c37b0fb3"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 452, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "0", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-21cbad9c8181", "req-7769718f689d", "req-c9c68150fc49"], "solver_correct": false, "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"answer": "NO", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 310, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e17eb8a67cc9", "req-58080751d71d", "req-8ed52cfdcf25"], "solver_correct": false, "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 313, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a9baa50686e5", "req-cc6ea3cf65fd", "req-227b7fb39d2b"], "solver_correct": false, "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 307, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-37ea63eefd03", "req-b85d0dc18305", "req-d66359bfdbbe"], "solver_correct": true, "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 307, "latency_ms": 17358.801, "output_tokens": 60, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-70b685f65f0b", "req-860f67e8afbd", "req-f3237002ab69"], "solver_correct": true, "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 307, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cec17ce47324", "req-7b4b7b448fb6", "req-5d082230ff9f"], "solver_correct": false, "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 301, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-de43c8a69443", "req-b81242a87049", "req-f9ad188d69c1"], "solver_correct": true, "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 310, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ecd4be7f7d3a", "req-292c0ee3a112", "req-a27c610dd4d1"], "solver_correct": false, "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 310, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dda71f4b973a", "req-3f771cabdce2", "req-0ee93357c5f2"], "solver_correct": false, "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 301, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4460f2e7cb2a", "req-3bc0e601bc8d", "req-468f4ef94be1"], "solver_correct": true, "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 307, "latency_ms": 4097.525, "output_tokens": 6, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8c3a461f4661", "req-e74f641bd4bb", "req-53fd6aaac12e"], "solver_correct": true, "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 432, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d4b62636aff1", "req-e328098619ff", "req-133f5e9278ff"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "C_hierarchical", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 448, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3c9ab2c96d4b", "req-9110477d75c3", "req-5d672020f84a"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 455, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b7f1817f0f1b", "req-fcd044b8817f", "req-1cb966699ada"], "solver_correct": true, "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 471, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5791f6f118e4", "req-3636a588ee2a", "req-fb5be49fe066"], "solver_correct": true, "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 438, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-425449a28f8c", "req-cdf98068dd67", "req-362c50bf2a9d"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 483, "latency_ms": 11301.71, "output_tokens": 60, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0063ce142466", "req-aa6aa15b6b3a", "req-9885700bca6e"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 453, "latency_ms": 8554.188, "output_tokens": 41, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f0dd613b1cd0", "req-6785d692ae87", "req-46e003b57226"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "4", "input_tokens": 458, "latency_ms": 8554.188, "output_tokens": 41, "parsed": "4", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8da0f6c53ebf", "req-9b5cd32500c8", "req-2b5ef813188d"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 447, "latency_ms": 8554.188, "output_tokens": 41, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bf8c446bc304", "req-dca7074e214c", "req-a7eddf2ec5eb"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "C_hierarchical", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "3", "input_tokens": 465, "latency_ms": 8554.188, "output_tokens": 41, "parsed": "3", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ecf59df55e49", "req-27f480e2e6e0", "req-e2914ed0d0b7"], "solver_correct": false, "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"all_agree": false, "answer": "214", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "254", "input_tokens": 282, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "254", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-727485adcc5e", "req-1779a16e16a8", "req-308d935e2630"], "sample_correct": [false, false, false], "samples": ["254", "554", "254"], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"all_agree": false, "answer": "125", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "104", "input_tokens": 279, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0dc568eb2bed", "req-58dadc66c60b", "req-30834dc1ad06"], "sample_correct": [false, false, false], "samples": ["168", "104", "114"], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"all_agree": false, "answer": "348", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1004", "input_tokens": 285, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "1004", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9a9e307470f8", "req-702a2bc78740", "req-e4c4f4c1a112"], "sample_correct": [false, false, false], "samples": ["1004", "408", "840"], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"all_agree": false, "answer": "470", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "462", "input_tokens": 282, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "462", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4ec92d12bf91", "req-3f9e10eb4b93", "req-aefa5db8f954"], "sample_correct": [false, false, false], "samples": ["462", "548", "544"], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"all_agree": false, "answer": "448", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1048", "input_tokens": 282, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "1048", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d23b42161acd", "req-0f8c4bbb6158", "req-588568e7d608"], "sample_correct": [false, false, false], "samples": ["930", "1048", "498"], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"all_agree": false, "answer": "474", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "502", "input_tokens": 285, "latency_ms": 6088.974, "output_tokens": 17, "parsed": "502", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2d1db8ad6aa1", "req-a230a5e34303", "req-7f96b2851857"], "sample_correct": [false, false, false], "samples": ["502", "756", "590"], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"all_agree": false, "answer": "261", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "148", "input_tokens": 282, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "148", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b0ed674f9e0f", "req-db6a98673500", "req-f5e90b299809"], "sample_correct": [false, false, false], "samples": ["786", "674", "148"], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"all_agree": false, "answer": "302", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "184", "input_tokens": 282, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "184", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f599c3d1b70d", "req-a66217d3ea20", "req-282b1454ab50"], "sample_correct": [false, false, false], "samples": ["840", "80", "184"], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"all_agree": false, "answer": "259", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "560", "input_tokens": 285, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "560", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-411c9fd8ecd4", "req-db56a3872451", "req-8ecf843cc181"], "sample_correct": [false, false, false], "samples": ["660", "560", "564"], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"all_agree": false, "answer": "577", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "504", "input_tokens": 285, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "504", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b3af4de796fa", "req-9a101f9daa5a", "req-696259a45e10"], "sample_correct": [false, false, false], "samples": ["504", "570", "504"], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"all_agree": false, "answer": "15", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 390, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "11", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3b1ac668f5d2", "req-158b2319392b", "req-c452cb84010b"], "sample_correct": [false, false, false], "samples": ["13", "11", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"all_agree": false, "answer": "15", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 390, "latency_ms": 7396.92, "output_tokens": 15, "parsed": "13", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9cfe93248e57", "req-6c7ab99e85be", "req-d462aeaa3042"], "sample_correct": [true, false, false], "samples": ["15", "14", "13"], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 417, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-312f39037c65", "req-44b8c93b4f2f", "req-05f0757157c8"], "sample_correct": [true, true, false], "samples": ["10", "10", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 417, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "11", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cc7fc27e6ac4", "req-d9e3bbcd2fd7", "req-62c36205e637"], "sample_correct": [true, false, true], "samples": ["11", "13", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 402, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "11", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c42d3e2ca7c9", "req-72d6b990afe1", "req-1dabd17428cb"], "sample_correct": [false, false, false], "samples": ["11", "14", "15"], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"all_agree": false, "answer": "9", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 387, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "14", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f8f8fbcfae2e", "req-17b7e83c75f9", "req-afba2c9eee03"], "sample_correct": [false, false, false], "samples": ["13", "14", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"all_agree": false, "answer": "9", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 387, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "14", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5301be85cbaa", "req-a1086958d828", "req-30fd160e4140"], "sample_correct": [false, false, false], "samples": ["14", "11", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 378, "latency_ms": 6107.155, "output_tokens": 9, "parsed": "14", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-90facde77cd9", "req-7aac36629010", "req-18b7dc8efed3"], "sample_correct": [true, false, false], "samples": ["11", "14", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 381, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b4a4471310bd", "req-6493dd9c0a64", "req-a6b8903e5bf4"], "sample_correct": [false, false, true], "samples": ["10", "13", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 369, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-93d9e66a7dae", "req-d0cfc555e641", "req-c2318e35655a"], "sample_correct": [false, true, true], "samples": ["13", "10", "10"], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 303, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8c3d5753696a", "req-24d3254db016", "req-655bea02b6f5"], "sample_correct": [false, false, true], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 291, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1a5fcecb15e8", "req-d8851c65d88f", "req-47046fbe791b"], "sample_correct": [false, false, false], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 291, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "3", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-64e71480f160", "req-6f58eb506298", "req-7cd3f675e1c6"], "sample_correct": [false, false, false], "samples": ["1", "3", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 285, "latency_ms": 5371.419, "output_tokens": 9, "parsed": "3", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a07b51d7f618", "req-36fa4544cb4e", "req-16ed2d05812a"], "sample_correct": [false, false, false], "samples": ["3", "3", "1"], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"all_agree": true, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 297, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-faf9d638ec49", "req-73cd31be878f", "req-013f902c73ff"], "sample_correct": [false, false, false], "samples": ["2", "2", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"all_agree": true, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "0", "input_tokens": 291, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fc89995ca78d", "req-c62e74bb708f", "req-464065ed403c"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 291, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "1", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b14480ba39b2", "req-36ac3e3a619a", "req-727cba370308"], "sample_correct": [false, false, false], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 297, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "2", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5da6b6f7c308", "req-ef0ec7857726", "req-7e471444588c"], "sample_correct": [false, false, true], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 291, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-622246291866", "req-b6b9eac6f503", "req-2dd3409cdfe5"], "sample_correct": [false, false, true], "samples": ["1", "3", "4"], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 291, "latency_ms": 4333.061, "output_tokens": 6, "parsed": "3", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3e19ed43772d", "req-a09d403d7d28", "req-04991b845fe0"], "sample_correct": [true, true, false], "samples": ["3", "3", "0"], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-484891b98a28", "req-91bf7f79ceee", "req-52ccd6e657bc"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7fa8de68d7d5", "req-4cefb0b77d51", "req-10c5121196b7"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f6ef8ce76d9c", "req-7c7684fbc706", "req-f0350ab660ce"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-af97e6d649c7", "req-301ba9122d49", "req-47884c0bc3c6"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b79951a98bb6", "req-5c045257b848", "req-a30a7bc75bc3"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5057.38, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5195e19246a2", "req-bd747e5ae9ac", "req-2d6c8aeca12f"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c054f3986cfa", "req-c586e02b32cb", "req-764fe7a78c3e"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-44053e236d0d", "req-009c24692b96", "req-57af957e9862"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 255, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-030f7129f98c", "req-4fce828ea76b", "req-ca98eddc2475"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-eb389b4940c5", "req-958835015b85", "req-7ba8a41cd2dd"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"all_agree": false, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 282, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bce216f4fccf", "req-2d1ef2710b83", "req-1125daf05d24"], "sample_correct": [false, true, true], "samples": ["DENY", "ALLOW", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"all_agree": false, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 276, "latency_ms": 5402.215, "output_tokens": 15, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-57503e00c6b0", "req-384d0cafb985", "req-7a3154006501"], "sample_correct": [true, false, true], "samples": ["ALLOW", "DENY", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"all_agree": true, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 288, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "DENY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-51835da2b10e", "req-59bcfd585915", "req-655b7f59ffbf"], "sample_correct": [false, false, false], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 276, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a237002a0a3c", "req-09c9e757370e", "req-a75a6f6ce385"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 264, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-52ec63c42212", "req-1875a3938972", "req-7dadf2bcfae1"], "sample_correct": [true, false, false], "samples": ["DENY", "ALLOW", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 288, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3c1f6e560538", "req-cf66861a11eb", "req-3971ebf16455"], "sample_correct": [true, false, true], "samples": ["DENY", "ALLOW", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"all_agree": true, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 294, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "DENY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9dfa1a03c9ea", "req-2b592cb159eb", "req-f5811d84eb9b"], "sample_correct": [false, false, false], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 276, "latency_ms": 4622.534, "output_tokens": 9, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4a38d4f73c1b", "req-9e7b5eb22462", "req-8a5a19ef9f5e"], "sample_correct": [false, true, false], "samples": ["ALLOW", "DENY", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 282, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-575ca90b7689", "req-12dbe7288219", "req-9fe00e46fde8"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 258, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a7f33c3e9b25", "req-e7e1982a821b", "req-399f541fd0b6"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"all_agree": true, "answer": "DEPLOY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "DEPLOY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-147e9853d597", "req-ca1bd63b3c47", "req-0066c27c25ab"], "sample_correct": [true, true, true], "samples": ["DEPLOY", "DEPLOY", "DEPLOY"], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"all_agree": false, "answer": "DOCUMENT", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-063f983fc213", "req-42e3a04c6a3d", "req-5c94685764c4"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"all_agree": false, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b9e839c3cfe3", "req-f04d45d3fd9e", "req-f54864182769"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "REVIEW", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"all_agree": true, "answer": "REVIEW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "BUILD", "input_tokens": 246, "latency_ms": 5101.061, "output_tokens": 11, "parsed": "BUILD", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-216c32283dd5", "req-de49ac13c788", "req-f27d4e6bb59e"], "sample_correct": [false, false, false], "samples": ["BUILD", "BUILD", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"all_agree": false, "answer": "DEPLOY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d7aec977207a", "req-e4305773068a", "req-dd71d7f7a548"], "sample_correct": [false, true, false], "samples": ["TEST", "DEPLOY", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"all_agree": true, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3d263def0233", "req-3c7a3d2c6bd9", "req-15c74cb63615"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "DEPLOY"], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"all_agree": true, "answer": "TEST", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2e0191872385", "req-fba08094d682", "req-e06f5c542b6e"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "DOCUMENT", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"all_agree": false, "answer": "DESIGN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-633477a1cf65", "req-6a900d8f7d47", "req-035235a5c575"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"all_agree": false, "answer": "REVIEW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "BUILD", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "BUILD", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-555f8b43a42e", "req-1e0223e78cb4", "req-b32a09c50068"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "DEPLOY", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"all_agree": false, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 5725.346, "output_tokens": 16, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cc4f35e51d2a", "req-afbdf9257000", "req-fde46991fd34"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "REVIEW", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 246, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "102", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e836afc0cf3d", "req-32f62bdcd3ee", "req-4419362ea883"], "sample_correct": [false, false, false], "samples": ["106", "102", "102"], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"all_agree": true, "answer": "70", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 252, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "82", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2542c4a0b376", "req-914c616e4b50", "req-a5aa7233f05e"], "sample_correct": [false, false, false], "samples": ["82", "82", "82"], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"all_agree": false, "answer": "45", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25", "input_tokens": 252, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "25", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-227fec1d8dc8", "req-336e922d113d", "req-bf77e1462f1a"], "sample_correct": [false, false, false], "samples": ["25", "65", "75"], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"all_agree": false, "answer": "42", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "35", "input_tokens": 252, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "35", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e95a6b1b6a1", "req-0ba182cb7eec", "req-d1c8b22d5ceb"], "sample_correct": [false, false, false], "samples": ["53", "35", "56"], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"all_agree": false, "answer": "74", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 249, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f5f8a486731e", "req-01444827978a", "req-ee66716f8931"], "sample_correct": [false, false, false], "samples": ["104", "194", "42"], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 246, "latency_ms": 5455.161, "output_tokens": 12, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a8c5195f2703", "req-d543244d888e", "req-06682fc54b4b"], "sample_correct": [false, false, false], "samples": ["104", "78", "108"], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "29", "input_tokens": 246, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "29", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b13181444bae", "req-6da2fdf7b967", "req-1f21aeae27f6"], "sample_correct": [false, false, false], "samples": ["29", "65", "57"], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"all_agree": true, "answer": "84", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 249, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "106", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-04da98f29261", "req-0a97808af939", "req-c40ecc393af6"], "sample_correct": [false, false, false], "samples": ["106", "106", "106"], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "47", "input_tokens": 246, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "47", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-267d6feab949", "req-a448be49156d", "req-7befe4f19dab"], "sample_correct": [false, false, false], "samples": ["47", "48", "70"], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "54", "input_tokens": 246, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "54", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1cfdda77663f", "req-137162c10479", "req-1a95a93479fe"], "sample_correct": [false, false, false], "samples": ["54", "38", "54"], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 348, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-54cc98341465", "req-1d15fa5fa83b", "req-58a29361a9aa"], "sample_correct": [false, true, true], "samples": ["1", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 11445.003, "output_tokens": 37, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b4dac69ca0bf", "req-562f11b1f8ea", "req-51300fc0f407"], "sample_correct": [true, false, false], "samples": ["2", "3", "0"], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"all_agree": true, "answer": "1", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 348, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1aee4cb4c0a6", "req-8f0bf31941bb", "req-7c0ed7f4a1eb"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"all_agree": true, "answer": "1", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0e62a70fabff", "req-bcc046e8d02e", "req-c7af55227f0b"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2b7e8aa5bf51", "req-77d4b9a96cf4", "req-ea50ef5455dc"], "sample_correct": [true, false, false], "samples": ["0", "1", "2"], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05861b15dc0d", "req-b983e26c4e1d", "req-c366f287b58e"], "sample_correct": [false, false, true], "samples": ["2", "0", "3"], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c2d4f3af5410", "req-d0434f3156ae", "req-5fa86912e041"], "sample_correct": [false, false, false], "samples": ["1", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 8023.475, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f486c99000cd", "req-9bee8846797f", "req-f2bc4876c924"], "sample_correct": [false, false, false], "samples": ["1", "3", "0"], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 360, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7532c86f41b2", "req-2ccbd7ff5570", "req-280e21f07300"], "sample_correct": [false, false, false], "samples": ["0", "0", "1"], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9d816fdd74e0", "req-ad4b020f9038", "req-335fd9f4cdbf"], "sample_correct": [false, false, false], "samples": ["0", "0", "1"], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"all_agree": false, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-535fee2c4c8f", "req-2469fda036ae", "req-9a4cf0b6aad8"], "sample_correct": [false, true, false], "samples": ["YES", "NO", "YES"], "split": "heldout", "task_id": "causal:heldout:9000"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 255, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "NO", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bbd8ed2e827e", "req-710227bfaea2", "req-f02c6710d40e"], "sample_correct": [true, true, true], "samples": ["NO", "NO", "NO"], "split": "heldout", "task_id": "causal:heldout:9001"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0e7bb420630c", "req-ece0a57dbf68", "req-a117d8761732"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9002"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 6913.76, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-539f80415deb", "req-d7634d16e128", "req-5ded60f8364c"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9003"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-50b5d150a2af", "req-d761cac68c1c", "req-9ce188f3010b"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9004"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 243, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bbccfc7ac2bf", "req-aeb7e5659680", "req-b84ba77fac4e"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9005"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bafdeef1de27", "req-c692eba777d2", "req-23f567035b38"], "sample_correct": [false, false, false], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9006"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6072361238cf", "req-9c4f603b5386", "req-c51eaf731bbc"], "sample_correct": [false, false, false], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9007"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 243, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-727b27c33c8a", "req-c58fdda3d584", "req-3a8cd2530c66"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9008"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 4385.735, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f96dc267fd9f", "req-f82efda54d4c", "req-4b5653534ef5"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9009"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 363, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-50749de9d84e", "req-945141625955", "req-9086cddf102a"], "sample_correct": [false, false, false], "samples": ["2", "2", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 378, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ee4db2cdb4fa", "req-cae046f89689", "req-e0502bfd03f3"], "sample_correct": [false, false, false], "samples": ["12", "2", "10"], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 384, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ae9c8af6d5f7", "req-9cdeeed32c6a", "req-b27a6d41c66c"], "sample_correct": [true, false, false], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 402, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "2", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-82b232854c0c", "req-26b3edd34090", "req-1179bc7a458d"], "sample_correct": [false, false, true], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 366, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-06d17271aa04", "req-d108abf968f6", "req-b30c238b47d2"], "sample_correct": [false, false, false], "samples": ["10", "3", "4"], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 369, "latency_ms": 6700.383, "output_tokens": 8, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-956e17b3c035", "req-a1badf3e113b", "req-addbb4b11ee5"], "sample_correct": [true, false, false], "samples": ["2", "3", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 384, "latency_ms": 7503.168, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5fd3d560b53a", "req-beb32f7dfa57", "req-4f05ce35538a"], "sample_correct": [false, false, false], "samples": ["2", "2", "14"], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 387, "latency_ms": 7503.168, "output_tokens": 9, "parsed": "1", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-05c7f77b0f42", "req-beb5d3383ca9", "req-b4fd4c3986dd"], "sample_correct": [false, false, false], "samples": ["12", "2", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 375, "latency_ms": 7503.168, "output_tokens": 9, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5c40cdc935f4", "req-e3cd7618d62c", "req-18cbb0ea5d98"], "sample_correct": [false, false, false], "samples": ["2", "10", "3"], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 372, "latency_ms": 7503.168, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cf84839f33a9", "req-fa76c1bff51e", "req-3abd99d3da5e"], "sample_correct": [false, false, false], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f4b1aba70d0c"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5d78cb67c7ff"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-377f49e5c378"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2d04e5ee8e5f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c9583cc63fad"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2721.852, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e7c97684f1ed"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d3035c207ae8"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b4e21d241682"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 85, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-652e3f74b633"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d27625f2ae6"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "DEPLOY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2cf2ae375a48"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 2676.47, "output_tokens": 5, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-840658d3bdbc"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-06497adf1497"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-76e2ebad922f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fb0bf28ff02c"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4ce75e3b0da9"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-07f95c6a5d64"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 2413.121, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8f5c7c347624"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document requires deploy first.", "input_tokens": 82, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1f7de928834d"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 82, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6dde94431045"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 82, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-03777f855eda"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 84, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9b23d9cdbb26"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25 crates", "input_tokens": 84, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "25", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-25e0b15c7949"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "52", "input_tokens": 84, "latency_ms": 2455.376, "output_tokens": 6, "parsed": "52", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ab7a547c9fbe"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 83, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "104", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9056d700259b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 82, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-026da0561aad"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "149", "input_tokens": 82, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "149", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-841395a7bc8c"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 83, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7507d2b2aa37"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 82, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-89831d78a646"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "58", "input_tokens": 82, "latency_ms": 1946.854, "output_tokens": 4, "parsed": "58", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5b854d218819"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d08a306d3820"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 85, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a93aebfb0a0a"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-962b94fc8534"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d837797bc679"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e92969aa86c7"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1577.267, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6d9d8927d52f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-643396ceabed"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ed163cb031ae"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66bb892705b3"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4ce3f51a54ff"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 121, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a787bb5b1352"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 126, "latency_ms": 2424.65, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c142b9574e58"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 128, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d3813f6b531"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 134, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5739a1ed93e1"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 122, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-97bb49a1ee0d"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 123, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-77dcd3ecc822"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 128, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dfcc194aa9db"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 129, "latency_ms": 2363.642, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-29717d15821e"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 125, "latency_ms": 2660.921, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c2ac54c34dd3"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 124, "latency_ms": 2660.921, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6d58b060de3c"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "214", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "214", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "125", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "125", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "348", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "348", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "470", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "470", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "448", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "448", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "474", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "474", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "261", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "261", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "302", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "302", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "259", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "259", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "577", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "577", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "15", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "15", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "9", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "9", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "9", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "9", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "5", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "5", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "0", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "0", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "0", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "1", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "1", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "1", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "1", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "0", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "0", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"all_agree": false, "answer": "214", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "254", "input_tokens": 282, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "254", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9869fa9164f5", "req-abfb4399443e", "req-e5542218322f"], "sample_correct": [false, false, false], "samples": ["254", "554", "254"], "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"all_agree": false, "answer": "125", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "104", "input_tokens": 279, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-71a34c935753", "req-c2e213ee6b03", "req-cd75c4581051"], "sample_correct": [false, false, false], "samples": ["168", "104", "114"], "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"all_agree": false, "answer": "348", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1004", "input_tokens": 285, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "1004", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bf1f75ff65e6", "req-f11a9d0e0f56", "req-3caf56b7b631"], "sample_correct": [false, false, false], "samples": ["1004", "408", "840"], "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"all_agree": false, "answer": "470", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "462", "input_tokens": 282, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "462", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e2104791b64d", "req-2a2dd5dfca85", "req-b0ed3417681c"], "sample_correct": [false, false, false], "samples": ["462", "548", "544"], "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"all_agree": false, "answer": "448", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "1048", "input_tokens": 282, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "1048", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1681eca51e26", "req-908c954032d2", "req-ddd4d671e6f3"], "sample_correct": [false, false, false], "samples": ["930", "1048", "498"], "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"all_agree": false, "answer": "474", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "502", "input_tokens": 285, "latency_ms": 11326.799, "output_tokens": 17, "parsed": "502", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2b22ff8cb808", "req-ee9a7022c248", "req-98f6356f0982"], "sample_correct": [false, false, false], "samples": ["502", "756", "590"], "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"all_agree": false, "answer": "261", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "148", "input_tokens": 282, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "148", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-10f72e85f699", "req-d0b7e66c8c40", "req-742952ce562b"], "sample_correct": [false, false, false], "samples": ["786", "674", "148"], "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"all_agree": false, "answer": "302", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "184", "input_tokens": 282, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "184", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-82058c14e3aa", "req-bac1844d7159", "req-434f12ae0d74"], "sample_correct": [false, false, false], "samples": ["840", "80", "184"], "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"all_agree": false, "answer": "259", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "560", "input_tokens": 285, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "560", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0371cb795e9b", "req-5c20be9ec683", "req-f33d389805de"], "sample_correct": [false, false, false], "samples": ["660", "560", "564"], "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"all_agree": false, "answer": "577", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "arith_finance", "final_text": "504", "input_tokens": 285, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "504", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5b9acd453790", "req-89dd7e22ce90", "req-c7366eaf7ef1"], "sample_correct": [false, false, false], "samples": ["504", "570", "504"], "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"all_agree": false, "answer": "15", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 390, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "11", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f1a912cce018", "req-c5b725e53167", "req-2cf17f98548d"], "sample_correct": [false, false, false], "samples": ["13", "11", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"all_agree": false, "answer": "15", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "13", "input_tokens": 390, "latency_ms": 9022.905, "output_tokens": 15, "parsed": "13", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bb9a374229d3", "req-5731e4f97a6a", "req-3156207e0ac4"], "sample_correct": [true, false, false], "samples": ["15", "14", "13"], "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 417, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0755c6c9955a", "req-74403ec2b3af", "req-6e1dc0615f55"], "sample_correct": [true, true, false], "samples": ["10", "10", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 417, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "11", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-abb4ba8f8dfc", "req-bdf09d677423", "req-6e4b5f109b9e"], "sample_correct": [true, false, true], "samples": ["11", "13", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 402, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "11", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4e5a18d55b1b", "req-bbd0bfa82f64", "req-c3b0392c8be4"], "sample_correct": [false, false, false], "samples": ["11", "14", "15"], "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"all_agree": false, "answer": "9", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 387, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "14", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-befd6012b77f", "req-b651871d7a9d", "req-62cdeabaa278"], "sample_correct": [false, false, false], "samples": ["13", "14", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"all_agree": false, "answer": "9", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 387, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "14", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d268f23a5319", "req-8b234c13d101", "req-bd3ae63c95a3"], "sample_correct": [false, false, false], "samples": ["14", "11", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "14", "input_tokens": 378, "latency_ms": 10672.246, "output_tokens": 9, "parsed": "14", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-66b30b08fc6e", "req-58a368353661", "req-2debbb4cc3e3"], "sample_correct": [true, false, false], "samples": ["11", "14", "14"], "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"all_agree": false, "answer": "11", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 381, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ca33c39e64ed", "req-34208009d7bd", "req-d717261b090a"], "sample_correct": [false, false, true], "samples": ["10", "13", "11"], "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"all_agree": false, "answer": "10", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 369, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "10", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-97d5124c7046", "req-b1a604981c60", "req-502f0595d4b1"], "sample_correct": [false, true, true], "samples": ["13", "10", "10"], "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 303, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2844d38586de", "req-6adfb567e9e2", "req-b3df29fc3688"], "sample_correct": [false, false, true], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 291, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cf1f13beb334", "req-559fd9ec11fd", "req-1a41192f7f23"], "sample_correct": [false, false, false], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 291, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "3", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9a33e5bafda1", "req-1c38c8dc9130", "req-cae4dcba6c2d"], "sample_correct": [false, false, false], "samples": ["1", "3", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 285, "latency_ms": 9360.484, "output_tokens": 9, "parsed": "3", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1827699a7bd4", "req-ed967ba16d44", "req-d32b7897582a"], "sample_correct": [false, false, false], "samples": ["3", "3", "1"], "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"all_agree": true, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 297, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-15afb53b5d46", "req-6053f13cd406", "req-7e4e73bd0a44"], "sample_correct": [false, false, false], "samples": ["2", "2", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"all_agree": true, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "0", "input_tokens": 291, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0392954899d4", "req-ce1711b3eb52", "req-84db603d0cd1"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 291, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "1", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-07efcb4e8bf4", "req-a598c1ab6348", "req-ea7e418fd660"], "sample_correct": [false, false, false], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 297, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "2", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-281ebf5f66f8", "req-75b4061fe88e", "req-d03d73928e76"], "sample_correct": [false, false, true], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "1", "input_tokens": 291, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d1b19345d7ff", "req-ee0f3988c57f", "req-c05ff2dc31f5"], "sample_correct": [false, false, true], "samples": ["1", "3", "4"], "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 291, "latency_ms": 6045.735, "output_tokens": 6, "parsed": "3", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-10aa33cfd119", "req-af2cfb12ed24", "req-8e12108076f7"], "sample_correct": [true, true, false], "samples": ["3", "3", "0"], "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8cca650550d1", "req-991dfb4ce2e3", "req-30d043ec3420"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-69f382358aa1", "req-ca1861c9359b", "req-848453bee214"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8669656346de", "req-6524ecc08c8a", "req-2048becd88a1"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0b19e0fbdd17", "req-5143bcc83619", "req-51682a93c33c"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c01b1a81a181", "req-a4a78b5a5ad5", "req-13b6b1d64c65"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 6819.638, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e4cf153d53bc", "req-d03886c4f8b3", "req-c663c910ea61"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-70c00ac9fde5", "req-c814ae3e36b3", "req-4aa234506b5c"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 258, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-64848277966c", "req-bdb55c4a1a3d", "req-3158bd161485"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"all_agree": true, "answer": "COMPLIANT", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 255, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5c5883300b04", "req-0fd50118a978", "req-9ae970c1e454"], "sample_correct": [true, true, true], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"all_agree": true, "answer": "VIOLATION", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 261, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "COMPLIANT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4faa8ffd3384", "req-f794588fd792", "req-a37a201791d1"], "sample_correct": [false, false, false], "samples": ["COMPLIANT", "COMPLIANT", "COMPLIANT"], "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"all_agree": false, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 282, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f0960665a655", "req-e0ed6de36db4", "req-6dc30d27f4cc"], "sample_correct": [false, true, true], "samples": ["DENY", "ALLOW", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"all_agree": false, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 276, "latency_ms": 7565.594, "output_tokens": 15, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cac9893ce2e7", "req-e81f22aac937", "req-2ace362b40ed"], "sample_correct": [true, false, true], "samples": ["ALLOW", "DENY", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"all_agree": true, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 288, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "DENY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c7e979f20e2d", "req-22886d3515e0", "req-63d6cc042c90"], "sample_correct": [false, false, false], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 276, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-efcf41900438", "req-d6332e1a2823", "req-0510ffecee88"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 264, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e382f4aadace", "req-5300be8696be", "req-29cb82b92dad"], "sample_correct": [true, false, false], "samples": ["DENY", "ALLOW", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 288, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cdbf447269c3", "req-8aeb15d013f7", "req-2993eda9a0e4"], "sample_correct": [true, false, true], "samples": ["DENY", "ALLOW", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"all_agree": true, "answer": "ALLOW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 294, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "DENY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ad5554101a8a", "req-5a261b5efca1", "req-1173ca21757f"], "sample_correct": [false, false, false], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"all_agree": false, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 276, "latency_ms": 6741.916, "output_tokens": 9, "parsed": "ALLOW", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c85ac426b194", "req-1bb477ca6369", "req-f3fa5f0085cc"], "sample_correct": [false, true, false], "samples": ["ALLOW", "DENY", "ALLOW"], "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 282, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c8d69ecd501d", "req-4c5448f0a625", "req-89d3f498298b"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"all_agree": true, "answer": "DENY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 258, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "DENY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2b7cbec206e0", "req-17d2d3fd838a", "req-b9f554ed3e20"], "sample_correct": [true, true, true], "samples": ["DENY", "DENY", "DENY"], "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"all_agree": true, "answer": "DEPLOY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "DEPLOY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8ffe7f235abb", "req-a051bd6e4a36", "req-e0e1d0548404"], "sample_correct": [true, true, true], "samples": ["DEPLOY", "DEPLOY", "DEPLOY"], "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"all_agree": false, "answer": "DOCUMENT", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a4c0e142c9c9", "req-1f2284419367", "req-e379b5dfb5b2"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"all_agree": false, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-58f8e54b734f", "req-19f2cf6cd713", "req-60059f712da8"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "REVIEW", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"all_agree": true, "answer": "REVIEW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "BUILD", "input_tokens": 246, "latency_ms": 6356.564, "output_tokens": 11, "parsed": "BUILD", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9846d37c5ab1", "req-4af82155ae51", "req-445d2da7c2e0"], "sample_correct": [false, false, false], "samples": ["BUILD", "BUILD", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"all_agree": false, "answer": "DEPLOY", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8a29c33ef6fc", "req-d856c1cbe36d", "req-6c2619151beb"], "sample_correct": [false, true, false], "samples": ["TEST", "DEPLOY", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"all_agree": true, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-20137f0e7f13", "req-5287c020d7b2", "req-75d0587bb50e"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "DEPLOY"], "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"all_agree": true, "answer": "TEST", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-30557d92f82f", "req-f2215410ea82", "req-df5ac5adcfd0"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "DOCUMENT", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"all_agree": false, "answer": "DESIGN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DEPLOY", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "DEPLOY", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d6f343d602e", "req-245ec7f32af4", "req-c481829b7c8c"], "sample_correct": [false, false, false], "samples": ["DEPLOY", "DEPLOY", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"all_agree": false, "answer": "REVIEW", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "BUILD", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "BUILD", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-93d250844d3c", "req-185e4ba893e4", "req-0997cd4c7bb6"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "DEPLOY", "BUILD"], "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"all_agree": false, "answer": "BUILD", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "DOCUMENT", "input_tokens": 246, "latency_ms": 6058.241, "output_tokens": 16, "parsed": "DOCUMENT", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f62abfc2ad9b", "req-a72b0afe1e1c", "req-c2b74be4ea5f"], "sample_correct": [false, false, false], "samples": ["DOCUMENT", "REVIEW", "DOCUMENT"], "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 246, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "102", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-28297af62727", "req-5e061cb547fa", "req-a08bb7b46fb8"], "sample_correct": [false, false, false], "samples": ["106", "102", "102"], "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"all_agree": true, "answer": "70", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 252, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "82", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f578e54f795e", "req-4e08fcec6d18", "req-0dbb319c10bd"], "sample_correct": [false, false, false], "samples": ["82", "82", "82"], "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"all_agree": false, "answer": "45", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25", "input_tokens": 252, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "25", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cde6789c24fc", "req-c8d29f067a8f", "req-4d3a09158cc2"], "sample_correct": [false, false, false], "samples": ["25", "65", "75"], "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"all_agree": false, "answer": "42", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "35", "input_tokens": 252, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "35", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cff4a92e9221", "req-cdc0ef58d2a2", "req-37087f3897ea"], "sample_correct": [false, false, false], "samples": ["53", "35", "56"], "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"all_agree": false, "answer": "74", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 249, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d3c468cb4ebb", "req-3778de4a08e2", "req-ac188a539201"], "sample_correct": [false, false, false], "samples": ["104", "194", "42"], "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 246, "latency_ms": 4787.932, "output_tokens": 12, "parsed": "104", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a0fb8b2f4cbb", "req-8492ee15cc0a", "req-14666f7d43d9"], "sample_correct": [false, false, false], "samples": ["104", "78", "108"], "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "29", "input_tokens": 246, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "29", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1df5c0cf0734", "req-61e3e450b292", "req-9df41d3d0853"], "sample_correct": [false, false, false], "samples": ["29", "65", "57"], "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"all_agree": true, "answer": "84", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 249, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "106", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e6e2090a87a6", "req-d9aef38a1297", "req-583578a95601"], "sample_correct": [false, false, false], "samples": ["106", "106", "106"], "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "47", "input_tokens": 246, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "47", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5d6888d1e230", "req-761e4800344d", "req-eb99161f2a3a"], "sample_correct": [false, false, false], "samples": ["47", "48", "70"], "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"all_agree": false, "answer": "UNKNOWN", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "54", "input_tokens": 246, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "54", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-6998f0d18d41", "req-25560d5c3f72", "req-52319b1412c3"], "sample_correct": [false, false, false], "samples": ["54", "38", "54"], "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 348, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7951c6d81b10", "req-be7323644a64", "req-6c80b25f1930"], "sample_correct": [false, true, true], "samples": ["1", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 9994.631, "output_tokens": 37, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-be01208cce78", "req-867f7cf8d717", "req-1843c3ecc9f7"], "sample_correct": [true, false, false], "samples": ["2", "3", "0"], "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"all_agree": true, "answer": "1", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 348, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b7a66feb8ed4", "req-363a5ada0f9b", "req-3c39f2bda64c"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"all_agree": true, "answer": "1", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f679a60b2ff5", "req-cccc8d92a073", "req-b427c3d111cb"], "sample_correct": [false, false, false], "samples": ["0", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bc4a79048c9e", "req-b8329753db03", "req-8036e00becba"], "sample_correct": [true, false, false], "samples": ["0", "1", "2"], "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-65f53d6c306a", "req-78995096b2a2", "req-3d0929a7b36a"], "sample_correct": [false, false, true], "samples": ["2", "0", "3"], "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2ca402793f4b", "req-6df13b718cc0", "req-89a8355a8672"], "sample_correct": [false, false, false], "samples": ["1", "0", "0"], "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 342, "latency_ms": 7082.422, "output_tokens": 18, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2754f6e21040", "req-516ab3599a17", "req-963d74449787"], "sample_correct": [false, false, false], "samples": ["1", "3", "0"], "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 360, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-12d840e5a9f0", "req-29e92601430f", "req-9b7ebda33699"], "sample_correct": [false, false, false], "samples": ["0", "0", "1"], "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "error_detect", "final_text": "0", "input_tokens": 339, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "0", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e58ca43dfdd", "req-0c75a54b856c", "req-e82d096e2e1f"], "sample_correct": [false, false, false], "samples": ["0", "0", "1"], "split": "heldout", "task_id": "error_detect:heldout:9009"}
{"all_agree": false, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-185e29928953", "req-cef533c47a7b", "req-4d9a5df1dea6"], "sample_correct": [false, true, false], "samples": ["YES", "NO", "YES"], "split": "heldout", "task_id": "causal:heldout:9000"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 255, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "NO", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8fed074fbe7c", "req-04b157f5b149", "req-3c592ff0b9a3"], "sample_correct": [true, true, true], "samples": ["NO", "NO", "NO"], "split": "heldout", "task_id": "causal:heldout:9001"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9d3d1ad60f6e", "req-6d002a0c8774", "req-2290e6269a94"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9002"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 5976.462, "output_tokens": 8, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-274f1e80e2c9", "req-6de004996212", "req-c0bb37078063"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9003"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-3c486f9ffd57", "req-84ed01df049d", "req-5c34d0b685c5"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9004"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 243, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f371138ba03d", "req-8deae7690a29", "req-d95a703dd2e0"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9005"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9c116f0a6480", "req-9b7fd3724358", "req-1d78a8a50381"], "sample_correct": [false, false, false], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9006"}
{"all_agree": true, "answer": "NO", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 252, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-46c592070fd0", "req-6310be89f4fa", "req-d196afe25810"], "sample_correct": [false, false, false], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9007"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 243, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-41e055a186d8", "req-4bc72c672c08", "req-5f81fb9fb32d"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9008"}
{"all_agree": true, "answer": "YES", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 249, "latency_ms": 4417.201, "output_tokens": 6, "parsed": "YES", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f269e970e4c3", "req-be8a6e775967", "req-59fc9db6aa46"], "sample_correct": [true, true, true], "samples": ["YES", "YES", "YES"], "split": "heldout", "task_id": "causal:heldout:9009"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 363, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c50a894005a0", "req-c5d60cd122e6", "req-a681b545e7e2"], "sample_correct": [false, false, false], "samples": ["2", "2", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"all_agree": false, "answer": "4", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 378, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b729a0537f71", "req-f9e641b4a0c7", "req-63c1265e3590"], "sample_correct": [false, false, false], "samples": ["12", "2", "10"], "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 384, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2f04339049fe", "req-36cea595df16", "req-72a90869dc54"], "sample_correct": [true, false, false], "samples": ["2", "1", "3"], "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 402, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "2", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-5686d10fcc88", "req-362c089c1801", "req-ec8b689639c6"], "sample_correct": [false, false, true], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 366, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-cc9c424cd074", "req-c7257bd972f9", "req-24b29b7cad59"], "sample_correct": [false, false, false], "samples": ["10", "3", "4"], "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"all_agree": false, "answer": "2", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 369, "latency_ms": 6856.063, "output_tokens": 8, "parsed": "1", "pass_at_k": true, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-381c46b8ab64", "req-48172195fac0", "req-6c0a1dd54dab"], "sample_correct": [true, false, false], "samples": ["2", "3", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 384, "latency_ms": 6927.854, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9eaf486b3c55", "req-e912a16b2042", "req-c8b768e6cdd9"], "sample_correct": [false, false, false], "samples": ["2", "2", "14"], "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"all_agree": false, "answer": "3", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 387, "latency_ms": 6927.854, "output_tokens": 9, "parsed": "1", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7a8cf5d37118", "req-376cf8296952", "req-8483018062a9"], "sample_correct": [false, false, false], "samples": ["12", "2", "1"], "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"all_agree": false, "answer": "5", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 375, "latency_ms": 6927.854, "output_tokens": 9, "parsed": "10", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0162e7234513", "req-68fdcf47751f", "req-a7d033dbd036"], "sample_correct": [false, false, false], "samples": ["2", "10", "3"], "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"all_agree": false, "answer": "0", "arm": "D_selfconsistency3", "calls": 3, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 372, "latency_ms": 6927.854, "output_tokens": 9, "parsed": "2", "pass_at_k": false, "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fc3c019f6717", "req-72b1abc089a2", "req-f37ee919844c"], "sample_correct": [false, false, false], "samples": ["4", "3", "2"], "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-fd84ccac8594"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9000"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-abea25ff41f6"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9001"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b2078e9edb71"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9002"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-edb06dae97bb"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9003"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c884500ee138"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9004"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1412.841, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1802d20019c7"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9005"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0309f4ab4cf1"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9006"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 86, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-7e04d97a4f79"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9007"}
{"answer": "COMPLIANT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 85, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a5696b2b99da"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9008"}
{"answer": "VIOLATION", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "contradiction", "final_text": "COMPLIANT", "input_tokens": 87, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "COMPLIANT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d9f699481ce8"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "contradiction:heldout:9009"}
{"answer": "DEPLOY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-46be0e2c8675"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9000"}
{"answer": "DOCUMENT", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1574.588, "output_tokens": 5, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-1b80203baef9"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9001"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document first.", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-0e74989cf02f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9002"}
{"answer": "REVIEW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Build", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "BUILD", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-42f6b5836cfd"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9003"}
{"answer": "DEPLOY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ff736f225265"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9004"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c458bf8939c1"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9005"}
{"answer": "TEST", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8d2c2c288805"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9006"}
{"answer": "DESIGN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "deploy", "input_tokens": 82, "latency_ms": 1415.957, "output_tokens": 4, "parsed": "DEPLOY", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e5551522544f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9007"}
{"answer": "REVIEW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Document requires deploy first.", "input_tokens": 82, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "DOCUMENT", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-dbcae01e22a4"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9008"}
{"answer": "BUILD", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "plan_order", "final_text": "Review", "input_tokens": 82, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "REVIEW", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e940f25d11ab"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "plan_order:heldout:9009"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "102", "input_tokens": 82, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "102", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2eb18897e63b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9000"}
{"answer": "70", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "82", "input_tokens": 84, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "82", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e42816bbbcd1"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9001"}
{"answer": "45", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "25 crates", "input_tokens": 84, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "25", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-d9fd5c4473f4"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9002"}
{"answer": "42", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "52", "input_tokens": 84, "latency_ms": 1674.218, "output_tokens": 6, "parsed": "52", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ba686635a8e9"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9003"}
{"answer": "74", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "104", "input_tokens": 83, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "104", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-e04ebf57900b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9004"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "108", "input_tokens": 82, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "108", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-2a4b29632b0a"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9005"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "149", "input_tokens": 82, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "149", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f0b5679e735b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9006"}
{"answer": "84", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "106", "input_tokens": 83, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "106", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-eff81495e849"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9007"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "48", "input_tokens": 82, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "48", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a629ca49b91f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9008"}
{"answer": "UNKNOWN", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "insufficient", "final_text": "58", "input_tokens": 82, "latency_ms": 1609.292, "output_tokens": 4, "parsed": "58", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b36f83ae71af"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "insufficient:heldout:9009"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 84, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f810bb8891da"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9000"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "NO", "input_tokens": 85, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "NO", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c24d6f4d3344"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9001"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-ba24275525b4"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9002"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-b55f9544c016"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9003"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-a81134bd265a"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9004"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1054.872, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-24b68f39344a"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9005"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-9fa046306338"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9006"}
{"answer": "NO", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 84, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-4d39db995964"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9007"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 81, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c8f0d7600fcd"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9008"}
{"answer": "YES", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "causal", "final_text": "YES", "input_tokens": 83, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "YES", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-8ce06bcba862"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "causal:heldout:9009"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 121, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c16c5b99a10b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9000"}
{"answer": "4", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 126, "latency_ms": 1675.734, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-95080bf022e6"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9001"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 128, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-380aa76e23df"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9002"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 134, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bbef7dce8dff"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9003"}
{"answer": "5", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "1", "input_tokens": 122, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "1", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-bc291f7f2969"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9004"}
{"answer": "2", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": true, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 123, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-72c6f7faa42b"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9005"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 128, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-c9ec8b039dd0"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9006"}
{"answer": "3", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "10", "input_tokens": 129, "latency_ms": 1703.936, "output_tokens": 3, "parsed": "10", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-51a8b75d3994"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9007"}
{"answer": "5", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 125, "latency_ms": 1821.379, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-62006932f5f8"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9008"}
{"answer": "0", "arm": "E_routed", "basis": "DEFAULT_NOT_MEASURED", "calls": 1, "correct": false, "execution_classes": ["REAL_LOCAL_MODEL"], "family": "transfer_supply", "final_text": "2", "input_tokens": 124, "latency_ms": 1821.379, "output_tokens": 2, "parsed": "2", "providers": ["qwen2.5-0.5b-instruct"], "request_ids": ["req-f70309f90c2f"], "routed_to": "qwen2.5-0.5b-instruct", "split": "heldout", "task_id": "transfer_supply:heldout:9009"}
{"answer": "214", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "214", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "214", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9000"}
{"answer": "125", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "125", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "125", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9001"}
{"answer": "348", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "348", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "348", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9002"}
{"answer": "470", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "470", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "470", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9003"}
{"answer": "448", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "448", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "448", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9004"}
{"answer": "474", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "474", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "474", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9005"}
{"answer": "261", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "261", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "261", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9006"}
{"answer": "302", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "302", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "302", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9007"}
{"answer": "259", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "259", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "259", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9008"}
{"answer": "577", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "arith_finance", "final_text": "577", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "577", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "arith_finance:heldout:9009"}
{"answer": "15", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "15", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9000"}
{"answer": "15", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "15", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "15", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9001"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9002"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9003"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9004"}
{"answer": "9", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "9", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "9", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9005"}
{"answer": "9", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "9", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "9", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9006"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9007"}
{"answer": "11", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "11", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "11", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9008"}
{"answer": "10", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "constraint_schedule", "final_text": "10", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "10", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "constraint_schedule:heldout:9009"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9000"}
{"answer": "5", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "5", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9001"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9002"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9003"}
{"answer": "5", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "5", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "5", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9004"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9005"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9006"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9007"}
{"answer": "4", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "4", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "4", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9008"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "dependency_impact", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "dependency_impact:heldout:9009"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9000"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9001"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9002"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9003"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9004"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9005"}
{"answer": "ALLOW", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "ALLOW", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "ALLOW", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9006"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9007"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9008"}
{"answer": "DENY", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "access_decision", "final_text": "DENY", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "DENY", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "access_decision:heldout:9009"}
{"answer": "0", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "0", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "0", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9000"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9001"}
{"answer": "1", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "1", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "1", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9002"}
{"answer": "1", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "1", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "1", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9003"}
{"answer": "0", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "0", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "0", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9004"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9005"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9006"}
{"answer": "2", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "2", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "2", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9007"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9008"}
{"answer": "3", "arm": "E_routed", "basis": "MEASURED", "calls": 1, "correct": true, "execution_classes": ["DETERMINISTIC_TOOL"], "family": "error_detect", "final_text": "3", "input_tokens": 0, "latency_ms": 0.0, "output_tokens": 0, "parsed": "3", "providers": ["deterministic-tools"], "request_ids": [""], "routed_to": "deterministic-tools", "split": "heldout", "task_id": "error_detect:heldout:9009"}
