{
 "heldout": [
  {
   "answer": "214",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 4 units at $16 each; 7 units at $18 each; 9 units at $6 each. A discount of $30 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 30,
    "items": [
     [
      4,
      16
     ],
     [
      7,
      18
     ],
     [
      9,
      6
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9000",
   "worked": "4*16=64 + 7*18=126 + 9*6=54 = 244; minus 30 = 214"
  },
  {
   "answer": "125",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 5 units at $4 each; 8 units at $9 each; 2 units at $30 each. A discount of $27 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 27,
    "items": [
     [
      5,
      4
     ],
     [
      8,
      9
     ],
     [
      2,
      30
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9001",
   "worked": "5*4=20 + 8*9=72 + 2*30=60 = 152; minus 27 = 125"
  },
  {
   "answer": "348",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 7 units at $19 each; 5 units at $32 each; 3 units at $27 each. A discount of $26 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 26,
    "items": [
     [
      7,
      19
     ],
     [
      5,
      32
     ],
     [
      3,
      27
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9002",
   "worked": "7*19=133 + 5*32=160 + 3*27=81 = 374; minus 26 = 348"
  },
  {
   "answer": "470",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 7 units at $7 each; 9 units at $11 each; 9 units at $38 each. A discount of $20 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 20,
    "items": [
     [
      7,
      7
     ],
     [
      9,
      11
     ],
     [
      9,
      38
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9003",
   "worked": "7*7=49 + 9*11=99 + 9*38=342 = 490; minus 20 = 470"
  },
  {
   "answer": "448",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 6 units at $17 each; 6 units at $28 each; 8 units at $23 each. A discount of $6 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 6,
    "items": [
     [
      6,
      17
     ],
     [
      6,
      28
     ],
     [
      8,
      23
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9004",
   "worked": "6*17=102 + 6*28=168 + 8*23=184 = 454; minus 6 = 448"
  },
  {
   "answer": "474",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 8 units at $38 each; 2 units at $13 each; 6 units at $28 each. A discount of $24 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 24,
    "items": [
     [
      8,
      38
     ],
     [
      2,
      13
     ],
     [
      6,
      28
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9005",
   "worked": "8*38=304 + 2*13=26 + 6*28=168 = 498; minus 24 = 474"
  },
  {
   "answer": "261",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 5 units at $20 each; 5 units at $11 each; 5 units at $23 each. A discount of $9 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 9,
    "items": [
     [
      5,
      20
     ],
     [
      5,
      11
     ],
     [
      5,
      23
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9006",
   "worked": "5*20=100 + 5*11=55 + 5*23=115 = 270; minus 9 = 261"
  },
  {
   "answer": "302",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 9 units at $6 each; 7 units at $31 each; 2 units at $22 each. A discount of $13 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 13,
    "items": [
     [
      9,
      6
     ],
     [
      7,
      31
     ],
     [
      2,
      22
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9007",
   "worked": "9*6=54 + 7*31=217 + 2*22=44 = 315; minus 13 = 302"
  },
  {
   "answer": "259",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 7 units at $12 each; 3 units at $28 each; 3 units at $37 each. A discount of $20 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 20,
    "items": [
     [
      7,
      12
     ],
     [
      3,
      28
     ],
     [
      3,
      37
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9008",
   "worked": "7*12=84 + 3*28=84 + 3*37=111 = 279; minus 20 = 259"
  },
  {
   "answer": "577",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 8 units at $36 each; 6 units at $24 each; 6 units at $27 each. A discount of $17 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 17,
    "items": [
     [
      8,
      36
     ],
     [
      6,
      24
     ],
     [
      6,
      27
     ]
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:heldout:9009",
   "worked": "8*36=288 + 6*24=144 + 6*27=162 = 594; minus 17 = 577"
  },
  {
   "answer": "15",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 12, 13, 14, 15, 16. Ben is free at hours 12, 14, 15, 16, 17. Cruz is free at hours 9, 11, 13, 15. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      12,
      13,
      14,
      15,
      16
     ],
     "Ben": [
      12,
      14,
      15,
      16,
      17
     ],
     "Cruz": [
      9,
      11,
      13,
      15
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9000",
   "worked": "hours free for all three: [15]; earliest = 15"
  },
  {
   "answer": "15",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 15, 16, 17. Ben is free at hours 11, 14, 15, 17. Cruz is free at hours 9, 10, 13, 15, 16, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      15,
      16,
      17
     ],
     "Ben": [
      11,
      14,
      15,
      17
     ],
     "Cruz": [
      9,
      10,
      13,
      15,
      16,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9001",
   "worked": "hours free for all three: [15, 17]; earliest = 15"
  },
  {
   "answer": "10",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 10, 12, 13, 15, 16. Ben is free at hours 9, 10, 11, 14, 16, 17. Cruz is free at hours 10, 12, 13, 14, 15, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      10,
      12,
      13,
      15,
      16
     ],
     "Ben": [
      9,
      10,
      11,
      14,
      16,
      17
     ],
     "Cruz": [
      10,
      12,
      13,
      14,
      15,
      16
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9002",
   "worked": "hours free for all three: [10, 16]; earliest = 10"
  },
  {
   "answer": "11",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 11, 12, 13, 14, 15. Ben is free at hours 10, 11, 12, 14, 15, 17. Cruz is free at hours 11, 12, 13, 15, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      11,
      12,
      13,
      14,
      15
     ],
     "Ben": [
      10,
      11,
      12,
      14,
      15,
      17
     ],
     "Cruz": [
      11,
      12,
      13,
      15,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9003",
   "worked": "hours free for all three: [11, 12, 15]; earliest = 11"
  },
  {
   "answer": "10",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 14, 16, 17. Ben is free at hours 9, 10, 11, 12, 13, 14. Cruz is free at hours 10, 12, 14, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      14,
      16,
      17
     ],
     "Ben": [
      9,
      10,
      11,
      12,
      13,
      14
     ],
     "Cruz": [
      10,
      12,
      14,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9004",
   "worked": "hours free for all three: [10, 14]; earliest = 10"
  },
  {
   "answer": "9",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 11, 13, 15, 17. Ben is free at hours 9, 12, 13, 14. Cruz is free at hours 9, 13, 14, 15, 16, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      11,
      13,
      15,
      17
     ],
     "Ben": [
      9,
      12,
      13,
      14
     ],
     "Cruz": [
      9,
      13,
      14,
      15,
      16,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9005",
   "worked": "hours free for all three: [9, 13]; earliest = 9"
  },
  {
   "answer": "9",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 14, 15, 17. Ben is free at hours 9, 11, 14, 17. Cruz is free at hours 9, 12, 15, 16, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      14,
      15,
      17
     ],
     "Ben": [
      9,
      11,
      14,
      17
     ],
     "Cruz": [
      9,
      12,
      15,
      16,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9006",
   "worked": "hours free for all three: [9, 17]; earliest = 9"
  },
  {
   "answer": "11",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 13, 14. Ben is free at hours 9, 10, 11, 13, 15. Cruz is free at hours 11, 12, 14, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      13,
      14
     ],
     "Ben": [
      9,
      10,
      11,
      13,
      15
     ],
     "Cruz": [
      11,
      12,
      14,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9007",
   "worked": "hours free for all three: [11]; earliest = 11"
  },
  {
   "answer": "11",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 15, 16. Ben is free at hours 11, 14, 16, 17. Cruz is free at hours 10, 11, 12, 13, 15. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      15,
      16
     ],
     "Ben": [
      11,
      14,
      16,
      17
     ],
     "Cruz": [
      10,
      11,
      12,
      13,
      15
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9008",
   "worked": "hours free for all three: [11]; earliest = 11"
  },
  {
   "answer": "10",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 13, 16. Ben is free at hours 10, 13, 15, 17. Cruz is free at hours 10, 11, 12, 14, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      13,
      16
     ],
     "Ben": [
      10,
      13,
      15,
      17
     ],
     "Cruz": [
      10,
      11,
      12,
      14,
      17
     ]
    }
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:heldout:9009",
   "worked": "hours free for all three: [10]; earliest = 10"
  },
  {
   "answer": "3",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: mail depends on nothing. search depends on mail. cache depends on mail and search. auth depends on cache and mail. gateway depends on cache and mail. billing depends on auth and mail. If cache fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "cache",
      "mail"
     ],
     "billing": [
      "auth",
      "mail"
     ],
     "cache": [
      "mail",
      "search"
     ],
     "gateway": [
      "cache",
      "mail"
     ],
     "mail": [],
     "search": [
      "mail"
     ]
    },
    "failed": "cache"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9000",
   "worked": "services that depend on cache directly or through others: auth, billing, gateway; count = 3"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: db depends on nothing. mail depends on db. billing depends on db. gateway depends on billing and db. cache depends on mail. queue depends on gateway and mail. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "billing": [
      "db"
     ],
     "cache": [
      "mail"
     ],
     "db": [],
     "gateway": [
      "billing",
      "db"
     ],
     "mail": [
      "db"
     ],
     "queue": [
      "gateway",
      "mail"
     ]
    },
    "failed": "db"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9001",
   "worked": "services that depend on db directly or through others: billing, cache, gateway, mail, queue; count = 5"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: search depends on nothing. auth depends on search. db depends on auth. cache depends on auth and db. billing depends on cache and search. queue depends on auth. If auth fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "search"
     ],
     "billing": [
      "cache",
      "search"
     ],
     "cache": [
      "auth",
      "db"
     ],
     "db": [
      "auth"
     ],
     "queue": [
      "auth"
     ],
     "search": []
    },
    "failed": "auth"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9002",
   "worked": "services that depend on auth directly or through others: billing, cache, db, queue; count = 4"
  },
  {
   "answer": "2",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: gateway depends on nothing. cache depends on gateway. auth depends on cache. search depends on cache. billing depends on auth and search. mail depends on auth. If auth fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "cache"
     ],
     "billing": [
      "auth",
      "search"
     ],
     "cache": [
      "gateway"
     ],
     "gateway": [],
     "mail": [
      "auth"
     ],
     "search": [
      "cache"
     ]
    },
    "failed": "auth"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9003",
   "worked": "services that depend on auth directly or through others: billing, mail; count = 2"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: billing depends on nothing. gateway depends on billing. db depends on gateway. cache depends on billing and db. auth depends on billing and db. mail depends on billing and gateway. If billing fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "billing",
      "db"
     ],
     "billing": [],
     "cache": [
      "billing",
      "db"
     ],
     "db": [
      "gateway"
     ],
     "gateway": [
      "billing"
     ],
     "mail": [
      "billing",
      "gateway"
     ]
    },
    "failed": "billing"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9004",
   "worked": "services that depend on billing directly or through others: auth, cache, db, gateway, mail; count = 5"
  },
  {
   "answer": "2",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: cache depends on nothing. search depends on cache. mail depends on cache and search. auth depends on mail and search. gateway depends on cache. billing depends on cache. If search fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "mail",
      "search"
     ],
     "billing": [
      "cache"
     ],
     "cache": [],
     "gateway": [
      "cache"
     ],
     "mail": [
      "cache",
      "search"
     ],
     "search": [
      "cache"
     ]
    },
    "failed": "search"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9005",
   "worked": "services that depend on search directly or through others: auth, mail; count = 2"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: queue depends on nothing. gateway depends on queue. billing depends on gateway. cache depends on billing. mail depends on cache and gateway. search depends on billing and gateway. If gateway fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "billing": [
      "gateway"
     ],
     "cache": [
      "billing"
     ],
     "gateway": [
      "queue"
     ],
     "mail": [
      "cache",
      "gateway"
     ],
     "queue": [],
     "search": [
      "billing",
      "gateway"
     ]
    },
    "failed": "gateway"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9006",
   "worked": "services that depend on gateway directly or through others: billing, cache, mail, search; count = 4"
  },
  {
   "answer": "2",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: search depends on nothing. mail depends on search. db depends on mail. auth depends on db and mail. queue depends on db and mail. billing depends on mail and search. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "db",
      "mail"
     ],
     "billing": [
      "mail",
      "search"
     ],
     "db": [
      "mail"
     ],
     "mail": [
      "search"
     ],
     "queue": [
      "db",
      "mail"
     ],
     "search": []
    },
    "failed": "db"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9007",
   "worked": "services that depend on db directly or through others: auth, queue; count = 2"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: mail depends on nothing. cache depends on mail. billing depends on cache and mail. gateway depends on cache and mail. search depends on gateway. queue depends on search. If cache fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "billing": [
      "cache",
      "mail"
     ],
     "cache": [
      "mail"
     ],
     "gateway": [
      "cache",
      "mail"
     ],
     "mail": [],
     "queue": [
      "search"
     ],
     "search": [
      "gateway"
     ]
    },
    "failed": "cache"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9008",
   "worked": "services that depend on cache directly or through others: billing, gateway, queue, search; count = 4"
  },
  {
   "answer": "3",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: db depends on nothing. auth depends on db. mail depends on auth and db. cache depends on db. gateway depends on auth and cache. search depends on auth. If auth fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "db"
     ],
     "cache": [
      "db"
     ],
     "db": [],
     "gateway": [
      "auth",
      "cache"
     ],
     "mail": [
      "auth",
      "db"
     ],
     "search": [
      "auth"
     ]
    },
    "failed": "auth"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:heldout:9009",
   "worked": "services that depend on auth directly or through others: gateway, mail, search; count = 3"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 332 days. Record: the audit logs were deleted after 196 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "contradiction detection",
   "spec": {
    "got": 196,
    "kind": "at least",
    "need": 332
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9000",
   "worked": "196 days is not at least 332 days -> VIOLATION"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 245 days. Record: the audit logs were deleted after 51 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "contradiction detection",
   "spec": {
    "got": 51,
    "kind": "at least",
    "need": 245
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9001",
   "worked": "51 days is not at least 245 days -> VIOLATION"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 267 days. Record: the audit logs were deleted after 77 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "contradiction detection",
   "spec": {
    "got": 77,
    "kind": "at most",
    "need": 267
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9002",
   "worked": "77 days is at most 267 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 321 days. Record: the audit logs were deleted after 240 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "contradiction detection",
   "spec": {
    "got": 240,
    "kind": "at least",
    "need": 321
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9003",
   "worked": "240 days is not at least 321 days -> VIOLATION"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 124 days. Record: the audit logs were deleted after 106 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "contradiction detection",
   "spec": {
    "got": 106,
    "kind": "at least",
    "need": 124
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9004",
   "worked": "106 days is not at least 124 days -> VIOLATION"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 173 days. Record: the audit logs were deleted after 295 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "contradiction detection",
   "spec": {
    "got": 295,
    "kind": "at least",
    "need": 173
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9005",
   "worked": "295 days is at least 173 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 150 days. Record: the audit logs were deleted after 304 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "contradiction detection",
   "spec": {
    "got": 304,
    "kind": "at most",
    "need": 150
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9006",
   "worked": "304 days is not at most 150 days -> VIOLATION"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 38 days. Record: the audit logs were deleted after 156 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "contradiction detection",
   "spec": {
    "got": 156,
    "kind": "at most",
    "need": 38
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9007",
   "worked": "156 days is not at most 38 days -> VIOLATION"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 57 days. Record: the audit logs were deleted after 39 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "contradiction detection",
   "spec": {
    "got": 39,
    "kind": "at most",
    "need": 57
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9008",
   "worked": "39 days is at most 57 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 164 days. Record: the audit logs were deleted after 285 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "contradiction detection",
   "spec": {
    "got": 285,
    "kind": "at most",
    "need": 164
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:heldout:9009",
   "worked": "285 days is not at most 164 days -> VIOLATION"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export, read, write. engineer may delete, write. auditor may export, read. intern may delete, write. Override: engineer may never read. Anything not listed is denied. Request: an analyst wants to read. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "rule application",
   "spec": {
    "act": "read",
    "allow": {
     "analyst": [
      "export",
      "read",
      "write"
     ],
     "auditor": [
      "export",
      "read"
     ],
     "engineer": [
      "delete",
      "write"
     ],
     "intern": [
      "delete",
      "write"
     ]
    },
    "deny": [
     "engineer",
     "read"
    ],
    "role": "analyst"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9000",
   "worked": "analyst listed actions: ['export', 'read', 'write']; override blocks engineer/read -> ALLOW"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may read, write. engineer may export. auditor may export, read, write. intern may delete, write. Override: engineer may never read. Anything not listed is denied. Request: an intern wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "read",
      "write"
     ],
     "auditor": [
      "export",
      "read",
      "write"
     ],
     "engineer": [
      "export"
     ],
     "intern": [
      "delete",
      "write"
     ]
    },
    "deny": [
     "engineer",
     "read"
    ],
    "role": "intern"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9001",
   "worked": "intern listed actions: ['delete', 'write']; override blocks engineer/read -> ALLOW"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, read, write. engineer may export, read. auditor may read, write. intern may delete, read, write. Override: intern may never export. Anything not listed is denied. Request: an analyst wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "delete",
      "read",
      "write"
     ],
     "auditor": [
      "read",
      "write"
     ],
     "engineer": [
      "export",
      "read"
     ],
     "intern": [
      "delete",
      "read",
      "write"
     ]
    },
    "deny": [
     "intern",
     "export"
    ],
    "role": "analyst"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9002",
   "worked": "analyst listed actions: ['delete', 'read', 'write']; override blocks intern/export -> ALLOW"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export. engineer may delete, export, write. auditor may export, read, write. intern may read. Override: intern may never write. Anything not listed is denied. Request: an intern wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "export"
     ],
     "auditor": [
      "export",
      "read",
      "write"
     ],
     "engineer": [
      "delete",
      "export",
      "write"
     ],
     "intern": [
      "read"
     ]
    },
    "deny": [
     "intern",
     "write"
    ],
    "role": "intern"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9003",
   "worked": "intern listed actions: ['read']; override blocks intern/write -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may read. engineer may read, write. auditor may delete, read. intern may write. Override: engineer may never export. Anything not listed is denied. Request: an auditor wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "read"
     ],
     "auditor": [
      "delete",
      "read"
     ],
     "engineer": [
      "read",
      "write"
     ],
     "intern": [
      "write"
     ]
    },
    "deny": [
     "engineer",
     "export"
    ],
    "role": "auditor"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9004",
   "worked": "auditor listed actions: ['delete', 'read']; override blocks engineer/export -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export, read. engineer may export, write. auditor may delete, read, write. intern may delete, export, write. Override: engineer may never export. Anything not listed is denied. Request: an engineer wants to read. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "rule application",
   "spec": {
    "act": "read",
    "allow": {
     "analyst": [
      "export",
      "read"
     ],
     "auditor": [
      "delete",
      "read",
      "write"
     ],
     "engineer": [
      "export",
      "write"
     ],
     "intern": [
      "delete",
      "export",
      "write"
     ]
    },
    "deny": [
     "engineer",
     "export"
    ],
    "role": "engineer"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9005",
   "worked": "engineer listed actions: ['export', 'write']; override blocks engineer/export -> DENY"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, write. engineer may delete, export, read. auditor may delete, export, write. intern may delete, read, write. Override: engineer may never export. Anything not listed is denied. Request: an auditor wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "delete",
      "write"
     ],
     "auditor": [
      "delete",
      "export",
      "write"
     ],
     "engineer": [
      "delete",
      "export",
      "read"
     ],
     "intern": [
      "delete",
      "read",
      "write"
     ]
    },
    "deny": [
     "engineer",
     "export"
    ],
    "role": "auditor"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9006",
   "worked": "auditor listed actions: ['delete', 'export', 'write']; override blocks engineer/export -> ALLOW"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, read. engineer may export. auditor may delete, export, write. intern may export, read. Override: auditor may never export. Anything not listed is denied. Request: an engineer wants to read. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "rule application",
   "spec": {
    "act": "read",
    "allow": {
     "analyst": [
      "delete",
      "read"
     ],
     "auditor": [
      "delete",
      "export",
      "write"
     ],
     "engineer": [
      "export"
     ],
     "intern": [
      "export",
      "read"
     ]
    },
    "deny": [
     "auditor",
     "export"
    ],
    "role": "engineer"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9007",
   "worked": "engineer listed actions: ['export']; override blocks auditor/export -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export, read. engineer may export. auditor may delete, read, write. intern may delete, export, write. Override: auditor may never delete. Anything not listed is denied. Request: an auditor wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "export",
      "read"
     ],
     "auditor": [
      "delete",
      "read",
      "write"
     ],
     "engineer": [
      "export"
     ],
     "intern": [
      "delete",
      "export",
      "write"
     ]
    },
    "deny": [
     "auditor",
     "delete"
    ],
    "role": "auditor"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9008",
   "worked": "auditor listed actions: ['delete', 'read', 'write']; override blocks auditor/delete -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may read. engineer may delete, export. auditor may export. intern may delete. Override: auditor may never export. Anything not listed is denied. Request: an intern wants to read. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "rule application",
   "spec": {
    "act": "read",
    "allow": {
     "analyst": [
      "read"
     ],
     "auditor": [
      "export"
     ],
     "engineer": [
      "delete",
      "export"
     ],
     "intern": [
      "delete"
     ]
    },
    "deny": [
     "auditor",
     "export"
    ],
    "role": "intern"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:heldout:9009",
   "worked": "intern listed actions: ['delete']; override blocks auditor/export -> DENY"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: document requires test first. test requires build first. build requires review first. review requires deploy first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "review",
     "document": "test",
     "review": "deploy",
     "test": "build"
    },
    "tasks": [
     "build",
     "deploy",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9000",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "DOCUMENT",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: deploy requires design first. design requires document first. build requires deploy first. review requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "deploy",
     "deploy": "design",
     "design": "document",
     "review": "build"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9001",
   "worked": "every step except document requires another step first -> document"
  },
  {
   "answer": "BUILD",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: review requires design first. test requires build first. design requires document first. document requires test first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "planning",
   "spec": {
    "prereq": {
     "design": "document",
     "document": "test",
     "review": "design",
     "test": "build"
    },
    "tasks": [
     "build",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9002",
   "worked": "every step except build requires another step first -> build"
  },
  {
   "answer": "REVIEW",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: document requires design first. test requires review first. build requires test first. design requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "test",
     "design": "build",
     "document": "design",
     "test": "review"
    },
    "tasks": [
     "build",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9003",
   "worked": "every step except review requires another step first -> review"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: review requires design first. document requires deploy first. design requires document first. test requires review first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "planning",
   "spec": {
    "prereq": {
     "design": "document",
     "document": "deploy",
     "review": "design",
     "test": "review"
    },
    "tasks": [
     "deploy",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9004",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "BUILD",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: design requires document first. document requires deploy first. deploy requires review first. review requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "planning",
   "spec": {
    "prereq": {
     "deploy": "review",
     "design": "document",
     "document": "deploy",
     "review": "build"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9005",
   "worked": "every step except build requires another step first -> build"
  },
  {
   "answer": "TEST",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: document requires test first. review requires deploy first. design requires review first. deploy requires document first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "planning",
   "spec": {
    "prereq": {
     "deploy": "document",
     "design": "review",
     "document": "test",
     "review": "deploy"
    },
    "tasks": [
     "deploy",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9006",
   "worked": "every step except test requires another step first -> test"
  },
  {
   "answer": "DESIGN",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: deploy requires review first. review requires test first. document requires deploy first. test requires design first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "planning",
   "spec": {
    "prereq": {
     "deploy": "review",
     "document": "deploy",
     "review": "test",
     "test": "design"
    },
    "tasks": [
     "deploy",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9007",
   "worked": "every step except design requires another step first -> design"
  },
  {
   "answer": "REVIEW",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: deploy requires design first. design requires review first. document requires deploy first. build requires document first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "document",
     "deploy": "design",
     "design": "review",
     "document": "deploy"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9008",
   "worked": "every step except review requires another step first -> review"
  },
  {
   "answer": "BUILD",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: document requires design first. test requires build first. review requires document first. design requires test first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "planning",
   "spec": {
    "prereq": {
     "design": "test",
     "document": "design",
     "review": "document",
     "test": "build"
    },
    "tasks": [
     "build",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:heldout:9009",
   "worked": "every step except build requires another step first -> build"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 73 crates. Some crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 73,
    "c": 9,
    "missing": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9000",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "70",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 71 crates. 10 crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 71,
    "b": 10,
    "c": 9,
    "missing": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9001",
   "worked": "71 - 10 + 9 = 70"
  },
  {
   "answer": "45",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 50 crates. 10 crates are shipped on Monday and 5 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 50,
    "b": 10,
    "c": 5,
    "missing": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9002",
   "worked": "50 - 10 + 5 = 45"
  },
  {
   "answer": "42",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 44 crates. 11 crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 44,
    "b": 11,
    "c": 9,
    "missing": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9003",
   "worked": "44 - 11 + 9 = 42"
  },
  {
   "answer": "74",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 80 crates. 8 crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 80,
    "b": 8,
    "c": 2,
    "missing": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9004",
   "worked": "80 - 8 + 2 = 74"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 80 crates. Some crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 80,
    "c": 2,
    "missing": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9005",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 57 crates. Some crates are shipped on Monday and 8 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 57,
    "c": 8,
    "missing": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9006",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "84",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 89 crates. 7 crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 89,
    "b": 7,
    "c": 2,
    "missing": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9007",
   "worked": "89 - 7 + 2 = 84"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 35 crates. Some crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 35,
    "c": 2,
    "missing": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9008",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 46 crates. Some crates are shipped on Monday and 8 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 46,
    "c": 8,
    "missing": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:heldout:9009",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "0",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 8 x 10 = 80. Step 2: 80 + 31 = 111. Step 3: 111 - 16 = 95. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "error detection",
   "spec": {
    "a": 8,
    "b": 10,
    "c": 31,
    "d": 16,
    "shown": [
     80,
     111,
     95
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9000",
   "worked": "true values: 8x10=80; recompute each shown step from its own left side -> first wrong step = 0"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 11 x 5 = 55. Step 2: 55 + 37 = 97. Step 3: 97 - 25 = 72. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "error detection",
   "spec": {
    "a": 11,
    "b": 5,
    "c": 37,
    "d": 25,
    "shown": [
     55,
     97,
     72
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9001",
   "worked": "true values: 11x5=55; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "1",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 11 x 5 = 65. Step 2: 65 + 37 = 102. Step 3: 102 - 28 = 74. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "error detection",
   "spec": {
    "a": 11,
    "b": 5,
    "c": 37,
    "d": 28,
    "shown": [
     65,
     102,
     74
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9002",
   "worked": "true values: 11x5=55; recompute each shown step from its own left side -> first wrong step = 1"
  },
  {
   "answer": "1",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 6 x 7 = 45. Step 2: 45 + 21 = 66. Step 3: 66 - 19 = 47. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "error detection",
   "spec": {
    "a": 6,
    "b": 7,
    "c": 21,
    "d": 19,
    "shown": [
     45,
     66,
     47
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9003",
   "worked": "true values: 6x7=42; recompute each shown step from its own left side -> first wrong step = 1"
  },
  {
   "answer": "0",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 10 x 5 = 50. Step 2: 50 + 48 = 98. Step 3: 98 - 36 = 62. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "error detection",
   "spec": {
    "a": 10,
    "b": 5,
    "c": 48,
    "d": 36,
    "shown": [
     50,
     98,
     62
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9004",
   "worked": "true values: 10x5=50; recompute each shown step from its own left side -> first wrong step = 0"
  },
  {
   "answer": "3",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 6 x 11 = 66. Step 2: 66 + 10 = 76. Step 3: 76 - 33 = 45. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "error detection",
   "spec": {
    "a": 6,
    "b": 11,
    "c": 10,
    "d": 33,
    "shown": [
     66,
     76,
     45
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9005",
   "worked": "true values: 6x11=66; recompute each shown step from its own left side -> first wrong step = 3"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 3 x 7 = 21. Step 2: 21 + 25 = 51. Step 3: 51 - 40 = 11. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "error detection",
   "spec": {
    "a": 3,
    "b": 7,
    "c": 25,
    "d": 40,
    "shown": [
     21,
     51,
     11
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9006",
   "worked": "true values: 3x7=21; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 12 x 5 = 60. Step 2: 60 + 28 = 87. Step 3: 87 - 18 = 69. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "error detection",
   "spec": {
    "a": 12,
    "b": 5,
    "c": 28,
    "d": 18,
    "shown": [
     60,
     87,
     69
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9007",
   "worked": "true values: 12x5=60; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "3",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 10 x 12 = 120. Step 2: 120 + 52 = 172. Step 3: 172 - 40 = 142. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "error detection",
   "spec": {
    "a": 10,
    "b": 12,
    "c": 52,
    "d": 40,
    "shown": [
     120,
     172,
     142
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9008",
   "worked": "true values: 10x12=120; recompute each shown step from its own left side -> first wrong step = 3"
  },
  {
   "answer": "3",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 5 x 3 = 15. Step 2: 15 + 40 = 55. Step 3: 55 - 39 = 13. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "error detection",
   "spec": {
    "a": 5,
    "b": 3,
    "c": 40,
    "d": 39,
    "shown": [
     15,
     55,
     13
    ]
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:heldout:9009",
   "worked": "true values: 5x3=15; recompute each shown step from its own left side -> first wrong step = 3"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event J always causes event X. Event K happens only when both X and N occur. Nothing else causes X or K. In this case J does not occur and N occurs. Does K happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": true,
    "need_both": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9000",
   "worked": "X does not occur; K needs only when both X and N occur -> NO"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event P always causes event Q. Event M happens when Q occurs or W occurs. Nothing else causes Q or M. In this case P does not occur and W does not occur. Does M happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": false,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9001",
   "worked": "Q does not occur; M needs when Q occurs or W occurs -> NO"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event F always causes event Y. Event M happens when Y occurs or E occurs. Nothing else causes Y or M. In this case F occurs and E does not occur. Does M happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9002",
   "worked": "Y occurs; M needs when Y occurs or E occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event J always causes event S. Event R happens when S occurs or W occurs. Nothing else causes S or R. In this case J occurs and W does not occur. Does R happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9003",
   "worked": "S occurs; R needs when S occurs or W occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event E always causes event H. Event Y happens when H occurs or M occurs. Nothing else causes H or Y. In this case E does not occur and M occurs. Does Y happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": true,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9004",
   "worked": "H does not occur; Y needs when H occurs or M occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event Y always causes event J. Event G happens when J occurs or X occurs. Nothing else causes J or G. In this case Y occurs and X occurs. Does G happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9005",
   "worked": "J occurs; G needs when J occurs or X occurs -> YES"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event N always causes event F. Event Q happens only when both F and S occur. Nothing else causes F or Q. In this case N occurs and S does not occur. Does Q happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9006",
   "worked": "F occurs; Q needs only when both F and S occur -> NO"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event L always causes event X. Event J happens only when both X and Q occur. Nothing else causes X or J. In this case L occurs and Q does not occur. Does J happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": true
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9007",
   "worked": "X occurs; J needs only when both X and Q occur -> NO"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event T always causes event J. Event W happens when J occurs or Q occurs. Nothing else causes J or W. In this case T occurs and Q occurs. Does W happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9008",
   "worked": "J occurs; W needs when J occurs or Q occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event E always causes event U. Event Z happens when U occurs or M occurs. Nothing else causes U or Z. In this case E occurs and M does not occur. Does Z happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": false
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:heldout:9009",
   "worked": "U occurs; Z needs when U occurs or M occurs -> YES"
  },
  {
   "answer": "3",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The press has no supplier. The warehouse receives material from the press. The foundry receives material from the warehouse. The smelter receives material from the press. The mine receives material from the press and the warehouse. The dock receives material from the mine. If the warehouse shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9000,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "dock": [
      "mine"
     ],
     "foundry": [
      "warehouse"
     ],
     "mine": [
      "press",
      "warehouse"
     ],
     "press": [],
     "smelter": [
      "press"
     ],
     "warehouse": [
      "press"
     ]
    },
    "failed": "warehouse"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9000",
   "worked": "sites supplied by the warehouse directly or through others: dock, foundry, mine; count = 3"
  },
  {
   "answer": "4",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The assembly has no supplier. The dock receives material from the assembly. The smelter receives material from the dock. The press receives material from the dock and the smelter. The mine receives material from the smelter. The foundry receives material from the dock and the press. If the dock shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9001,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [],
     "dock": [
      "assembly"
     ],
     "foundry": [
      "dock",
      "press"
     ],
     "mine": [
      "smelter"
     ],
     "press": [
      "dock",
      "smelter"
     ],
     "smelter": [
      "dock"
     ]
    },
    "failed": "dock"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9001",
   "worked": "sites supplied by the dock directly or through others: foundry, mine, press, smelter; count = 4"
  },
  {
   "answer": "2",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The foundry has no supplier. The mine receives material from the foundry. The paint shop receives material from the foundry. The press receives material from the paint shop. The dock receives material from the foundry and the mine. The assembly receives material from the dock and the press. If the mine shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9002,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [
      "dock",
      "press"
     ],
     "dock": [
      "foundry",
      "mine"
     ],
     "foundry": [],
     "mine": [
      "foundry"
     ],
     "paint shop": [
      "foundry"
     ],
     "press": [
      "paint shop"
     ]
    },
    "failed": "mine"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9002",
   "worked": "sites supplied by the mine directly or through others: assembly, dock; count = 2"
  },
  {
   "answer": "2",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The smelter has no supplier. The paint shop receives material from the smelter. The press receives material from the smelter. The warehouse receives material from the paint shop and the smelter. The dock receives material from the press and the smelter. The mine receives material from the paint shop and the warehouse. If the paint shop shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9003,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "dock": [
      "press",
      "smelter"
     ],
     "mine": [
      "paint shop",
      "warehouse"
     ],
     "paint shop": [
      "smelter"
     ],
     "press": [
      "smelter"
     ],
     "smelter": [],
     "warehouse": [
      "paint shop",
      "smelter"
     ]
    },
    "failed": "paint shop"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9003",
   "worked": "sites supplied by the paint shop directly or through others: mine, warehouse; count = 2"
  },
  {
   "answer": "5",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The press has no supplier. The assembly receives material from the press. The warehouse receives material from the press. The foundry receives material from the assembly. The paint shop receives material from the warehouse. The smelter receives material from the press and the warehouse. If the press shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9004,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [
      "press"
     ],
     "foundry": [
      "assembly"
     ],
     "paint shop": [
      "warehouse"
     ],
     "press": [],
     "smelter": [
      "press",
      "warehouse"
     ],
     "warehouse": [
      "press"
     ]
    },
    "failed": "press"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9004",
   "worked": "sites supplied by the press directly or through others: assembly, foundry, paint shop, smelter, warehouse; count = 5"
  },
  {
   "answer": "2",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The warehouse has no supplier. The mine receives material from the warehouse. The assembly receives material from the mine and the warehouse. The dock receives material from the warehouse. The foundry receives material from the dock and the warehouse. The press receives material from the assembly. If the mine shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9005,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [
      "mine",
      "warehouse"
     ],
     "dock": [
      "warehouse"
     ],
     "foundry": [
      "dock",
      "warehouse"
     ],
     "mine": [
      "warehouse"
     ],
     "press": [
      "assembly"
     ],
     "warehouse": []
    },
    "failed": "mine"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9005",
   "worked": "sites supplied by the mine directly or through others: assembly, press; count = 2"
  },
  {
   "answer": "3",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The dock has no supplier. The smelter receives material from the dock. The assembly receives material from the smelter. The press receives material from the assembly and the dock. The mine receives material from the assembly and the smelter. The warehouse receives material from the assembly and the dock. If the assembly shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9006,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [
      "smelter"
     ],
     "dock": [],
     "mine": [
      "assembly",
      "smelter"
     ],
     "press": [
      "assembly",
      "dock"
     ],
     "smelter": [
      "dock"
     ],
     "warehouse": [
      "assembly",
      "dock"
     ]
    },
    "failed": "assembly"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9006",
   "worked": "sites supplied by the assembly directly or through others: mine, press, warehouse; count = 3"
  },
  {
   "answer": "3",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The foundry has no supplier. The dock receives material from the foundry. The paint shop receives material from the dock and the foundry. The mine receives material from the dock and the foundry. The smelter receives material from the mine. The press receives material from the foundry. If the dock shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9007,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "dock": [
      "foundry"
     ],
     "foundry": [],
     "mine": [
      "dock",
      "foundry"
     ],
     "paint shop": [
      "dock",
      "foundry"
     ],
     "press": [
      "foundry"
     ],
     "smelter": [
      "mine"
     ]
    },
    "failed": "dock"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9007",
   "worked": "sites supplied by the dock directly or through others: mine, paint shop, smelter; count = 3"
  },
  {
   "answer": "5",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The mine has no supplier. The assembly receives material from the mine. The paint shop receives material from the mine. The warehouse receives material from the assembly and the mine. The foundry receives material from the warehouse. The dock receives material from the foundry and the mine. If the mine shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9008,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "assembly": [
      "mine"
     ],
     "dock": [
      "foundry",
      "mine"
     ],
     "foundry": [
      "warehouse"
     ],
     "mine": [],
     "paint shop": [
      "mine"
     ],
     "warehouse": [
      "assembly",
      "mine"
     ]
    },
    "failed": "mine"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9008",
   "worked": "sites supplied by the mine directly or through others: assembly, dock, foundry, paint shop, warehouse; count = 5"
  },
  {
   "answer": "0",
   "domain": "supply chains (unseen)",
   "family": "transfer_supply",
   "prompt": "Supply chain: The paint shop has no supplier. The dock receives material from the paint shop. The mine receives material from the paint shop. The warehouse receives material from the dock. The foundry receives material from the dock and the paint shop. The press receives material from the warehouse. If the mine shuts down, how many OTHER sites run out of material, counting sites further down the chain? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 9009,
   "skill": "cross-domain transfer",
   "spec": {
    "deps": {
     "dock": [
      "paint shop"
     ],
     "foundry": [
      "dock",
      "paint shop"
     ],
     "mine": [
      "paint shop"
     ],
     "paint shop": [],
     "press": [
      "warehouse"
     ],
     "warehouse": [
      "dock"
     ]
    },
    "failed": "mine"
   },
   "split": "heldout",
   "suite": "e85-xdomain-v1",
   "task_id": "transfer_supply:heldout:9009",
   "worked": "sites supplied by the mine directly or through others: none; count = 0"
  }
 ],
 "train": [
  {
   "answer": "145",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 3 units at $3 each; 6 units at $16 each; 4 units at $12 each. A discount of $8 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 8,
    "items": [
     [
      3,
      3
     ],
     [
      6,
      16
     ],
     [
      4,
      12
     ]
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:train:1000",
   "worked": "3*3=9 + 6*16=96 + 4*12=48 = 153; minus 8 = 145"
  },
  {
   "answer": "172",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 3 units at $12 each; 8 units at $10 each; 5 units at $15 each. A discount of $19 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 19,
    "items": [
     [
      3,
      12
     ],
     [
      8,
      10
     ],
     [
      5,
      15
     ]
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:train:1001",
   "worked": "3*12=36 + 8*10=80 + 5*15=75 = 191; minus 19 = 172"
  },
  {
   "answer": "278",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 9 units at $7 each; 3 units at $40 each; 6 units at $17 each. A discount of $7 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 7,
    "items": [
     [
      9,
      7
     ],
     [
      3,
      40
     ],
     [
      6,
      17
     ]
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:train:1002",
   "worked": "9*7=63 + 3*40=120 + 6*17=102 = 285; minus 7 = 278"
  },
  {
   "answer": "286",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 7 units at $21 each; 5 units at $24 each; 2 units at $18 each. A discount of $17 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 17,
    "items": [
     [
      7,
      21
     ],
     [
      5,
      24
     ],
     [
      2,
      18
     ]
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:train:1003",
   "worked": "7*21=147 + 5*24=120 + 2*18=36 = 303; minus 17 = 286"
  },
  {
   "answer": "10",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 12, 14, 17. Ben is free at hours 10, 11, 12, 14, 15. Cruz is free at hours 10, 11, 14, 15. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      12,
      14,
      17
     ],
     "Ben": [
      10,
      11,
      12,
      14,
      15
     ],
     "Cruz": [
      10,
      11,
      14,
      15
     ]
    }
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:train:1000",
   "worked": "hours free for all three: [10, 11, 14]; earliest = 10"
  },
  {
   "answer": "10",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 10, 11, 12, 13, 15, 17. Ben is free at hours 10, 11, 16, 17. Cruz is free at hours 9, 10, 14, 15, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      10,
      11,
      12,
      13,
      15,
      17
     ],
     "Ben": [
      10,
      11,
      16,
      17
     ],
     "Cruz": [
      9,
      10,
      14,
      15,
      17
     ]
    }
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:train:1001",
   "worked": "hours free for all three: [10, 17]; earliest = 10"
  },
  {
   "answer": "14",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 13, 14, 15, 17. Ben is free at hours 10, 12, 13, 14, 17. Cruz is free at hours 9, 12, 14, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      13,
      14,
      15,
      17
     ],
     "Ben": [
      10,
      12,
      13,
      14,
      17
     ],
     "Cruz": [
      9,
      12,
      14,
      16
     ]
    }
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:train:1002",
   "worked": "hours free for all three: [14]; earliest = 14"
  },
  {
   "answer": "9",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 15, 17. Ben is free at hours 9, 11, 12, 13, 16. Cruz is free at hours 9, 12, 15, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      15,
      17
     ],
     "Ben": [
      9,
      11,
      12,
      13,
      16
     ],
     "Cruz": [
      9,
      12,
      15,
      16
     ]
    }
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:train:1003",
   "worked": "hours free for all three: [9]; earliest = 9"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: queue depends on nothing. gateway depends on queue. mail depends on gateway and queue. db depends on mail and queue. auth depends on gateway and mail. billing depends on gateway. If queue fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "gateway",
      "mail"
     ],
     "billing": [
      "gateway"
     ],
     "db": [
      "mail",
      "queue"
     ],
     "gateway": [
      "queue"
     ],
     "mail": [
      "gateway",
      "queue"
     ],
     "queue": []
    },
    "failed": "queue"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:train:1000",
   "worked": "services that depend on queue directly or through others: auth, billing, db, gateway, mail; count = 5"
  },
  {
   "answer": "2",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: queue depends on nothing. db depends on queue. cache depends on queue. auth depends on db. billing depends on queue. gateway depends on auth and billing. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "db"
     ],
     "billing": [
      "queue"
     ],
     "cache": [
      "queue"
     ],
     "db": [
      "queue"
     ],
     "gateway": [
      "auth",
      "billing"
     ],
     "queue": []
    },
    "failed": "db"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:train:1001",
   "worked": "services that depend on db directly or through others: auth, gateway; count = 2"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: search depends on nothing. queue depends on search. db depends on queue and search. cache depends on db and search. mail depends on db. billing depends on db and mail. If search fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "billing": [
      "db",
      "mail"
     ],
     "cache": [
      "db",
      "search"
     ],
     "db": [
      "queue",
      "search"
     ],
     "mail": [
      "db"
     ],
     "queue": [
      "search"
     ],
     "search": []
    },
    "failed": "search"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:train:1002",
   "worked": "services that depend on search directly or through others: billing, cache, db, mail, queue; count = 5"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: queue depends on nothing. billing depends on queue. db depends on billing and queue. mail depends on billing. auth depends on billing. search depends on auth and queue. If billing fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "billing"
     ],
     "billing": [
      "queue"
     ],
     "db": [
      "billing",
      "queue"
     ],
     "mail": [
      "billing"
     ],
     "queue": [],
     "search": [
      "auth",
      "queue"
     ]
    },
    "failed": "billing"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:train:1003",
   "worked": "services that depend on billing directly or through others: auth, db, mail, search; count = 4"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 176 days. Record: the audit logs were deleted after 70 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "contradiction detection",
   "spec": {
    "got": 70,
    "kind": "at most",
    "need": 176
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:train:1000",
   "worked": "70 days is at most 176 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 53 days. Record: the audit logs were deleted after 366 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "contradiction detection",
   "spec": {
    "got": 366,
    "kind": "at most",
    "need": 53
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:train:1001",
   "worked": "366 days is not at most 53 days -> VIOLATION"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 260 days. Record: the audit logs were deleted after 154 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "contradiction detection",
   "spec": {
    "got": 154,
    "kind": "at least",
    "need": 260
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:train:1002",
   "worked": "154 days is not at least 260 days -> VIOLATION"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 321 days. Record: the audit logs were deleted after 313 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "contradiction detection",
   "spec": {
    "got": 313,
    "kind": "at most",
    "need": 321
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:train:1003",
   "worked": "313 days is at most 321 days -> COMPLIANT"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export. engineer may export, read, write. auditor may delete. intern may delete. Override: auditor may never read. Anything not listed is denied. Request: an engineer wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "export"
     ],
     "auditor": [
      "delete"
     ],
     "engineer": [
      "export",
      "read",
      "write"
     ],
     "intern": [
      "delete"
     ]
    },
    "deny": [
     "auditor",
     "read"
    ],
    "role": "engineer"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:train:1000",
   "worked": "engineer listed actions: ['export', 'read', 'write']; override blocks auditor/read -> DENY"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, read, write. engineer may export, read. auditor may export, write. intern may export, read, write. Override: intern may never export. Anything not listed is denied. Request: an engineer wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "delete",
      "read",
      "write"
     ],
     "auditor": [
      "export",
      "write"
     ],
     "engineer": [
      "export",
      "read"
     ],
     "intern": [
      "export",
      "read",
      "write"
     ]
    },
    "deny": [
     "intern",
     "export"
    ],
    "role": "engineer"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:train:1001",
   "worked": "engineer listed actions: ['export', 'read']; override blocks intern/export -> ALLOW"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may write. engineer may delete, export, write. auditor may write. intern may delete. Override: auditor may never export. Anything not listed is denied. Request: an intern wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "write"
     ],
     "auditor": [
      "write"
     ],
     "engineer": [
      "delete",
      "export",
      "write"
     ],
     "intern": [
      "delete"
     ]
    },
    "deny": [
     "auditor",
     "export"
    ],
    "role": "intern"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:train:1002",
   "worked": "intern listed actions: ['delete']; override blocks auditor/export -> ALLOW"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, export. engineer may delete, write. auditor may export, read, write. intern may delete, read, write. Override: auditor may never export. Anything not listed is denied. Request: an analyst wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "delete",
      "export"
     ],
     "auditor": [
      "export",
      "read",
      "write"
     ],
     "engineer": [
      "delete",
      "write"
     ],
     "intern": [
      "delete",
      "read",
      "write"
     ]
    },
    "deny": [
     "auditor",
     "export"
    ],
    "role": "analyst"
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:train:1003",
   "worked": "analyst listed actions: ['delete', 'export']; override blocks auditor/export -> ALLOW"
  },
  {
   "answer": "TEST",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: review requires build first. design requires test first. build requires design first. document requires review first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "design",
     "design": "test",
     "document": "review",
     "review": "build"
    },
    "tasks": [
     "build",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:train:1000",
   "worked": "every step except test requires another step first -> test"
  },
  {
   "answer": "DOCUMENT",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: deploy requires document first. build requires deploy first. review requires build first. test requires review first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "deploy",
     "deploy": "document",
     "review": "build",
     "test": "review"
    },
    "tasks": [
     "build",
     "deploy",
     "document",
     "review",
     "test"
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:train:1001",
   "worked": "every step except document requires another step first -> document"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: design requires document first. test requires deploy first. document requires test first. review requires design first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "planning",
   "spec": {
    "prereq": {
     "design": "document",
     "document": "test",
     "review": "design",
     "test": "deploy"
    },
    "tasks": [
     "deploy",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:train:1002",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "DESIGN",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: build requires review first. document requires test first. review requires design first. test requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "review",
     "document": "test",
     "review": "design",
     "test": "build"
    },
    "tasks": [
     "build",
     "design",
     "document",
     "review",
     "test"
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:train:1003",
   "worked": "every step except design requires another step first -> design"
  },
  {
   "answer": "40",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 44 crates. 6 crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 44,
    "b": 6,
    "c": 2,
    "missing": false
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:train:1000",
   "worked": "44 - 6 + 2 = 40"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 59 crates. Some crates are shipped on Monday and 6 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 59,
    "c": 6,
    "missing": true
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:train:1001",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 81 crates. Some crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 81,
    "c": 9,
    "missing": true
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:train:1002",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "24",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 28 crates. 13 crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 28,
    "b": 13,
    "c": 9,
    "missing": false
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:train:1003",
   "worked": "28 - 13 + 9 = 24"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 5 x 7 = 35. Step 2: 35 + 52 = 92. Step 3: 92 - 32 = 60. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "error detection",
   "spec": {
    "a": 5,
    "b": 7,
    "c": 52,
    "d": 32,
    "shown": [
     35,
     92,
     60
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:train:1000",
   "worked": "true values: 5x7=35; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 8 x 4 = 32. Step 2: 32 + 43 = 74. Step 3: 74 - 33 = 41. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "error detection",
   "spec": {
    "a": 8,
    "b": 4,
    "c": 43,
    "d": 33,
    "shown": [
     32,
     74,
     41
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:train:1001",
   "worked": "true values: 8x4=32; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 10 x 7 = 70. Step 2: 70 + 20 = 95. Step 3: 95 - 8 = 87. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "error detection",
   "spec": {
    "a": 10,
    "b": 7,
    "c": 20,
    "d": 8,
    "shown": [
     70,
     95,
     87
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:train:1002",
   "worked": "true values: 10x7=70; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "3",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 9 x 9 = 81. Step 2: 81 + 51 = 132. Step 3: 132 - 29 = 105. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "error detection",
   "spec": {
    "a": 9,
    "b": 9,
    "c": 51,
    "d": 29,
    "shown": [
     81,
     132,
     105
    ]
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:train:1003",
   "worked": "true values: 9x9=81; recompute each shown step from its own left side -> first wrong step = 3"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event P always causes event Y. Event R happens only when both Y and Q occur. Nothing else causes Y or R. In this case P does not occur and Q does not occur. Does R happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 1000,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": false,
    "need_both": true
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:train:1000",
   "worked": "Y does not occur; R needs only when both Y and Q occur -> NO"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event S always causes event U. Event E happens when U occurs or N occurs. Nothing else causes U or E. In this case S occurs and N occurs. Does E happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 1001,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": false
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:train:1001",
   "worked": "U occurs; E needs when U occurs or N occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event P always causes event S. Event Y happens when S occurs or R occurs. Nothing else causes S or Y. In this case P occurs and R occurs. Does Y happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 1002,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": false
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:train:1002",
   "worked": "S occurs; Y needs when S occurs or R occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event N always causes event Q. Event F happens only when both Q and P occur. Nothing else causes Q or F. In this case N occurs and P occurs. Does F happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 1003,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": true
   },
   "split": "train",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:train:1003",
   "worked": "Q occurs; F needs only when both Q and P occur -> YES"
  }
 ],
 "validation": [
  {
   "answer": "350",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 5 units at $38 each; 7 units at $16 each; 2 units at $31 each. A discount of $14 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 14,
    "items": [
     [
      5,
      38
     ],
     [
      7,
      16
     ],
     [
      2,
      31
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2000",
   "worked": "5*38=190 + 7*16=112 + 2*31=62 = 364; minus 14 = 350"
  },
  {
   "answer": "260",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 7 units at $15 each; 2 units at $4 each; 9 units at $17 each. A discount of $6 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 6,
    "items": [
     [
      7,
      15
     ],
     [
      2,
      4
     ],
     [
      9,
      17
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2001",
   "worked": "7*15=105 + 2*4=8 + 9*17=153 = 266; minus 6 = 260"
  },
  {
   "answer": "238",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 2 units at $4 each; 2 units at $39 each; 8 units at $20 each. A discount of $8 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 8,
    "items": [
     [
      2,
      4
     ],
     [
      2,
      39
     ],
     [
      8,
      20
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2002",
   "worked": "2*4=8 + 2*39=78 + 8*20=160 = 246; minus 8 = 238"
  },
  {
   "answer": "260",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 2 units at $25 each; 4 units at $26 each; 5 units at $27 each. A discount of $29 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 29,
    "items": [
     [
      2,
      25
     ],
     [
      4,
      26
     ],
     [
      5,
      27
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2003",
   "worked": "2*25=50 + 4*26=104 + 5*27=135 = 289; minus 29 = 260"
  },
  {
   "answer": "571",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 2 units at $40 each; 8 units at $26 each; 8 units at $39 each. A discount of $29 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 29,
    "items": [
     [
      2,
      40
     ],
     [
      8,
      26
     ],
     [
      8,
      39
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2004",
   "worked": "2*40=80 + 8*26=208 + 8*39=312 = 600; minus 29 = 571"
  },
  {
   "answer": "440",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 2 units at $26 each; 9 units at $38 each; 2 units at $27 each. A discount of $8 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 8,
    "items": [
     [
      2,
      26
     ],
     [
      9,
      38
     ],
     [
      2,
      27
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2005",
   "worked": "2*26=52 + 9*38=342 + 2*27=54 = 448; minus 8 = 440"
  },
  {
   "answer": "111",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 2 units at $26 each; 7 units at $7 each; 4 units at $4 each. A discount of $6 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 6,
    "items": [
     [
      2,
      26
     ],
     [
      7,
      7
     ],
     [
      4,
      4
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2006",
   "worked": "2*26=52 + 7*7=49 + 4*4=16 = 117; minus 6 = 111"
  },
  {
   "answer": "215",
   "domain": "finance and accounting",
   "family": "arith_finance",
   "prompt": "An invoice lists: 3 units at $34 each; 9 units at $6 each; 7 units at $12 each. A discount of $25 is then subtracted. What is the invoice total in dollars? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "multi-step arithmetic",
   "spec": {
    "discount": 25,
    "items": [
     [
      3,
      34
     ],
     [
      9,
      6
     ],
     [
      7,
      12
     ]
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "arith_finance:validation:2007",
   "worked": "3*34=102 + 9*6=54 + 7*12=84 = 240; minus 25 = 215"
  },
  {
   "answer": "12",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 11, 12, 14, 17. Ben is free at hours 9, 10, 12, 14, 15, 17. Cruz is free at hours 10, 11, 12, 13, 14, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      11,
      12,
      14,
      17
     ],
     "Ben": [
      9,
      10,
      12,
      14,
      15,
      17
     ],
     "Cruz": [
      10,
      11,
      12,
      13,
      14,
      16
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2000",
   "worked": "hours free for all three: [12, 14]; earliest = 12"
  },
  {
   "answer": "12",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 11, 12, 13, 15, 16. Ben is free at hours 10, 11, 12, 14, 15, 16. Cruz is free at hours 12, 13, 14, 15, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      11,
      12,
      13,
      15,
      16
     ],
     "Ben": [
      10,
      11,
      12,
      14,
      15,
      16
     ],
     "Cruz": [
      12,
      13,
      14,
      15,
      17
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2001",
   "worked": "hours free for all three: [12, 15]; earliest = 12"
  },
  {
   "answer": "9",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 13, 16, 17. Ben is free at hours 9, 10, 11, 12, 16. Cruz is free at hours 9, 11, 12, 13, 14, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      13,
      16,
      17
     ],
     "Ben": [
      9,
      10,
      11,
      12,
      16
     ],
     "Cruz": [
      9,
      11,
      12,
      13,
      14,
      17
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2002",
   "worked": "hours free for all three: [9]; earliest = 9"
  },
  {
   "answer": "17",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 10, 11, 12, 16, 17. Ben is free at hours 9, 10, 14, 17. Cruz is free at hours 12, 14, 16, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      10,
      11,
      12,
      16,
      17
     ],
     "Ben": [
      9,
      10,
      14,
      17
     ],
     "Cruz": [
      12,
      14,
      16,
      17
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2003",
   "worked": "hours free for all three: [17]; earliest = 17"
  },
  {
   "answer": "11",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 11, 12, 13, 16, 17. Ben is free at hours 9, 10, 11, 12, 16, 17. Cruz is free at hours 10, 11, 14, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      11,
      12,
      13,
      16,
      17
     ],
     "Ben": [
      9,
      10,
      11,
      12,
      16,
      17
     ],
     "Cruz": [
      10,
      11,
      14,
      16
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2004",
   "worked": "hours free for all three: [11, 16]; earliest = 11"
  },
  {
   "answer": "12",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 9, 12, 13, 14, 15. Ben is free at hours 10, 12, 13, 14, 15, 17. Cruz is free at hours 9, 10, 12, 14, 15, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      9,
      12,
      13,
      14,
      15
     ],
     "Ben": [
      10,
      12,
      13,
      14,
      15,
      17
     ],
     "Cruz": [
      9,
      10,
      12,
      14,
      15,
      17
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2005",
   "worked": "hours free for all three: [12, 14, 15]; earliest = 12"
  },
  {
   "answer": "15",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 10, 12, 13, 14, 15, 17. Ben is free at hours 11, 13, 15, 16, 17. Cruz is free at hours 9, 15, 16, 17. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      10,
      12,
      13,
      14,
      15,
      17
     ],
     "Ben": [
      11,
      13,
      15,
      16,
      17
     ],
     "Cruz": [
      9,
      15,
      16,
      17
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2006",
   "worked": "hours free for all three: [15, 17]; earliest = 15"
  },
  {
   "answer": "11",
   "domain": "business operations",
   "family": "constraint_schedule",
   "prompt": "Ana is free at hours 11, 13, 14, 15, 17. Ben is free at hours 10, 11, 14, 16. Cruz is free at hours 9, 10, 11, 12, 16. What is the earliest hour at which all three are free? Answer with the hour number. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "constraint satisfaction",
   "spec": {
    "availability": {
     "Ana": [
      11,
      13,
      14,
      15,
      17
     ],
     "Ben": [
      10,
      11,
      14,
      16
     ],
     "Cruz": [
      9,
      10,
      11,
      12,
      16
     ]
    }
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "constraint_schedule:validation:2007",
   "worked": "hours free for all three: [11]; earliest = 11"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: cache depends on nothing. gateway depends on cache. mail depends on cache and gateway. billing depends on gateway and mail. auth depends on mail. queue depends on billing and mail. If gateway fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "mail"
     ],
     "billing": [
      "gateway",
      "mail"
     ],
     "cache": [],
     "gateway": [
      "cache"
     ],
     "mail": [
      "cache",
      "gateway"
     ],
     "queue": [
      "billing",
      "mail"
     ]
    },
    "failed": "gateway"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2000",
   "worked": "services that depend on gateway directly or through others: auth, billing, mail, queue; count = 4"
  },
  {
   "answer": "2",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: cache depends on nothing. queue depends on cache. mail depends on cache and queue. auth depends on cache. search depends on auth and mail. db depends on auth. If queue fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "cache"
     ],
     "cache": [],
     "db": [
      "auth"
     ],
     "mail": [
      "cache",
      "queue"
     ],
     "queue": [
      "cache"
     ],
     "search": [
      "auth",
      "mail"
     ]
    },
    "failed": "queue"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2001",
   "worked": "services that depend on queue directly or through others: mail, search; count = 2"
  },
  {
   "answer": "4",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: queue depends on nothing. db depends on queue. billing depends on db. cache depends on db and queue. auth depends on cache. search depends on cache. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "cache"
     ],
     "billing": [
      "db"
     ],
     "cache": [
      "db",
      "queue"
     ],
     "db": [
      "queue"
     ],
     "queue": [],
     "search": [
      "cache"
     ]
    },
    "failed": "db"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2002",
   "worked": "services that depend on db directly or through others: auth, billing, cache, search; count = 4"
  },
  {
   "answer": "3",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: billing depends on nothing. gateway depends on billing. cache depends on gateway. db depends on billing. mail depends on cache. queue depends on billing and gateway. If gateway fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "billing": [],
     "cache": [
      "gateway"
     ],
     "db": [
      "billing"
     ],
     "gateway": [
      "billing"
     ],
     "mail": [
      "cache"
     ],
     "queue": [
      "billing",
      "gateway"
     ]
    },
    "failed": "gateway"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2003",
   "worked": "services that depend on gateway directly or through others: cache, mail, queue; count = 3"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: db depends on nothing. cache depends on db. gateway depends on db. queue depends on cache and db. mail depends on db and gateway. auth depends on db. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "db"
     ],
     "cache": [
      "db"
     ],
     "db": [],
     "gateway": [
      "db"
     ],
     "mail": [
      "db",
      "gateway"
     ],
     "queue": [
      "cache",
      "db"
     ]
    },
    "failed": "db"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2004",
   "worked": "services that depend on db directly or through others: auth, cache, gateway, mail, queue; count = 5"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: auth depends on nothing. cache depends on auth. queue depends on auth and cache. search depends on cache. billing depends on cache. gateway depends on queue and search. If auth fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [],
     "billing": [
      "cache"
     ],
     "cache": [
      "auth"
     ],
     "gateway": [
      "queue",
      "search"
     ],
     "queue": [
      "auth",
      "cache"
     ],
     "search": [
      "cache"
     ]
    },
    "failed": "auth"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2005",
   "worked": "services that depend on auth directly or through others: billing, cache, gateway, queue, search; count = 5"
  },
  {
   "answer": "3",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: mail depends on nothing. queue depends on mail. cache depends on mail and queue. db depends on cache and mail. auth depends on db and mail. search depends on db. If cache fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "db",
      "mail"
     ],
     "cache": [
      "mail",
      "queue"
     ],
     "db": [
      "cache",
      "mail"
     ],
     "mail": [],
     "queue": [
      "mail"
     ],
     "search": [
      "db"
     ]
    },
    "failed": "cache"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2006",
   "worked": "services that depend on cache directly or through others: auth, db, search; count = 3"
  },
  {
   "answer": "5",
   "domain": "infrastructure",
   "family": "dependency_impact",
   "prompt": "Services: db depends on nothing. mail depends on db. queue depends on db. cache depends on mail and queue. search depends on queue. auth depends on mail. If db fails, how many OTHER services stop working, counting indirect dependents? Answer with a whole number. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "transitive reasoning",
   "spec": {
    "deps": {
     "auth": [
      "mail"
     ],
     "cache": [
      "mail",
      "queue"
     ],
     "db": [],
     "mail": [
      "db"
     ],
     "queue": [
      "db"
     ],
     "search": [
      "queue"
     ]
    },
    "failed": "db"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "dependency_impact:validation:2007",
   "worked": "services that depend on db directly or through others: auth, cache, mail, queue, search; count = 5"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 41 days. Record: the audit logs were deleted after 188 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "contradiction detection",
   "spec": {
    "got": 188,
    "kind": "at least",
    "need": 41
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2000",
   "worked": "188 days is at least 41 days -> COMPLIANT"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 233 days. Record: the audit logs were deleted after 201 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "contradiction detection",
   "spec": {
    "got": 201,
    "kind": "at most",
    "need": 233
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2001",
   "worked": "201 days is at most 233 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 358 days. Record: the audit logs were deleted after 313 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "contradiction detection",
   "spec": {
    "got": 313,
    "kind": "at least",
    "need": 358
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2002",
   "worked": "313 days is not at least 358 days -> VIOLATION"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 61 days. Record: the audit logs were deleted after 259 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "contradiction detection",
   "spec": {
    "got": 259,
    "kind": "at most",
    "need": 61
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2003",
   "worked": "259 days is not at most 61 days -> VIOLATION"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 119 days. Record: the audit logs were deleted after 91 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "contradiction detection",
   "spec": {
    "got": 91,
    "kind": "at most",
    "need": 119
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2004",
   "worked": "91 days is at most 119 days -> COMPLIANT"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 358 days. Record: the audit logs were deleted after 325 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "contradiction detection",
   "spec": {
    "got": 325,
    "kind": "at most",
    "need": 358
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2005",
   "worked": "325 days is at most 358 days -> COMPLIANT"
  },
  {
   "answer": "COMPLIANT",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at most 321 days. Record: the audit logs were deleted after 61 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "contradiction detection",
   "spec": {
    "got": 61,
    "kind": "at most",
    "need": 321
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2006",
   "worked": "61 days is at most 321 days -> COMPLIANT"
  },
  {
   "answer": "VIOLATION",
   "domain": "compliance and auditing",
   "family": "contradiction",
   "prompt": "Policy: audit logs must be kept for at least 255 days. Record: the audit logs were deleted after 202 days. Is the record COMPLIANT or a VIOLATION? Answer with one word. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "contradiction detection",
   "spec": {
    "got": 202,
    "kind": "at least",
    "need": 255
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "contradiction:validation:2007",
   "worked": "202 days is not at least 255 days -> VIOLATION"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete. engineer may read, write. auditor may export, read, write. intern may delete, export, write. Override: intern may never export. Anything not listed is denied. Request: an analyst wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "delete"
     ],
     "auditor": [
      "export",
      "read",
      "write"
     ],
     "engineer": [
      "read",
      "write"
     ],
     "intern": [
      "delete",
      "export",
      "write"
     ]
    },
    "deny": [
     "intern",
     "export"
    ],
    "role": "analyst"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2000",
   "worked": "analyst listed actions: ['delete']; override blocks intern/export -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export, read. engineer may delete, read, write. auditor may delete, export, read. intern may delete, read. Override: auditor may never delete. Anything not listed is denied. Request: an intern wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "export",
      "read"
     ],
     "auditor": [
      "delete",
      "export",
      "read"
     ],
     "engineer": [
      "delete",
      "read",
      "write"
     ],
     "intern": [
      "delete",
      "read"
     ]
    },
    "deny": [
     "auditor",
     "delete"
    ],
    "role": "intern"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2001",
   "worked": "intern listed actions: ['delete', 'read']; override blocks auditor/delete -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may export, write. engineer may delete, write. auditor may read. intern may export. Override: analyst may never export. Anything not listed is denied. Request: an auditor wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "export",
      "write"
     ],
     "auditor": [
      "read"
     ],
     "engineer": [
      "delete",
      "write"
     ],
     "intern": [
      "export"
     ]
    },
    "deny": [
     "analyst",
     "export"
    ],
    "role": "auditor"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2002",
   "worked": "auditor listed actions: ['read']; override blocks analyst/export -> DENY"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may write. engineer may read. auditor may delete, read, write. intern may delete, read. Override: analyst may never delete. Anything not listed is denied. Request: an intern wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "write"
     ],
     "auditor": [
      "delete",
      "read",
      "write"
     ],
     "engineer": [
      "read"
     ],
     "intern": [
      "delete",
      "read"
     ]
    },
    "deny": [
     "analyst",
     "delete"
    ],
    "role": "intern"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2003",
   "worked": "intern listed actions: ['delete', 'read']; override blocks analyst/delete -> ALLOW"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete. engineer may delete, write. auditor may delete, export, write. intern may delete, read. Override: auditor may never delete. Anything not listed is denied. Request: an intern wants to read. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "rule application",
   "spec": {
    "act": "read",
    "allow": {
     "analyst": [
      "delete"
     ],
     "auditor": [
      "delete",
      "export",
      "write"
     ],
     "engineer": [
      "delete",
      "write"
     ],
     "intern": [
      "delete",
      "read"
     ]
    },
    "deny": [
     "auditor",
     "delete"
    ],
    "role": "intern"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2004",
   "worked": "intern listed actions: ['delete', 'read']; override blocks auditor/delete -> ALLOW"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may write. engineer may delete, read, write. auditor may export, read. intern may delete. Override: auditor may never write. Anything not listed is denied. Request: an auditor wants to delete. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "rule application",
   "spec": {
    "act": "delete",
    "allow": {
     "analyst": [
      "write"
     ],
     "auditor": [
      "export",
      "read"
     ],
     "engineer": [
      "delete",
      "read",
      "write"
     ],
     "intern": [
      "delete"
     ]
    },
    "deny": [
     "auditor",
     "write"
    ],
    "role": "auditor"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2005",
   "worked": "auditor listed actions: ['export', 'read']; override blocks auditor/write -> DENY"
  },
  {
   "answer": "DENY",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may read. engineer may delete, export. auditor may write. intern may export, read, write. Override: analyst may never delete. Anything not listed is denied. Request: an auditor wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "read"
     ],
     "auditor": [
      "write"
     ],
     "engineer": [
      "delete",
      "export"
     ],
     "intern": [
      "export",
      "read",
      "write"
     ]
    },
    "deny": [
     "analyst",
     "delete"
    ],
    "role": "auditor"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2006",
   "worked": "auditor listed actions: ['write']; override blocks analyst/delete -> DENY"
  },
  {
   "answer": "ALLOW",
   "domain": "cybersecurity",
   "family": "access_decision",
   "prompt": "Rules: analyst may delete, export, write. engineer may export, write. auditor may export, read, write. intern may delete, export, read. Override: auditor may never export. Anything not listed is denied. Request: an engineer wants to export. Answer ALLOW or DENY. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "rule application",
   "spec": {
    "act": "export",
    "allow": {
     "analyst": [
      "delete",
      "export",
      "write"
     ],
     "auditor": [
      "export",
      "read",
      "write"
     ],
     "engineer": [
      "export",
      "write"
     ],
     "intern": [
      "delete",
      "export",
      "read"
     ]
    },
    "deny": [
     "auditor",
     "export"
    ],
    "role": "engineer"
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "access_decision:validation:2007",
   "worked": "engineer listed actions: ['export', 'write']; override blocks auditor/export -> ALLOW"
  },
  {
   "answer": "TEST",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: build requires deploy first. design requires build first. deploy requires test first. document requires design first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "deploy",
     "deploy": "test",
     "design": "build",
     "document": "design"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "test"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2000",
   "worked": "every step except test requires another step first -> test"
  },
  {
   "answer": "BUILD",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: design requires build first. review requires deploy first. deploy requires design first. document requires review first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "planning",
   "spec": {
    "prereq": {
     "deploy": "design",
     "design": "build",
     "document": "review",
     "review": "deploy"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2001",
   "worked": "every step except build requires another step first -> build"
  },
  {
   "answer": "DESIGN",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: deploy requires document first. build requires design first. document requires build first. review requires deploy first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "design",
     "deploy": "document",
     "document": "build",
     "review": "deploy"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2002",
   "worked": "every step except design requires another step first -> design"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: design requires review first. review requires deploy first. build requires design first. document requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "design",
     "design": "review",
     "document": "build",
     "review": "deploy"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "document",
     "review"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2003",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "REVIEW",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: test requires design first. build requires review first. design requires build first. deploy requires test first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "review",
     "deploy": "test",
     "design": "build",
     "test": "design"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "review",
     "test"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2004",
   "worked": "every step except review requires another step first -> review"
  },
  {
   "answer": "TEST",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: build requires review first. deploy requires test first. review requires document first. document requires deploy first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "review",
     "deploy": "test",
     "document": "deploy",
     "review": "document"
    },
    "tasks": [
     "build",
     "deploy",
     "document",
     "review",
     "test"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2005",
   "worked": "every step except test requires another step first -> test"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: review requires deploy first. build requires review first. test requires design first. design requires build first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "review",
     "design": "build",
     "review": "deploy",
     "test": "design"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "review",
     "test"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2006",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "DEPLOY",
   "domain": "software engineering",
   "family": "plan_order",
   "prompt": "Project steps: build requires deploy first. design requires build first. review requires design first. test requires review first. Exactly one step has no prerequisite. Which step must be done first? Answer with the step name. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "planning",
   "spec": {
    "prereq": {
     "build": "deploy",
     "design": "build",
     "review": "design",
     "test": "review"
    },
    "tasks": [
     "build",
     "deploy",
     "design",
     "review",
     "test"
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "plan_order:validation:2007",
   "worked": "every step except deploy requires another step first -> deploy"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 34 crates. Some crates are shipped on Monday and 2 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 34,
    "c": 2,
    "missing": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2000",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "81",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 84 crates. 6 crates are shipped on Monday and 3 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 84,
    "b": 6,
    "c": 3,
    "missing": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2001",
   "worked": "84 - 6 + 3 = 81"
  },
  {
   "answer": "71",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 71 crates. 9 crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 71,
    "b": 9,
    "c": 9,
    "missing": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2002",
   "worked": "71 - 9 + 9 = 71"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 89 crates. Some crates are shipped on Monday and 5 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 89,
    "c": 5,
    "missing": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2003",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "UNKNOWN",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 59 crates. Some crates are shipped on Monday and 9 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 59,
    "c": 9,
    "missing": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2004",
   "worked": "the number shipped on Monday is not given -> UNKNOWN"
  },
  {
   "answer": "30",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 36 crates. 11 crates are shipped on Monday and 5 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 36,
    "b": 11,
    "c": 5,
    "missing": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2005",
   "worked": "36 - 11 + 5 = 30"
  },
  {
   "answer": "62",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 74 crates. 19 crates are shipped on Monday and 7 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 74,
    "b": 19,
    "c": 7,
    "missing": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2006",
   "worked": "74 - 19 + 7 = 62"
  },
  {
   "answer": "12",
   "domain": "logistics",
   "family": "insufficient",
   "prompt": "A warehouse holds 22 crates. 14 crates are shipped on Monday and 4 more arrive on Tuesday. How many crates are in the warehouse after Tuesday? If the information given is not enough, answer UNKNOWN. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "uncertainty / incomplete information",
   "spec": {
    "a": 22,
    "b": 14,
    "c": 4,
    "missing": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "insufficient:validation:2007",
   "worked": "22 - 14 + 4 = 12"
  },
  {
   "answer": "1",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 9 x 3 = 37. Step 2: 37 + 40 = 77. Step 3: 77 - 21 = 56. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "error detection",
   "spec": {
    "a": 9,
    "b": 3,
    "c": 40,
    "d": 21,
    "shown": [
     37,
     77,
     56
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2000",
   "worked": "true values: 9x3=27; recompute each shown step from its own left side -> first wrong step = 1"
  },
  {
   "answer": "0",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 5 x 9 = 45. Step 2: 45 + 52 = 97. Step 3: 97 - 18 = 79. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "error detection",
   "spec": {
    "a": 5,
    "b": 9,
    "c": 52,
    "d": 18,
    "shown": [
     45,
     97,
     79
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2001",
   "worked": "true values: 5x9=45; recompute each shown step from its own left side -> first wrong step = 0"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 4 x 8 = 32. Step 2: 32 + 51 = 82. Step 3: 82 - 32 = 50. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "error detection",
   "spec": {
    "a": 4,
    "b": 8,
    "c": 51,
    "d": 32,
    "shown": [
     32,
     82,
     50
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2002",
   "worked": "true values: 4x8=32; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 8 x 6 = 48. Step 2: 48 + 27 = 85. Step 3: 85 - 32 = 53. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "error detection",
   "spec": {
    "a": 8,
    "b": 6,
    "c": 27,
    "d": 32,
    "shown": [
     48,
     85,
     53
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2003",
   "worked": "true values: 8x6=48; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "0",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 6 x 7 = 42. Step 2: 42 + 56 = 98. Step 3: 98 - 32 = 66. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "error detection",
   "spec": {
    "a": 6,
    "b": 7,
    "c": 56,
    "d": 32,
    "shown": [
     42,
     98,
     66
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2004",
   "worked": "true values: 6x7=42; recompute each shown step from its own left side -> first wrong step = 0"
  },
  {
   "answer": "3",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 7 x 11 = 77. Step 2: 77 + 19 = 96. Step 3: 96 - 6 = 92. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "error detection",
   "spec": {
    "a": 7,
    "b": 11,
    "c": 19,
    "d": 6,
    "shown": [
     77,
     96,
     92
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2005",
   "worked": "true values: 7x11=77; recompute each shown step from its own left side -> first wrong step = 3"
  },
  {
   "answer": "0",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 12 x 4 = 48. Step 2: 48 + 51 = 99. Step 3: 99 - 24 = 75. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "error detection",
   "spec": {
    "a": 12,
    "b": 4,
    "c": 51,
    "d": 24,
    "shown": [
     48,
     99,
     75
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2006",
   "worked": "true values: 12x4=48; recompute each shown step from its own left side -> first wrong step = 0"
  },
  {
   "answer": "2",
   "domain": "mathematics",
   "family": "error_detect",
   "prompt": "Check this calculation. Step 1: 8 x 11 = 88. Step 2: 88 + 15 = 113. Step 3: 113 - 11 = 102. Which step number contains the first arithmetic error? If every step is correct, answer 0. Answer with one digit. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "error detection",
   "spec": {
    "a": 8,
    "b": 11,
    "c": 15,
    "d": 11,
    "shown": [
     88,
     113,
     102
    ]
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "error_detect:validation:2007",
   "worked": "true values: 8x11=88; recompute each shown step from its own left side -> first wrong step = 2"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event J always causes event P. Event R happens only when both P and G occur. Nothing else causes P or R. In this case J does not occur and G does not occur. Does R happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2000,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": false,
    "need_both": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2000",
   "worked": "P does not occur; R needs only when both P and G occur -> NO"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event H always causes event U. Event Z happens only when both U and Q occur. Nothing else causes U or Z. In this case H does not occur and Q occurs. Does Z happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2001,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": true,
    "need_both": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2001",
   "worked": "U does not occur; Z needs only when both U and Q occur -> NO"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event S always causes event W. Event P happens only when both W and Z occur. Nothing else causes W or P. In this case S occurs and Z occurs. Does P happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2002,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2002",
   "worked": "W occurs; P needs only when both W and Z occur -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event G always causes event K. Event P happens when K occurs or W occurs. Nothing else causes K or P. In this case G occurs and W occurs. Does P happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2003,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2003",
   "worked": "K occurs; P needs when K occurs or W occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event V always causes event R. Event Z happens when R occurs or F occurs. Nothing else causes R or Z. In this case V does not occur and F occurs. Does Z happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2004,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": true,
    "need_both": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2004",
   "worked": "R does not occur; Z needs when R occurs or F occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event L always causes event J. Event Z happens when J occurs or Y occurs. Nothing else causes J or Z. In this case L does not occur and Y occurs. Does Z happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2005,
   "skill": "causal reasoning",
   "spec": {
    "a": false,
    "d": true,
    "need_both": false
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2005",
   "worked": "J does not occur; Z needs when J occurs or Y occurs -> YES"
  },
  {
   "answer": "YES",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event R always causes event J. Event V happens only when both J and N occur. Nothing else causes J or V. In this case R occurs and N occurs. Does V happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2006,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": true,
    "need_both": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2006",
   "worked": "J occurs; V needs only when both J and N occur -> YES"
  },
  {
   "answer": "NO",
   "domain": "scientific reasoning",
   "family": "causal",
   "prompt": "Event R always causes event H. Event E happens only when both H and Q occur. Nothing else causes H or E. In this case R occurs and Q does not occur. Does E happen? Answer YES or NO. Reply with only the final answer on one line, no explanation.",
   "seed": 2007,
   "skill": "causal reasoning",
   "spec": {
    "a": true,
    "d": false,
    "need_both": true
   },
   "split": "validation",
   "suite": "e85-xdomain-v1",
   "task_id": "causal:validation:2007",
   "worked": "H occurs; E needs only when both H and Q occur -> NO"
  }
 ]
}