{
 "product": "FALSEWORK",
 "slug": "falsework",
 "base": "https://cainstudio.online",
 "page": "https://cainstudio.online/top25-falsework.html",
 "claims": [
  {
   "id": "C01",
   "kind": "attack",
   "claim": "Sample: A signature check that never checks the signature -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "forged-token"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C02",
   "kind": "attack",
   "claim": "Sample: A metric that reports a random number as accuracy -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "random-accuracy"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C03",
   "kind": "attack",
   "claim": "Sample: A webhook signature compared with == (timing leak) -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "timing-leak"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C04",
   "kind": "attack",
   "claim": "Sample: An alert function that says 'sent' and sends nothing -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "fake-alert"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C05",
   "kind": "control",
   "claim": "Your code: honest constant-time compare -> Real code must grade REAL or nobody trusts the alarms.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan-source",
     "body": {
      "source": "import hmac\n\ndef same(a: bytes, b: bytes) -> bool:\n    return hmac.compare_digest(a, b)\n",
      "name": "compare.py"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "REAL"
   ]
  },
  {
   "id": "C06",
   "kind": "attack",
   "claim": "Your code: verify() that always says yes -> A security check that cannot fail is fabrication.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan-source",
     "body": {
      "source": "def verify_token(token: str) -> bool:\n    # TODO: check the signature\n    return True\n",
      "name": "auth.py"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "STUB",
    "PARTIAL"
   ]
  },
  {
   "id": "C07",
   "kind": "attack",
   "claim": "Your code: encryption that is base64 -> Looks like encryption; is encoding.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan-source",
     "body": {
      "source": "import base64\n\ndef encrypt(data: bytes, key: bytes) -> bytes:\n    return base64.b64encode(data)\n",
      "name": "crypto.py"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "STUB",
    "PARTIAL"
   ]
  }
 ]
}