{
 "product": "KEYSTONE",
 "slug": "keystone",
 "base": "https://cainstudio.online",
 "page": "https://cainstudio.online/top25-keystone.html",
 "claims": [
  {
   "id": "C01",
   "kind": "attack",
   "claim": "Sample: A signature check that never checks the signature -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "forged-token"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C02",
   "kind": "attack",
   "claim": "Sample: A metric that reports a random number as accuracy -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "random-accuracy"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C03",
   "kind": "attack",
   "claim": "Sample: A webhook signature compared with == (timing leak) -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "timing-leak"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C04",
   "kind": "attack",
   "claim": "Sample: An alert function that says 'sent' and sends nothing -> A deterministic detector, not the model, finds the defect the AI is then asked to fix.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan",
     "body": {
      "sample_id": "fake-alert"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "PARTIAL",
    "STUB"
   ]
  },
  {
   "id": "C05",
   "kind": "control",
   "claim": "Your code: honest constant-time compare -> Real code must grade REAL or nobody trusts the alarms.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan-source",
     "body": {
      "source": "import hmac\n\ndef same(a: bytes, b: bytes) -> bool:\n    return hmac.compare_digest(a, b)\n",
      "name": "compare.py"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "REAL"
   ]
  },
  {
   "id": "C06",
   "kind": "attack",
   "claim": "Your code: verify() that always says yes -> A security check that cannot fail is fabrication.",
   "requests": [
    {
     "method": "POST",
     "url": "/keystone/scan-source",
     "body": {
      "source": "def verify_token(token: str) -> bool:\n    # TODO: check the signature\n    return True\n",
      "name": "auth.py"
     }
    }
   ],
   "path": "grade",
   "expect": [
    "STUB",
    "PARTIAL"
   ]
  },
  {
   "id": "C07",
   "kind": "control",
   "claim": "Ledger: latest record re-verifies -> The signed fix record verifies against the pinned key.",
   "requests": [
    {
     "method": "GET",
     "url": "/keystone/runs/latest"
    },
    {
     "method": "POST",
     "url": "/keystone/verify",
     "body": {
      "record": "${0.record}"
     }
    }
   ],
   "path": "valid",
   "expect": [
    true
   ]
  },
  {
   "id": "C08",
   "kind": "attack",
   "claim": "Ledger: latest record with its verdict edited to NOT_VERIFIED -> Changing any signed field breaks the record hash and signature.",
   "requests": [
    {
     "method": "GET",
     "url": "/keystone/runs/latest"
    },
    {
     "method": "POST",
     "url": "/keystone/verify",
     "body": {
      "record": "${0.record}"
     },
     "set": {
      "record.verdict": "NOT_VERIFIED"
     }
    }
   ],
   "path": "valid",
   "expect": [
    false
   ]
  }
 ]
}