{
  "schema": "cain42.premier-products.auditor-run.v1",
  "base": "https://cainstudio.online",
  "summary": {
    "PASS": 43,
    "MISMATCH": 0,
    "NOT_RUN": 0
  },
  "results": [
    {
      "product": "FALSEWORK",
      "check": "a call with no API key is refused (401)",
      "result": "PASS",
      "status": 401
    },
    {
      "product": "FALSEWORK",
      "check": "a random number returned as a score is graded STUB, with the line it is on",
      "result": "PASS",
      "status": 200,
      "grade": "STUB",
      "findings": [
        "hardcoded_success",
        "rng_fabrication"
      ]
    },
    {
      "product": "FALSEWORK",
      "check": "code that computes its result is graded REAL with no findings",
      "result": "PASS",
      "status": 200,
      "grade": "REAL"
    },
    {
      "product": "FALSEWORK",
      "check": "a swallowed import error and a bare except-pass are both found, on the right lines",
      "result": "PASS",
      "status": 200,
      "grade": "PARTIAL",
      "findings": [
        [
          "bare_except_pass",
          9
        ],
        [
          "swallowed_import_error",
          3
        ]
      ]
    },
    {
      "product": "FALSEWORK",
      "check": "a route calling a method that does not exist is graded STUB",
      "result": "PASS",
      "status": 200,
      "grade": "STUB"
    },
    {
      "product": "FALSEWORK",
      "check": "an inherited method is not reported as missing (no false STUB)",
      "result": "PASS",
      "status": 200,
      "grade": "REAL"
    },
    {
      "product": "FALSEWORK",
      "check": "an endpoint that answers two different inputs identically is flagged",
      "result": "PASS",
      "status": 200,
      "flag": "possible canned response"
    },
    {
      "product": "FALSEWORK",
      "check": "a live audit aimed at http://127.0.0.1:8392 is refused (400)",
      "result": "PASS",
      "status": 400
    },
    {
      "product": "FALSEWORK",
      "check": "a live audit aimed at http://169.254.169.254 is refused (400)",
      "result": "PASS",
      "status": 400
    },
    {
      "product": "FALSEWORK",
      "check": "a live audit aimed at http://10.0.0.1 is refused (400)",
      "result": "PASS",
      "status": 400
    },
    {
      "product": "FALSEWORK",
      "check": "an empty audit is refused (422)",
      "result": "PASS",
      "status": 422
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade: duplicate call at steps [0, 1]",
      "result": "PASS",
      "status": 200
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade: steps [1, 2, 3, 4] not needed, wasted cost 8 of 10",
      "result": "PASS",
      "wasted": 8.0,
      "total": 10.0
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade: risky step 2 corrected by step 3",
      "result": "PASS"
    },
    {
      "product": "SCENIC",
      "check": "the same run gets the same verdict twice, and with its steps listed in reverse",
      "result": "PASS"
    },
    {
      "product": "SCENIC",
      "check": "a run with nothing wasted scores 1.0",
      "result": "PASS",
      "score": 1.0
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade refuses 2,001 steps (limit 2,000) (422)",
      "result": "PASS",
      "status": 422
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade refuses a dependency on a step that does not exist (422)",
      "result": "PASS",
      "status": 422
    },
    {
      "product": "SCENIC",
      "check": "trajectory grade refuses two steps with the same index (422)",
      "result": "PASS",
      "status": 422
    },
    {
      "product": "SCENIC",
      "check": "root-cause tracer names the first broken step, 5 times out of 5",
      "result": "PASS",
      "statuses": [
        200,
        200,
        200,
        200,
        200
      ],
      "named": [
        "a",
        "a",
        "a",
        "a",
        "a"
      ]
    },
    {
      "product": "SCENIC",
      "check": "root-cause tracer names nothing when nothing broke",
      "result": "PASS",
      "status": 200
    },
    {
      "product": "SCENIC",
      "check": "cainbench is reachable (health only; its behaviour is not checked here)",
      "result": "PASS",
      "status": 200
    },
    {
      "product": "SCENIC",
      "check": "contamination-scanner is reachable (health only; its behaviour is not checked here)",
      "result": "PASS",
      "status": 200
    },
    {
      "product": "SCENIC",
      "check": "contamination scan aimed at an internal address is refused (422)",
      "result": "PASS",
      "status": 422
    },
    {
      "product": "NULLIUS",
      "check": "the downloaded verifier imports no CAIN code",
      "result": "PASS",
      "imports": []
    },
    {
      "product": "NULLIUS",
      "check": "the verifier accepts the published journal (exit 0, VERIFIED)",
      "result": "PASS",
      "exit": 0
    },
    {
      "product": "NULLIUS",
      "check": "25 random single-character changes to the journal (your own random choices): none accepted",
      "result": "PASS",
      "accepted": 0
    },
    {
      "product": "SALLYPORT",
      "check": "a session with your two files is created",
      "result": "PASS",
      "status": 200
    },
    {
      "product": "SALLYPORT",
      "check": "'ls' is allowed, runs, and lists your two files",
      "result": "PASS",
      "verdict": "ALLOW"
    },
    {
      "product": "SALLYPORT",
      "check": "'cat notes.txt' is allowed and returns the file",
      "result": "PASS",
      "verdict": "ALLOW"
    },
    {
      "product": "SALLYPORT",
      "check": "each allowed command has its own decision id and a SHA-256 digest of the exact request, and is recorded",
      "result": "PASS"
    },
    {
      "product": "SALLYPORT",
      "check": "'python3 test_probe.py' is refused and never runs",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "COMMAND_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'curl https://example.com' is refused and never runs",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "COMMAND_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'bash -c id' is refused and never runs",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "COMMAND_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'rm -rf /' is refused and never runs",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "COMMAND_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'cat /etc/passwd' is refused: an allowed program may not reach outside the session",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "ARGUMENT_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'cat ../../../etc/passwd' is refused: an allowed program may not reach outside the session",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "ARGUMENT_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "'ls /root' is refused: an allowed program may not reach outside the session",
      "result": "PASS",
      "verdict": "DENY",
      "reason": "ARGUMENT_NOT_IN_POLICY"
    },
    {
      "product": "SALLYPORT",
      "check": "your probe runs inside the sandbox",
      "result": "PASS",
      "verdict": "ALLOW"
    },
    {
      "product": "SALLYPORT",
      "check": "sandbox: the probe is not root",
      "result": "PASS",
      "uid": 64100
    },
    {
      "product": "SALLYPORT",
      "check": "sandbox: no network",
      "result": "PASS",
      "network": "blocked"
    },
    {
      "product": "SALLYPORT",
      "check": "sandbox: cannot write to /tmp, /etc or /srv",
      "result": "PASS",
      "writes": {
        "write /etc/sallyport_probe": "blocked",
        "write /srv/sallyport_probe": "blocked",
        "write /tmp/sallyport_probe": "blocked"
      }
    },
    {
      "product": "SALLYPORT",
      "check": "sandbox: cannot read /etc/shadow or list /root",
      "result": "PASS"
    }
  ]
}