#!/usr/bin/env python3
"""Check the CAIN-42 premier products yourself. Python 3 standard library only; no CAIN code.

  1. Get a key:  curl -s -X POST https://cainstudio.online/subscribe/free/signup -H 'content-type: application/json' -d '{}'
     (a free signup carries a 7-day Full-plan trial; no card)
  2. Run:        python3 verify_premier_products.py --key <api_key>

Every check sends a request to the public site, or runs a verifier it downloads from the public site, and compares the
answer with a result you can work out by hand from the input shown in this file. Nothing is taken from the developer's
own test results. What each product's checks cover, and what they cannot cover, is printed at the end.

  FALSEWORK   the audit engine grades source you send it and refuses internal targets
  SCENIC      the trajectory grader, the root-cause tracer and the reachability of the other two engines
  NULLIUS     a standalone verifier accepts a published journal and refuses it after any single change
  SALLYPORT   the enforcement gate running on the developer's host decides each command you send, runs the allowed
              ones in a sandbox and refuses the rest; a probe you send reports what the sandbox can reach

The NULLIUS step downloads one Python file from the site and runs it with `python3 -I`. Read it first if you prefer
(--keep DIR leaves it on disk; --skip-nullius leaves the step out). It needs the `cryptography` package.
Exit code: 0 if every answer matched, 1 otherwise.
"""
from __future__ import annotations

import argparse
import json
import os
import random
import subprocess
import sys
import tempfile
import urllib.error
import urllib.request

RESULTS: list = []


def record(product: str, name: str, ok, **detail) -> None:
    result = ok if isinstance(ok, str) else ("PASS" if ok else "MISMATCH")
    RESULTS.append({"product": product, "check": name, "result": result, **detail})
    print(f"  {result:12s} {product:10s} {name}" + (f"   {detail}" if result == "MISMATCH" else ""))


class Site:
    def __init__(self, base: str, key: str):
        self.base, self.key = base.rstrip("/"), key

    def call(self, path: str, body=None, key: bool = True):
        headers = {"content-type": "application/json", "user-agent": "cain-premier-audit/1"}
        if key:
            headers["X-API-Key"] = self.key
        req = urllib.request.Request(self.base + path, data=None if body is None else json.dumps(body).encode(),
                                     headers=headers)
        try:
            with urllib.request.urlopen(req, timeout=60) as r:
                raw = r.read()
                status = r.status
        except urllib.error.HTTPError as e:
            raw, status = e.read(), e.code
        except Exception as e:
            return 0, {"error": f"{type(e).__name__}: {e}"}
        try:
            return status, json.loads(raw)
        except ValueError:
            return status, {"raw": raw[:200].decode(errors="replace")}

    def fetch(self, path: str) -> bytes:
        req = urllib.request.Request(self.base + path, headers={"user-agent": "cain-premier-audit/1"})
        with urllib.request.urlopen(req, timeout=60) as r:
            return r.read()


def falsework(s: Site) -> None:
    P = "FALSEWORK"
    st, _ = s.call("/cainwitness/audit/source", {"files": {"a.py": "x = 1\n"}}, key=False)
    record(P, "a call with no API key is refused (401)", st == 401, status=st)

    # A 'score' that is a random number, and a result that is always "ok": fabricated. Expected grade STUB.
    fake = "import random\n\ndef risk_score(x):\n    return random.uniform(0.7, 0.99)\n\ndef run(x):\n    return {\"status\": \"ok\"}\n"
    st, r = s.call("/cainwitness/audit/source", {"files": {"fake.py": fake}})
    found = sorted({f.get("check") for f in r.get("findings", [])})
    record(P, "a random number returned as a score is graded STUB, with the line it is on",
           st == 200 and r.get("grade") == "STUB" and any(f.get("check") == "rng_fabrication" and f.get("line") == 4
                                                          for f in r.get("findings", [])), status=st, grade=r.get("grade"), findings=found)

    # The result is computed from the input. Expected grade REAL, no findings.
    real = "import hashlib\n\ndef digest(b: bytes) -> str:\n    return hashlib.sha256(b).hexdigest()\n"
    st, r = s.call("/cainwitness/audit/source", {"files": {"real.py": real}})
    record(P, "code that computes its result is graded REAL with no findings",
           st == 200 and r.get("grade") == "REAL" and r.get("total_findings") == 0, status=st, grade=r.get("grade"))

    # A missing dependency silently ignored (line 3) and every error swallowed (line 9). Expected PARTIAL, both found.
    swallow = "try:\n    import realengine\nexcept ImportError:\n    pass\n\ndef f():\n    try:\n        g()\n    except Exception:\n        pass\n"
    st, r = s.call("/cainwitness/audit/source", {"files": {"m.py": swallow}})
    got = sorted((f.get("check"), f.get("line")) for f in r.get("findings", []))
    record(P, "a swallowed import error and a bare except-pass are both found, on the right lines",
           st == 200 and r.get("grade") == "PARTIAL" and got == [("bare_except_pass", 9), ("swallowed_import_error", 3)],
           status=st, grade=r.get("grade"), findings=got)

    # A route calls a method its engine class does not define (STUB); an inherited method must not be reported.
    st, r = s.call("/cainwitness/audit/source", {"files": {
        "engine.py": "class Engine:\n    def run(self):\n        return 1\n",
        "main.py": "from engine import Engine\nengine = Engine()\n\ndef route():\n    return engine.launch()\n"}})
    record(P, "a route calling a method that does not exist is graded STUB", st == 200 and r.get("grade") == "STUB",
           status=st, grade=r.get("grade"))
    st, r = s.call("/cainwitness/audit/source", {"files": {
        "srv.py": "import socketserver\n\nclass S(socketserver.TCPServer):\n    pass\n\nsrv = S(('', 0), None)\nsrv.serve_forever()\n"}})
    record(P, "an inherited method is not reported as missing (no false STUB)", st == 200 and r.get("grade") == "REAL",
           status=st, grade=r.get("grade"))

    # Live differential: this public endpoint is documented as accepting any body and storing nothing, so it answers two
    # different inputs identically. Expected: flagged.
    st, r = s.call("/cainwitness/audit/live", {"base_url": "https://analytics.mcpgate.online", "endpoints": [
        {"path": "/api/v1/telemetry/beacon", "method": "POST", "payload_a": {"path": "/a"}, "payload_b": {"path": "/b", "t": 2}}]})
    res = (r.get("results") or [{}])[0]
    record(P, "an endpoint that answers two different inputs identically is flagged",
           st == 200 and res.get("flag") == "possible canned response", status=st, flag=res.get("flag"))

    for target in ("http://127.0.0.1:8392", "http://169.254.169.254", "http://10.0.0.1"):
        st, r = s.call("/cainwitness/audit/live", {"base_url": target, "endpoints": [{"path": "/"}]})
        record(P, f"a live audit aimed at {target} is refused (400)", st == 400, status=st)
    st, _ = s.call("/cainwitness/audit/source", {"files": {}})
    record(P, "an empty audit is refused (422)", st == 422, status=st)


def scenic(s: Site) -> None:
    P = "SCENIC"

    def step(i, tool, args=None, cost=1.0, deps=(), **kw):
        return {"index": i, "tool": tool, "args": args or {}, "cost": cost, "depends_on": list(deps), **kw}

    # Six steps, total cost 10. Step 1 repeats step 0. The answer (step 5) depends only on step 0, so steps 1-4
    # (cost 1+2+2+3 = 8) were not needed. Step 2 is high-risk and step 3 restores the same resource.
    run = [step(0, "search", {"q": "a"}), step(1, "search", {"q": "a"}),
           step(2, "db.delete", {"t": "x"}, 2, risk_tier="high", resource="db:x"),
           step(3, "db.restore", {"t": "x"}, 2, resource="db:x"), step(4, "fetch", {"u": "z"}, 3),
           step(5, "answer", cost=1, deps=[0])]
    st, a = s.call("/trajectorygate/trajectory/grade", {"steps": run, "terminal_step": 5})
    record(P, "trajectory grade: duplicate call at steps [0, 1]", st == 200 and
           a.get("redundant_calls") == [{"tool": "search", "repeated_at_steps": [0, 1], "extra_calls": 1}], status=st)
    record(P, "trajectory grade: steps [1, 2, 3, 4] not needed, wasted cost 8 of 10",
           a.get("unreachable_from_terminal") == [1, 2, 3, 4] and a.get("wasted_cost") == 8.0 and a.get("total_cost") == 10.0,
           wasted=a.get("wasted_cost"), total=a.get("total_cost"))
    record(P, "trajectory grade: risky step 2 corrected by step 3",
           [(x.get("risky_step"), x.get("correction_step")) for x in a.get("risky_then_corrected", [])] == [(2, 3)])
    strip = lambda d: {k: v for k, v in d.items() if k != "grading_id"}
    st, b = s.call("/trajectorygate/trajectory/grade", {"steps": run, "terminal_step": 5})
    st2, c = s.call("/trajectorygate/trajectory/grade", {"steps": list(reversed(run)), "terminal_step": 5})
    record(P, "the same run gets the same verdict twice, and with its steps listed in reverse",
           st == 200 and st2 == 200 and strip(a) == strip(b) == strip(c))
    clean = [step(i, f"t{i}", deps=[i - 1] if i else []) for i in range(5)]
    st, r = s.call("/trajectorygate/trajectory/grade", {"steps": clean, "terminal_step": 4})
    record(P, "a run with nothing wasted scores 1.0", st == 200 and r.get("score") == 1.0, score=r.get("score"))
    for label, body in (("2,001 steps (limit 2,000)", {"steps": [step(i, "t", {"i": i}) for i in range(2001)]}),
                        ("a dependency on a step that does not exist", {"steps": [step(0, "a", deps=[9])], "terminal_step": 0}),
                        ("two steps with the same index", {"steps": [step(0, "a"), step(0, "b")]})):
        st, _ = s.call("/trajectorygate/trajectory/grade", body)
        record(P, f"trajectory grade refuses {label} (422)", st == 422, status=st)

    # a -> b -> c all broke: the first one is the cause. Asked 5 times; the answer must not change.
    chain = {"steps": [{"step_id": "a", "agent_id": "planner", "broken": True},
                       {"step_id": "b", "agent_id": "coder", "broken": True, "depends_on": ["a"]},
                       {"step_id": "c", "agent_id": "tester", "broken": True, "depends_on": ["b"]}]}
    tops, codes = [], []
    for _ in range(5):
        st, r = s.call("/multi-agent-incident-tracer/v1/trace/attribute", chain)
        codes.append(st)
        tops.append((r.get("ranked_causes") or [{}])[0].get("step_id"))
    record(P, "root-cause tracer names the first broken step, 5 times out of 5", codes == [200] * 5 and tops == ["a"] * 5,
           statuses=codes, named=tops)
    st, r = s.call("/multi-agent-incident-tracer/v1/trace/attribute", {"steps": [
        {"step_id": "a", "agent_id": "x"}, {"step_id": "b", "agent_id": "y", "depends_on": ["a"]}]})
    record(P, "root-cause tracer names nothing when nothing broke", st == 200 and r.get("ranked_causes") == [], status=st)
    for slug in ("cainbench", "contamination-scanner"):
        st, _ = s.call(f"/{slug}/health")
        record(P, f"{slug} is reachable (health only; its behaviour is not checked here)", st == 200, status=st)
    st, _ = s.call("/contamination-scanner/scan", {"model_endpoint": "http://127.0.0.1:8660/health", "benchmark_items": [
        {"id": "1", "prompt_prefix": "a", "canonical_continuation": "b"}]})
    record(P, "contamination scan aimed at an internal address is refused (422)", st == 422, status=st)


def nullius(s: Site, keep: str, mutations: int) -> None:
    P = "NULLIUS"
    try:
        import cryptography  # noqa: F401
    except ImportError:
        record(P, "standalone verifier on a published journal", "NOT_RUN", reason="install the 'cryptography' package")
        return
    B = "/evidence/evolution-16-durable-mission-runtime"
    ctx = tempfile.TemporaryDirectory() if not keep else None
    d = keep or ctx.name
    os.makedirs(d, exist_ok=True)
    try:
        for name, path in (("verify_mission_journal.py", f"{B}/verify_mission_journal.py"),
                           ("mission.journal.jsonl", f"{B}/sample/mission.journal.jsonl"),
                           ("trust.public.json", f"{B}/sample/trust.public.json"), ("head.json", f"{B}/sample/head.json")):
            with open(os.path.join(d, name), "wb") as f:
                f.write(s.fetch(path))
        src = open(os.path.join(d, "verify_mission_journal.py")).read()
        imports = [l for l in src.splitlines() if l.startswith(("import cain", "from cain"))]
        record(P, "the downloaded verifier imports no CAIN code", not imports, imports=imports)

        def verify(journal: str):
            r = subprocess.run([sys.executable, "-I", os.path.join(d, "verify_mission_journal.py"), journal, "--trust",
                                os.path.join(d, "trust.public.json"), "--head", os.path.join(d, "head.json")],
                               capture_output=True, text=True, timeout=120)
            return r.returncode, '"VERIFIED"' in r.stdout

        code, ok = verify(os.path.join(d, "mission.journal.jsonl"))
        record(P, "the verifier accepts the published journal (exit 0, VERIFIED)", code == 0 and ok, exit=code)
        lines = open(os.path.join(d, "mission.journal.jsonl")).read().split("\n")
        rng, accepted = random.Random(), 0
        body = [i for i, l in enumerate(lines) if l.strip()]
        for _ in range(mutations):
            m = list(lines)
            i = rng.choice(body)
            pos = [j for j, ch in enumerate(m[i]) if ch in "0123456789abcdef"]
            j = rng.choice(pos)
            ch = m[i][j]
            m[i] = m[i][:j] + rng.choice([c for c in ("0123456789" if ch.isdigit() else "abcdef") if c != ch]) + m[i][j + 1:]
            t = os.path.join(d, "tampered.jsonl")
            open(t, "w").write("\n".join(m))
            code, ok = verify(t)
            accepted += 1 if code == 0 else 0
        record(P, f"{mutations} random single-character changes to the journal (your own random choices): none accepted",
               accepted == 0, accepted=accepted)
    finally:
        if ctx:
            ctx.cleanup()


PROBE = '''import json, os, socket


def test_probe():
    f = {"uid": os.getuid()}
    try:
        socket.create_connection(("1.1.1.1", 80), 3)
        f["network"] = "REACHED"
    except OSError:
        f["network"] = "blocked"
    for p in ("/tmp/sallyport_probe", "/etc/sallyport_probe", "/srv/sallyport_probe"):
        try:
            open(p, "w").write("x")
            f["write " + p] = "WROTE"
        except OSError:
            f["write " + p] = "blocked"
    try:
        open("/etc/shadow").read()
        f["read /etc/shadow"] = "READ"
    except OSError:
        f["read /etc/shadow"] = "blocked"
    try:
        os.listdir("/root")
        f["list /root"] = "LISTED"
    except OSError:
        f["list /root"] = "blocked"
    print("SALLYPORT_FACTS " + json.dumps(f, sort_keys=True))
'''


def sallyport(s: Site) -> None:
    """Commands go to the TALOS gate through the talos-coder service. The gate decides each one against its command
    policy, signs an authorization bound to that exact command, re-checks it and only then runs it in the sandbox."""
    P = "SALLYPORT"
    st, r = s.call("/talos-coder/v1/coder/sessions", {"seed_files": {"test_probe.py": PROBE, "notes.txt": "hello\n"}})
    sid = r.get("session_id")
    record(P, "a session with your two files is created", st == 200 and bool(sid), status=st)
    if not sid:
        return

    def run(cmd: str):
        return s.call(f"/talos-coder/v1/coder/sessions/{sid}/check", {"command": cmd})[1]

    a = run("ls")
    record(P, "'ls' is allowed, runs, and lists your two files", a.get("verdict") == "ALLOW" and a.get("execution_state") ==
           "COMPLETED" and {"notes.txt", "test_probe.py"} <= set((a.get("stdout") or "").split()), verdict=a.get("verdict"))
    b = run("cat notes.txt")
    record(P, "'cat notes.txt' is allowed and returns the file", b.get("verdict") == "ALLOW" and b.get("stdout") == "hello\n",
           verdict=b.get("verdict"))
    hexd = lambda x: isinstance(x, str) and len(x) == 64 and all(c in "0123456789abcdef" for c in x)
    record(P, "each allowed command has its own decision id and a SHA-256 digest of the exact request, and is recorded",
           bool(a.get("decision_id")) and a.get("decision_id") != b.get("decision_id") and hexd(a.get("parameters_digest"))
           and hexd(b.get("parameters_digest")) and a.get("parameters_digest") != b.get("parameters_digest")
           and a.get("evidence_recorded") is True and b.get("evidence_recorded") is True)
    for cmd in ("python3 test_probe.py", "curl https://example.com", "bash -c id", "rm -rf /"):
        r = run(cmd)
        record(P, f"{cmd!r} is refused and never runs", r.get("verdict") == "DENY" and r.get("reason") ==
               "COMMAND_NOT_IN_POLICY" and r.get("execution_state") is None and r.get("stdout") is None,
               verdict=r.get("verdict"), reason=r.get("reason"))
    for cmd in ("cat /etc/passwd", "cat ../../../etc/passwd", "ls /root"):
        r = run(cmd)
        record(P, f"{cmd!r} is refused: an allowed program may not reach outside the session", r.get("verdict") == "DENY"
               and r.get("reason") == "ARGUMENT_NOT_IN_POLICY" and r.get("execution_state") is None,
               verdict=r.get("verdict"), reason=r.get("reason"))
    # The one way to run your own code is the allowed test runner. The probe reports what the sandbox let it do.
    r = run("pytest -q -s test_probe.py")
    line = next((l for l in (r.get("stdout") or "").splitlines() if l.startswith("SALLYPORT_FACTS ")), "")
    facts = json.loads(line[len("SALLYPORT_FACTS "):]) if line else {}
    record(P, "your probe runs inside the sandbox", r.get("verdict") == "ALLOW" and bool(facts), verdict=r.get("verdict"))
    record(P, "sandbox: the probe is not root", isinstance(facts.get("uid"), int) and facts.get("uid") != 0, uid=facts.get("uid"))
    record(P, "sandbox: no network", facts.get("network") == "blocked", network=facts.get("network"))
    writes = {k: v for k, v in facts.items() if k.startswith("write ")}
    record(P, "sandbox: cannot write to /tmp, /etc or /srv", len(writes) == 3 and set(writes.values()) == {"blocked"}, writes=writes)
    record(P, "sandbox: cannot read /etc/shadow or list /root", facts.get("read /etc/shadow") == "blocked"
           and facts.get("list /root") == "blocked")


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--key", default=os.environ.get("CAIN_API_KEY", ""), help="API key from a free signup (or CAIN_API_KEY)")
    ap.add_argument("--base", default="https://cainstudio.online")
    ap.add_argument("--skip-nullius", action="store_true")
    ap.add_argument("--keep", default="", help="directory to leave the downloaded verifier and journal in")
    ap.add_argument("--mutations", type=int, default=25)
    ap.add_argument("--out", default="", help="write the results as JSON (the key is never written)")
    a = ap.parse_args()
    if not a.key:
        print("an API key is needed: see the first lines of this file", file=sys.stderr)
        return 2
    s = Site(a.base, a.key)
    print(f"CAIN-42 premier products, checked against {a.base}")
    falsework(s)
    scenic(s)
    if a.skip_nullius:
        record("NULLIUS", "standalone verifier on a published journal", "NOT_RUN", reason="--skip-nullius")
    else:
        nullius(s, a.keep, a.mutations)
    sallyport(s)
    counts = {r: sum(1 for x in RESULTS if x["result"] == r) for r in ("PASS", "MISMATCH", "NOT_RUN")}
    print(f"\n{counts}")
    print("What a clean run shows: the public FALSEWORK and SCENIC engines returned the hand-checkable answers above for\n"
          "these inputs, the NULLIUS verifier refused every change you made, and the SALLYPORT gate allowed, refused\n"
          "and confined the commands you sent as stated. What it does not show: behaviour on\n"
          "other inputs, capacity, the two engines checked for health only, SALLYPORT installed in a customer's own\n"
          "cluster (the gate checked here runs on the developer's host), or that the source behind the engines is\n"
          "what the developer says it is.")
    if a.out:
        with open(a.out, "w") as f:
            json.dump({"schema": "cain42.premier-products.auditor-run.v1", "base": a.base, "summary": counts,
                       "results": RESULTS}, f, indent=2)
    return 1 if counts["MISMATCH"] else 0


if __name__ == "__main__":
    sys.exit(main())