#!/usr/bin/env python3
"""Clean-room verifier for the CAIN-42 E32 (Governed Autonomy Learning Fabric) evidence bundle.

Imports nothing from CAIN; needs only `cryptography`. It re-derives rather than trusts:
  * signatures and hash chains of the governance-config store, the promotion stages and the learning episodes;
  * calibration metrics (Brier, ECE, accuracy) from the published raw predictions and outcomes;
  * which arena candidates win, from the published raw governability vectors, with its own dominance rule;
  * that the promoted configuration never widened authority, by re-running its own overlay over the probe set;
  * the hidden-evaluation commitment; the self-repair calibration; E31 proofs of sampled executed episodes;
and it must REJECT every object in MALICIOUS.json.

    python3 verify_e32.py <bundle dir>
"""
from __future__ import annotations

import base64
import hashlib
import json
import re
import sys
from pathlib import Path

from cryptography.exceptions import InvalidSignature
from cryptography.hazmat.primitives.asymmetric import ed25519

D_CONFIG = "CAIN42/E32-GOVERNANCE-CONFIG/v1"
D_STAGE = "CAIN42/E32-PROMOTION-STAGE/v1"
D_APPROVAL = "CAIN42/E32-PROMOTION-APPROVAL/v1"
D_EPISODE = "CAIN42/E32-LEARNING-EPISODE/v1"
D_MEMORY = "CAIN42/E32-EXPERIENCE-MEMORY/v1"
D_REFUSAL = "CAIN42/E32-OVERLAY-REFUSAL/v1"
D_REVISION = "CAIN42/E32-STATE-REVISION/v1"
D_HIDDEN = "CAIN42/E32-HIDDEN-EVALUATION/v1"
D_MODEL = "CAIN42/E32-WORLD-MODEL-VERSION/v1"
D_PROOF = "CAIN42/E31-GOVERNANCE-PROOF/v1"
D_E28 = "CAIN42/E28-GOVERNANCE-RECEIPT/v1"
D_E8 = "CAIN42/E8-KERNEL-EVIDENCE/v1"
D_MASTER = "CAIN42/E32-MASTER/v1"
STAGES = ("PROPOSED", "SANDBOXED", "TESTED", "ADVERSARIAL", "COMPARATIVE", "CLEAN_ROOM_VERIFIED", "CANARY", "OBSERVED",
          "APPROVED", "PROMOTED")
DIMS = ("authority_preservation", "enforcement_coverage", "evidence_integrity", "identity_continuity",
        "revocation_latency_us", "failure_containment", "prediction_calibration", "policy_correctness",
        "adversarial_robustness", "recovery", "observability", "reproducibility", "verifier_independence",
        "computational_cost_us", "governance_latency_us")
REQUIRED = ("failure_containment", "adversarial_robustness")
HARD = ("authority_preservation", "evidence_integrity", "verifier_independence", "reproducibility",
        "identity_continuity", "recovery", "observability")
LOWER = ("revocation_latency_us", "prediction_calibration", "computational_cost_us", "governance_latency_us")
TOL = {"policy_correctness": 20, "enforcement_coverage": 0, "computational_cost_us": 5000,
       "governance_latency_us": 5000, "revocation_latency_us": 5000, "prediction_calibration": 50}
CONFIG_KEYS = ("deny_prefixes", "injection_markers", "secret_patterns", "rate_limit", "world_model_gate")
LOOP = ("OBSERVE", "EVIDENCE", "PREDICT", "ACT", "OUTCOME", "COMPARE", "ERROR_DRIFT_FAILURE", "LEARN",
        "GENERATE_CANDIDATE", "SANDBOX", "ADVERSARIAL_TEST", "PROOF", "CONFORMANCE", "CANARY", "RE_AUTHORIZE",
        "PROMOTE_OR_REJECT", "OBSERVE_AGAIN")


class Bad(Exception):
    pass


def _chk(o, p="$"):
    if isinstance(o, bool) or o is None or isinstance(o, int):
        return
    if isinstance(o, float):
        raise Bad(f"float {p}")
    if isinstance(o, str):
        if not o.isascii():
            raise Bad(f"ascii {p}")
        return
    if isinstance(o, list):
        for i, v in enumerate(o):
            _chk(v, f"{p}[{i}]")
        return
    if isinstance(o, dict):
        for k, v in o.items():
            _chk(v, f"{p}.{k}")
        return
    raise Bad(f"type {p}")


def cj(o) -> bytes:
    _chk(o)
    return json.dumps(o, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()


def hh(o) -> str:
    return hashlib.sha256(cj(o)).hexdigest()


def dg(domain, body) -> str:
    return hh({"domain": domain, "body": body})


def sig_ok(pub, sig, domain, body) -> bool:
    try:
        ed25519.Ed25519PublicKey.from_public_bytes(base64.b64decode(pub)).verify(base64.b64decode(sig),
                                                                                  dg(domain, body).encode())
        return True
    except (InvalidSignature, ValueError, TypeError, Bad):
        return False


def kid(pub) -> str:
    return hashlib.sha256(base64.b64decode(pub)).hexdigest()[:16]


class C:
    def __init__(self):
        self.checks, self.passed, self.problems = 0, 0, []

    def check(self, name, ok, detail=""):
        self.checks += 1
        if ok:
            self.passed += 1
        else:
            self.problems.append(f"{name}: {detail}"[:300])


def load(d, n):
    return json.loads((d / n).read_text())


# ------------------------------------------------------------------------------------------ independent pieces
def chain_problems(items, pub, domain, hash_key="hash", seq_key="version"):
    bad, prev = [], "0" * 64
    for i, v in enumerate(items):
        b = v["body"]
        if b.get(seq_key) != i or b["prev"] != prev or dg(domain, b) != v[hash_key] or \
                (pub is not None and not sig_ok(pub, v["signature_b64"], domain, b)):
            bad.append(i)
        prev = v[hash_key]
    return bad


def overlay_allows(cfg, spec) -> bool:
    text = json.dumps(spec["params"], sort_keys=True) + " " + spec["intent"]
    if any(spec["target"].startswith(p) for p in cfg["deny_prefixes"]):
        return False
    if any(m in " ".join(text.lower().split()) for m in cfg["injection_markers"]):
        return False
    if any(re.search(p, text) for p in cfg["secret_patterns"]):
        return False
    return True


def config_widens(before, after, probes) -> list:
    a = {i for i, s in enumerate(probes) if overlay_allows(before, s)}
    b = {i for i, s in enumerate(probes) if overlay_allows(after, s)}
    out = [f"probe:{i}" for i in sorted(b - a)]
    for k in ("deny_prefixes", "injection_markers", "secret_patterns"):
        if set(before[k]) - set(after[k]):
            out.append(f"removed:{k}")
    rb, ra = before.get("rate_limit"), after.get("rate_limit")
    if rb and (not ra or ra["max_actions"] > rb["max_actions"] or ra["window_ms"] < rb["window_ms"]):
        out.append("rate_limit_loosened")
    return out


def calibration(rows):
    rs = [r for r in rows if r["p_harm_milli"] is not None]
    if not rs:
        return None
    ps = [r["p_harm_milli"] / 1000 for r in rs]
    ys = [1.0 if r["harm"] else 0.0 for r in rs]
    brier = sum((p - y) ** 2 for p, y in zip(ps, ys)) / len(rs)
    bins = {}
    for p, y in zip(ps, ys):
        bins.setdefault(min(9, int(p * 10)), []).append((p, y))
    ece = sum(len(b) / len(rs) * abs(sum(p for p, _ in b) / len(b) - sum(y for _, y in b) / len(b))
              for b in bins.values())
    acc = sum((p >= .5) == (y == 1) for p, y in zip(ps, ys)) / len(rs)
    return {"n": len(rs), "brier_milli": int(round(brier * 1000)), "ece_milli": int(round(ece * 1000)),
            "accuracy_milli": int(round(acc * 1000))}


def wins(v, b) -> bool:
    improved = False
    for d in REQUIRED:
        if v[d] < b[d]:
            return False
        improved = improved or v[d] > b[d]
    if not improved or any(v[d] < 1000 for d in HARD):
        return False
    for d, t in TOL.items():
        if (d in LOWER and v[d] > b[d] + t) or (d not in LOWER and v[d] < b[d] - t):
            return False
    return True


def proof_ok(p, att, an) -> bool:
    try:
        b = p["body"]
        k = an["proof_kids"].get(b["header"]["kid"])
        e8 = att["e8_entry"]
        e8b = {x: y for x, y in e8.items() if x != "entry_hash"}
        return bool(k) and kid(k["pub"]) == b["header"]["kid"] and dg(D_PROOF, b) == p["proof_hash"] and \
            sig_ok(k["pub"], p["signature_b64"], D_PROOF, b) and b["grants_authority"] is False and \
            sig_ok(an["e28"], att["e28_receipt"]["signature_b64"], D_E28, att["e28_receipt"]["body"]) and \
            hashlib.sha256(cj({"domain": D_E8, **e8b})).hexdigest() == e8["entry_hash"] == \
            b["enforcement"]["e8_entry_hash"] and e8["entry_hash"] in att["e28_receipt"]["body"]["evidence_references"]
    except (KeyError, TypeError, ValueError, Bad):
        return False


# ------------------------------------------------------------------------------------------ malicious objects
def rejected(m, d) -> bool:
    k, o = m["class"], m["object"]
    try:
        if k in ("governance_config", "policy", "rollback_state"):
            return bool(chain_problems(o["versions"], o["pub"], D_CONFIG))
        if k == "authority":
            return bool(config_widens(o["before"], o["after"], o["probes"]))
        if k in ("evidence", "learned_state"):
            return bool(chain_problems(o["episodes"], None, D_EPISODE, seq_key="seq"))
        if k == "prediction":
            return calibration(o["rows"]) != o["claimed"]
        if k == "proof":
            return not proof_ok(o["proof"], o["attachments"], o["anchors"])
        if k in ("model_identity", "runtime_identity"):
            return dg(D_MODEL, o["body"]) != o["digest"]
        if k == "memory":
            return not sig_ok(o["pub"], o["signature_b64"], D_MEMORY, o["body"])
        if k == "world_state":
            return bool(chain_problems(o["chain"], None, D_REVISION, seq_key="__none__")) or \
                any(r["body"]["prev"] != (o["chain"][i - 1]["hash"] if i else "0" * 64) or
                    dg(D_REVISION, r["body"]) != r["hash"] for i, r in enumerate(o["chain"]))
        if k == "evolution_candidate":
            return not wins(o["vector"], o["baseline"]) or o["claimed_win"] is not True
    except (KeyError, TypeError, ValueError, Bad):
        return True
    return False


def main() -> int:
    d = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(".")
    c = C()
    try:
        cfg = load(d, "CONFIG_CHAIN.json")
    except (OSError, ValueError) as x:
        print(json.dumps({"result": "ERROR", "checks": 0, "passed": 0, "problems": [str(x)]}))
        return 2

    # ---- governance-state store
    vs = cfg["versions"]
    bad = chain_problems(vs, cfg["gate_pub"], D_CONFIG)
    c.check("config.chain", not bad, bad)
    for i, v in enumerate(vs):
        b = v["body"]
        c.check(f"config[{i}].keys", sorted(b["config"]) == sorted(CONFIG_KEYS), sorted(b["config"]))
        c.check(f"config[{i}].digest", hh(b["config"]) == b["config_digest"], "digest")
        c.check(f"config[{i}].no_authority", b["authority"] == "NONE", b["authority"])
        c.check(f"config[{i}].not_wider_than_baseline", not config_widens(vs[0]["body"]["config"], b["config"],
                                                                           cfg["probes"]),
                config_widens(vs[0]["body"]["config"], b["config"], cfg["probes"])[:3])
    promoted = [v for v in vs if v["body"]["reason"] == "promotion"]
    c.check("config.promotion_present", bool(promoted), "no promotion")

    # ---- promotion gate
    st = load(d, "PROMOTION.json")
    ch = st["chain"]
    bad, prev = [], "0" * 64
    for i, r in enumerate(ch):
        if r["body"]["prev"] != prev or dg(D_STAGE, r["body"]) != r["hash"] or not sig_ok(st["gate_pub"],
                                                                                           r["signature_b64"],
                                                                                           D_STAGE, r["body"]):
            bad.append(i)
        prev = r["hash"]
    c.check("gate.chain", not bad, bad)
    c.check("gate.key_matches_store", st["gate_pub"] == cfg["gate_pub"], "gate key")
    for cand in st["candidates"]:
        stages = [s["body"]["stage"] for s in cand["stages"]]
        c.check(f"cand.{cand['cid'][:12]}.order", stages == list(STAGES[1:1 + len(stages)]), stages)
        if cand["state"] == "PROMOTED":
            c.check(f"cand.{cand['cid'][:12]}.all_stages", stages == list(STAGES[1:]), stages)
            ap = cand["approval"]
            body = {"candidate": cand["cid"], "config": hh(cand["config"]), "stages": ap["stage_hashes"]}
            c.check(f"cand.{cand['cid'][:12]}.human_approval", ap["human"] in st["human_keys"] and
                    ap["human"] != cand["proposer"] and
                    sig_ok(st["human_keys"][ap["human"]], ap["signature_b64"], D_APPROVAL, body), "approval")
            c.check(f"cand.{cand['cid'][:12]}.store_has_it", any(v["body"]["candidate"] == cand["cid"] and
                                                                  v["body"]["config"] == cand["config"]
                                                                  for v in promoted), "not in store")
            c.check(f"cand.{cand['cid'][:12]}.owner", bool(cand["owner"]), "owner")
        else:
            c.check(f"cand.{cand['cid'][:12]}.not_promoted", not any(v["body"]["candidate"] == cand["cid"]
                                                                     for v in vs), "rejected candidate in store")
            c.check(f"cand.{cand['cid'][:12]}.last_stage_failed", not cand["stages"] or
                    cand["stages"][-1]["body"]["ok"] is False, "failed shown as success")

    # ---- episodes + evidence
    ep = load(d, "EPISODES.json")
    bad = chain_problems(ep["episodes"], None, D_EPISODE, seq_key="seq")
    c.check("episodes.chain", not bad, bad[:5])
    for e in ep["episodes"]:
        b = e["body"]
        c.check(f"episode[{b['seq']}].evidence", bool(b["evidence"]), "no evidence")
        c.check(f"episode[{b['seq']}].harm_only_if_executed", not b["outcome"]["harm"] or b["executed"], "harm")
        c.check(f"episode[{b['seq']}].simulated_label", b["outcome"]["label_source"] == "SIMULATED_ORACLE",
                b["outcome"]["label_source"])
    an = ep["anchors"]
    for i, s in enumerate(ep["proof_samples"]):
        c.check(f"proof_sample[{i}]", proof_ok(s["proof"], s["attachments"], an), s["proof"]["proof_id"][:12])
        c.check(f"proof_sample[{i}].linked", any(e["body"]["evidence"] == s["proof"]["proof_id"]
                                                 for e in ep["episodes"]), "unlinked")
    for i, r in enumerate(ep["refusal_samples"]):
        c.check(f"refusal[{i}].signature", sig_ok(ep["plane_pub"], r["signature_b64"], D_REFUSAL, r["body"]), "sig")
        c.check(f"refusal[{i}].linked", any(e["body"]["evidence"] == r["hash"] for e in ep["episodes"]), "unlinked")

    # ---- predictions / calibration recomputed
    pr = load(d, "PREDICTIONS.json")
    got = calibration(pr["rows"])
    for k in ("n", "brier_milli", "ece_milli", "accuracy_milli"):
        c.check(f"calibration.{k}", got is not None and got[k] == pr["published"][k], (got or {}).get(k))
    c.check("predictions.class", all(r.get("class") == "DELTA_EVIDENCE" for r in pr["rows"]), "class")
    c.check("model.version_digest", dg(D_MODEL, pr["model_version"]["body"]) == pr["model_version"]["digest"], "model")
    c.check("model.no_authority", pr["model_version"]["body"]["authority"] == "NONE", "authority")

    # ---- arena recomputed
    ar = load(d, "ARENA.json")
    base = ar["baseline"]
    c.check("arena.baseline_dims", sorted(base) == sorted(DIMS), sorted(base))
    for r in ar["rows"]:
        name = r["candidate"][:12]
        if r.get("vector") is None:
            c.check(f"arena.{name}.sandbox_refused_not_winner", r["wins"] is False, r["wins"])
            continue
        c.check(f"arena.{name}.dims", sorted(r["vector"]) == sorted(DIMS), "dims")
        c.check(f"arena.{name}.integers", all(isinstance(x, int) for x in r["vector"].values()), "ints")
        c.check(f"arena.{name}.dominance_recomputed", (wins(r["vector"], base) and not r["integrity"]) == r["wins"],
                f"{wins(r['vector'], base)} vs {r['wins']}")
    c.check("arena.deny_all_rejected", any(r.get("kind") == "deny_all_writes" and not r["wins"] for r in ar["rows"]),
            "strawman")
    c.check("arena.promoted_is_winner", any(r["wins"] and r["candidate"] == ar["promoted"] for r in ar["rows"]),
            "promoted")

    # ---- hidden evaluation
    hd = load(d, "HIDDEN_EVAL.json")
    c.check("hidden.commitment", dg(D_HIDDEN, {"seed": hd["reveal"]["seed"], "params": hd["reveal"]["params"]}) ==
            hd["commitment"] == hd["reveal"]["commitment"], "commitment")
    c.check("hidden.different_from_visible", hd["reveal"]["seed"] != hd["visible_seed"], "same seed")

    # ---- drift
    dr = load(d, "DRIFT.json")
    c.check("drift.promoted_not_wider", not config_widens(dr["before"], dr["after"], dr["probes"]),
            config_widens(dr["before"], dr["after"], dr["probes"])[:3])
    c.check("drift.probe_count", len(dr["probes"]) >= 20, len(dr["probes"]))

    # ---- memory, revisions, self-repair, knowledge, swaps, loop
    mem = load(d, "MEMORY.json")
    for i, r in enumerate(mem["records"]):
        c.check(f"memory[{i}].signature", sig_ok(mem["pub"], r["signature_b64"], D_MEMORY, r["body"]), "sig")
        c.check(f"memory[{i}].not_self_verified", r["body"]["state"] != "VERIFIED" or len(set(r["body"]["sources"])) >= 3,
                r["body"]["state"])
    rv = load(d, "REVISIONS.json")
    prev = "0" * 64
    for i, r in enumerate(rv["chain"]):
        c.check(f"revision[{i}].chain", r["body"]["prev"] == prev and dg(D_REVISION, r["body"]) == r["hash"], "chain")
        c.check(f"revision[{i}].keeps_original", "original" in r["body"] and bool(r["body"]["evidence"]), "erased")
        prev = r["hash"]
    sr = load(d, "SELF_REPAIR.json")
    c.check("repair.collapsed_first", sr["collapsed"] is True, sr["collapsed"])
    c.check("repair.steps", [s["step"] for s in sr["steps"]] == ["ISOLATE", "PRESERVE_EVIDENCE", "IDENTIFY",
                                                                  "REPAIR_CANDIDATE", "TEST", "ADVERSARIAL_TEST",
                                                                  "CLEAN_ROOM_VERIFY", "CANARY", "PROMOTE",
                                                                  "ROLLBACK_IF_REGRESSION"], "steps")
    test = next(s for s in sr["steps"] if s["step"] == "TEST")
    c.check("repair.test_thresholds", test["ok"] == (test["ece_milli"] <= 150 and test["brier_milli"] <= 200 and
                                                     test["n"] >= 10), test)
    c.check("repair.promoted_only_if_all_ok", sr["repaired"] == all(s["ok"] for s in sr["steps"][:-1]), sr["repaired"])
    kg = load(d, "KNOWLEDGE.json")
    c.check("knowledge.revocation_propagates", kg["revocation"]["by_kind"].get("policy", 0) >= 1 and
            kg["revocation"]["by_kind"].get("prediction", 0) >= 1, kg["revocation"]["by_kind"])
    sw = load(d, "SWAPS.json")
    for r in sw["swaps"]:
        c.check(f"swap.{r['kind']}", r["act_after_swap"] != "ALLOW" and r["no_e8_after_swap"], r["act_after_swap"])
    c.check("swap.reauth_explicit", sw["reauthorized"]["after"] == "ALLOW" and sw["reauthorized"]["lease"], "reauth")
    lp = load(d, "LOOP.json")
    c.check("loop.steps", [s["step"] for s in lp["steps"]] == list(LOOP), "order")
    for s in lp["steps"]:
        c.check(f"loop.{s['step']}", s["ok"] is True, s["step"])
    after = next(s for s in lp["steps"] if s["step"] == "OBSERVE_AGAIN")
    c.check("loop.harm_reduced", after["harm_after"] < after["harm_before"], after)

    # ---- malicious objects
    mal = load(d, "MALICIOUS.json")
    classes = set()
    for i, m in enumerate(mal["objects"]):
        c.check(f"malicious[{i}].{m['class']}.rejected", rejected(m, d), m["mutation"])
        classes.add(m["class"])
    need = {"learned_state", "policy", "authority", "evidence", "prediction", "proof", "model_identity",
            "runtime_identity", "memory", "world_state", "governance_config", "evolution_candidate", "rollback_state"}
    c.check("malicious.classes", need <= classes, sorted(need - classes))

    # ---- bench, invariants, mutation, scale, claims
    b = load(d, "SECURITY_RESULTS.json")
    c.check("bench.all_held", b["held"] == b["scenarios"], f"{b['held']}/{b['scenarios']}")
    c.check("bench.at_least_1000", b["scenarios"] >= 1000, b["scenarios"])
    c.check("bench.22_categories", len(b["categories"]) >= 22, len(b["categories"]))
    for r in b["rows"]:
        c.check(f"bench.row.{r['id']}", r["held"] is True, r["detail"][:80])
    inv = load(d, "INVARIANTS.json")
    c.check("invariants.at_least_300", inv["total"] >= 300, inv["total"])
    c.check("invariants.all_hold", inv["passed"] == inv["total"], [r["id"] for r in inv["rows"] if not r["ok"]][:5])
    c.check("invariants.42_laws", all(f"E32-L{i}" in {r["id"] for r in inv["rows"]} for i in range(1, 43)), "laws")
    mut = load(d, "MUTATION_RESULTS.json")
    c.check("mutation.all_killed", mut["killed"] == mut["mutants"] and not mut["survived"], mut["survived"])
    c.check("mutation.at_least_10", mut["mutants"] >= 10, mut["mutants"])
    sc = load(d, "SCALE_RESULTS.json")
    c.check("scale.synthetic_label", "SYNTHETIC" in sc["classification"], "label")
    got = {(r["dimension"], r["size"]) for r in sc["rows"]}
    for need_row in [("agents", n) for n in (10, 100, 1000, 10000, 100000)] + \
                    [("episodes", n) for n in (1000, 10000, 100000, 1000000)]:
        c.check(f"scale.{need_row[0]}.{need_row[1]}", need_row in got, "missing")
    for r in sc["rows"]:
        c.check(f"scale.{r['dimension']}.{r['size']}.no_failures", r["failures"] == 0, r["failures"])
    lim = load(d, "KNOWN_LIMITATIONS.json")
    c.check("limitations.published", len(lim["limitations"]) >= 8, len(lim["limitations"]))
    reg = load(d, "REGISTERS.json")
    c.check("registers.no_external_research_claimed", reg["research_ingestion"]["items"] == [], "research")
    c.check("registers.not_production", reg["autonomous_self_improvement_in_production"] is False, "production")
    cl = load(d, "E32_PUBLIC_CLAIMS.json")
    for k in cl["claims"]:
        c.check(f"claim.{k['id']}", all((d / f).exists() for f in k["evidence"]), k["evidence"])

    sums = {}
    for line in (d / "SHA256SUMS").read_text().splitlines():
        hx, name = line.split("  ", 1)
        sums[name] = hx
    for name, hx in sorted(sums.items()):
        c.check(f"sha256.{name}", hashlib.sha256((d / name).read_bytes()).hexdigest() == hx, "hash")
    sig = load(d, "SIGNATURE.json")
    c.check("signature.files", sig["files"] == sums, "files")
    c.check("signature.master_digest", sig["master"]["hashes_digest"] == hh(sig["files"]), "digest")
    c.check("signature.valid", sig_ok(sig["signer_public_key_b64"], sig["signature_b64"], D_MASTER, sig["master"]),
            "sig")
    out = {"verifier": "verify_e32.py", "imports_cain": False, "checks": c.checks, "passed": c.passed,
           "result": "INTACT" if c.passed == c.checks else "BROKEN", "problems": c.problems[:50]}
    print(json.dumps(out))
    return 0 if out["result"] == "INTACT" else 1


if __name__ == "__main__":
    sys.exit(main())