#!/usr/bin/env python3 """Evolution 80 standalone offline verifier. python3 -I verify_e80_offline.py Imports no CAIN code: standard library plus ``cryptography`` (Ed25519). It does not trust any summary. Accuracies, paired tests, routing decisions, call accounting, signatures and hash chains are recomputed from raw rows and compared with what the signed bundle states. It verifies a record. It cannot make the record true and it is not a third-party audit. """ from __future__ import annotations import base64 import hashlib import json import math import re import sys from pathlib import Path from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey PRIMS = ("det_exec", "m05_direct", "m05_pot", "m15_direct", "m15_pot") COORD = ("c01_single", "c02_parallel_vote", "c03_pairwise_debate", "c04_critic_reviser", "c05_planner_executor", "c06_hierarchical", "c07_specialist_committee", "c08_evidence_first", "c09_parallel_synthesis", "c10_adaptive_graph") def canonical(o): return json.dumps(o, sort_keys=True, separators=(",", ":"), default=str).encode() def sha(o): if isinstance(o, str): return hashlib.sha256(o.encode()).hexdigest() return hashlib.sha256(canonical(o)).hexdigest() def merkle(leaves): if not leaves: return hashlib.sha256(b"").hexdigest() level = [hashlib.sha256(b"\x00" + bytes.fromhex(h)).digest() for h in leaves] while len(level) > 1: level = [hashlib.sha256(b"\x01" + level[i] + (level[i + 1] if i + 1 < len(level) else level[i])).digest() for i in range(0, len(level), 2)] return level[0].hex() def norm(t): return re.sub(r"[^a-z0-9_\-]", "", str(t).strip().lower().replace(" ", "_")) def verify_sig(pub_b64, payload, sig_b64): try: Ed25519PublicKey.from_public_bytes(base64.b64decode(pub_b64)).verify(base64.b64decode(sig_b64), canonical(payload)) return True except Exception: return False def mcnemar(a, b): bo = sum(1 for x, y in zip(a, b) if (not x) and y) ao = sum(1 for x, y in zip(a, b) if x and (not y)) n = bo + ao p = 1.0 if n == 0 else min(1.0, 2 * sum(math.comb(n, i) for i in range(min(bo, ao) + 1)) / (2 ** n)) return round(p, 6) def jl(p): return [json.loads(x) for x in p.read_text().splitlines() if x.strip()] if p.is_file() else [] def chain_ok(rows): """Hash-chained log written by several processes: each process starts a new segment at seq 0.""" prev = "0" * 64 for r in rows: if r.get("seq") == 0: prev = "0" * 64 body = {k: v for k, v in r.items() if k != "hash"} if body.get("prev") != prev or sha(body) != r.get("hash"): return False prev = r["hash"] return True def main(): root = Path(sys.argv[1] if len(sys.argv) > 1 else ".") checks = [] def ck(name, ok, detail=""): checks.append({"check": name, "ok": bool(ok), "detail": str(detail)[:300]}) def J(rel): p = root / rel return json.loads(p.read_text()) if p.is_file() else None # ---- 1. signature, manifest, files ---- ev = J("evidence.json") ck("evidence:present", ev is not None) if ev is None: print(json.dumps({"verified": False, "passed": 0, "total": 1, "failed": checks})) return man = ev["manifest"] ck("evidence:signature_valid", verify_sig(ev["public_key_b64"], man, ev["signature_b64"])) ck("evidence:fingerprint_matches_key", hashlib.sha256(base64.b64decode(ev["public_key_b64"])).hexdigest() == ev["fingerprint_sha256"]) bad = [k for k, h in man["artifact_sha256"].items() if not (root / k).is_file() or hashlib.sha256((root / k).read_bytes()).hexdigest() != h] ck("evidence:all_artifact_hashes_match", not bad, bad[:5]) ck("evidence:merkle_root", merkle([man["artifact_sha256"][k] for k in sorted(man["artifact_sha256"])]) == man["merkle_root"]) ck("evidence:not_deployed_no_cloud_spend", man["deployed"] is False and man["production_modified"] is False and man["cloud_spend_usd"] == 0.0) ck("evidence:trust_note_states_self_generated_key", "Self-generated" in ev.get("trust_note", "")) cb = J("collective_benchmark.json") pol = J("routing_policy.json") pre = J("preregistration.json") tasks = {s: J(f"tasks/{s}.json") for s in ("train", "test", "replication")} calls = jl(root / "model_calls.jsonl") by_id = {c["request_id"]: c for c in calls} # ---- 2. task splits ---- H = {s: {t["question_sha256"] for t in tasks[s]} for s in tasks} ck("tasks:splits_disjoint", not (H["train"] & H["test"]) and not (H["train"] & H["replication"]) and not (H["test"] & H["replication"])) ck("tasks:question_hashes_recompute", all(hashlib.sha256(t["question"].encode()).hexdigest() == t["question_sha256"] for s in tasks for t in tasks[s])) ck("tasks:sizes", [len(tasks[s]) for s in ("train", "test", "replication")] == [40, 48, 24], [len(tasks[s]) for s in tasks]) # ---- 3. policy freeze and pre-registration ---- ck("policy:hash_recomputes", sha(pol["families"]) == pol["policy_sha256"]) ck("policy:names_only_measured_paths", all(v["path"] in PRIMS for v in pol["families"].values())) ck("prereg:binds_policy_hash", pre["policy_sha256"] == pol["policy_sha256"]) first_test = min((c["utc"] for c in calls if c["stage"].startswith(("test.", "replication."))), default=None) ck("prereg:written_before_first_test_call", first_test is not None and pre["written_utc"] < first_test, f"{pre['written_utc']} < {first_test}") ck("prereg:no_test_stage_file_existed", pre["test_or_replication_stage_files_existing_when_written"] == []) ck("policy:frozen_before_first_test_call", first_test is not None and pol["frozen_utc"] < first_test) # refit the policy from train rows refit = {} fams = sorted({t["family"] for t in tasks["train"]}) tr = {arm: jl(root / f"bench/train/{arm}.jsonl") for arm in PRIMS} for fam in fams: st = {} for arm in PRIMS: rs = [r for r in tr[arm] if r["family"] == fam and r["applicable"]] if rs: st[arm] = (sum(r["correct"] for r in rs), len(rs), round(sum(r["latency_ms"] for r in rs) / len(rs), 1)) best = max(c / n for c, n, _ in st.values()) n = max(n for _, n, _ in st.values()) okp = [a for a, (c, m, _) in st.items() if c / m >= best - 1.0 / n - 1e-9] refit[fam] = min(okp, key=lambda a: (st[a][2], a)) ck("policy:refit_from_train_rows_matches", refit == {f: v["path"] for f, v in pol["families"].items()}, refit) best_fixed = max(PRIMS, key=lambda arm: sum(r["correct"] for r in tr[arm])) ck("prereg:best_fixed_path_recomputes", best_fixed == pre["best_fixed_path_selected_on_train"], best_fixed) # ---- 4. raw rows: regrade, recount, call linkage ---- all_rows = {} for split in ("train", "test", "replication"): arms_stated = cb["splits"][split].get("arms", {}) for arm, stated in arms_stated.items(): rows = jl(root / f"bench/{split}/{arm}.jsonl") all_rows[(split, arm)] = rows regrade = all(r["correct"] == bool(r["applicable"] and r["answer"] is not None and norm(r["answer"]) == norm(r["expected"])) for r in rows) truth = {t["task_id"]: t["answer"] for t in tasks[split]} exp_ok = all(r["expected"] == truth.get(r["task_id"]) for r in rows) and len({r["task_id"] for r in rows}) == len(tasks[split]) k = sum(r["correct"] for r in rows) ck(f"rows:{split}:{arm}:accuracy_recomputes", regrade and exp_ok and k == stated["correct"] and len(rows) == stated["tasks"] and abs(k / len(rows) - stated["accuracy"]) < 1e-4, f"{k}/{len(rows)} stated {stated['correct']}/{stated['tasks']}") ck(f"rows:{split}:{arm}:calls_and_tokens_recompute", sum(r["n_model_calls"] for r in rows) == stated["model_calls"] and sum(r["tokens"] for r in rows) == stated["tokens"] and all(len(r["call_ids"]) == r["n_model_calls"] for r in rows) and all(cid in by_id for r in rows for cid in r["call_ids"]) and all(sum(by_id[c]["total_tokens"] for c in r["call_ids"]) == r["tokens"] for r in rows)) # ---- 5. routing: every routed row follows the frozen policy ---- for split in ("test", "replication"): rr = all_rows.get((split, "router"), []) prim = {arm: {r["task_id"]: r for r in all_rows.get((split, arm), [])} for arm in PRIMS} ok = bool(rr) and all(r["trace"]["selected_path"] == pol["families"].get(r["family"], {"path": "m05_direct"})["path"] and r["answer"] == prim[r["trace"]["selected_path"]][r["task_id"]]["answer"] for r in rr) ck(f"routing:{split}:rows_follow_frozen_policy", ok) blk = cb["splits"][split] if rr: a = {r["task_id"]: r["correct"] for r in all_rows[(split, best_fixed)]} b = {r["task_id"]: r["correct"] for r in rr} ids = sorted(a) st = blk["router_vs_best_fixed_on_train"] ck(f"routing:{split}:paired_test_recomputes", st["a_correct"] == sum(a.values()) and st["b_correct"] == sum(b.values()) and abs(st["mcnemar"]["p_two_sided"] - mcnemar([a[i] for i in ids], [b[i] for i in ids])) < 1e-6, f"{sum(b.values())} vs {sum(a.values())}") orc = all_rows[(split, "oracle_any_path")] ck(f"routing:{split}:oracle_recomputes", sum(r["correct"] for r in orc) == sum(any(prim[arm][t["task_id"]]["correct"] for arm in PRIMS) for t in tasks[split])) # ---- 6. hypotheses ---- te = cb["splits"]["test"] h1 = cb["hypotheses"]["H1_router_beats_best_fixed_path"] ck("hypothesis:H1_verdict_follows_from_numbers", h1["supported"] == (h1["diff"] > 0 and h1["mcnemar"]["p_two_sided"] < 0.05)) single = {r["task_id"]: r["correct"] for r in all_rows[("test", "c01_single")]} ids = sorted(single) ps = {} for arm in COORD[1:]: o = {r["task_id"]: r["correct"] for r in all_rows[("test", arm)]} p = mcnemar([single[i] for i in ids], [o[i] for i in ids]) st = te["coordination_vs_single"][arm] ck(f"coordination:{arm}:paired_test_recomputes", abs(st["mcnemar"]["p_two_sided"] - p) < 1e-6 and st["b_correct"] == sum(o.values()) and st["a_correct"] == sum(single.values())) if arm != "c10_adaptive_graph": ps[arm] = (p, sum(o.values()) - sum(single.values())) order = sorted((p, a) for a, (p, _) in ps.items()) holm = {a: min(1.0, round(p * (len(order) - i), 6)) for i, (p, a) in enumerate(order)} sig = sorted(a for a in ps if ps[a][1] > 0 and holm[a] < 0.05) h2 = cb["hypotheses"]["H2_some_multi_agent_arm_beats_single"] ck("hypothesis:H2_verdict_recomputes", sorted(h2["significant_arms"]) == sig and h2["supported"] == bool(sig), sig) ck("hypothesis:H2_worse_arms_recompute", sorted(h2["arms_worse_than_single"]) == sorted(a for a in ps if ps[a][1] < 0)) h3 = cb["hypotheses"]["H3_direction_replicates"] if all_rows.get(("replication", "router")): ck("hypothesis:H3_verdict_follows_from_numbers", h3["supported"] == (h3["diff"] > 0)) ck("coordination:same_task_ids_for_every_arm", all({r["task_id"] for r in all_rows[("test", arm)]} == set(ids) for arm in COORD)) # ---- 7. model call log ---- acc = cb["inference_accounting"] ck("calls:count_and_tokens_recompute", acc["model_calls"] == len(calls) and acc["tokens"] == sum(c["total_tokens"] for c in calls), f"{len(calls)} calls") ck("calls:all_local_inference_zero_cost", all(c["execution_class"] == "LOCAL_MODEL_INFERENCE" and c["cost_usd"] == 0.0 and c["provider"] == "self_hosted_hf" for c in calls)) ck("calls:request_ids_unique", len(by_id) == len(calls)) ck("calls:hashes_bind_text", all(sha(c["input_text"]) == c["input_text_sha256"] for c in calls)) ck("calls:never_grant_authority", all(c["grants_authority"] is False for c in calls)) keys = {} for i in (J("specialists.json") or {"identities": []})["identities"]: keys.setdefault(i["agent_id"], set()).add(i["public_key_b64"]) for reg in jl(root / "identity_registries.jsonl"): for i in reg["identities"]: keys.setdefault(i["agent_id"], set()).add(i["public_key_b64"]) def signed(c): body = {k: c[k] for k in ("request_id", "task_id", "stage", "agent_id", "model_key", "input_text_sha256", "output_text")} return any(verify_sig(k, body, c["agent_signature_b64"]) for k in keys.get(c["agent_id"], ())) unsigned = [c["request_id"] for c in calls if not signed(c)] ck("calls:every_call_signed_by_a_registered_identity", not unsigned, f"{len(unsigned)} unsigned") ck("calls:one_weight_file_per_model", all(len({c["model_file_sha256"] for c in calls if c["model_key"] == m}) == 1 for m in {c["model_key"] for c in calls})) ids_reg = (J("specialists.json") or {"identities": []})["identities"] ck("identities:20_specialists_self_signed", len(ids_reg) == 20 and all(verify_sig(i["public_key_b64"], {k: i[k] for k in ("agent_id", "tenant", "public_key_b64", "manifest", "grants_authority")}, i["self_signature_b64"]) for i in ids_reg)) ck("identities:none_grants_authority", all(i["grants_authority"] is False for i in ids_reg)) # ---- 8. gate audit ---- ga = jl(root / "gate_audit.jsonl") ck("gate:audit_chain_intact", bool(ga) and chain_ok(ga), len(ga)) exe = [g for g in ga if g["event"] == "EXECUTE" and g["decision"] == "EXECUTED"] allow = {g.get("grant_id") for g in ga if g["event"] == "DECIDE" and g["decision"] == "ALLOW"} ck("gate:every_execution_has_an_allow_grant", all(g.get("grant_id") in allow for g in exe)) ck("gate:no_grant_executed_twice", len({g["grant_id"] for g in exe}) == len(exe)) # ---- 9. security lab ---- sec = J("security_lab.json") if sec: ck("security:counts_recompute", sec["scenarios"] == len(sec["rows"]) and sec["blocked"] == sum(r["blocked"] for r in sec["rows"]) and sec["bypassed"] == sum(not r["blocked"] for r in sec["rows"]), f"{sec['blocked']}/{sec['scenarios']}") ck("security:states_limits", "not a third-party audit" in sec["limits"].lower()) # ---- 10. execution drill ---- dr = J("execution_drill.json") if dr: ev_c = {} for e in dr["events"]: ev_c[e["event"]] = ev_c.get(e["event"], 0) + 1 ck("drill:event_counts_recompute", ev_c == dr["reconcile"]["events"]) ck("drill:done_equals_tasks_iff_all_done", dr["all_done"] == (dr["reconcile"]["done"] == dr["reconcile"]["tasks"])) ck("drill:no_duplicate_effect_and_no_done_without_effect", (dr["reconcile"]["duplicate_effects"] == 0 and dr["reconcile"]["done_without_effect"] == 0) == dr["zero_duplicate_effects"] or not dr["all_done"]) ck("drill:commits_equal_effects", ev_c.get("COMMIT", 0) == dr["reconcile"]["effects"]) ck("drill:not_evaluated_items_listed", "network_partition" in dr["not_evaluated"]) # ---- 11. memory ---- mem = J("memory/memory_evaluation.json") if mem: mr = jl(root / "memory/rows.jsonl") def g(row): out = row["output_text"] or "" m = re.search(r"ANSWER\s*[:=]\s*(.+)", out, re.I) got = (m.group(1) if m else out).strip().splitlines()[0].strip(" .`'\"").lower() if out.strip() else "" want = row["expected"].lower() return got.startswith("unknown") if want == "unknown" else (re.search(rf"\b{re.escape(want)}\b", got) is not None and "mallory" not in got) ok = all(r["correct"] == g(r) for r in mr) for arm, st in mem["arms"].items(): rs = [r for r in mr if r["arm"] == arm] ok = ok and st["correct"] == sum(r["correct"] for r in rs) and st["questions"] == len(rs) ck("memory:answer_accuracy_regrades_from_outputs", ok) ck("memory:public_benchmarks_marked_not_evaluated", all(v.startswith("NOT_EVALUATED") for v in mem["public_benchmarks"].values())) # ---- 12. skill ---- sk = J("skill/skill_report.json") if sk: er = jl(root / "skill/eval_rows.jsonl") ok = True for name, st in sk["results"].items(): for adapter, key in ((False, "baseline"), (True, "candidate")): rs = [r for r in er if r["set"] == name and r["adapter"] == adapter] ok = ok and st[key]["correct"] == sum(r["correct"] for r in rs) and st[key]["tasks"] == len(rs) ck("skill:accuracies_recompute_from_eval_rows", ok) ck("skill:not_promoted_without_human", sk["promotion"]["promoted"] is False and sk["promotion"]["activated"] is False and sk["versioning"]["store_active"] is None) ck("skill:self_approval_refused", sk["promotion"]["self_approval_attempt"]["reason"] == "SELF_APPROVAL_REFUSED") ck("skill:training_data_disjoint_from_evaluation", sk["data"]["disjoint_from_all_evaluation_sets"] is True) # ---- 13. mission ---- mi = J("mission.json") if mi: ck("mission:journal_chain_intact", chain_ok(mi["journal"])) st = {} for n in mi["node_table"]: st[n["state"]] = st.get(n["state"], 0) + 1 ck("mission:node_states_recompute", st == mi["node_states"] and mi["verified_nodes"] == st.get("VERIFIED", 0)) ck("mission:verifier_differs_from_producer", all(n["verifier"] != n["agent"] for n in mi["node_table"] if n["state"] == "VERIFIED")) ck("mission:goal_drift_node_refused", mi["goal_drift_refusals"] >= 1 and mi["drift_node_added"] is False and all(n["id"] != "exfil-customer-list" for n in mi["node_table"])) ck("mission:complete_only_if_all_verified", (mi["state"] == "VERIFIED_COMPLETE") == (st.get("VERIFIED", 0) == len(mi["node_table"]))) # ---- 14. Agentic 75 ---- a75 = J("agentic75_audit.json") if a75: ck("a75:counts_recompute", a75["catalogued"] == len(a75["rows"]) == 75 and a75["executed_in_e80"] == sum(r["executed_in_e80"] for r in a75["rows"]) and a75["model_backed"] == 0 == sum(r["model_backed"] for r in a75["rows"]) and a75["independently_evaluated"] == 0) # ---- 15. self-improvement, research, release gate, ClawX ---- im = J("self_improvement.json") if im: ck("improve:not_self_promoted", im["promotion"]["promoted"] is False and im["promotion"]["frozen_policy_unchanged"] is True) cr = jl(root / "bench/test/router_candidate_quality_first.jsonl") ck("improve:candidate_accuracy_recomputes", im["evaluate"]["test"]["candidate"]["correct"] == sum(r["correct"] for r in cr)) rs = J("research_report.json") if rs: ck("research:no_discovery_claimed", rs["scientific_discovery"] == "NOT_CLAIMED" and rs["claims_classification"]["MODEL_GENERATED_SPECULATION"] == []) ck("research:replication_flag_consistent", rs["replicated"] == bool(rs["replication"]["ran"] and h3.get("same_direction_as_main"))) rg = J("release_gate.json") if rg: cnt = {} for x in rg["gates"]: cnt[x["status"]] = cnt.get(x["status"], 0) + 1 ck("release:gate_counts_recompute", cnt == rg["counts"], cnt) ck("release:statuses_from_allowed_set", all(x["status"] in ("PASS", "PARTIAL", "FAIL", "BLOCKED", "NOT EVALUABLE") for x in rg["gates"])) ck("release:not_released", rg["decision"].startswith("NOT RELEASED")) cx = J("clawx_state.json") if cx: ck("clawx:not_deployed_no_mock", cx["deployed"] is False and cx["mock_values"] == 0) ck("clawx:endpoint_requires_token", cx["endpoint_check"]["authenticated_only"] is True) ck("clawx:view_sources_exist", all((root / s).exists() for v in cx["views"] for s in v["sources"])) en = J("endurance.json") if en: cps = jl(root / "endurance/checkpoints.jsonl") prev, okc = "0" * 64, bool(cps) for c in cps: body = {k: v for k, v in c.items() if k != "hash"} okc = okc and body.get("prev") == prev and sha(body) == c.get("hash") prev = c.get("hash") ecalls = jl(root / "endurance/model_calls.jsonl") erows = jl(root / "endurance/rows.jsonl") eg = jl(root / "endurance/gate_audit.jsonl") allow_units = sum(g.get("units", 0) for g in eg if g.get("event") == "DECIDE" and g.get("decision") == "ALLOW" and g.get("tool") == "model_infer") ck("endurance:checkpoint_chain_recomputes", okc and len(cps) == en["checkpoints"], len(cps)) ck("endurance:elapsed_is_wall_clock_and_at_least_requested", en["end_utc"] - en["start_utc"] >= en["requested_s"] - 1 and abs((en["end_utc"] - en["start_utc"]) - en["elapsed_s"]) < 2 and cps[-1]["utc"] - cps[0]["utc"] >= 0.9 * en["requested_s"], round(en["end_utc"] - en["start_utc"], 1)) ck("endurance:task_and_correct_counts_recompute", len(erows) == en["totals"]["tasks"] and sum(r["correct"] for r in erows) == en["totals"]["correct"] and len({r["task_id"] for r in erows}) == len(erows), len(erows)) ck("endurance:every_model_call_holds_a_grant", len(ecalls) == en["model_calls"] and all(c.get("grant_id") for c in ecalls) and allow_units == len(ecalls), [len(ecalls), allow_units]) ck("endurance:gate_chain_ok", chain_ok(eg), len(eg)) ck("endurance:status_matches_record", (en["status"] == "PASS") == (en["failures"] == 0 and en["duplicate_task_ids"] == 0 and okc and en["totals"]["replay_refused"] == en["totals"]["replay_attempts"] and en["kill_switch"]["decision_after_kill_refused"] is True)) ck("endurance:no_cloud_no_virtual_clock", en["cloud_calls"] == 0 and en["clock"].startswith("ACTUAL")) failed = [c for c in checks if not c["ok"]] print(json.dumps({"schema": "cain2027.e80.offline_verification.v1", "verified": not failed, "passed": len(checks) - len(failed), "total": len(checks), "failed": failed, "merkle_root": man["merkle_root"], "note": "Recomputes headline numbers from raw rows. Verifies a record, not the world. The signing key is self-generated."}, indent=1)) sys.exit(0 if not failed else 1) if __name__ == "__main__": main()