#!/usr/bin/env python3
"""Offline verifier for the VAGI Behavior Suite model baseline (2026-11-18).

Usage: python3 -I verify_offline.py <bundle-root>

It recomputes, from the raw rows and the published suite generator:
  * task identity and the Fable stratified selection,
  * every score with the published suite checker,
  * overall and per-domain counts, Wilson intervals, invalid-JSON counts,
  * token and USD cost from the published list prices,
  * that no prompt in model_tasks.json contains `fixes`, `hidden`, `bad` or a reference answer,
  * that the prompt hash in each result matches the prompt builder.

It imports only the suite source bundled next to it. It does not import CAIN.
This is an offline recomputation with the *published* checker, not an independent
reimplementation and not a third-party audit.
"""
from __future__ import annotations

import hashlib
import json
import math
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parent
sys.path.insert(0, str(ROOT))

from vagi_suite import suite  # noqa: E402
from vagi_suite import model_arm  # noqa: E402

CHECKS: list[tuple[str, bool, str]] = []


def check(name: str, ok: bool, detail: str = "") -> None:
    CHECKS.append((name, bool(ok), detail))


def jl(p: Path) -> list[dict]:
    return [json.loads(x) for x in p.read_text().splitlines() if x.strip()]


def sha(o) -> str:
    return hashlib.sha256(json.dumps(o, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()).hexdigest()


def close(a, b, tol=1e-9) -> bool:
    return abs(float(a) - float(b)) <= tol


def wilson(k: int, n: int, z: float = 1.959964) -> list[float]:
    if n == 0:
        return [0.0, 0.0]
    p = k / n
    d = 1 + z * z / n
    c = (p + z * z / (2 * n)) / d
    h = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
    return [round(c - h, 4), round(c + h, 4)]


def main() -> int:
    root = Path(sys.argv[1] if len(sys.argv) > 1 else ".")
    summary = json.loads((root / "SUMMARY.json").read_text())
    tasks = {t["task_id"]: t for t in suite.make_tasks(9402, 50)}
    check("suite has 500 held-out tasks", len(tasks) == 500, str(len(tasks)))
    total_cost = 0.0
    total_rows = 0
    for arm in summary["arms"]:
        sub = root / arm["dir"]
        rows = jl(sub / "rows.jsonl")
        res = json.loads((sub / "results.json").read_text())
        check(f"{arm['id']}: rows = {len(rows)}", len(rows) == res["n_tasks"], str(len(rows)))
        check(f"{arm['id']}: model id recorded", res["model"] == arm["model"], res["model"])
        if arm["id"].startswith("fable"):
            expected = model_arm.select_tasks(10, 9411)
            check(f"{arm['id']}: stratified selection reproducible",
                  sorted(t["task_id"] for t in expected) == sorted(r["task_id"] for r in rows))
            check(f"{arm['id']}: 10 per domain",
                  all(sum(1 for r in rows if r["domain"] == d) == 10 for d in suite.DOMAINS))
        # no answer fields in the model view
        mt = json.loads((sub / "model_tasks.json").read_text())
        blob = json.dumps(mt)
        for field in ("fixes", "hidden", '"bad"', "reference"):
            check(f"{arm['id']}: prompt view has no {field}", field not in blob)
        # prompt hash
        prompt_tasks = [tasks[t["task_id"]] for t in mt]
        check(f"{arm['id']}: prompt hash matches", sha([model_arm.prompt_for(t) for t in prompt_tasks])
              == res["protocol"]["prompt_sha256"])
        # re-score every row
        bad = 0
        for r in rows:
            t = dict(tasks[r["task_id"]])
            if r.get("skipped"):
                continue
            ans = r.get("answer")
            if "INVALID" in str(r.get("reason")):
                ok = False
            elif t["domain"] == "software_repair":
                ok = bool(isinstance(ans, str) and ans.strip()) and not model_arm.static_screen(ans) \
                    and bool(suite.check(t["domain"], t, ans))
            elif t["domain"] == "planning":
                ok = isinstance(ans, dict) and isinstance(ans.get("order"), list) \
                    and isinstance(ans.get("makespan"), (int, float)) \
                    and not isinstance(ans.get("makespan"), bool) \
                    and bool(suite.check(t["domain"], t, (ans["order"], int(ans["makespan"]))))
            else:
                ok = bool(suite.check(t["domain"], t, ans))
            if ok != bool(r["accepted"]):
                bad += 1
        check(f"{arm['id']}: every row re-scores identically", bad == 0, f"mismatches={bad}")
        # counts
        done = [r for r in rows if not r.get("skipped")]
        k = sum(1 for r in done if r["accepted"])
        check(f"{arm['id']}: accepted count", k == res["accepted"], f"{k}/{len(done)}")
        check(f"{arm['id']}: Wilson interval", wilson(k, len(done)) == res["wilson95"])
        for d in suite.DOMAINS:
            got = sum(1 for r in done if r["domain"] == d and r["accepted"])
            n = sum(1 for r in done if r["domain"] == d)
            check(f"{arm['id']}: {d} = {got}/{n}", got == res["per_domain"][d]["accepted"])
        # cost
        pin, pout = model_arm.PRICES[res["model"]]
        cost = sum(int(r.get("input_tokens") or 0) / 1e6 * pin + int(r.get("output_tokens") or 0) / 1e6 * pout
                   for r in done)
        check(f"{arm['id']}: cost arithmetic", close(cost, res["cost"]["total_usd"], 1e-6),
              f"{cost:.6f} vs {res['cost']['total_usd']}")
        check(f"{arm['id']}: under budget cap", res["cost"]["total_usd"] <= res["protocol"]["budget_cap_usd"],
              str(res["cost"]["total_usd"]))
        total_cost += cost
        total_rows += len(done)
    check("total spend matches summary", close(total_cost, summary["total_cost_usd"], 1e-6),
          f"{total_cost:.6f} vs {summary['total_cost_usd']}")
    check("no task was scored twice", total_rows == sum(a["n_tasks"] for a in summary["arms"]), str(total_rows))
    passed = sum(1 for _, ok, _ in CHECKS if ok)
    for name, ok, detail in CHECKS:
        print(("PASS " if ok else "FAIL ") + name + ((" :: " + detail) if detail and not ok else ""))
    print(f"\n{passed}/{len(CHECKS)} checks passed")
    return 0 if passed == len(CHECKS) else 1


if __name__ == "__main__":
    raise SystemExit(main())
