{
  "schema": "cain42.cluster.restore.v1",
  "generated_at": "2026-11-16T11:52:00Z",
  "evidence_class": "SELF_OBSERVED_SAME_ADMIN_DOMAIN",
  "not_claimed": [
    "third-party audit",
    "certification",
    "SOC 2",
    "48-hour soak passed",
    "independent witness",
    "root cause of the WAL disk I/O error"
  ],
  "clusters": [
    {
      "cluster_id": "cain-mr-03",
      "role": "live",
      "n": 4,
      "f": 1,
      "quorum": 3,
      "regions": [
        "atl",
        "lax",
        "mia",
        "sjc"
      ],
      "before": {
        "replicas_healthy": 3,
        "replicas_expected": 4,
        "down_replica": "cain-mr3-node-1",
        "exit_code": 3,
        "exit_error": "[Errno 98] error while attempting to bind on address ('10.88.0.1', 29811): address already in use",
        "quorum_still_met": true
      },
      "after": {
        "replicas_healthy": 4,
        "same_commit_index": true,
        "same_application_state_hash": true,
        "write_probe_qc": "VALID",
        "write_probe_signers": 4,
        "gateway_verdict": "HEALTHY"
      }
    },
    {
      "cluster_id": "cain-mr-01",
      "role": "monitored_second_cluster",
      "n": 4,
      "f": 1,
      "quorum": 3,
      "before": {
        "replicas_healthy": 3,
        "replicas_expected": 4,
        "safe_mode_replica": "cain-mr-node-4",
        "safe_mode_reason": "wal_write_failure: sequence 143035: OperationalError: disk I/O error",
        "commit_index_spread": {
          "node-1": 143271,
          "node-2": 143230,
          "node-3": 143230,
          "node-4": 143035
        },
        "write_probe_status": "COMMITTED",
        "write_probe_qc": "INVALID: INSUFFICIENT_QUORUM",
        "write_probe_signers": 1,
        "quorum_size": 3,
        "gateway_verdict": "DEGRADED"
      },
      "after": {
        "replicas_healthy": 4,
        "commit_index_spread": {
          "node-1": 143278,
          "node-2": 143278,
          "node-3": 143278,
          "node-4": 143278
        },
        "write_probe_qc": "VALID",
        "write_probe_signers": 4,
        "gateway_verdict": "HEALTHY"
      }
    },
    {
      "cluster_id": "cain-mr-02",
      "role": "retired_quarantined_2026-11-15",
      "after": {
        "running": false,
        "container_preserved": true,
        "gateway_verdict": "HALTED",
        "replicas_reachable": 0
      }
    }
  ],
  "findings": [
    {
      "id": "F1",
      "severity": "HIGH",
      "title": "The Moat #7 guardian revived a retired, quarantined replica, which stole the live replica's overlay address",
      "detail": "cain-mr2-node-1 was in the sentinel's required-container list. It was restarted at 2026-11-16T10:54:29Z and bound 10.88.0.1:29811, which cain-mr-03 assigns to cain-mr3-node-1. cain-mr3-node-1 then exited with EADDRINUSE (code 3) at 11:04:17Z.",
      "evidence": [
        "pre-fix/sentinel-mr2-restarts.log",
        "pre-fix/mr3-node1-state.json",
        "pre-fix/mr3-node1.log"
      ]
    },
    {
      "id": "F2",
      "severity": "CRITICAL",
      "title": "A replica reported COMMITTED while its quorum certificate was INVALID with one signer",
      "detail": "cain-mr-01/node-1 returned status=COMMITTED, decision=ALLOW with quorum_certificate_status=INVALID: INSUFFICIENT_QUORUM and 1 accepted signer against a quorum of 3. The response exposed this only in an adjacent string field.",
      "evidence": [
        "mr01/invalid-quorum-response.json"
      ]
    },
    {
      "id": "F3",
      "severity": "HIGH",
      "title": "cain-mr-01 node-4 was in SAFE_MODE and two replicas were 41-236 sequences behind",
      "detail": "node-4 recorded a WAL disk I/O error at sequence 143035 and never left SAFE_MODE (no restart since 2026-11-03). nodes 2 and 3 stopped at 143230.",
      "evidence": [
        "mr01/node-4-status.json",
        "mr01/node-2-status.json"
      ]
    },
    {
      "id": "F4",
      "severity": "HIGH",
      "title": "The gateway attributed the live cluster's replicas to the retired cluster",
      "detail": "GET /api/v1/live-cluster/health?cluster=cain-mr-02 returned '3 of 4 replicas reachable, height 11468' because it counted whatever answered on 10.88.0.x. Those were cain-mr-03 replicas. A quarantined cluster was being reported as a healthy live cluster.",
      "evidence": [
        "pre-fix/mr02-mislabel-observation.txt",
        "post-fix/gateway-health-mr02.json"
      ]
    },
    {
      "id": "F5",
      "severity": "MEDIUM",
      "title": "The soak verdict ignored check_errors, so a run with a replica down could still exit 0",
      "detail": "multi_region_soak.py now returns a verdict() from (divergences, max_unexpected_check_streak, final cluster health/height, disk floor) instead of `0 if divergences == 0`. Expected fault-window errors are counted separately from unexpected ones. Tests: tests/test_soak_verdict.py 8/8.",
      "evidence": [
        "pre-fix/soak-tail.log",
        "INVALID-RUN marker in ../../soak-multiregion-48h-2026-11-15-mr03b/",
        "followups/soak-verdict-tests.txt"
      ],
      "status": "FIXED 2026-11-16"
    }
  ],
  "changes": [
    {
      "file": "deploy/scripts/cain_sentinel_guardian.py",
      "status": "live",
      "summary": "required container cain-mr2-node-1 -> cain-mr3-node-1; RETIRED_CONTAINERS refuses to start any cain-mr2-node-*; cluster monitor cain-mr-02 -> cain-mr-03"
    },
    {
      "file": "platform-gateway/routers/live_cluster.py",
      "status": "live",
      "summary": "registers cain-mr-03; a replica whose reported node_id/cluster_id does not match the requested cluster is unreachable, never a healthy member"
    },
    {
      "file": "platform-gateway/cain_pbft_engine_33.py",
      "status": "staged_source_only",
      "summary": "adds outcome_certified and outcome_final (both false when the quorum certificate is invalid); not live until the node image is rebuilt"
    },
    {
      "file": "tests/test_cain_sentinel_guardian.py",
      "status": "live",
      "summary": "expects cluster_cain-mr-03 instead of the retired cluster_cain-mr-02"
    }
  ],
  "soak": {
    "previous_run": "soak-multiregion-48h-2026-11-15-mr03b",
    "previous_verdict": "INVALID \u2014 stopped after 18 h with a replica down; not a 48-hour run",
    "new_run": "soak-multiregion-48h-2026-11-16-mr03d",
    "new_run_started": "2026-11-16T12:05:15Z",
    "checkpoint_0000": "VALID, 48 certificates verified",
    "elapsed_hours_at_capture": 0.009,
    "planned_hours": 48,
    "verdict": "RUNNING \u2014 not a passed soak",
    "superseded_runs": [
      "soak-multiregion-48h-2026-11-15-mr03b (INVALID, 18 h, replica down)",
      "soak-multiregion-48h-2026-11-16-mr03c (SUPERSEDED at 0.28 h, old verdict logic)"
    ]
  },
  "team": [
    "Vincent Noriega (founder)",
    "Lena Noriega",
    "Monica Diaz"
  ],
  "followups": {
    "soak_verdict": "fixed \u2014 verdict() fails on a sustained unexpected outage, unhealthy/split final state, or a host below --min-free-gb-run",
    "disk": "lax 45.76.169.191 reclaimed 9.23 GB -> 14.94 GB (+5.73 GB) by removing four clean reproducible verification clones; quarantined cain-mr-02 data (20.5 GB) untouched",
    "evidence": "followups/"
  }
}