# Cluster restore 2026-11-16 — commands run, in order (UTC unless noted) # Diagnosis docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.Image}}' | grep -E 'mr2|mr3|mr-node' docker inspect cain-mr3-node-1 --format '{{json .State}}' docker logs --tail 200 cain-mr3-node-1 journalctl -u cain-sentinel-guardian.service --since '2026-11-16 10:40' tail -60 /root/cain-mr-soak-48h-2026-11-15-mr03b/soak.log for n in 1 2 3 4; do curl -s http://10.88.0.$n:$((29810+n))/api/v1/cluster/pbft/status; done # Containment (11:35Z) systemctl stop cain-mr03-soak-48h-2026-11-15b.service docker stop cain-mr2-node-1 # Guardian fix (11:36Z) — cain-mr2-node-1 was required and auto-healed; now retired # deploy/scripts/cain_sentinel_guardian.py # RETIRED_CONTAINERS = {cain-mr2-node-1..4}; required = [cain-traefik, cain-mr3-node-1, cain-mr-node-1, cain-opa] # check_consensus_clusters: cain-mr-02 -> cain-mr-03 systemctl restart cain-sentinel-guardian.service # Live replica restore (11:38Z) docker start cain-mr3-node-1 # gateway: register cain-mr-03 + identity check in routers/live_cluster.py systemctl restart cain-platform-gateway.service # ready in ~138 s # cain-mr-01 repair (11:43-11:46Z) curl -s -X POST http://10.42.0.1:29801/api/v1/cluster/pbft/request -H 'content-type: application/json' \ -d '{"client_id":"ops-check","request_id":"mr01-liveness-2026-11-16","operation":{"action":"write","resource":"ops/0","data":{}}}' # -> COMMITTED but quorum_certificate_status INVALID: INSUFFICIENT_QUORUM, accepted_signers 1 ssh -i ~/.ssh/cain42_cluster root@207.246.66.130 'docker restart cain-mr-node-4' # -> node-4: HEALTHY, WAL ok, commit 143035 -> 143275 curl -s -X POST http://10.42.0.1:29801/api/v1/cluster/pbft/request ... # x3 -> VALID, 4 signers # New clean soak (11:47Z first attempt failed: transient unit had no working directory) systemd-run --unit=cain-mr03-soak-48h-2026-11-16c --collect \ --working-directory=/root/cain-studio-platform \ /usr/bin/python3 /root/cain-studio-platform/scripts/cain42_bft/multi_region_soak.py \ --hours 48 --interval 6 --fault-every 1200 --publish-every 3600 --min-free-gb 8 \ --name soak-multiregion-48h-2026-11-16-mr03c \ --work /root/cain-mr-soak-48h-2026-11-16-mr03c \ --record /root/cain-studio-platform/deploy/multi-region/cain_mr03_deploy_record.json # Verification curl -s 'http://127.0.0.1:8088/api/v1/live-cluster/health?cluster=cain-mr-03' curl -s 'http://127.0.0.1:8088/api/v1/live-cluster/health?cluster=cain-mr-01' curl -s 'http://127.0.0.1:8088/api/v1/live-cluster/health?cluster=cain-mr-02' python3 /root/cain-studio-platform/scripts/cain42_bft/verify_soak.py \ /root/cain-studio-platform/platform-gateway/frontend/proof/bundle/soak-multiregion-48h-2026-11-16-mr03c