diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 4be925a..0b978be 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -239,7 +239,7 @@ | task | RPF-WP-0070-T01 | done | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T02 | done | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T03 | done | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | -| task | RPF-WP-0070-T04 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | +| task | RPF-WP-0070-T04 | progress | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T05 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | intake | RPF-IN-0001 | closed | — | intakes/intakes.md | | intake | RPF-IN-0002 | open | — | intakes/intakes.md | diff --git a/scripts/mas_evidence_restore_rehearsal.py b/scripts/mas_evidence_restore_rehearsal.py new file mode 100644 index 0000000..0a748f8 --- /dev/null +++ b/scripts/mas_evidence_restore_rehearsal.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Rehearse MAS evidence recovery from the offsite copy on the workstation. + +Run only inside the attended OpenBao platform-admin login: + + warden access openbao-platform-admin-login --exec -- \ + python3 scripts/mas_evidence_restore_rehearsal.py --identity /abs/path/to/identity + +Independent credentials: the founder-held supplier key on the MAS IAM +application, not the node's workload pair. The object is fetched at the exact +version named in the T03 evidence receipt and checked against its hash. +The identity must derive to the recorded recipient before it is used. The +packet is decrypted into a private scratch directory, every file is checked +against the packet's own hashes, SQLite integrity is checked, and every event +line must parse. Nothing is restored onto the facility, no receiver is +enabled, and no mail is sent. Plaintext scratch files are removed after the +checks. Stdout and stderr stay silent; the receipt records names, counts, +hashes and timings only. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import shutil +import subprocess +import sys +import tarfile +import time +from datetime import UTC, datetime +from pathlib import Path + +ROOT = Path("/home/worsch/railiance-platform") +EVIDENCE = ROOT / "docs/evidence/mas-evidence-first-offsite-2026-11-03.json" +RECIPIENT = (ROOT / "deploy/mas-backup/age-recipient").read_text().strip() if ( + ROOT / "deploy/mas-backup/age-recipient").exists() else "" +RECEIPT_DIR = Path("/home/worsch/.local/state/railiance/receipts") +SCRATCH_ROOT = Path("/home/worsch/.local/state/railiance/mas-restore-rehearsal") +RECEIPT_NAME = "mas-evidence-offsite-rehearsal.json" + + +def _load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +sys.path.insert(0, str(ROOT / "scripts")) +gate = _load("mas_evidence_publisher_gate", ROOT / "scripts/mas_evidence_publisher_gate.py") +archive = _load("mas_evidence_archive", ROOT / "scripts/mas_evidence_archive.py") +primary = _load("primary_application_archive", ROOT / "scripts/primary_application_archive.py") + + +class RehearsalError(RuntimeError): + def __init__(self, code, detail=None): + super().__init__(code) + self.code, self.detail = code, detail + + +def identity_matches(identity, recipient=None): + recipient = recipient or RECIPIENT + path = Path(identity) + if not path.is_absolute() or not path.is_file() or path.stat().st_mode & 0o077: + raise RehearsalError("private_absolute_identity_required") + result = subprocess.run(["age-keygen", "-y", str(path)], capture_output=True, timeout=30) + public = result.stdout.decode().strip().splitlines() + if result.returncode or not recipient or public[:1] != [recipient]: + raise RehearsalError("identity_recipient_mismatch") + return True + + +def offsite_receipt(path=EVIDENCE): + body = json.loads(path.read_text(encoding="utf-8")) + receipt = dict(body["receipt"]) + if receipt.get("schema") != "platform.application-primary-archive.v1": + raise RehearsalError("offsite_receipt_rejected") + return receipt + + +def verify_restore(encrypted, identity, scratch): + """Decrypt, extract, and check. Returns counts and hashes; never values.""" + scratch.mkdir(mode=0o700, parents=False) + packet = scratch / "packet.tar" + result = subprocess.run( + ["age", "-d", "-i", str(identity), "-o", str(packet), str(encrypted)], + capture_output=True, timeout=120, + ) + if result.returncode: + raise RehearsalError("decryption_failed") + os.chmod(packet, 0o600) + with tarfile.open(packet, "r:") as tar: + entries = tar.getmembers() + if "receipt.json" not in {e.name for e in entries} or any( + not e.isfile() or "/" in e.name or e.name.startswith(".") for e in entries + ): + raise RehearsalError("invalid_packet_members") + for entry in entries: + target = scratch / entry.name + with tar.extractfile(entry) as source, target.open("xb") as out: + os.chmod(target, 0o600) + out.write(source.read()) + inner = json.loads((scratch / "receipt.json").read_text()) + if inner.get("application") != "mas" or inner.get("mail_sent") is not False: + raise RehearsalError("capture_receipt_rejected") + files = {} + for name, meta in inner["files"].items(): + path = scratch / name + if not meta.get("present"): + files[name] = {"present": False, "restored": path.exists()} + continue + if archive.digest(path) != meta["sha256"] or path.stat().st_size != meta["bytes"]: + raise RehearsalError("restored_hash_mismatch", {"file": name}) + entry = {"present": True, "bytes": meta["bytes"], "sha256_matches_capture": True} + if name.endswith(".sqlite3"): + fingerprint = archive._sqlite_fingerprint(path) # integrity_check must be ok + entry["integrity_check"] = "ok" + entry["tables"] = {t: v["rows"] for t, v in fingerprint["tables"].items()} + if "tables" in meta: + entry["fingerprint_matches_capture"] = fingerprint["tables"] == meta["tables"] + else: + count = 0 + with path.open("rb") as stream: + for line in stream: + if line.strip(): + json.loads(line) + count += 1 + entry["events"] = count + files[name] = entry + return {"captured_at": inner.get("captured_at"), "image": inner.get("image"), "files": files} + + +def rehearse(identity, client, receipt, scratch_root=SCRATCH_ROOT, clock=time.monotonic): + started = clock() + started_at = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + identity_matches(identity) + scratch_root.mkdir(mode=0o700, parents=True, exist_ok=True) + run = scratch_root / started_at.replace(":", "") + run.mkdir(mode=0o700) + try: + encrypted = run / "fetched.tar.age" + fetched = primary.retrieve(client, "mas", encrypted, receipt) + fetched_seconds = round(clock() - started, 2) + restored = verify_restore(encrypted, identity, run / "restored") + finally: + shutil.rmtree(run, ignore_errors=True) + elapsed = round(clock() - started, 2) + sqlite_ok = all( + meta.get("integrity_check") == "ok" and meta.get("fingerprint_matches_capture", True) + for name, meta in restored["files"].items() if name.endswith(".sqlite3") and meta.get("present") + ) + return { + "schema": "platform.mas-evidence-offsite-rehearsal.v1", + "status": "passed" if sqlite_ok else "failed", + "started_at": started_at, + "object_key": fetched["object_key"], + "version_id": fetched["version_id"], + "ciphertext_sha256": fetched["ciphertext_sha256"], + "ciphertext_bytes": fetched["ciphertext_bytes"], + "fetch_verified": fetched["status"] == "retrieved_verified", + "credential": "supplier key on IAM application 25a4cbca (independent of the node workload pair)", + "identity_matches_recipient": True, + "restored": restored, + "fetch_seconds": fetched_seconds, + "elapsed_seconds": elapsed, + "rto_minutes_provisional": 60, + "within_rto": elapsed <= 3600, + "scratch_removed": not run.exists(), + "production_restored": False, + "receiver_enabled": False, + "mail_sent": False, + "provider_mutations": False, + "values_recorded": False, + } + + +def _supplier_client(): + pair = gate.supplier_pair(gate._kv_data(gate.SUPPLIER), gate.SUPPLIER) + scoped, _ = gate.scoped_supplier_pair(pair, gate.PROJECT_ID) + return gate._client(scoped) + + +def main(argv=None): + args = list(sys.argv[1:] if argv is None else argv) + sys.stdout = open(os.devnull, "w") + sys.stderr = open(os.devnull, "w") + path = RECEIPT_DIR / RECEIPT_NAME + try: + if len(args) != 2 or args[0] != "--identity": + raise RehearsalError("usage_refused") + gate._ensure_boto() + RECEIPT_DIR.mkdir(mode=0o700, exist_ok=True) + result = rehearse(args[1], _supplier_client(), offsite_receipt()) + except (RehearsalError, gate.GateError) as exc: + result = {"status": "failed", "code": exc.code, "values_recorded": False} + if exc.detail: + result["detail"] = exc.detail + except Exception as exc: + result = {"status": "failed", "code": "contained_rehearsal_failure", + "error_type": type(exc).__name__, "values_recorded": False} + gate._write_json(path, result) + return 0 if result.get("status") == "passed" else 71 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_mas_evidence_restore_rehearsal.py b/tests/test_mas_evidence_restore_rehearsal.py new file mode 100644 index 0000000..f067fd2 --- /dev/null +++ b/tests/test_mas_evidence_restore_rehearsal.py @@ -0,0 +1,99 @@ +"""Offsite restore rehearsal with a real age key and a synthetic provider.""" + +import hashlib +import importlib.util +import io +import json +import shutil +import sqlite3 +import subprocess +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.skipif(not shutil.which("age"), reason="age required") +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("rehearsal", ROOT / "scripts/mas_evidence_restore_rehearsal.py") +rh = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rh) +KEY = "platform-pg/application-archives/mas/2026/11/03/20261103T064731Z-45cf967d431d7f72143bb0c7.tar.age" + + +class Provider: + def __init__(self, data): + self.data, self.calls = data, [] + + def get_object(self, Bucket, Key, VersionId): + self.calls.append((Key, VersionId)) + return {"ContentLength": len(self.data), "VersionId": VersionId, "Body": io.BytesIO(self.data)} + + +@pytest.fixture +def packet(tmp_path): + identity = tmp_path / "id.txt" + subprocess.check_call(["age-keygen", "-o", str(identity)], stderr=subprocess.DEVNULL) + identity.chmod(0o600) + recipient = subprocess.check_output(["age-keygen", "-y", str(identity)], text=True).strip() + source = tmp_path / "volume" + source.mkdir() + (source / "events.jsonl").write_text('{"e":1}\n{"e":2}\n{"e":3}\n') + for name in ("tickets.sqlite3", "tickets-outbox.sqlite3"): + with sqlite3.connect(source / name) as db: + db.execute("create table t(x)") + db.executemany("insert into t values (?)", [(1,), (2,)]) + run = tmp_path / "run" + rh.archive.capture(source, run, captured_at="2026-11-03T06:47:31Z", image="img", + writers_quiesced=True, replicas={"mas-mx": 1, "mas-statute": 1}) + sealed = rh.archive.seal(run, run, recipient) + data = (run / "packet.tar.age").read_bytes() + receipt = {"schema": "platform.application-primary-archive.v1", "application": "mas", + "status": "offsite_fetched_verified", "remote_retrieval_verified": True, + "destination": "s3://railiance-platform-pg-backup/" + KEY, "object_key": KEY, + "version_id": "v1", "ciphertext_sha256": sealed["ciphertext_sha256"], + "ciphertext_bytes": sealed["ciphertext_bytes"]} + return identity, recipient, data, receipt + + +def test_rehearsal_restores_verifies_and_removes_plaintext(packet, tmp_path, monkeypatch): + identity, recipient, data, receipt = packet + monkeypatch.setattr(rh, "RECIPIENT", recipient) + provider = Provider(data) + scratch = tmp_path / "scratch" + result = rh.rehearse(str(identity), provider, receipt, scratch) + assert result["status"] == "passed" and result["fetch_verified"] and result["within_rto"] + assert provider.calls == [(KEY, "v1")] + files = result["restored"]["files"] + assert files["events.jsonl"]["events"] == 3 + assert files["tickets.sqlite3"]["integrity_check"] == "ok" + assert files["tickets.sqlite3"]["tables"] == {"t": 2} + assert files["tickets.sqlite3"]["fingerprint_matches_capture"] is True + assert result["scratch_removed"] and list(scratch.iterdir()) == [] + + +def test_wrong_identity_refuses_before_fetch(packet, tmp_path, monkeypatch): + identity, recipient, data, receipt = packet + monkeypatch.setattr(rh, "RECIPIENT", "age1someoneelse") + provider = Provider(data) + with pytest.raises(rh.RehearsalError) as caught: + rh.rehearse(str(identity), provider, receipt, tmp_path / "s") + assert caught.value.code == "identity_recipient_mismatch" and provider.calls == [] + + +def test_tampered_ciphertext_fails_fetch(packet, tmp_path, monkeypatch): + identity, recipient, data, receipt = packet + monkeypatch.setattr(rh, "RECIPIENT", recipient) + with pytest.raises(ValueError): + rh.rehearse(str(identity), Provider(data[:-1] + b"X"), receipt, tmp_path / "s") + assert list((tmp_path / "s").iterdir()) == [] + + +def test_identity_file_must_be_private(packet, monkeypatch): + identity, recipient, _, _ = packet + identity.chmod(0o644) + with pytest.raises(rh.RehearsalError): + rh.identity_matches(str(identity), recipient) + + +def test_evidence_receipt_loads_the_committed_t03_object(): + receipt = rh.offsite_receipt() + assert receipt["object_key"] == KEY and receipt["version_id"] == "1793688453584021" diff --git a/workplans/RPF-WP-0070-mas-evidence-recovery.md b/workplans/RPF-WP-0070-mas-evidence-recovery.md index 2feb853..9ef47f9 100644 --- a/workplans/RPF-WP-0070-mas-evidence-recovery.md +++ b/workplans/RPF-WP-0070-mas-evidence-recovery.md @@ -245,7 +245,7 @@ hourly freshness timer. T03 is done. Receipt: ```task id: RPF-WP-0070-T04 -status: todo +status: progress priority: medium depends_on: [RPF-WP-0070-T03] state_hub_task_id: "9e6987a5-9cbc-5d73-bd2e-9a4e1e63ecdc" @@ -257,6 +257,18 @@ evidence files into an isolated scratch location. Verify SQLite integrity and the event count. Do not restore the facility, enable a receiver, or send or resend mail. Record the timed receipt under `docs/evidence/`. +2026-11-03: `scripts/mas_evidence_restore_rehearsal.py` runs inside the +attended platform-admin login. It uses the founder-held supplier key, which +is independent of the node workload pair. It fetches the exact T03 version +and checks its hash, and accepts the identity file only if it derives to +`age17sxu…xdxs05w`. It decrypts into a private scratch directory and checks +every file against the packet's capture hashes. It runs SQLite +`integrity_check` and compares table fingerprints, and every event line +must parse. It times the run against the provisional 60-minute RTO, then +removes the plaintext. The workstation key `~/.config/age/railiance-backup.key` +derives to a different recipient (`age1zvry…`), so the founder names the +MAS identity path for the run. + ## Escrow the age identity and get owner acceptance ```task