diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index c6e3903..caf711f 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -49,7 +49,7 @@ | workplan | RPF-WP-0067 | active | — | workplans/RPF-WP-0067-netkingdom-identity-receipt-and-metadata-answers.md | | workplan | RPF-WP-0068 | finished | — | workplans/RPF-WP-0068-email-connect-platform-smtp.md | | workplan | RPF-WP-0069 | active | — | workplans/RPF-WP-0069-bounded-executor-lanes.md | -| workplan | RPF-WP-0070 | ready | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | +| workplan | RPF-WP-0070 | active | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-ADHOC-2026-10-10-T01 | done | — | workplans/ADHOC-2026-10-10.md | | task | RPF-WP-ADHOC-2026-10-11-T01 | done | — | workplans/ADHOC-2026-10-11.md | | task | RPF-WP-ADHOC-2026-10-11-T02 | done | — | workplans/ADHOC-2026-10-11.md | @@ -236,7 +236,7 @@ | task | RPF-WP-0069-T07 | todo | — | workplans/RPF-WP-0069-bounded-executor-lanes.md | | task | RPF-WP-0069-T08 | done | — | workplans/RPF-WP-0069-bounded-executor-lanes.md | | task | RPF-WP-0069-T09 | wait | — | workplans/RPF-WP-0069-bounded-executor-lanes.md | -| task | RPF-WP-0070-T01 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | +| task | RPF-WP-0070-T01 | progress | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | intake | RPF-IN-0001 | closed | — | intakes/intakes.md | | intake | RPF-IN-0002 | open | — | intakes/intakes.md | | intake | RPF-IN-0003 | open | — | intakes/intakes.md | diff --git a/docs/evidence/mas-evidence-local-capture-2026-11-02.json b/docs/evidence/mas-evidence-local-capture-2026-11-02.json new file mode 100644 index 0000000..b48fa18 --- /dev/null +++ b/docs/evidence/mas-evidence-local-capture-2026-11-02.json @@ -0,0 +1,39 @@ +{ + "status": "sealed", + "captured_at": "2026-11-02T01:18:19Z", + "image": "docker.io/library/mas-live:v0.2-480e69a", + "volume": "pvc-415825c2-35db-4fc6-b58e-726dbb82c5d3", + "replicas": { + "mas-mx": 1, + "mas-statute": 1 + }, + "facility_scaled": false, + "relay_read": false, + "mail_sent": false, + "files": { + "events.jsonl": { + "present": true, + "bytes": 51382 + }, + "tickets.sqlite3": { + "present": true, + "bytes": 45056 + }, + "tickets-outbox.sqlite3": { + "present": true, + "bytes": 20480 + } + }, + "ciphertext_sha256": "72bef875cdbe577b2733227448619c1914357552ab1de04b1f47ba13fd6c80aa", + "ciphertext_bytes": 133352, + "rehearsal": "rehearsal_matched", + "production_restored": false, + "offsite_published": false, + "age_identity_on_node": false, + "schema": "platform.mas-evidence-local-capture.v1", + "age_recipient": "age17sxuqajpt3vvfep64j5vwlhshrn3g02uhm49par9pmec7q23u3xqdxs05w", + "age_identity_escrowed": false, + "prefix": "platform-pg/application-archives/mas/", + "node_ciphertext": "/var/lib/mas-evidence-backup/20261102T011818Z/packet.tar.age", + "pods_restarted": false +} diff --git a/scripts/mas_evidence_archive.py b/scripts/mas_evidence_archive.py new file mode 100644 index 0000000..f00bdeb --- /dev/null +++ b/scripts/mas_evidence_archive.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""MAS evidence capture and restore rehearsal. No SMTP, no replica changes.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sqlite3 +import stat +import subprocess +import tarfile +import tempfile +from contextlib import closing +from pathlib import Path + +FILES = ("events.jsonl", "tickets.sqlite3", "tickets-outbox.sqlite3") +PREFIX = "platform-pg/application-archives/mas/" +MAX_BYTES = 64 * 1024**2 + + +class Refused(RuntimeError): + pass + + +def digest(path): + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def checkpoint(path, value): + fd, name = tempfile.mkstemp(prefix="receipt-", dir=path.parent) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(name, 0o600) + Path(name).replace(path) + + +def _regular(path): + info = path.lstat() + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + raise Refused("evidence_file_required") + if info.st_size > MAX_BYTES or "mas-relay" in path.parts: + raise Refused("evidence_path_outside_contract") + + +def _sqlite_fingerprint(path): + with closing(sqlite3.connect(path.absolute().as_uri() + "?mode=ro", uri=True)) as database: + database.execute("PRAGMA trusted_schema=OFF") + if database.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + raise Refused("sqlite_integrity_failed") + tables = sorted( + row[0] + for row in database.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + ) + ) + result = {"tables": {}} + for name in tables: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise Refused("unexpected_sqlite_table") + rows = [ + json.dumps(row, separators=(",", ":"), default=lambda value: value.hex()).encode() + for row in database.execute(f'SELECT * FROM "{name}"') + ] + result["tables"][name] = { + "rows": len(rows), + "sha256": hashlib.sha256(b"\n".join(sorted(rows))).hexdigest(), + } + return result + + +def _backup_sqlite(source, output): + _regular(source) + if output.exists() or output.is_symlink(): + raise Refused("capture_destination_exists") + fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + os.close(fd) + with closing(sqlite3.connect(source.absolute().as_uri() + "?mode=ro", uri=True)) as src, closing( + sqlite3.connect(output) + ) as dst: + src.backup(dst, pages=64) + return {"sha256": digest(output), "bytes": output.stat().st_size, **_sqlite_fingerprint(output)} + + +def capture(source, destination, *, captured_at, image, writers_quiesced, replicas): + """Copy the three evidence files. Caller must already have quiesced writers.""" + if writers_quiesced is not True or replicas != {"mas-mx": 1, "mas-statute": 1}: + raise Refused("writers_not_quiesced") + destination.mkdir(mode=0o700, exist_ok=False) + files = {} + for name in FILES: + path = source / name + if name == "tickets-outbox.sqlite3" and not path.exists(): + files[name] = {"present": False} + continue + if name.endswith(".sqlite3"): + files[name] = {"present": True, **_backup_sqlite(path, destination / name)} + else: + _regular(path) + target = destination / name + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "wb") as out, path.open("rb") as src: + while block := src.read(1024**2): + out.write(block) + files[name] = {"present": True, "sha256": digest(target), "bytes": target.stat().st_size} + receipt = { + "status": "captured", + "application": "mas", + "captured_at": captured_at, + "image": image, + "writers_quiesced": True, + "replicas": replicas, + "mail_sent": False, + "files": files, + } + checkpoint(destination / "receipt.json", receipt) + return receipt + + +def seal(directory, stage, recipient): + stage.mkdir(mode=0o700, exist_ok=True) + encrypted = stage / "packet.tar.age" + if encrypted.exists() or encrypted.is_symlink(): + raise Refused("partial_encryption_requires_review") + archive = stage / "packet.tar" + with archive.open("xb") as stream: + os.chmod(archive, 0o600) + with tarfile.open(fileobj=stream, mode="w") as packet: + for name in (*FILES, "receipt.json"): + path = directory / name + if path.exists(): + packet.add(path, arcname=name, recursive=False) + subprocess.run(["age", "-r", recipient, "-o", str(encrypted), str(archive)], check=True, timeout=120) + encrypted.chmod(0o600) + archive.unlink() + return {"ciphertext_sha256": digest(encrypted), "ciphertext_bytes": encrypted.stat().st_size} + + +def rehearse(encrypted, identity, destination): + """Decrypt and compare hashes. Do not restore onto the facility or send mail.""" + if destination.exists() or destination.is_symlink(): + raise Refused("fresh_destination_required") + destination.mkdir(mode=0o700) + packet = destination / "packet.tar" + result = subprocess.run( + ["age", "-d", "-i", str(identity), "-o", str(packet), str(encrypted)], + capture_output=True, + timeout=120, + ) + if result.returncode: + raise Refused("decryption_failed") + os.chmod(packet, 0o600) + with tarfile.open(packet, "r:") as archive: + entries = archive.getmembers() + names = {entry.name for entry in entries} + if ( + "receipt.json" not in names + or any(not entry.isfile() or "/" in entry.name or entry.name.startswith(".") for entry in entries) + ): + raise Refused("invalid_packet_members") + for entry in entries: + target = destination / entry.name + with archive.extractfile(entry) as source, target.open("xb") as out: + os.chmod(target, 0o600) + out.write(source.read()) + receipt = json.loads((destination / "receipt.json").read_text()) + if receipt.get("application") != "mas" or receipt.get("mail_sent") is not False: + raise Refused("capture_receipt_rejected") + for name, meta in receipt["files"].items(): + path = destination / name + if not meta.get("present"): + if path.exists(): + raise Refused("unexpected_absent_file") + continue + if digest(path) != meta["sha256"] or path.stat().st_size != meta["bytes"]: + raise Refused("rehearsal_mismatch") + path.unlink() + packet.unlink() + (destination / "receipt.json").unlink() + proof = { + "status": "rehearsal_matched", + "application": "mas", + "ciphertext_sha256": digest(encrypted), + "mail_sent": False, + "production_restored": False, + "facility_scaled": False, + } + checkpoint(destination / "rehearsal.json", proof) + return proof diff --git a/scripts/mas_evidence_capture.py b/scripts/mas_evidence_capture.py new file mode 100644 index 0000000..c75647a --- /dev/null +++ b/scripts/mas_evidence_capture.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Quiesce mas-mx in place, capture the evidence files, and rehearse locally. + +Does not change Deployment replicas, roll the image, delete the namespace, +or read the TEM relay files. Offsite publication stays a separate grant. +""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import mas_evidence_archive as archive + +STATE = Path("/var/lib/mas-evidence-backup") + + +def kubectl(*args): + return json.loads( + subprocess.check_output(["k3s", "kubectl", *args, "-o", "json"], text=True) + ) + + +def volume(): + claim = kubectl("-n", "mas", "get", "pvc", "mas-evidence") + name = claim["spec"]["volumeName"] + body = kubectl("get", "pv", name) + local = body["spec"].get("local") or {} + host = body["spec"].get("hostPath") or {} + path = local.get("path") or host.get("path") + if not path or not Path(path).is_dir(): + raise archive.Refused("evidence_volume_path_required") + return name, Path(path) + + +def deployments(): + found = {} + for name in ("mas-mx", "mas-statute"): + body = kubectl("-n", "mas", "get", "deploy", name) + image = body["spec"]["template"]["spec"]["containers"][0]["image"] + found[name] = {"replicas": body["spec"]["replicas"], "image": image} + return found + + +def container_init_pid(): + pods = kubectl("-n", "mas", "get", "pods", "-l", "app.kubernetes.io/name=mas-mx") + items = [item for item in pods["items"] if item["status"]["phase"] == "Running"] + if len(items) != 1: + raise archive.Refused("mas_mx_pod_required") + statuses = items[0]["status"]["containerStatuses"] + if len(statuses) != 1 or not statuses[0].get("containerID", "").startswith("containerd://"): + raise archive.Refused("mas_mx_container_required") + container = statuses[0]["containerID"].removeprefix("containerd://") + info = json.loads( + subprocess.check_output(["k3s", "crictl", "inspect", container], text=True) + ) + return int(info["info"]["pid"]) + + +def namespace_pids(init_pid): + host = os.readlink("/proc/self/ns/pid") + target = os.readlink(f"/proc/{init_pid}/ns/pid") + if host == target: + raise archive.Refused("refusing_host_pid_namespace") + found = [] + for entry in os.listdir("/proc"): + if not entry.isdigit(): + continue + try: + if os.readlink(f"/proc/{entry}/ns/pid") == target: + found.append(int(entry)) + except OSError: + continue + if init_pid not in found: + raise archive.Refused("container_process_missing") + return found + + +def quiesced_copy(destination): + before = deployments() + volume_name, volume_path = volume() + if {name: item["replicas"] for name, item in before.items()} != {"mas-mx": 1, "mas-statute": 1}: + raise archive.Refused("facility_replica_drift") + if before["mas-mx"]["image"] != before["mas-statute"]["image"]: + raise archive.Refused("facility_image_drift") + pids = namespace_pids(container_init_pid()) + stopped = [] + try: + for pid in pids: + os.kill(pid, signal.SIGSTOP) + stopped.append(pid) + receipt = archive.capture( + volume_path, + destination, + captured_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + image=before["mas-mx"]["image"], + writers_quiesced=True, + replicas={"mas-mx": 1, "mas-statute": 1}, + ) + finally: + for pid in stopped: + try: + os.kill(pid, signal.SIGCONT) + except OSError: + pass + after = deployments() + if after != before: + raise archive.Refused("facility_changed_during_capture") + receipt["volume"] = volume_name + receipt["facility_scaled"] = False + receipt["relay_read"] = False + archive.checkpoint(destination / "receipt.json", receipt) + return receipt + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--execute", action="store_true") + parser.add_argument("--recipient") + args = parser.parse_args() + if not args.execute: + print(json.dumps({"status": "preview", "offsite": False, "facility_scaled": False})) + return + if os.geteuid() != 0: + raise archive.Refused("root_required") + if not isinstance(args.recipient, str) or not args.recipient.startswith("age1"): + raise archive.Refused("age_recipient_required") + STATE.mkdir(mode=0o700, exist_ok=True) + stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + destination = STATE / stamp + receipt = quiesced_copy(destination) + sealed = archive.seal(destination, destination, args.recipient) + for name in archive.FILES: + path = destination / name + if path.exists(): + path.unlink() + public = { + "status": "sealed", + "application": "mas", + "captured_at": receipt["captured_at"], + "image": receipt["image"], + "volume": receipt["volume"], + "files": { + name: {"present": meta.get("present"), "bytes": meta.get("bytes")} + for name, meta in receipt["files"].items() + }, + "ciphertext_sha256": sealed["ciphertext_sha256"], + "ciphertext_bytes": sealed["ciphertext_bytes"], + "mail_sent": False, + "production_restored": False, + "facility_scaled": False, + "relay_read": False, + "offsite_published": False, + "age_identity_on_node": False, + } + archive.checkpoint(STATE / "latest-public.json", public) + print(json.dumps(public)) + + +if __name__ == "__main__": + main() diff --git a/scripts/primary_application_archive.py b/scripts/primary_application_archive.py index 3976511..4ebb7a9 100644 --- a/scripts/primary_application_archive.py +++ b/scripts/primary_application_archive.py @@ -23,6 +23,7 @@ BUCKET = "railiance-platform-pg-backup" PROFILES = { "coulomb-social": "platform-pg/application-archives/coulomb-social/", "email-connect": "platform-pg/application-archives/email-connect/", + "mas": "platform-pg/application-archives/mas/", } PART_SIZE = 64 * 1024**2 MAX_BYTES = 20 * 1024**3 diff --git a/tests/test_mas_evidence_archive.py b/tests/test_mas_evidence_archive.py new file mode 100644 index 0000000..3606366 --- /dev/null +++ b/tests/test_mas_evidence_archive.py @@ -0,0 +1,80 @@ +"""MAS evidence capture and rehearsal. No cluster and no provider.""" + +import importlib.util +import sqlite3 +from pathlib import Path + +import pytest + + +def load(name): + path = Path(__file__).resolve().parents[1] / "scripts" / name + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +archive = load("mas_evidence_archive.py") +primary = load("primary_application_archive.py") + + +def tree(tmp_path): + source = tmp_path / "evidence" + source.mkdir() + (source / "events.jsonl").write_text('{"kind":"accepted"}\n') + for name in ("tickets.sqlite3", "tickets-outbox.sqlite3"): + connection = sqlite3.connect(source / name) + connection.execute("CREATE TABLE item (id INTEGER PRIMARY KEY, note TEXT)") + connection.execute("INSERT INTO item (note) VALUES ('held')") + connection.commit() + connection.close() + return source + + +def test_profile_is_separate_from_email_connect(): + key = primary.key_for("mas", "2026-11-02T01:20:00Z") + assert key.startswith("platform-pg/application-archives/mas/") + with pytest.raises(ValueError): + primary.validate_key("email-connect", key) + + +def test_capture_refuses_unless_writers_stay_at_one_replica(tmp_path): + with pytest.raises(archive.Refused, match="writers_not_quiesced"): + archive.capture( + tree(tmp_path), + tmp_path / "out", + captured_at="2026-11-02T01:20:00Z", + image="mas-live:test", + writers_quiesced=False, + replicas={"mas-mx": 1, "mas-statute": 1}, + ) + + +def test_seal_and_rehearse_do_not_restore_or_send(tmp_path): + source = tree(tmp_path) + captured = archive.capture( + source, + tmp_path / "captured", + captured_at="2026-11-02T01:20:00Z", + image="mas-live:test", + writers_quiesced=True, + replicas={"mas-mx": 1, "mas-statute": 1}, + ) + identity = tmp_path / "identity.txt" + __import__("subprocess").check_call(["age-keygen", "-o", str(identity)]) + recipient = __import__("subprocess").check_output( + ["age-keygen", "-y", str(identity)], text=True + ).strip() + sealed = archive.seal(tmp_path / "captured", tmp_path / "stage", recipient) + proof = archive.rehearse(tmp_path / "stage" / "packet.tar.age", identity, tmp_path / "rehearsal") + assert captured["mail_sent"] is False + assert proof == { + "status": "rehearsal_matched", + "application": "mas", + "ciphertext_sha256": sealed["ciphertext_sha256"], + "mail_sent": False, + "production_restored": False, + "facility_scaled": False, + } + assert [path.name for path in (tmp_path / "rehearsal").iterdir()] == ["rehearsal.json"] diff --git a/workplans/RPF-WP-0070-mas-evidence-recovery.md b/workplans/RPF-WP-0070-mas-evidence-recovery.md index d0759a2..bb29bb3 100644 --- a/workplans/RPF-WP-0070-mas-evidence-recovery.md +++ b/workplans/RPF-WP-0070-mas-evidence-recovery.md @@ -4,7 +4,7 @@ type: workplan title: "Recover the MAS evidence volume the way the email-connect ledger is recovered" domain: financials repo: railiance-platform -status: ready +status: active owner: codex topic_slug: coulomb-social created: "2026-11-01" @@ -25,7 +25,8 @@ owned by RPF-WP-0053-T05. The MAS evidence volume is still only the node-local PVC `mas-evidence` in namespace `mas`: `events.jsonl`, `tickets.sqlite3`, and the outbox database. Give that volume the same kind of recovery. RPF-WP-0053-T05 accepted that path on 2026-11-02. This workplan -is ready to copy the shape. The copy has not started. +is in progress. A local quiesced capture and rehearsal are recorded. +Offsite publication has not started. The MAS facility can be powered off without touching platform mail. This backup must not scale it, roll its image, or delete the namespace. SMTP and @@ -35,9 +36,11 @@ TEM credentials stay out of the backup worker. ```task id: RPF-WP-0070-T01 -status: todo +status: progress priority: medium +needs_human: true depends_on: [RPF-WP-0053-T05] +intervention_note: "Local quiesced capture and restore rehearsal are done. Prefix-scoped publisher custody and age-identity escrow are still required before offsite publication." state_hub_task_id: "4f6484fb-8303-5dd5-94cb-b2622e1322c8" ``` @@ -45,4 +48,16 @@ RPF-WP-0053-T05 is done. Copy its accepted shape for the MAS evidence files: prefix-scoped custody, a quiesced capture, an independent receipt, and a restore rehearsal that does not resend mail. Do not scale the MAS facility, roll its image, or delete the namespace. SMTP and TEM credentials -stay out of the backup worker. The copy has not started. +stay out of the backup worker. + +2026-11-02: writers were stopped in place and continued. Replicas stayed 1, +the image stayed `docker.io/library/mas-live:v0.2-480e69a`, and both pods +remained the same Running pods. The TEM relay was not read. The three +evidence files were sealed and the plaintext copies removed. A separate +workstation rehearsal matched the ciphertext and did not restore the +facility or send mail. Receipt: +`docs/evidence/mas-evidence-local-capture-2026-11-02.json`. +The age identity is only on the operator workstation. It is not escrowed, +and nothing was published to `platform-pg/application-archives/mas/`. +That prefix still needs its own publisher grant. The profile is now +accepted by `scripts/primary_application_archive.py`.