Record the first quiesced MAS evidence capture.

The facility stayed at one replica and the relay credential was not read.
Offsite publication still waits for a prefix-scoped publisher grant.

Assistant: grok
Assistant-Session: 01a198fa-fd27-7e41-a0a2-57355c3e4efd

WORK-RECORDS.md

@@ -49,7 +49,7 @@
 | workplan | RPF-WP-0067 | active | — | workplans/RPF-WP-0067-netkingdom-identity-receipt-and-metadata-answers.md |
 | workplan | RPF-WP-0068 | finished | — | workplans/RPF-WP-0068-email-connect-platform-smtp.md |
 | workplan | RPF-WP-0069 | active | — | workplans/RPF-WP-0069-bounded-executor-lanes.md |
-| workplan | RPF-WP-0070 | ready | — | workplans/RPF-WP-0070-mas-evidence-recovery.md |
+| workplan | RPF-WP-0070 | active | — | workplans/RPF-WP-0070-mas-evidence-recovery.md |
 | task | RPF-WP-ADHOC-2026-10-10-T01 | done | — | workplans/ADHOC-2026-10-10.md |
 | task | RPF-WP-ADHOC-2026-10-11-T01 | done | — | workplans/ADHOC-2026-10-11.md |
 | task | RPF-WP-ADHOC-2026-10-11-T02 | done | — | workplans/ADHOC-2026-10-11.md |
@@ -236,7 +236,7 @@
 | task | RPF-WP-0069-T07 | todo | — | workplans/RPF-WP-0069-bounded-executor-lanes.md |
 | task | RPF-WP-0069-T08 | done | — | workplans/RPF-WP-0069-bounded-executor-lanes.md |
 | task | RPF-WP-0069-T09 | wait | — | workplans/RPF-WP-0069-bounded-executor-lanes.md |
-| task | RPF-WP-0070-T01 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md |
+| task | RPF-WP-0070-T01 | progress | — | workplans/RPF-WP-0070-mas-evidence-recovery.md |
 | intake | RPF-IN-0001 | closed | — | intakes/intakes.md |
 | intake | RPF-IN-0002 | open | — | intakes/intakes.md |
 | intake | RPF-IN-0003 | open | — | intakes/intakes.md |

docs/evidence/mas-evidence-local-capture-2026-11-02.json

@@ -0,0 +1,39 @@
+{
+  "status": "sealed",
+  "captured_at": "2026-11-02T01:18:19Z",
+  "image": "docker.io/library/mas-live:v0.2-480e69a",
+  "volume": "pvc-415825c2-35db-4fc6-b58e-726dbb82c5d3",
+  "replicas": {
+    "mas-mx": 1,
+    "mas-statute": 1
+  },
+  "facility_scaled": false,
+  "relay_read": false,
+  "mail_sent": false,
+  "files": {
+    "events.jsonl": {
+      "present": true,
+      "bytes": 51382
+    },
+    "tickets.sqlite3": {
+      "present": true,
+      "bytes": 45056
+    },
+    "tickets-outbox.sqlite3": {
+      "present": true,
+      "bytes": 20480
+    }
+  },
+  "ciphertext_sha256": "72bef875cdbe577b2733227448619c1914357552ab1de04b1f47ba13fd6c80aa",
+  "ciphertext_bytes": 133352,
+  "rehearsal": "rehearsal_matched",
+  "production_restored": false,
+  "offsite_published": false,
+  "age_identity_on_node": false,
+  "schema": "platform.mas-evidence-local-capture.v1",
+  "age_recipient": "age17sxuqajpt3vvfep64j5vwlhshrn3g02uhm49par9pmec7q23u3xqdxs05w",
+  "age_identity_escrowed": false,
+  "prefix": "platform-pg/application-archives/mas/",
+  "node_ciphertext": "/var/lib/mas-evidence-backup/20261102T011818Z/packet.tar.age",
+  "pods_restarted": false
+}

scripts/mas_evidence_archive.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+"""MAS evidence capture and restore rehearsal. No SMTP, no replica changes."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import sqlite3
+import stat
+import subprocess
+import tarfile
+import tempfile
+from contextlib import closing
+from pathlib import Path
+
+FILES = ("events.jsonl", "tickets.sqlite3", "tickets-outbox.sqlite3")
+PREFIX = "platform-pg/application-archives/mas/"
+MAX_BYTES = 64 * 1024**2
+
+
+class Refused(RuntimeError):
+    pass
+
+
+def digest(path):
+    with path.open("rb") as stream:
+        return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def checkpoint(path, value):
+    fd, name = tempfile.mkstemp(prefix="receipt-", dir=path.parent)
+    with os.fdopen(fd, "w") as stream:
+        json.dump(value, stream, indent=2)
+        stream.flush()
+        os.fsync(stream.fileno())
+    os.chmod(name, 0o600)
+    Path(name).replace(path)
+
+
+def _regular(path):
+    info = path.lstat()
+    if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
+        raise Refused("evidence_file_required")
+    if info.st_size > MAX_BYTES or "mas-relay" in path.parts:
+        raise Refused("evidence_path_outside_contract")
+
+
+def _sqlite_fingerprint(path):
+    with closing(sqlite3.connect(path.absolute().as_uri() + "?mode=ro", uri=True)) as database:
+        database.execute("PRAGMA trusted_schema=OFF")
+        if database.execute("PRAGMA integrity_check").fetchall() != [("ok",)]:
+            raise Refused("sqlite_integrity_failed")
+        tables = sorted(
+            row[0]
+            for row in database.execute(
+                "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
+            )
+        )
+        result = {"tables": {}}
+        for name in tables:
+            if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name):
+                raise Refused("unexpected_sqlite_table")
+            rows = [
+                json.dumps(row, separators=(",", ":"), default=lambda value: value.hex()).encode()
+                for row in database.execute(f'SELECT * FROM "{name}"')
+            ]
+            result["tables"][name] = {
+                "rows": len(rows),
+                "sha256": hashlib.sha256(b"\n".join(sorted(rows))).hexdigest(),
+            }
+        return result
+
+
+def _backup_sqlite(source, output):
+    _regular(source)
+    if output.exists() or output.is_symlink():
+        raise Refused("capture_destination_exists")
+    fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    os.close(fd)
+    with closing(sqlite3.connect(source.absolute().as_uri() + "?mode=ro", uri=True)) as src, closing(
+        sqlite3.connect(output)
+    ) as dst:
+        src.backup(dst, pages=64)
+    return {"sha256": digest(output), "bytes": output.stat().st_size, **_sqlite_fingerprint(output)}
+
+
+def capture(source, destination, *, captured_at, image, writers_quiesced, replicas):
+    """Copy the three evidence files. Caller must already have quiesced writers."""
+    if writers_quiesced is not True or replicas != {"mas-mx": 1, "mas-statute": 1}:
+        raise Refused("writers_not_quiesced")
+    destination.mkdir(mode=0o700, exist_ok=False)
+    files = {}
+    for name in FILES:
+        path = source / name
+        if name == "tickets-outbox.sqlite3" and not path.exists():
+            files[name] = {"present": False}
+            continue
+        if name.endswith(".sqlite3"):
+            files[name] = {"present": True, **_backup_sqlite(path, destination / name)}
+        else:
+            _regular(path)
+            target = destination / name
+            fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+            with os.fdopen(fd, "wb") as out, path.open("rb") as src:
+                while block := src.read(1024**2):
+                    out.write(block)
+            files[name] = {"present": True, "sha256": digest(target), "bytes": target.stat().st_size}
+    receipt = {
+        "status": "captured",
+        "application": "mas",
+        "captured_at": captured_at,
+        "image": image,
+        "writers_quiesced": True,
+        "replicas": replicas,
+        "mail_sent": False,
+        "files": files,
+    }
+    checkpoint(destination / "receipt.json", receipt)
+    return receipt
+
+
+def seal(directory, stage, recipient):
+    stage.mkdir(mode=0o700, exist_ok=True)
+    encrypted = stage / "packet.tar.age"
+    if encrypted.exists() or encrypted.is_symlink():
+        raise Refused("partial_encryption_requires_review")
+    archive = stage / "packet.tar"
+    with archive.open("xb") as stream:
+        os.chmod(archive, 0o600)
+        with tarfile.open(fileobj=stream, mode="w") as packet:
+            for name in (*FILES, "receipt.json"):
+                path = directory / name
+                if path.exists():
+                    packet.add(path, arcname=name, recursive=False)
+    subprocess.run(["age", "-r", recipient, "-o", str(encrypted), str(archive)], check=True, timeout=120)
+    encrypted.chmod(0o600)
+    archive.unlink()
+    return {"ciphertext_sha256": digest(encrypted), "ciphertext_bytes": encrypted.stat().st_size}
+
+
+def rehearse(encrypted, identity, destination):
+    """Decrypt and compare hashes. Do not restore onto the facility or send mail."""
+    if destination.exists() or destination.is_symlink():
+        raise Refused("fresh_destination_required")
+    destination.mkdir(mode=0o700)
+    packet = destination / "packet.tar"
+    result = subprocess.run(
+        ["age", "-d", "-i", str(identity), "-o", str(packet), str(encrypted)],
+        capture_output=True,
+        timeout=120,
+    )
+    if result.returncode:
+        raise Refused("decryption_failed")
+    os.chmod(packet, 0o600)
+    with tarfile.open(packet, "r:") as archive:
+        entries = archive.getmembers()
+        names = {entry.name for entry in entries}
+        if (
+            "receipt.json" not in names
+            or any(not entry.isfile() or "/" in entry.name or entry.name.startswith(".") for entry in entries)
+        ):
+            raise Refused("invalid_packet_members")
+        for entry in entries:
+            target = destination / entry.name
+            with archive.extractfile(entry) as source, target.open("xb") as out:
+                os.chmod(target, 0o600)
+                out.write(source.read())
+    receipt = json.loads((destination / "receipt.json").read_text())
+    if receipt.get("application") != "mas" or receipt.get("mail_sent") is not False:
+        raise Refused("capture_receipt_rejected")
+    for name, meta in receipt["files"].items():
+        path = destination / name
+        if not meta.get("present"):
+            if path.exists():
+                raise Refused("unexpected_absent_file")
+            continue
+        if digest(path) != meta["sha256"] or path.stat().st_size != meta["bytes"]:
+            raise Refused("rehearsal_mismatch")
+        path.unlink()
+    packet.unlink()
+    (destination / "receipt.json").unlink()
+    proof = {
+        "status": "rehearsal_matched",
+        "application": "mas",
+        "ciphertext_sha256": digest(encrypted),
+        "mail_sent": False,
+        "production_restored": False,
+        "facility_scaled": False,
+    }
+    checkpoint(destination / "rehearsal.json", proof)
+    return proof

scripts/mas_evidence_capture.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Quiesce mas-mx in place, capture the evidence files, and rehearse locally.
+
+Does not change Deployment replicas, roll the image, delete the namespace,
+or read the TEM relay files. Offsite publication stays a separate grant.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import subprocess
+from datetime import UTC, datetime
+from pathlib import Path
+
+import mas_evidence_archive as archive
+
+STATE = Path("/var/lib/mas-evidence-backup")
+
+
+def kubectl(*args):
+    return json.loads(
+        subprocess.check_output(["k3s", "kubectl", *args, "-o", "json"], text=True)
+    )
+
+
+def volume():
+    claim = kubectl("-n", "mas", "get", "pvc", "mas-evidence")
+    name = claim["spec"]["volumeName"]
+    body = kubectl("get", "pv", name)
+    local = body["spec"].get("local") or {}
+    host = body["spec"].get("hostPath") or {}
+    path = local.get("path") or host.get("path")
+    if not path or not Path(path).is_dir():
+        raise archive.Refused("evidence_volume_path_required")
+    return name, Path(path)
+
+
+def deployments():
+    found = {}
+    for name in ("mas-mx", "mas-statute"):
+        body = kubectl("-n", "mas", "get", "deploy", name)
+        image = body["spec"]["template"]["spec"]["containers"][0]["image"]
+        found[name] = {"replicas": body["spec"]["replicas"], "image": image}
+    return found
+
+
+def container_init_pid():
+    pods = kubectl("-n", "mas", "get", "pods", "-l", "app.kubernetes.io/name=mas-mx")
+    items = [item for item in pods["items"] if item["status"]["phase"] == "Running"]
+    if len(items) != 1:
+        raise archive.Refused("mas_mx_pod_required")
+    statuses = items[0]["status"]["containerStatuses"]
+    if len(statuses) != 1 or not statuses[0].get("containerID", "").startswith("containerd://"):
+        raise archive.Refused("mas_mx_container_required")
+    container = statuses[0]["containerID"].removeprefix("containerd://")
+    info = json.loads(
+        subprocess.check_output(["k3s", "crictl", "inspect", container], text=True)
+    )
+    return int(info["info"]["pid"])
+
+
+def namespace_pids(init_pid):
+    host = os.readlink("/proc/self/ns/pid")
+    target = os.readlink(f"/proc/{init_pid}/ns/pid")
+    if host == target:
+        raise archive.Refused("refusing_host_pid_namespace")
+    found = []
+    for entry in os.listdir("/proc"):
+        if not entry.isdigit():
+            continue
+        try:
+            if os.readlink(f"/proc/{entry}/ns/pid") == target:
+                found.append(int(entry))
+        except OSError:
+            continue
+    if init_pid not in found:
+        raise archive.Refused("container_process_missing")
+    return found
+
+
+def quiesced_copy(destination):
+    before = deployments()
+    volume_name, volume_path = volume()
+    if {name: item["replicas"] for name, item in before.items()} != {"mas-mx": 1, "mas-statute": 1}:
+        raise archive.Refused("facility_replica_drift")
+    if before["mas-mx"]["image"] != before["mas-statute"]["image"]:
+        raise archive.Refused("facility_image_drift")
+    pids = namespace_pids(container_init_pid())
+    stopped = []
+    try:
+        for pid in pids:
+            os.kill(pid, signal.SIGSTOP)
+            stopped.append(pid)
+        receipt = archive.capture(
+            volume_path,
+            destination,
+            captured_at=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            image=before["mas-mx"]["image"],
+            writers_quiesced=True,
+            replicas={"mas-mx": 1, "mas-statute": 1},
+        )
+    finally:
+        for pid in stopped:
+            try:
+                os.kill(pid, signal.SIGCONT)
+            except OSError:
+                pass
+    after = deployments()
+    if after != before:
+        raise archive.Refused("facility_changed_during_capture")
+    receipt["volume"] = volume_name
+    receipt["facility_scaled"] = False
+    receipt["relay_read"] = False
+    archive.checkpoint(destination / "receipt.json", receipt)
+    return receipt
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--execute", action="store_true")
+    parser.add_argument("--recipient")
+    args = parser.parse_args()
+    if not args.execute:
+        print(json.dumps({"status": "preview", "offsite": False, "facility_scaled": False}))
+        return
+    if os.geteuid() != 0:
+        raise archive.Refused("root_required")
+    if not isinstance(args.recipient, str) or not args.recipient.startswith("age1"):
+        raise archive.Refused("age_recipient_required")
+    STATE.mkdir(mode=0o700, exist_ok=True)
+    stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    destination = STATE / stamp
+    receipt = quiesced_copy(destination)
+    sealed = archive.seal(destination, destination, args.recipient)
+    for name in archive.FILES:
+        path = destination / name
+        if path.exists():
+            path.unlink()
+    public = {
+        "status": "sealed",
+        "application": "mas",
+        "captured_at": receipt["captured_at"],
+        "image": receipt["image"],
+        "volume": receipt["volume"],
+        "files": {
+            name: {"present": meta.get("present"), "bytes": meta.get("bytes")}
+            for name, meta in receipt["files"].items()
+        },
+        "ciphertext_sha256": sealed["ciphertext_sha256"],
+        "ciphertext_bytes": sealed["ciphertext_bytes"],
+        "mail_sent": False,
+        "production_restored": False,
+        "facility_scaled": False,
+        "relay_read": False,
+        "offsite_published": False,
+        "age_identity_on_node": False,
+    }
+    archive.checkpoint(STATE / "latest-public.json", public)
+    print(json.dumps(public))
+
+
+if __name__ == "__main__":
+    main()

scripts/primary_application_archive.py

@@ -23,6 +23,7 @@ BUCKET = "railiance-platform-pg-backup"
 PROFILES = {
     "coulomb-social": "platform-pg/application-archives/coulomb-social/",
     "email-connect": "platform-pg/application-archives/email-connect/",
+    "mas": "platform-pg/application-archives/mas/",
 }
 PART_SIZE = 64 * 1024**2
 MAX_BYTES = 20 * 1024**3

tests/test_mas_evidence_archive.py

@@ -0,0 +1,80 @@
+"""MAS evidence capture and rehearsal. No cluster and no provider."""
+
+import importlib.util
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+
+def load(name):
+    path = Path(__file__).resolve().parents[1] / "scripts" / name
+    spec = importlib.util.spec_from_file_location(name, path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+archive = load("mas_evidence_archive.py")
+primary = load("primary_application_archive.py")
+
+
+def tree(tmp_path):
+    source = tmp_path / "evidence"
+    source.mkdir()
+    (source / "events.jsonl").write_text('{"kind":"accepted"}\n')
+    for name in ("tickets.sqlite3", "tickets-outbox.sqlite3"):
+        connection = sqlite3.connect(source / name)
+        connection.execute("CREATE TABLE item (id INTEGER PRIMARY KEY, note TEXT)")
+        connection.execute("INSERT INTO item (note) VALUES ('held')")
+        connection.commit()
+        connection.close()
+    return source
+
+
+def test_profile_is_separate_from_email_connect():
+    key = primary.key_for("mas", "2026-11-02T01:20:00Z")
+    assert key.startswith("platform-pg/application-archives/mas/")
+    with pytest.raises(ValueError):
+        primary.validate_key("email-connect", key)
+
+
+def test_capture_refuses_unless_writers_stay_at_one_replica(tmp_path):
+    with pytest.raises(archive.Refused, match="writers_not_quiesced"):
+        archive.capture(
+            tree(tmp_path),
+            tmp_path / "out",
+            captured_at="2026-11-02T01:20:00Z",
+            image="mas-live:test",
+            writers_quiesced=False,
+            replicas={"mas-mx": 1, "mas-statute": 1},
+        )
+
+
+def test_seal_and_rehearse_do_not_restore_or_send(tmp_path):
+    source = tree(tmp_path)
+    captured = archive.capture(
+        source,
+        tmp_path / "captured",
+        captured_at="2026-11-02T01:20:00Z",
+        image="mas-live:test",
+        writers_quiesced=True,
+        replicas={"mas-mx": 1, "mas-statute": 1},
+    )
+    identity = tmp_path / "identity.txt"
+    __import__("subprocess").check_call(["age-keygen", "-o", str(identity)])
+    recipient = __import__("subprocess").check_output(
+        ["age-keygen", "-y", str(identity)], text=True
+    ).strip()
+    sealed = archive.seal(tmp_path / "captured", tmp_path / "stage", recipient)
+    proof = archive.rehearse(tmp_path / "stage" / "packet.tar.age", identity, tmp_path / "rehearsal")
+    assert captured["mail_sent"] is False
+    assert proof == {
+        "status": "rehearsal_matched",
+        "application": "mas",
+        "ciphertext_sha256": sealed["ciphertext_sha256"],
+        "mail_sent": False,
+        "production_restored": False,
+        "facility_scaled": False,
+    }
+    assert [path.name for path in (tmp_path / "rehearsal").iterdir()] == ["rehearsal.json"]

workplans/RPF-WP-0070-mas-evidence-recovery.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Recover the MAS evidence volume the way the email-connect ledger is recovered"
 domain: financials
 repo: railiance-platform
-status: ready
+status: active
 owner: codex
 topic_slug: coulomb-social
 created: "2026-11-01"
@@ -25,7 +25,8 @@ owned by RPF-WP-0053-T05. The MAS evidence volume is still only the
 node-local PVC `mas-evidence` in namespace `mas`: `events.jsonl`,
 `tickets.sqlite3`, and the outbox database. Give that volume the same kind
 of recovery. RPF-WP-0053-T05 accepted that path on 2026-11-02. This workplan
-is ready to copy the shape. The copy has not started.
+is in progress. A local quiesced capture and rehearsal are recorded.
+Offsite publication has not started.
 
 The MAS facility can be powered off without touching platform mail. This
 backup must not scale it, roll its image, or delete the namespace. SMTP and
@@ -35,9 +36,11 @@ TEM credentials stay out of the backup worker.
 
 ```task
 id: RPF-WP-0070-T01
-status: todo
+status: progress
 priority: medium
+needs_human: true
 depends_on: [RPF-WP-0053-T05]
+intervention_note: "Local quiesced capture and restore rehearsal are done. Prefix-scoped publisher custody and age-identity escrow are still required before offsite publication."
 state_hub_task_id: "4f6484fb-8303-5dd5-94cb-b2622e1322c8"
 ```
 
@@ -45,4 +48,16 @@ RPF-WP-0053-T05 is done. Copy its accepted shape for the MAS evidence
 files: prefix-scoped custody, a quiesced capture, an independent receipt,
 and a restore rehearsal that does not resend mail. Do not scale the MAS
 facility, roll its image, or delete the namespace. SMTP and TEM credentials
-stay out of the backup worker. The copy has not started.
+stay out of the backup worker.
+
+2026-11-02: writers were stopped in place and continued. Replicas stayed 1,
+the image stayed `docker.io/library/mas-live:v0.2-480e69a`, and both pods
+remained the same Running pods. The TEM relay was not read. The three
+evidence files were sealed and the plaintext copies removed. A separate
+workstation rehearsal matched the ciphertext and did not restore the
+facility or send mail. Receipt:
+`docs/evidence/mas-evidence-local-capture-2026-11-02.json`.
+The age identity is only on the operator workstation. It is not escrowed,
+and nothing was published to `platform-pg/application-archives/mas/`.
+That prefix still needs its own publisher grant. The profile is now
+accepted by `scripts/primary_application_archive.py`.