diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 5428e0e..37a3eb6 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -238,7 +238,7 @@ | task | RPF-WP-0069-T09 | wait | — | workplans/RPF-WP-0069-bounded-executor-lanes.md | | task | RPF-WP-0070-T01 | done | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T02 | done | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | -| task | RPF-WP-0070-T03 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | +| task | RPF-WP-0070-T03 | progress | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T04 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | task | RPF-WP-0070-T05 | todo | — | workplans/RPF-WP-0070-mas-evidence-recovery.md | | intake | RPF-IN-0001 | closed | — | intakes/intakes.md | diff --git a/deploy/mas-backup/age-recipient b/deploy/mas-backup/age-recipient new file mode 100644 index 0000000..e367aff --- /dev/null +++ b/deploy/mas-backup/age-recipient @@ -0,0 +1 @@ +age17sxuqajpt3vvfep64j5vwlhshrn3g02uhm49par9pmec7q23u3xqdxs05w diff --git a/deploy/mas-backup/install.sh b/deploy/mas-backup/install.sh new file mode 100755 index 0000000..aa69b88 --- /dev/null +++ b/deploy/mas-backup/install.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Install the MAS evidence backup worker on railiance01 and run the first upload. +# Run as root from a directory holding exactly the files named in SHA256SUMS. +# Effects: /opt/railiance-platform/mas-backup/, four systemd units, +# /etc/mas-backup/age-recipient (public key). Timers are enabled only after +# the first backup and freshness check pass. Does not scale, roll or restart +# the MAS facility and never deletes retained objects. +set -eu +[ "$(id -u)" -eq 0 ] || { echo '{"status":"failed","code":"root_required"}'; exit 1; } +sha256sum -c --quiet SHA256SUMS +[ -f /etc/mas-backup/primary.env ] || { echo '{"status":"failed","code":"primary_env_missing"}'; exit 1; } +recipient=$(cat age-recipient) +case "$recipient" in age1*) ;; *) echo '{"status":"failed","code":"age_recipient_required"}'; exit 1;; esac + +install -d -m 0755 -o root -g root /opt/railiance-platform/mas-backup +for f in mas_evidence_backup.py mas_evidence_capture.py mas_evidence_archive.py primary_application_archive.py; do + install -m 0644 -o root -g root "$f" /opt/railiance-platform/mas-backup/"$f" +done +install -m 0644 -o root -g root age-recipient /etc/mas-backup/age-recipient +for u in mas-evidence-backup.service mas-evidence-backup.timer mas-evidence-freshness.service mas-evidence-freshness.timer; do + install -m 0644 -o root -g root "$u" /etc/systemd/system/"$u" +done +systemctl daemon-reload + +systemctl start mas-evidence-backup.service +systemctl start mas-evidence-freshness.service +systemctl enable --now mas-evidence-backup.timer mas-evidence-freshness.timer >/dev/null 2>&1 + +python3 - <<'PY' +import json, pathlib +latest = json.loads(pathlib.Path("/var/lib/mas-evidence-backup/latest-offsite.json").read_text()) +keep = ("status", "captured_at", "object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes", + "remote_retrieval_verified", "immutable_create", "retained_objects_deleted", "run", "capture") +print(json.dumps({k: latest.get(k) for k in keep}, indent=2, sort_keys=True)) +PY +systemctl list-timers --all --no-pager | grep mas-evidence diff --git a/deploy/mas-backup/mas-evidence-backup.service b/deploy/mas-backup/mas-evidence-backup.service new file mode 100644 index 0000000..b9189ac --- /dev/null +++ b/deploy/mas-backup/mas-evidence-backup.service @@ -0,0 +1,21 @@ +[Unit] +Description=Quiesced, age-sealed MAS evidence backup to the primary archive (RPF-WP-0070) +After=network-online.target k3s.service +Wants=network-online.target + +[Service] +Type=oneshot +User=root +UMask=0077 +StateDirectory=mas-evidence-backup +StateDirectoryMode=0700 +EnvironmentFile=/etc/mas-backup/primary.env +ExecStart=/opt/railiance-platform/backup-venv/bin/python /opt/railiance-platform/mas-backup/mas_evidence_backup.py backup --execute +TimeoutStartSec=20min +KillMode=mixed +NoNewPrivileges=true +PrivateTmp=true +# full, not strict: the SQLite read-only open of the stopped writers' WAL +# databases on the local PV may still need their -shm file. +ProtectSystem=full +ProtectHome=read-only diff --git a/deploy/mas-backup/mas-evidence-backup.timer b/deploy/mas-backup/mas-evidence-backup.timer new file mode 100644 index 0000000..647d3cc --- /dev/null +++ b/deploy/mas-backup/mas-evidence-backup.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Daily MAS evidence recovery point (provisional RPO 24h) + +[Timer] +OnCalendar=*-*-* 04:15:00 UTC +Persistent=true +RandomizedDelaySec=5min +Unit=mas-evidence-backup.service + +[Install] +WantedBy=timers.target diff --git a/deploy/mas-backup/mas-evidence-freshness.service b/deploy/mas-backup/mas-evidence-freshness.service new file mode 100644 index 0000000..73c55a0 --- /dev/null +++ b/deploy/mas-backup/mas-evidence-freshness.service @@ -0,0 +1,13 @@ +[Unit] +Description=Check MAS evidence archive age (stale after 26h) and local ciphertext hash + +[Service] +Type=oneshot +User=root +UMask=0077 +ExecStart=/opt/railiance-platform/backup-venv/bin/python /opt/railiance-platform/mas-backup/mas_evidence_backup.py freshness +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadOnlyPaths=/var/lib/mas-evidence-backup diff --git a/deploy/mas-backup/mas-evidence-freshness.timer b/deploy/mas-backup/mas-evidence-freshness.timer new file mode 100644 index 0000000..3554632 --- /dev/null +++ b/deploy/mas-backup/mas-evidence-freshness.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Hourly MAS evidence backup freshness check + +[Timer] +OnCalendar=hourly +Persistent=true +Unit=mas-evidence-freshness.service + +[Install] +WantedBy=timers.target diff --git a/scripts/mas_evidence_backup.py b/scripts/mas_evidence_backup.py new file mode 100644 index 0000000..a0b6966 --- /dev/null +++ b/scripts/mas_evidence_backup.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Daily MAS evidence backup worker on railiance01, plus its freshness check. + +backup --execute + Quiesce the MAS writers in place (stop and continue, never scale or roll), + capture the three evidence files, seal them to the workstation age + recipient, delete the plaintext copies, and publish the ciphertext to + platform-pg/application-archives/mas/ with an immutable create and a + version-pinned hash readback. Writes a public receipt per run and + latest-offsite.json. Never deletes a retained object. + +freshness + Fails when the latest verified upload is older than STALE_HOURS, its + receipt is not verified, or the local ciphertext no longer matches it. + +Credentials come only from /etc/mas-backup/primary.env through the systemd +EnvironmentFile. No TEM or SMTP credential and no age identity is read here. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import mas_evidence_archive as archive # noqa: E402 +import primary_application_archive as primary # noqa: E402 + +STATE = Path("/var/lib/mas-evidence-backup") +RECIPIENT_FILE = Path("/etc/mas-backup/age-recipient") +LATEST = "latest-offsite.json" +STALE_HOURS = 26 +APPLICATION = "mas" + + +def recipient(path=None): + value = (path or RECIPIENT_FILE).read_text(encoding="utf-8").strip() + if not value.startswith("age1") or len(value) > 128 or any(c.isspace() for c in value): + raise archive.Refused("age_recipient_required") + return value + + +def publish_run(run_dir, captured_at, client, save_latest): + """Upload run_dir/packet.tar.age and verify it by version-pinned readback.""" + source = run_dir / "packet.tar.age" + readback = run_dir / "readback.tar.age" + receipt_path = run_dir / "offsite-receipt.json" + receipt = { + "schema": "platform.application-primary-archive.v1", + "status": "pending", + "application": APPLICATION, + "captured_at": captured_at, + "object_key": primary.key_for(APPLICATION, captured_at), + } + primary.checkpoint(receipt_path, receipt) + try: + primary.transfer( + client, APPLICATION, source, readback, receipt, + lambda: primary.checkpoint(receipt_path, receipt), + ) + except Exception as error: + receipt.update(status="failed", failure_type=type(error).__name__) + primary.checkpoint(receipt_path, receipt) + raise + # The readback matched by hash; keep one local ciphertext only. + readback.unlink() + receipt["local_ciphertext"] = str(source) + receipt["retained_objects_deleted"] = False + primary.checkpoint(receipt_path, receipt) + save_latest(receipt) + return receipt + + +def backup(state=STATE, capture=None, client_factory=None, now=None): + if capture is None: + import mas_evidence_capture as capture # needs k3s, crictl and root + if client_factory is None: + client_factory = primary.runtime_client + state.mkdir(mode=0o700, exist_ok=True) + primary.private_directory(state) + age_recipient = recipient() + client = client_factory() # refuse before quiescing when custody is missing + stamp = (now or datetime.now(UTC)).strftime("%Y%m%dT%H%M%SZ") + run_dir = state / stamp + try: + captured = capture.quiesced_copy(run_dir) + sealed = archive.seal(run_dir, run_dir, age_recipient) + finally: + for name in archive.FILES: + path = run_dir / name + if path.exists(): + path.unlink() + if (run_dir / "receipt.json").exists(): + (run_dir / "receipt.json").unlink() + public = { + "schema": "platform.mas-evidence-run.v1", + "captured_at": captured["captured_at"], + "image": captured["image"], + "volume": captured.get("volume"), + "files": { + name: {"present": meta.get("present"), "bytes": meta.get("bytes")} + for name, meta in captured["files"].items() + }, + "ciphertext_sha256": sealed["ciphertext_sha256"], + "ciphertext_bytes": sealed["ciphertext_bytes"], + "facility_scaled": False, + "relay_read": False, + "mail_sent": False, + "age_identity_on_node": False, + } + archive.checkpoint(run_dir / "capture-public.json", public) + + def save_latest(receipt): + archive.checkpoint(state / LATEST, {**receipt, "run": stamp, "capture": public}) + + receipt = publish_run(run_dir, captured["captured_at"], client, save_latest) + return {**receipt, "run": stamp, "capture": public} + + +def freshness(state=STATE, now=None): + now = now or datetime.now(UTC) + path = state / LATEST + primary.regular_file(path) + latest = json.loads(path.read_text(encoding="utf-8")) + captured = datetime.fromisoformat(latest["captured_at"].replace("Z", "+00:00")) + local = Path(latest.get("local_ciphertext", "")) + checks = { + "verified": latest.get("status") == "offsite_fetched_verified" + and latest.get("remote_retrieval_verified") is True + and latest.get("immutable_create") is True, + "fresh": now - captured <= timedelta(hours=STALE_HOURS), + "local_ciphertext_matches": local.is_file() + and local.parent.parent == state + and primary.digest(local) == latest.get("ciphertext_sha256"), + } + result = { + "status": "fresh" if all(checks.values()) else "stale_or_unverified", + "captured_at": latest["captured_at"], + "age_hours": round((now - captured).total_seconds() / 3600, 2), + "stale_after_hours": STALE_HOURS, + "object_key": latest.get("object_key"), + "version_id_present": bool(latest.get("version_id")), + "checks": checks, + } + return result + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("action", choices=["backup", "freshness"]) + parser.add_argument("--execute", action="store_true") + args = parser.parse_args(argv) + try: + if args.action == "freshness": + result = freshness() + print(json.dumps(result, sort_keys=True)) + return 0 if result["status"] == "fresh" else 1 + if not args.execute: + print(json.dumps({"status": "preview", "prefix": primary.PROFILES[APPLICATION], + "facility_scaled": False, "effects": []})) + return 0 + if os.geteuid() != 0: + raise archive.Refused("root_required") + print(json.dumps(backup(), sort_keys=True)) + return 0 + except Exception as error: # never print provider bodies or values + code = str(error) if isinstance(error, (archive.Refused, ValueError)) else "" + print(json.dumps({"status": "failed", "failure_type": type(error).__name__, + "code": code[:80]})) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_mas_evidence_backup.py b/tests/test_mas_evidence_backup.py new file mode 100644 index 0000000..1cd925c --- /dev/null +++ b/tests/test_mas_evidence_backup.py @@ -0,0 +1,138 @@ +"""MAS backup worker: capture, seal, publish with readback, freshness. Synthetic only.""" + +import hashlib +import importlib.util +import io +import json +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +spec = importlib.util.spec_from_file_location("mas_backup", ROOT / "scripts/mas_evidence_backup.py") +worker = importlib.util.module_from_spec(spec) +spec.loader.exec_module(worker) + +NOW = datetime(2026, 11, 3, 4, 15, tzinfo=UTC) +CAPTURED = "2026-11-03T04:15:00Z" + + +class NotFound(Exception): + response = {"ResponseMetadata": {"HTTPStatusCode": 404}} + + +class Provider: + def __init__(self, fail_complete=False): + self.objects, self.calls, self.fail_complete = {}, [], fail_complete + self.parts = [] + + def head_object(self, Bucket, Key): + self.calls.append("head") + raise NotFound() + + def create_multipart_upload(self, **kw): + self.calls.append("start") + self.meta = kw["Metadata"] + return {"UploadId": "u1"} + + def upload_part(self, **kw): + self.parts.append(kw["Body"]) + return {"ETag": "e"} + + def complete_multipart_upload(self, **kw): + self.calls.append("complete") + assert kw["IfNoneMatch"] == "*" + if self.fail_complete: + raise TimeoutError("synthetic") + self.objects[kw["Key"]] = b"".join(self.parts) + return {"VersionId": "v1"} + + def abort_multipart_upload(self, **kw): + self.calls.append("abort") + + def get_object(self, Bucket, Key, VersionId): + data = self.objects[Key] + return {"ContentLength": len(data), "VersionId": VersionId, "Body": io.BytesIO(data)} + + def delete_object(self, **kw): # pragma: no cover - must never be called + raise AssertionError("worker must never delete") + + +class FakeCapture: + def quiesced_copy(self, destination): + destination.mkdir(mode=0o700) + for name in worker.archive.FILES: + (destination / name).write_bytes(b"plaintext " + name.encode()) + (destination / "receipt.json").write_text("{}") + return {"captured_at": CAPTURED, "image": "mas-live:test", "volume": "pvc-x", + "files": {n: {"present": True, "bytes": 10} for n in worker.archive.FILES}} + + +@pytest.fixture +def env(tmp_path, monkeypatch): + state = tmp_path / "state" + recipient = tmp_path / "age-recipient" + recipient.write_text("age1examplerecipient\n") + monkeypatch.setattr(worker, "RECIPIENT_FILE", recipient) + + def seal(directory, stage, age_recipient): + assert age_recipient == "age1examplerecipient" + out = stage / "packet.tar.age" + out.write_bytes(b"age-encryption.org/v1\nsealed") + out.chmod(0o600) + return {"ciphertext_sha256": hashlib.sha256(out.read_bytes()).hexdigest(), + "ciphertext_bytes": out.stat().st_size} + + monkeypatch.setattr(worker.archive, "seal", seal) + return state + + +def test_backup_publishes_verified_and_leaves_no_plaintext(env): + provider = Provider() + result = worker.backup(env, FakeCapture(), lambda: provider, now=NOW) + assert result["status"] == "offsite_fetched_verified" and result["remote_retrieval_verified"] + assert result["object_key"].startswith("platform-pg/application-archives/mas/2026/11/03/") + assert result["version_id"] == "v1" and result["retained_objects_deleted"] is False + run = env / result["run"] + assert sorted(p.name for p in run.iterdir()) == ["capture-public.json", "offsite-receipt.json", "packet.tar.age"] + latest = json.loads((env / worker.LATEST).read_text()) + assert latest["version_id"] == "v1" and latest["capture"]["mail_sent"] is False + assert worker.freshness(env, now=NOW + timedelta(hours=1))["status"] == "fresh" + + +def test_failed_upload_keeps_no_plaintext_and_no_latest(env): + with pytest.raises(TimeoutError): + worker.backup(env, FakeCapture(), lambda: Provider(fail_complete=True), now=NOW) + run = env / "20261103T041500Z" + assert not any((run / name).exists() for name in worker.archive.FILES) + assert json.loads((run / "offsite-receipt.json").read_text())["status"] == "failed" + assert not (env / worker.LATEST).exists() + + +def test_missing_custody_refuses_before_quiescing(env): + class Boom(FakeCapture): + def quiesced_copy(self, destination): # pragma: no cover + raise AssertionError("must not quiesce") + + def no_custody(): + raise ValueError("owner_delivered_primary_custody_required") + + with pytest.raises(ValueError): + worker.backup(env, Boom(), no_custody, now=NOW) + + +def test_freshness_fails_when_stale_or_tampered(env): + result = worker.backup(env, FakeCapture(), lambda: Provider(), now=NOW) + assert worker.freshness(env, now=NOW + timedelta(hours=27))["checks"]["fresh"] is False + (env / result["run"] / "packet.tar.age").write_bytes(b"age-encryption.org/v1\nchanged") + assert worker.freshness(env, now=NOW)["checks"]["local_ciphertext_matches"] is False + + +def test_recipient_must_be_a_public_age_key(tmp_path): + bad = tmp_path / "r" + bad.write_text("AGE-SECRET-KEY-1XYZ") + with pytest.raises(worker.archive.Refused): + worker.recipient(bad) diff --git a/workplans/RPF-WP-0070-mas-evidence-recovery.md b/workplans/RPF-WP-0070-mas-evidence-recovery.md index 78bde02..e88d8a2 100644 --- a/workplans/RPF-WP-0070-mas-evidence-recovery.md +++ b/workplans/RPF-WP-0070-mas-evidence-recovery.md @@ -195,7 +195,7 @@ status `delivered`, outcome `created`, KV version 1, `readback_equal`, ```task id: RPF-WP-0070-T03 -status: todo +status: progress priority: medium depends_on: [RPF-WP-0070-T02] state_hub_task_id: "e290e07a-9d7a-5876-a2ce-e0c6fc4c8035" @@ -210,6 +210,25 @@ after 26h). Done when one verified upload has a hash readback and a receipt. Do not scale the facility, roll its image or delete namespace `mas`. The worker never deletes retained objects. +2026-11-03: the source is ready. `scripts/mas_evidence_backup.py` +`backup --execute` refuses before quiescing if the custody env is missing. +It then runs the existing stop-and-continue capture, seals the packet to +the recipient in `/etc/mas-backup/age-recipient` (the workstation key +`age17sxu…xdxs05w`), deletes the plaintext copies, and publishes through +`primary_application_archive.transfer`: conditional multipart create, then +a version-pinned readback compared by hash. It keeps one local ciphertext +and writes `latest-offsite.json`. `freshness` fails when the last verified +upload is older than 26h or the local ciphertext hash no longer matches. +Units live in `deploy/mas-backup/`: a daily backup timer at 04:15 UTC and +an hourly freshness timer. The worker runs from the node's +hash-locked `backup-venv` (Python 3.12, boto3 1.40, conditional multipart +supported) out of a dedicated `/opt/railiance-platform/mas-backup/`, so +the shared scripts are unchanged. `deploy/mas-backup/install.sh` checks +`SHA256SUMS`, installs the files, runs one backup and one freshness check, +and enables the timers only if both pass. Known residual: a deliberately +powered-off facility (replicas 0) makes the daily run fail by design. No +alert unit exists yet beyond the failed systemd state. + ## Rehearse recovery from the offsite copy ```task