#!/usr/bin/env python3 """Daily MAS evidence backup worker on railiance01, plus its freshness check. backup --execute Quiesce the MAS writers in place (stop and continue, never scale or roll), capture the three evidence files, seal them to the workstation age recipient, delete the plaintext copies, and publish the ciphertext to platform-pg/application-archives/mas/ with an immutable create and a version-pinned hash readback. Writes a public receipt per run and latest-offsite.json. Never deletes a retained object. freshness Fails when the latest verified upload is older than STALE_HOURS, its receipt is not verified, or the local ciphertext no longer matches it. After each verified upload, local run directories older than LOCAL_RETENTION_DAYS are pruned. The current run is never pruned, and nothing at the provider is ever deleted. Credentials come only from /etc/mas-backup/primary.env through the systemd EnvironmentFile. No TEM or SMTP credential and no age identity is read here. """ from __future__ import annotations import argparse import json import os import re import shutil import sys from datetime import UTC, datetime, timedelta from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) import mas_evidence_archive as archive # noqa: E402 import primary_application_archive as primary # noqa: E402 STATE = Path("/var/lib/mas-evidence-backup") RECIPIENT_FILE = Path("/etc/mas-backup/age-recipient") LATEST = "latest-offsite.json" STALE_HOURS = 26 LOCAL_RETENTION_DAYS = 30 # owner-accepted 2026-11-03 _RUN_NAME = re.compile(r"\A\d{8}T\d{6}Z\Z") APPLICATION = "mas" def recipient(path=None): value = (path or RECIPIENT_FILE).read_text(encoding="utf-8").strip() if not value.startswith("age1") or len(value) > 128 or any(c.isspace() for c in value): raise archive.Refused("age_recipient_required") return value def publish_run(run_dir, captured_at, client, save_latest): """Upload run_dir/packet.tar.age and verify it by version-pinned readback.""" source = run_dir / "packet.tar.age" readback = run_dir / "readback.tar.age" receipt_path = run_dir / "offsite-receipt.json" receipt = { "schema": "platform.application-primary-archive.v1", "status": "pending", "application": APPLICATION, "captured_at": captured_at, "object_key": primary.key_for(APPLICATION, captured_at), } primary.checkpoint(receipt_path, receipt) try: primary.transfer( client, APPLICATION, source, readback, receipt, lambda: primary.checkpoint(receipt_path, receipt), ) except Exception as error: receipt.update(status="failed", failure_type=type(error).__name__) primary.checkpoint(receipt_path, receipt) raise # The readback matched by hash; keep one local ciphertext only. readback.unlink() receipt["local_ciphertext"] = str(source) receipt["retained_objects_deleted"] = False primary.checkpoint(receipt_path, receipt) save_latest(receipt) return receipt def backup(state=STATE, capture=None, client_factory=None, now=None): if capture is None: import mas_evidence_capture as capture # needs k3s, crictl and root if client_factory is None: client_factory = primary.runtime_client state.mkdir(mode=0o700, exist_ok=True) primary.private_directory(state) age_recipient = recipient() client = client_factory() # refuse before quiescing when custody is missing stamp = (now or datetime.now(UTC)).strftime("%Y%m%dT%H%M%SZ") run_dir = state / stamp try: captured = capture.quiesced_copy(run_dir) sealed = archive.seal(run_dir, run_dir, age_recipient) finally: for name in archive.FILES: path = run_dir / name if path.exists(): path.unlink() if (run_dir / "receipt.json").exists(): (run_dir / "receipt.json").unlink() public = { "schema": "platform.mas-evidence-run.v1", "captured_at": captured["captured_at"], "image": captured["image"], "volume": captured.get("volume"), "files": { name: {"present": meta.get("present"), "bytes": meta.get("bytes")} for name, meta in captured["files"].items() }, "ciphertext_sha256": sealed["ciphertext_sha256"], "ciphertext_bytes": sealed["ciphertext_bytes"], "facility_scaled": False, "relay_read": False, "mail_sent": False, "age_identity_on_node": False, } archive.checkpoint(run_dir / "capture-public.json", public) def save_latest(receipt): archive.checkpoint(state / LATEST, {**receipt, "run": stamp, "capture": public}) receipt = publish_run(run_dir, captured["captured_at"], client, save_latest) pruned = prune_local(state, stamp, now or datetime.now(UTC)) return {**receipt, "run": stamp, "capture": public, "local_retention_days": LOCAL_RETENTION_DAYS, "local_runs_pruned": pruned} def prune_local(state, keep, now, days=LOCAL_RETENTION_DAYS): """Remove dated run directories older than `days`, never `keep`. Local only.""" cutoff = now - timedelta(days=days) removed = [] for entry in sorted(state.iterdir()): if entry.name == keep or not _RUN_NAME.fullmatch(entry.name): continue if entry.is_symlink() or not entry.is_dir() or entry.parent != state: continue stamp = datetime.strptime(entry.name, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC) if stamp < cutoff: shutil.rmtree(entry) removed.append(entry.name) return removed def freshness(state=STATE, now=None): now = now or datetime.now(UTC) path = state / LATEST primary.regular_file(path) latest = json.loads(path.read_text(encoding="utf-8")) captured = datetime.fromisoformat(latest["captured_at"].replace("Z", "+00:00")) local = Path(latest.get("local_ciphertext", "")) checks = { "verified": latest.get("status") == "offsite_fetched_verified" and latest.get("remote_retrieval_verified") is True and latest.get("immutable_create") is True, "fresh": now - captured <= timedelta(hours=STALE_HOURS), "local_ciphertext_matches": local.is_file() and local.parent.parent == state and primary.digest(local) == latest.get("ciphertext_sha256"), } result = { "status": "fresh" if all(checks.values()) else "stale_or_unverified", "captured_at": latest["captured_at"], "age_hours": round((now - captured).total_seconds() / 3600, 2), "stale_after_hours": STALE_HOURS, "object_key": latest.get("object_key"), "version_id_present": bool(latest.get("version_id")), "checks": checks, } return result def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("action", choices=["backup", "freshness"]) parser.add_argument("--execute", action="store_true") args = parser.parse_args(argv) try: if args.action == "freshness": result = freshness() print(json.dumps(result, sort_keys=True)) return 0 if result["status"] == "fresh" else 1 if not args.execute: print(json.dumps({"status": "preview", "prefix": primary.PROFILES[APPLICATION], "facility_scaled": False, "effects": []})) return 0 if os.geteuid() != 0: raise archive.Refused("root_required") print(json.dumps(backup(), sort_keys=True)) return 0 except Exception as error: # never print provider bodies or values code = str(error) if isinstance(error, (archive.Refused, ValueError)) else "" print(json.dumps({"status": "failed", "failure_type": type(error).__name__, "code": code[:80]})) return 1 if __name__ == "__main__": sys.exit(main())