railiance-platform/scripts/mas_evidence_backup.py

#!/usr/bin/env python3
"""Daily MAS evidence backup worker on railiance01, plus its freshness check.

backup --execute
    Quiesce the MAS writers in place (stop and continue, never scale or roll),
    capture the three evidence files, seal them to the workstation age
    recipient, delete the plaintext copies, and publish the ciphertext to
    platform-pg/application-archives/mas/ with an immutable create and a
    version-pinned hash readback. Writes a public receipt per run and
    latest-offsite.json. Never deletes a retained object.

freshness
    Fails when the latest verified upload is older than STALE_HOURS, its
    receipt is not verified, or the local ciphertext no longer matches it.

After each verified upload, local run directories older than
LOCAL_RETENTION_DAYS are pruned. The current run is never pruned, and
nothing at the provider is ever deleted.

Credentials come only from /etc/mas-backup/primary.env through the systemd
EnvironmentFile. No TEM or SMTP credential and no age identity is read here.
"""

from __future__ import annotations

import argparse
import json
import os
import re
import shutil
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path

HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))

import mas_evidence_archive as archive  # noqa: E402
import primary_application_archive as primary  # noqa: E402

STATE = Path("/var/lib/mas-evidence-backup")
RECIPIENT_FILE = Path("/etc/mas-backup/age-recipient")
LATEST = "latest-offsite.json"
STALE_HOURS = 26
LOCAL_RETENTION_DAYS = 30  # owner-accepted 2026-11-03
_RUN_NAME = re.compile(r"\A\d{8}T\d{6}Z\Z")
APPLICATION = "mas"


def recipient(path=None):
    value = (path or RECIPIENT_FILE).read_text(encoding="utf-8").strip()
    if not value.startswith("age1") or len(value) > 128 or any(c.isspace() for c in value):
        raise archive.Refused("age_recipient_required")
    return value


def publish_run(run_dir, captured_at, client, save_latest):
    """Upload run_dir/packet.tar.age and verify it by version-pinned readback."""
    source = run_dir / "packet.tar.age"
    readback = run_dir / "readback.tar.age"
    receipt_path = run_dir / "offsite-receipt.json"
    receipt = {
        "schema": "platform.application-primary-archive.v1",
        "status": "pending",
        "application": APPLICATION,
        "captured_at": captured_at,
        "object_key": primary.key_for(APPLICATION, captured_at),
    }
    primary.checkpoint(receipt_path, receipt)
    try:
        primary.transfer(
            client, APPLICATION, source, readback, receipt,
            lambda: primary.checkpoint(receipt_path, receipt),
        )
    except Exception as error:
        receipt.update(status="failed", failure_type=type(error).__name__)
        primary.checkpoint(receipt_path, receipt)
        raise
    # The readback matched by hash; keep one local ciphertext only.
    readback.unlink()
    receipt["local_ciphertext"] = str(source)
    receipt["retained_objects_deleted"] = False
    primary.checkpoint(receipt_path, receipt)
    save_latest(receipt)
    return receipt


def backup(state=STATE, capture=None, client_factory=None, now=None):
    if capture is None:
        import mas_evidence_capture as capture  # needs k3s, crictl and root
    if client_factory is None:
        client_factory = primary.runtime_client
    state.mkdir(mode=0o700, exist_ok=True)
    primary.private_directory(state)
    age_recipient = recipient()
    client = client_factory()  # refuse before quiescing when custody is missing
    stamp = (now or datetime.now(UTC)).strftime("%Y%m%dT%H%M%SZ")
    run_dir = state / stamp
    try:
        captured = capture.quiesced_copy(run_dir)
        sealed = archive.seal(run_dir, run_dir, age_recipient)
    finally:
        for name in archive.FILES:
            path = run_dir / name
            if path.exists():
                path.unlink()
    if (run_dir / "receipt.json").exists():
        (run_dir / "receipt.json").unlink()
    public = {
        "schema": "platform.mas-evidence-run.v1",
        "captured_at": captured["captured_at"],
        "image": captured["image"],
        "volume": captured.get("volume"),
        "files": {
            name: {"present": meta.get("present"), "bytes": meta.get("bytes")}
            for name, meta in captured["files"].items()
        },
        "ciphertext_sha256": sealed["ciphertext_sha256"],
        "ciphertext_bytes": sealed["ciphertext_bytes"],
        "facility_scaled": False,
        "relay_read": False,
        "mail_sent": False,
        "age_identity_on_node": False,
    }
    archive.checkpoint(run_dir / "capture-public.json", public)

    def save_latest(receipt):
        archive.checkpoint(state / LATEST, {**receipt, "run": stamp, "capture": public})

    receipt = publish_run(run_dir, captured["captured_at"], client, save_latest)
    pruned = prune_local(state, stamp, now or datetime.now(UTC))
    return {**receipt, "run": stamp, "capture": public,
            "local_retention_days": LOCAL_RETENTION_DAYS, "local_runs_pruned": pruned}


def prune_local(state, keep, now, days=LOCAL_RETENTION_DAYS):
    """Remove dated run directories older than `days`, never `keep`. Local only."""
    cutoff = now - timedelta(days=days)
    removed = []
    for entry in sorted(state.iterdir()):
        if entry.name == keep or not _RUN_NAME.fullmatch(entry.name):
            continue
        if entry.is_symlink() or not entry.is_dir() or entry.parent != state:
            continue
        stamp = datetime.strptime(entry.name, "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
        if stamp < cutoff:
            shutil.rmtree(entry)
            removed.append(entry.name)
    return removed


def freshness(state=STATE, now=None):
    now = now or datetime.now(UTC)
    path = state / LATEST
    primary.regular_file(path)
    latest = json.loads(path.read_text(encoding="utf-8"))
    captured = datetime.fromisoformat(latest["captured_at"].replace("Z", "+00:00"))
    local = Path(latest.get("local_ciphertext", ""))
    checks = {
        "verified": latest.get("status") == "offsite_fetched_verified"
        and latest.get("remote_retrieval_verified") is True
        and latest.get("immutable_create") is True,
        "fresh": now - captured <= timedelta(hours=STALE_HOURS),
        "local_ciphertext_matches": local.is_file()
        and local.parent.parent == state
        and primary.digest(local) == latest.get("ciphertext_sha256"),
    }
    result = {
        "status": "fresh" if all(checks.values()) else "stale_or_unverified",
        "captured_at": latest["captured_at"],
        "age_hours": round((now - captured).total_seconds() / 3600, 2),
        "stale_after_hours": STALE_HOURS,
        "object_key": latest.get("object_key"),
        "version_id_present": bool(latest.get("version_id")),
        "checks": checks,
    }
    return result


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("action", choices=["backup", "freshness"])
    parser.add_argument("--execute", action="store_true")
    args = parser.parse_args(argv)
    try:
        if args.action == "freshness":
            result = freshness()
            print(json.dumps(result, sort_keys=True))
            return 0 if result["status"] == "fresh" else 1
        if not args.execute:
            print(json.dumps({"status": "preview", "prefix": primary.PROFILES[APPLICATION],
                              "facility_scaled": False, "effects": []}))
            return 0
        if os.geteuid() != 0:
            raise archive.Refused("root_required")
        print(json.dumps(backup(), sort_keys=True))
        return 0
    except Exception as error:  # never print provider bodies or values
        code = str(error) if isinstance(error, (archive.Refused, ValueError)) else ""
        print(json.dumps({"status": "failed", "failure_type": type(error).__name__,
                          "code": code[:80]}))
        return 1


if __name__ == "__main__":
    sys.exit(main())