Alert platform-root when the MAS evidence backup fails or goes stale.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 4541@bnt-lap001
Assistant-Session: 2099b3aa-d7d9-49a9-ab4e-fa8b7d0e39e8

scripts/mas_evidence_notify.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Redacted MAS evidence backup failure notices to the platform-root State Hub inbox.
+
+Started by systemd as OnFailure=mas-evidence-alert@%n.service for the backup
+and freshness units, and with no --unit by the freshness unit to replay any
+notice still pending. A notice is queued privately before sending, so a hub
+outage loses nothing; delivery counts only when the hub echoes the exact
+message back. Bodies carry the unit name and next steps, never values.
+"""
+
+from __future__ import annotations
+
+import argparse
+import fcntl
+import json
+import os
+import stat
+import tempfile
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+from urllib.request import ProxyHandler, Request, build_opener
+
+UNITS = {"mas-evidence-backup.service", "mas-evidence-freshness.service"}
+TEST_UNIT = "mas-evidence-alert-test.service"
+SENDER = "railiance-platform"
+RECIPIENT = "platform-root"
+STATE = Path("/var/lib/mas-evidence-backup/notifications")
+API_BASE = "http://10.43.68.154:8000"
+
+
+class Refused(RuntimeError):
+    pass
+
+
+def _private_directory(path):
+    info = path.lstat()
+    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077:
+        raise Refused("private_notification_directory_required")
+
+
+def _private_file(path):
+    info = path.lstat()
+    if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077:
+        raise Refused("private_notification_file_required")
+
+
+def _checkpoint(path, value):
+    fd, name = tempfile.mkstemp(prefix=".notice-", dir=path.parent)
+    with os.fdopen(fd, "w") as out:
+        json.dump(value, out, indent=2, sort_keys=True)
+        out.flush()
+        os.fsync(out.fileno())
+    os.chmod(name, 0o600)
+    Path(name).replace(path)
+
+
+def post(base, message):
+    request = Request(base.rstrip("/") + "/messages/", data=json.dumps(message).encode(),
+                      headers={"Content-Type": "application/json"}, method="POST")
+    with build_opener(ProxyHandler({})).open(request, timeout=15) as response:
+        result = json.loads(response.read(65536))
+    # A queue receipt is pending, never evidence of a human acknowledgement.
+    if any(result.get(key) != message[key] for key in ("from_agent", "to_agent", "subject", "body")):
+        raise Refused("inbox_receipt_not_verified")
+    uuid.UUID(result["id"])
+    return result["id"]
+
+
+def message_for(event, unit, test):
+    failure = unit == "mas-evidence-backup.service"
+    action = (
+        "Confirm inbox delivery only." if test else
+        "Inspect `journalctl -u mas-evidence-backup.service` on railiance01. Check that "
+        "mas-mx and mas-statute run at 1 replica (a deliberately powered-off facility fails "
+        "the capture), /etc/mas-backup/primary.env exists, and the provider is reachable. "
+        "Do not scale or restart the facility from this notice and never delete retained objects."
+        if failure else
+        "The last verified MAS evidence upload is older than 26h or its local ciphertext no "
+        "longer matches. Inspect mas-evidence-backup.service and "
+        "/var/lib/mas-evidence-backup/latest-offsite.json on railiance01."
+    )
+    kind = "notification test" if test else ("backup failure" if failure else "backup stale")
+    return {
+        "from_agent": SENDER,
+        "to_agent": RECIPIENT,
+        "subject": "MAS evidence " + kind,
+        "body": json.dumps({
+            "event_id": event, "unit": unit, "test": test, "host": "railiance01",
+            "workplan": "RPF-WP-0070", "rpo_hours": 24, "stale_after_hours": 26,
+            "action": action,
+        }, sort_keys=True),
+    }
+
+
+def notify(state=STATE, base=API_BASE, unit=None, test=False, sender=post, now=None):
+    if unit not in UNITS | {None, TEST_UNIT} or (test and unit is None):
+        raise Refused("invalid_notification_unit")
+    test = test or unit == TEST_UNIT
+    state.mkdir(mode=0o700, parents=True, exist_ok=True)
+    _private_directory(state)
+    fd = os.open(state / "notify.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
+    with os.fdopen(fd, "w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        if unit:
+            event = str(uuid.uuid4())
+            _checkpoint(state / (event + ".json"), {
+                "event_id": event, "unit": unit, "test": test, "status": "pending",
+                "observed_at": (now or datetime.now(UTC)).isoformat(),
+                "message": message_for(event, unit, test),
+            })
+        delivered, pending = [], []
+        for path in sorted(state.glob("*.json")):
+            _private_file(path)
+            record = json.loads(path.read_text())
+            if record.get("status") != "pending":
+                continue
+            message = record["message"]
+            if message.get("from_agent") != SENDER or message.get("to_agent") != RECIPIENT:
+                raise Refused("invalid_pending_notification")
+            try:
+                message_id = sender(base, message)
+            except Exception:
+                pending.append(record["event_id"])
+                continue
+            record.update(status="delivered", message_id=message_id,
+                          delivered_at=datetime.now(UTC).isoformat())
+            _checkpoint(path, record)
+            delivered.append(message_id)
+    return {
+        "status": "pending_notification_requires_replay" if pending else "inbox_delivery_verified",
+        "recipient": RECIPIENT, "message_ids": delivered, "pending_event_ids": pending,
+        "human_acknowledged": False,
+    }
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--unit", choices=sorted(UNITS | {TEST_UNIT}))
+    parser.add_argument("--test", action="store_true")
+    args = parser.parse_args(argv)
+    try:
+        result = notify(unit=args.unit, test=args.test)
+    except Exception as error:
+        result = {"status": "notification_failed", "failure_type": type(error).__name__}
+    print(json.dumps(result, sort_keys=True))
+    return 0 if result["status"] == "inbox_delivery_verified" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())