#!/usr/bin/env python3 """Redacted MAS evidence backup failure notices to the platform-root State Hub inbox. Started by systemd as OnFailure=mas-evidence-alert@%n.service for the backup and freshness units, and with no --unit by the freshness unit to replay any notice still pending. A notice is queued privately before sending, so a hub outage loses nothing; delivery counts only when the hub echoes the exact message back. Bodies carry the unit name and next steps, never values. """ from __future__ import annotations import argparse import fcntl import json import os import stat import tempfile import uuid from datetime import UTC, datetime from pathlib import Path from urllib.request import ProxyHandler, Request, build_opener UNITS = {"mas-evidence-backup.service", "mas-evidence-freshness.service"} TEST_UNIT = "mas-evidence-alert-test.service" SENDER = "railiance-platform" RECIPIENT = "platform-root" STATE = Path("/var/lib/mas-evidence-backup/notifications") API_BASE = "http://10.43.68.154:8000" class Refused(RuntimeError): pass def _private_directory(path): info = path.lstat() if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: raise Refused("private_notification_directory_required") def _private_file(path): info = path.lstat() if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: raise Refused("private_notification_file_required") def _checkpoint(path, value): fd, name = tempfile.mkstemp(prefix=".notice-", dir=path.parent) with os.fdopen(fd, "w") as out: json.dump(value, out, indent=2, sort_keys=True) out.flush() os.fsync(out.fileno()) os.chmod(name, 0o600) Path(name).replace(path) def post(base, message): request = Request(base.rstrip("/") + "/messages/", data=json.dumps(message).encode(), headers={"Content-Type": "application/json"}, method="POST") with build_opener(ProxyHandler({})).open(request, timeout=15) as response: result = json.loads(response.read(65536)) # A queue receipt is pending, never evidence of a human acknowledgement. if any(result.get(key) != message[key] for key in ("from_agent", "to_agent", "subject", "body")): raise Refused("inbox_receipt_not_verified") uuid.UUID(result["id"]) return result["id"] def message_for(event, unit, test): failure = unit == "mas-evidence-backup.service" action = ( "Confirm inbox delivery only." if test else "Inspect `journalctl -u mas-evidence-backup.service` on railiance01. Check that " "mas-mx and mas-statute run at 1 replica (a deliberately powered-off facility fails " "the capture), /etc/mas-backup/primary.env exists, and the provider is reachable. " "Do not scale or restart the facility from this notice and never delete retained objects." if failure else "The last verified MAS evidence upload is older than 26h or its local ciphertext no " "longer matches. Inspect mas-evidence-backup.service and " "/var/lib/mas-evidence-backup/latest-offsite.json on railiance01." ) kind = "notification test" if test else ("backup failure" if failure else "backup stale") return { "from_agent": SENDER, "to_agent": RECIPIENT, "subject": "MAS evidence " + kind, "body": json.dumps({ "event_id": event, "unit": unit, "test": test, "host": "railiance01", "workplan": "RPF-WP-0070", "rpo_hours": 24, "stale_after_hours": 26, "action": action, }, sort_keys=True), } def notify(state=STATE, base=API_BASE, unit=None, test=False, sender=post, now=None): if unit not in UNITS | {None, TEST_UNIT} or (test and unit is None): raise Refused("invalid_notification_unit") test = test or unit == TEST_UNIT state.mkdir(mode=0o700, parents=True, exist_ok=True) _private_directory(state) fd = os.open(state / "notify.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) if unit: event = str(uuid.uuid4()) _checkpoint(state / (event + ".json"), { "event_id": event, "unit": unit, "test": test, "status": "pending", "observed_at": (now or datetime.now(UTC)).isoformat(), "message": message_for(event, unit, test), }) delivered, pending = [], [] for path in sorted(state.glob("*.json")): _private_file(path) record = json.loads(path.read_text()) if record.get("status") != "pending": continue message = record["message"] if message.get("from_agent") != SENDER or message.get("to_agent") != RECIPIENT: raise Refused("invalid_pending_notification") try: message_id = sender(base, message) except Exception: pending.append(record["event_id"]) continue record.update(status="delivered", message_id=message_id, delivered_at=datetime.now(UTC).isoformat()) _checkpoint(path, record) delivered.append(message_id) return { "status": "pending_notification_requires_replay" if pending else "inbox_delivery_verified", "recipient": RECIPIENT, "message_ids": delivered, "pending_event_ids": pending, "human_acknowledged": False, } def main(argv=None): parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--unit", choices=sorted(UNITS | {TEST_UNIT})) parser.add_argument("--test", action="store_true") args = parser.parse_args(argv) try: result = notify(unit=args.unit, test=args.test) except Exception as error: result = {"status": "notification_failed", "failure_type": type(error).__name__} print(json.dumps(result, sort_keys=True)) return 0 if result["status"] == "inbox_delivery_verified" else 1 if __name__ == "__main__": raise SystemExit(main())