railiance-platform/scripts/mas_evidence_notify.py

#!/usr/bin/env python3
"""Redacted MAS evidence backup failure notices to the platform-root State Hub inbox.

Started by systemd as OnFailure=mas-evidence-alert@%n.service for the backup
and freshness units, and with no --unit by the freshness unit to replay any
notice still pending. A notice is queued privately before sending, so a hub
outage loses nothing; delivery counts only when the hub echoes the exact
message back. Bodies carry the unit name and next steps, never values.
"""

from __future__ import annotations

import argparse
import fcntl
import json
import os
import stat
import tempfile
import uuid
from datetime import UTC, datetime
from pathlib import Path
from urllib.request import ProxyHandler, Request, build_opener

UNITS = {"mas-evidence-backup.service", "mas-evidence-freshness.service"}
TEST_UNIT = "mas-evidence-alert-test.service"
SENDER = "railiance-platform"
RECIPIENT = "platform-root"
STATE = Path("/var/lib/mas-evidence-backup/notifications")
API_BASE = "http://10.43.68.154:8000"


class Refused(RuntimeError):
    pass


def _private_directory(path):
    info = path.lstat()
    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077:
        raise Refused("private_notification_directory_required")


def _private_file(path):
    info = path.lstat()
    if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077:
        raise Refused("private_notification_file_required")


def _checkpoint(path, value):
    fd, name = tempfile.mkstemp(prefix=".notice-", dir=path.parent)
    with os.fdopen(fd, "w") as out:
        json.dump(value, out, indent=2, sort_keys=True)
        out.flush()
        os.fsync(out.fileno())
    os.chmod(name, 0o600)
    Path(name).replace(path)


def post(base, message):
    request = Request(base.rstrip("/") + "/messages/", data=json.dumps(message).encode(),
                      headers={"Content-Type": "application/json"}, method="POST")
    with build_opener(ProxyHandler({})).open(request, timeout=15) as response:
        result = json.loads(response.read(65536))
    # A queue receipt is pending, never evidence of a human acknowledgement.
    if any(result.get(key) != message[key] for key in ("from_agent", "to_agent", "subject", "body")):
        raise Refused("inbox_receipt_not_verified")
    uuid.UUID(result["id"])
    return result["id"]


def message_for(event, unit, test):
    failure = unit == "mas-evidence-backup.service"
    action = (
        "Confirm inbox delivery only." if test else
        "Inspect `journalctl -u mas-evidence-backup.service` on railiance01. Check that "
        "mas-mx and mas-statute run at 1 replica (a deliberately powered-off facility fails "
        "the capture), /etc/mas-backup/primary.env exists, and the provider is reachable. "
        "Do not scale or restart the facility from this notice and never delete retained objects."
        if failure else
        "The last verified MAS evidence upload is older than 26h or its local ciphertext no "
        "longer matches. Inspect mas-evidence-backup.service and "
        "/var/lib/mas-evidence-backup/latest-offsite.json on railiance01."
    )
    kind = "notification test" if test else ("backup failure" if failure else "backup stale")
    return {
        "from_agent": SENDER,
        "to_agent": RECIPIENT,
        "subject": "MAS evidence " + kind,
        "body": json.dumps({
            "event_id": event, "unit": unit, "test": test, "host": "railiance01",
            "workplan": "RPF-WP-0070", "rpo_hours": 24, "stale_after_hours": 26,
            "action": action,
        }, sort_keys=True),
    }


def notify(state=STATE, base=API_BASE, unit=None, test=False, sender=post, now=None):
    if unit not in UNITS | {None, TEST_UNIT} or (test and unit is None):
        raise Refused("invalid_notification_unit")
    test = test or unit == TEST_UNIT
    state.mkdir(mode=0o700, parents=True, exist_ok=True)
    _private_directory(state)
    fd = os.open(state / "notify.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
    with os.fdopen(fd, "w") as lock:
        fcntl.flock(lock, fcntl.LOCK_EX)
        if unit:
            event = str(uuid.uuid4())
            _checkpoint(state / (event + ".json"), {
                "event_id": event, "unit": unit, "test": test, "status": "pending",
                "observed_at": (now or datetime.now(UTC)).isoformat(),
                "message": message_for(event, unit, test),
            })
        delivered, pending = [], []
        for path in sorted(state.glob("*.json")):
            _private_file(path)
            record = json.loads(path.read_text())
            if record.get("status") != "pending":
                continue
            message = record["message"]
            if message.get("from_agent") != SENDER or message.get("to_agent") != RECIPIENT:
                raise Refused("invalid_pending_notification")
            try:
                message_id = sender(base, message)
            except Exception:
                pending.append(record["event_id"])
                continue
            record.update(status="delivered", message_id=message_id,
                          delivered_at=datetime.now(UTC).isoformat())
            _checkpoint(path, record)
            delivered.append(message_id)
    return {
        "status": "pending_notification_requires_replay" if pending else "inbox_delivery_verified",
        "recipient": RECIPIENT, "message_ids": delivered, "pending_event_ids": pending,
        "human_acknowledged": False,
    }


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--unit", choices=sorted(UNITS | {TEST_UNIT}))
    parser.add_argument("--test", action="store_true")
    args = parser.parse_args(argv)
    try:
        result = notify(unit=args.unit, test=args.test)
    except Exception as error:
        result = {"status": "notification_failed", "failure_type": type(error).__name__}
    print(json.dumps(result, sort_keys=True))
    return 0 if result["status"] == "inbox_delivery_verified" else 1


if __name__ == "__main__":
    raise SystemExit(main())