diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 18b042a..3c0512d 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -11,6 +11,7 @@ | workplan | RPF-WP-ADHOC-2026-10-10 | finished | — | workplans/ADHOC-2026-10-10.md | | workplan | RPF-WP-ADHOC-2026-10-11 | finished | — | workplans/ADHOC-2026-10-11.md | | workplan | RPF-WP-ADHOC-2026-10-19 | finished | — | workplans/ADHOC-2026-10-19.md | +| workplan | RPF-WP-ADHOC-2026-11-03 | active | — | workplans/ADHOC-2026-11-03.md | | workplan | RPF-WP-0015 | active | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md | | workplan | RPF-WP-0025 | finished | — | workplans/RPF-WP-0025-openbao-operator-only-access.md | | workplan | RPF-WP-0027 | blocked | — | workplans/RPF-WP-0027-keycape-live-secret-exposure-recovery.md | @@ -57,6 +58,7 @@ | task | RPF-WP-ADHOC-2026-10-19-T01 | done | — | workplans/ADHOC-2026-10-19.md | | task | RPF-WP-ADHOC-2026-10-19-T02 | done | — | workplans/ADHOC-2026-10-19.md | | task | RPF-WP-ADHOC-2026-10-19-T03 | done | — | workplans/ADHOC-2026-10-19.md | +| task | RPF-WP-ADHOC-2026-11-03-T01 | progress | — | workplans/ADHOC-2026-11-03.md | | task | RPF-WP-0015-T01 | done | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md | | task | RPF-WP-0015-T02 | done | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md | | task | RPF-WP-0015-T03 | wait | — | workplans/RPF-WP-0015-audit-core-custody-and-recovery-coordination.md | diff --git a/deploy/mas-backup/install.sh b/deploy/mas-backup/install.sh index aa69b88..2b9e1c9 100755 --- a/deploy/mas-backup/install.sh +++ b/deploy/mas-backup/install.sh @@ -13,11 +13,11 @@ recipient=$(cat age-recipient) case "$recipient" in age1*) ;; *) echo '{"status":"failed","code":"age_recipient_required"}'; exit 1;; esac install -d -m 0755 -o root -g root /opt/railiance-platform/mas-backup -for f in mas_evidence_backup.py mas_evidence_capture.py mas_evidence_archive.py primary_application_archive.py; do +for f in mas_evidence_backup.py mas_evidence_capture.py mas_evidence_archive.py primary_application_archive.py mas_evidence_notify.py; do install -m 0644 -o root -g root "$f" /opt/railiance-platform/mas-backup/"$f" done install -m 0644 -o root -g root age-recipient /etc/mas-backup/age-recipient -for u in mas-evidence-backup.service mas-evidence-backup.timer mas-evidence-freshness.service mas-evidence-freshness.timer; do +for u in mas-evidence-backup.service mas-evidence-backup.timer mas-evidence-freshness.service mas-evidence-freshness.timer mas-evidence-alert@.service; do install -m 0644 -o root -g root "$u" /etc/systemd/system/"$u" done systemctl daemon-reload diff --git a/deploy/mas-backup/mas-evidence-alert@.service b/deploy/mas-backup/mas-evidence-alert@.service new file mode 100644 index 0000000..6152f5d --- /dev/null +++ b/deploy/mas-backup/mas-evidence-alert@.service @@ -0,0 +1,17 @@ +[Unit] +Description=Notify platform-root of MAS evidence backup failure in %i +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +UMask=0077 +StateDirectory=mas-evidence-backup/notifications +StateDirectoryMode=0700 +ExecStart=/usr/bin/python3 /opt/railiance-platform/mas-backup/mas_evidence_notify.py --unit %i +TimeoutStartSec=2min +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true diff --git a/deploy/mas-backup/mas-evidence-backup.service b/deploy/mas-backup/mas-evidence-backup.service index b9189ac..597aad2 100644 --- a/deploy/mas-backup/mas-evidence-backup.service +++ b/deploy/mas-backup/mas-evidence-backup.service @@ -2,6 +2,7 @@ Description=Quiesced, age-sealed MAS evidence backup to the primary archive (RPF-WP-0070) After=network-online.target k3s.service Wants=network-online.target +OnFailure=mas-evidence-alert@%n.service [Service] Type=oneshot diff --git a/deploy/mas-backup/mas-evidence-freshness.service b/deploy/mas-backup/mas-evidence-freshness.service index 73c55a0..bdc6718 100644 --- a/deploy/mas-backup/mas-evidence-freshness.service +++ b/deploy/mas-backup/mas-evidence-freshness.service @@ -1,13 +1,17 @@ [Unit] Description=Check MAS evidence archive age (stale after 26h) and local ciphertext hash +OnFailure=mas-evidence-alert@%n.service [Service] Type=oneshot User=root UMask=0077 ExecStart=/opt/railiance-platform/backup-venv/bin/python /opt/railiance-platform/mas-backup/mas_evidence_backup.py freshness +# Replay any notice still pending after a hub outage. Never fails the check. +ExecStartPost=-/usr/bin/python3 /opt/railiance-platform/mas-backup/mas_evidence_notify.py NoNewPrivileges=true PrivateTmp=true ProtectSystem=strict ProtectHome=true -ReadOnlyPaths=/var/lib/mas-evidence-backup +StateDirectory=mas-evidence-backup/notifications +StateDirectoryMode=0700 diff --git a/scripts/mas_evidence_notify.py b/scripts/mas_evidence_notify.py new file mode 100644 index 0000000..075c72b --- /dev/null +++ b/scripts/mas_evidence_notify.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Redacted MAS evidence backup failure notices to the platform-root State Hub inbox. + +Started by systemd as OnFailure=mas-evidence-alert@%n.service for the backup +and freshness units, and with no --unit by the freshness unit to replay any +notice still pending. A notice is queued privately before sending, so a hub +outage loses nothing; delivery counts only when the hub echoes the exact +message back. Bodies carry the unit name and next steps, never values. +""" + +from __future__ import annotations + +import argparse +import fcntl +import json +import os +import stat +import tempfile +import uuid +from datetime import UTC, datetime +from pathlib import Path +from urllib.request import ProxyHandler, Request, build_opener + +UNITS = {"mas-evidence-backup.service", "mas-evidence-freshness.service"} +TEST_UNIT = "mas-evidence-alert-test.service" +SENDER = "railiance-platform" +RECIPIENT = "platform-root" +STATE = Path("/var/lib/mas-evidence-backup/notifications") +API_BASE = "http://10.43.68.154:8000" + + +class Refused(RuntimeError): + pass + + +def _private_directory(path): + info = path.lstat() + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise Refused("private_notification_directory_required") + + +def _private_file(path): + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077: + raise Refused("private_notification_file_required") + + +def _checkpoint(path, value): + fd, name = tempfile.mkstemp(prefix=".notice-", dir=path.parent) + with os.fdopen(fd, "w") as out: + json.dump(value, out, indent=2, sort_keys=True) + out.flush() + os.fsync(out.fileno()) + os.chmod(name, 0o600) + Path(name).replace(path) + + +def post(base, message): + request = Request(base.rstrip("/") + "/messages/", data=json.dumps(message).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with build_opener(ProxyHandler({})).open(request, timeout=15) as response: + result = json.loads(response.read(65536)) + # A queue receipt is pending, never evidence of a human acknowledgement. + if any(result.get(key) != message[key] for key in ("from_agent", "to_agent", "subject", "body")): + raise Refused("inbox_receipt_not_verified") + uuid.UUID(result["id"]) + return result["id"] + + +def message_for(event, unit, test): + failure = unit == "mas-evidence-backup.service" + action = ( + "Confirm inbox delivery only." if test else + "Inspect `journalctl -u mas-evidence-backup.service` on railiance01. Check that " + "mas-mx and mas-statute run at 1 replica (a deliberately powered-off facility fails " + "the capture), /etc/mas-backup/primary.env exists, and the provider is reachable. " + "Do not scale or restart the facility from this notice and never delete retained objects." + if failure else + "The last verified MAS evidence upload is older than 26h or its local ciphertext no " + "longer matches. Inspect mas-evidence-backup.service and " + "/var/lib/mas-evidence-backup/latest-offsite.json on railiance01." + ) + kind = "notification test" if test else ("backup failure" if failure else "backup stale") + return { + "from_agent": SENDER, + "to_agent": RECIPIENT, + "subject": "MAS evidence " + kind, + "body": json.dumps({ + "event_id": event, "unit": unit, "test": test, "host": "railiance01", + "workplan": "RPF-WP-0070", "rpo_hours": 24, "stale_after_hours": 26, + "action": action, + }, sort_keys=True), + } + + +def notify(state=STATE, base=API_BASE, unit=None, test=False, sender=post, now=None): + if unit not in UNITS | {None, TEST_UNIT} or (test and unit is None): + raise Refused("invalid_notification_unit") + test = test or unit == TEST_UNIT + state.mkdir(mode=0o700, parents=True, exist_ok=True) + _private_directory(state) + fd = os.open(state / "notify.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + if unit: + event = str(uuid.uuid4()) + _checkpoint(state / (event + ".json"), { + "event_id": event, "unit": unit, "test": test, "status": "pending", + "observed_at": (now or datetime.now(UTC)).isoformat(), + "message": message_for(event, unit, test), + }) + delivered, pending = [], [] + for path in sorted(state.glob("*.json")): + _private_file(path) + record = json.loads(path.read_text()) + if record.get("status") != "pending": + continue + message = record["message"] + if message.get("from_agent") != SENDER or message.get("to_agent") != RECIPIENT: + raise Refused("invalid_pending_notification") + try: + message_id = sender(base, message) + except Exception: + pending.append(record["event_id"]) + continue + record.update(status="delivered", message_id=message_id, + delivered_at=datetime.now(UTC).isoformat()) + _checkpoint(path, record) + delivered.append(message_id) + return { + "status": "pending_notification_requires_replay" if pending else "inbox_delivery_verified", + "recipient": RECIPIENT, "message_ids": delivered, "pending_event_ids": pending, + "human_acknowledged": False, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--unit", choices=sorted(UNITS | {TEST_UNIT})) + parser.add_argument("--test", action="store_true") + args = parser.parse_args(argv) + try: + result = notify(unit=args.unit, test=args.test) + except Exception as error: + result = {"status": "notification_failed", "failure_type": type(error).__name__} + print(json.dumps(result, sort_keys=True)) + return 0 if result["status"] == "inbox_delivery_verified" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_mas_evidence_notify.py b/tests/test_mas_evidence_notify.py new file mode 100644 index 0000000..48a62c7 --- /dev/null +++ b/tests/test_mas_evidence_notify.py @@ -0,0 +1,74 @@ +"""MAS backup failure notices: queued privately, delivered with a verified echo, replayed.""" + +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("notify", ROOT / "scripts/mas_evidence_notify.py") +nt = importlib.util.module_from_spec(spec) +spec.loader.exec_module(nt) + + +class Hub: + def __init__(self, down=False): + self.down, self.sent = down, [] + + def __call__(self, base, message): + if self.down: + raise OSError("hub unreachable") + self.sent.append(message) + return f"00000000-0000-0000-0000-{len(self.sent):012d}" + + +def test_backup_failure_is_delivered_to_platform_root(tmp_path): + hub = Hub() + result = nt.notify(tmp_path / "n", unit="mas-evidence-backup.service", sender=hub) + assert result["status"] == "inbox_delivery_verified" and len(result["message_ids"]) == 1 + message = hub.sent[0] + assert message["to_agent"] == "platform-root" and message["subject"] == "MAS evidence backup failure" + body = json.loads(message["body"]) + assert body["unit"] == "mas-evidence-backup.service" and body["test"] is False + assert "never delete retained objects" in body["action"] + + +def test_stale_notice_has_its_own_subject(tmp_path): + hub = Hub() + nt.notify(tmp_path / "n", unit="mas-evidence-freshness.service", sender=hub) + assert hub.sent[0]["subject"] == "MAS evidence backup stale" + + +def test_hub_outage_keeps_the_notice_and_replay_delivers_it_once(tmp_path): + state = tmp_path / "n" + down = nt.notify(state, unit="mas-evidence-backup.service", sender=Hub(down=True)) + assert down["status"] == "pending_notification_requires_replay" and len(down["pending_event_ids"]) == 1 + hub = Hub() + replay = nt.notify(state, sender=hub) + assert replay["status"] == "inbox_delivery_verified" and len(hub.sent) == 1 + again = Hub() + nt.notify(state, sender=again) + assert again.sent == [] + + +def test_test_unit_marks_the_notice_as_a_test(tmp_path): + hub = Hub() + nt.notify(tmp_path / "n", unit="mas-evidence-alert-test.service", sender=hub) + assert hub.sent[0]["subject"] == "MAS evidence notification test" + assert json.loads(hub.sent[0]["body"])["test"] is True + + +def test_unknown_unit_is_refused(tmp_path): + with pytest.raises(nt.Refused): + nt.notify(tmp_path / "n", unit="sshd.service", sender=Hub()) + + +def test_echo_mismatch_is_not_counted_as_delivered(tmp_path): + def tampering(base, message): + raise nt.Refused("inbox_receipt_not_verified") + + result = nt.notify(tmp_path / "n", unit="mas-evidence-backup.service", sender=tampering) + assert result["status"] == "pending_notification_requires_replay" + record = json.loads(next((tmp_path / "n").glob("*.json")).read_text()) + assert record["status"] == "pending" diff --git a/workplans/ADHOC-2026-11-03.md b/workplans/ADHOC-2026-11-03.md new file mode 100644 index 0000000..f706a23 --- /dev/null +++ b/workplans/ADHOC-2026-11-03.md @@ -0,0 +1,43 @@ +--- +id: RPF-WP-ADHOC-2026-11-03 +type: workplan +title: "Alert platform-root when the MAS evidence backup fails or goes stale" +domain: financials +repo: railiance-platform +status: active +flavor: implementation +owner: claude-code +topic_slug: coulomb-social +created: "2026-11-03" +updated: "2026-11-03" +related: + - RPF-WP-0070 +--- + +## Alert on failed or stale MAS evidence backups + +```task +id: RPF-WP-ADHOC-2026-11-03-T01 +status: progress +priority: medium +``` + +RPF-WP-0070 closed with one residual: a failed MAS backup only showed up as a +failed systemd unit. The fix copies the email-connect ledger pattern. +`OnFailure=mas-evidence-alert@%n.service` on `mas-evidence-backup.service` +and `mas-evidence-freshness.service` starts `scripts/mas_evidence_notify.py`. +That script queues a notice with no values in a private directory +(`/var/lib/mas-evidence-backup/notifications`, 0700), then posts it to the +`platform-root` State Hub inbox at the in-cluster address `10.43.68.154:8000`. +Delivery counts only when the hub echoes back the exact sender, recipient, +subject and body. A notice that cannot be sent stays pending, and the next +successful freshness run replays it (`ExecStartPost=-`, which never fails the +check). The subjects are `MAS evidence backup failure`, `MAS evidence backup +stale` and `MAS evidence notification test`. Each body carries the unit, +host, workplan and next steps, and a failure notice says not to scale or +restart the facility and never to delete retained objects. While a backup +stays stale, the hourly check repeats the notice once per hour. + +Done when the bundle is installed on railiance01 and a test notice from +`mas-evidence-alert@mas-evidence-alert-test.service.service` arrives in the +`platform-root` inbox.