"""Durable consumer outbox for the existing State Hub and Alertmanager owners. No credentials or SMTP implementation. HTTP acceptance by Alertmanager is a handoff, never proof of email delivery. Active events must be refreshed hourly; their three-hour leases cannot replace independent monitoring of this producer. """ from __future__ import annotations import fcntl import json import os import stat import tempfile import uuid from datetime import UTC, datetime, timedelta from pathlib import Path from urllib.parse import urlencode from urllib.request import Request, urlopen CONDITIONS = {"backup_failed", "pending_backup", "backup_stale"} RECEIVER = "coulomb-backup-platform-root-email" MAX_EVENTS = 1024 MAX_BYTES = 16384 class Refused(ValueError): pass def private(path, directory=False): info = path.lstat() valid = stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode) if (not valid or info.st_uid != os.geteuid() or info.st_mode & 0o077 or (not directory and info.st_nlink != 1)): raise Refused("unsafe_alert_state") def checkpoint(path, record): data = json.dumps(record, sort_keys=True).encode() if len(data) > MAX_BYTES: raise Refused("alert_record_too_large") fd, name = tempfile.mkstemp(prefix=".checkpoint-", dir=path.parent) try: with os.fdopen(fd, "wb") as stream: stream.write(data) stream.flush() os.fsync(stream.fileno()) os.replace(name, path) directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory) finally: os.close(directory) finally: Path(name).unlink(missing_ok=True) def timestamp(value): parsed = datetime.fromisoformat(value) if parsed.tzinfo is None: raise Refused("timezone_required") return parsed.astimezone(UTC) def labels(record): return {"alertname": "CoulombBackupProblem", "application": "coulomb-social", "owner": "platform-root", "instance": "railiance01", "condition": record["condition"], "event_id": record["event_id"], "verification": "true" if record["test"] else "false"} def message(record): return { "from_agent": "coulomb-social", "to_agent": "platform-root", "subject": "Coulomb backup " + ("TEST " if record["test"] else "") + record["condition"], "body": json.dumps({ "event_id": record["event_id"], "condition": record["condition"], "observed_at": record["observed_at"], "test": record["test"], "application": "coulomb-social", "host": "railiance01", "workplan": "CSOC-WP-0012", "action": "Inspect backup/freshness service and retained receipts. " "Do not automatically restore, delete or discard a pending capture.", }, sort_keys=True), } def request_json(url, payload=None): request = Request(url, data=None if payload is None else json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) with urlopen(request, timeout=15) as response: if response.status not in (200, 201): raise Refused("unexpected_http_status") data = response.read(65537) if len(data) > 65536: raise Refused("oversized_transport_response") return json.loads(data) if data else None def deliver_hub(base, record): expected = message(record) receipt = request_json(base.rstrip("/") + "/messages/", expected) if not isinstance(receipt, dict) or any(receipt.get(k) != v for k, v in expected.items()): raise Refused("central_inbox_receipt_required") return str(uuid.UUID(receipt["id"])) def deliver_alertmanager(base, record, now): # Alertmanager's v2 API returns millisecond precision. Submit that same # precision so exact readback does not falsely reject a successful handoff. end = timestamp(record["resolved_at"]) if record.get("resolved_at") else ( now + timedelta(hours=3)) end = end.isoformat(timespec="milliseconds") start = timestamp(record["observed_at"]).isoformat(timespec="milliseconds") alert = {"labels": labels(record), "startsAt": start, "endsAt": end, "annotations": {"summary": "Coulomb backup " + record["condition"], "description": message(record)["body"]}} endpoint = base.rstrip("/") + "/api/v2/alerts" request_json(endpoint, [alert]) if not record.get("resolved_at"): query = urlencode({"filter": 'event_id="' + record["event_id"] + '"'}) active = request_json(endpoint + "?" + query) if not isinstance(active, list) or not any( item.get("labels") == alert["labels"] and timestamp(item["startsAt"]) == timestamp(alert["startsAt"]) and timestamp(item["endsAt"]) == timestamp(end) and {r["name"] for r in item.get("receivers", [])} == {RECEIVER} for item in active ): raise Refused("reviewed_email_route_readback_required") return {"status": "resolution_accepted" if record.get("resolved_at") else "active_email_route_verified", "accepted_at": now.isoformat(), "email_delivery_proven": False} def load(path): private(path) if path.stat().st_size > MAX_BYTES: raise Refused("alert_record_too_large") with path.open() as stream: record = json.load(stream) if (record.get("schema") != "coulomb.backup-alert-event.v1" or str(uuid.UUID(record["event_id"])) + ".json" != path.name or record["condition"] not in CONDITIONS or type(record["test"]) is not bool): raise Refused("invalid_alert_record") timestamp(record["observed_at"]) if record.get("resolved_at"): if timestamp(record["resolved_at"]) < timestamp(record["observed_at"]): raise Refused("resolution_before_event") return record def drain(state, hub_base, alertmanager_base, *, event=None, resolve=None, now=None, observations=None, recovered_capture=None, hub_sender=deliver_hub, email_sender=deliver_alertmanager): """Serialize producers, checkpoint before sends, and attempt channels independently. event is {event_id, condition, test}; a repeated ID must match. resolve names an existing UUID. No automatic deletion: a full journal refuses new events but still allows replay of existing events. Archive disposition is explicit. """ now = now or datetime.now(UTC) if now.tzinfo is None: raise Refused("timezone_required") if event: event = dict(event) event["event_id"] = str(uuid.UUID(event["event_id"])) if set(event) != {"event_id", "condition", "test"} or ( event["condition"] not in CONDITIONS or type(event["test"]) is not bool ): raise Refused("invalid_event") if resolve: resolve = str(uuid.UUID(resolve)) if observations is not None and (set(observations) != CONDITIONS or any(type(v) is not bool for v in observations.values())): raise Refused("complete_health_observations_required") if recovered_capture: recovered_capture = timestamp(recovered_capture) if observations is None or any(observations.values()) or recovered_capture > now: raise Refused("healthy_new_capture_required") state.mkdir(mode=0o700, parents=True, exist_ok=True) private(state, directory=True) if state.resolve() != state.absolute(): raise Refused("symlinked_alert_state") fd = os.open(state / "notify.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) with os.fdopen(fd, "w") as lock: private(state / "notify.lock") fcntl.flock(lock, fcntl.LOCK_EX) paths = sorted(state.glob("*.json")) records = {p.stem: (p, load(p)) for p in paths} if event: key = event["event_id"] if key in records: if any(records[key][1][k] != v for k, v in event.items()): raise Refused("event_id_reused_with_different_content") else: if len(records) >= MAX_EVENTS: raise Refused("alert_journal_full_requires_archival") record = {"schema": "coulomb.backup-alert-event.v1", **event, "origin": "explicit", "observed_at": now.isoformat(), "hub_message_id": None, "alertmanager": None} path = state / (key + ".json") checkpoint(path, record) records[key] = (path, record) if observations is not None: for condition, firing in sorted(observations.items()): active = [(p, r) for p, r in records.values() if r.get("origin") == "health" and r["condition"] == condition and not r["test"] and not r.get("resolved_at")] if firing and not active: if len(records) >= MAX_EVENTS: raise Refused("alert_journal_full_requires_archival") key = str(uuid.uuid4()) record = {"schema": "coulomb.backup-alert-event.v1", "event_id": key, "condition": condition, "test": False, "origin": "health", "observed_at": now.isoformat(), "hub_message_id": None, "alertmanager": None} path = state / (key + ".json") checkpoint(path, record) records[key] = (path, record) elif not firing: for path, record in active: record["resolved_at"] = now.isoformat() checkpoint(path, record) if recovered_capture: for path, record in records.values(): if (record.get("origin") == "explicit" and not record["test"] and record["condition"] == "backup_failed" and not record.get("resolved_at") and timestamp(record["observed_at"]) < recovered_capture): record["resolved_at"] = now.isoformat() checkpoint(path, record) if resolve: path, record = records[resolve] if not record.get("resolved_at"): record["resolved_at"] = now.isoformat() checkpoint(path, record) failures = [] # Rotate unfinished work across bounded systemd invocations: a slow or # unreachable channel must not permanently starve later events. ordered = sorted(records.items(), key=lambda item: item[1][1].get("last_attempt_at", "")) for key, (path, record) in ordered: if (record.get("hub_message_id") and (record.get("alertmanager") or {}).get("status") == "resolution_accepted"): continue record["last_attempt_at"] = now.isoformat() checkpoint(path, record) if not record.get("hub_message_id"): try: record["hub_message_id"] = hub_sender(hub_base, record) checkpoint(path, record) except Exception: failures.append({"event_id": key, "channel": "state_hub"}) accepted = record.get("alertmanager") or {} if accepted.get("status") == "resolution_accepted": continue try: record["alertmanager"] = email_sender(alertmanager_base, record, now) checkpoint(path, record) except Exception: failures.append({"event_id": key, "channel": "email_handoff"}) return {"status": "pending_channel_retry" if failures else "channel_handoffs_verified", "failures": failures, "events": len(records), "email_delivery_proven": False, "automatic_monitoring_active": False}