coulomb-social/scripts/backup_alerts.py

"""Durable consumer outbox for the existing State Hub and Alertmanager owners.

No credentials or SMTP implementation. HTTP acceptance by Alertmanager is a
handoff, never proof of email delivery. Active events must be refreshed hourly;
their three-hour leases cannot replace independent monitoring of this producer.
"""

from __future__ import annotations

import fcntl
import json
import os
import stat
import tempfile
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from urllib.parse import urlencode
from urllib.request import Request, urlopen

CONDITIONS = {"backup_failed", "pending_backup", "backup_stale"}
RECEIVER = "coulomb-backup-platform-root-email"
MAX_EVENTS = 1024
MAX_BYTES = 16384


class Refused(ValueError):
    pass


def private(path, directory=False):
    info = path.lstat()
    valid = stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode)
    if (not valid or info.st_uid != os.geteuid() or info.st_mode & 0o077
            or (not directory and info.st_nlink != 1)):
        raise Refused("unsafe_alert_state")


def checkpoint(path, record):
    data = json.dumps(record, sort_keys=True).encode()
    if len(data) > MAX_BYTES:
        raise Refused("alert_record_too_large")
    fd, name = tempfile.mkstemp(prefix=".checkpoint-", dir=path.parent)
    try:
        with os.fdopen(fd, "wb") as stream:
            stream.write(data)
            stream.flush()
            os.fsync(stream.fileno())
        os.replace(name, path)
        directory = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
        try:
            os.fsync(directory)
        finally:
            os.close(directory)
    finally:
        Path(name).unlink(missing_ok=True)


def timestamp(value):
    parsed = datetime.fromisoformat(value)
    if parsed.tzinfo is None:
        raise Refused("timezone_required")
    return parsed.astimezone(UTC)


def labels(record):
    return {"alertname": "CoulombBackupProblem", "application": "coulomb-social",
            "owner": "platform-root", "instance": "railiance01",
            "condition": record["condition"], "event_id": record["event_id"],
            "verification": "true" if record["test"] else "false"}


def message(record):
    return {
        "from_agent": "coulomb-social", "to_agent": "platform-root",
        "subject": "Coulomb backup " + ("TEST " if record["test"] else "")
        + record["condition"],
        "body": json.dumps({
            "event_id": record["event_id"], "condition": record["condition"],
            "observed_at": record["observed_at"], "test": record["test"],
            "application": "coulomb-social", "host": "railiance01",
            "workplan": "CSOC-WP-0012",
            "action": "Inspect backup/freshness service and retained receipts. "
            "Do not automatically restore, delete or discard a pending capture.",
        }, sort_keys=True),
    }


def request_json(url, payload=None):
    request = Request(url, data=None if payload is None else json.dumps(payload).encode(),
                      headers={"Content-Type": "application/json"})
    with urlopen(request, timeout=15) as response:
        if response.status not in (200, 201):
            raise Refused("unexpected_http_status")
        data = response.read(65537)
    if len(data) > 65536:
        raise Refused("oversized_transport_response")
    return json.loads(data) if data else None


def deliver_hub(base, record):
    expected = message(record)
    receipt = request_json(base.rstrip("/") + "/messages/", expected)
    if not isinstance(receipt, dict) or any(receipt.get(k) != v for k, v in expected.items()):
        raise Refused("central_inbox_receipt_required")
    return str(uuid.UUID(receipt["id"]))


def deliver_alertmanager(base, record, now):
    # Alertmanager's v2 API returns millisecond precision. Submit that same
    # precision so exact readback does not falsely reject a successful handoff.
    end = timestamp(record["resolved_at"]) if record.get("resolved_at") else (
        now + timedelta(hours=3))
    end = end.isoformat(timespec="milliseconds")
    start = timestamp(record["observed_at"]).isoformat(timespec="milliseconds")
    alert = {"labels": labels(record), "startsAt": start, "endsAt": end,
             "annotations": {"summary": "Coulomb backup " + record["condition"],
                             "description": message(record)["body"]}}
    endpoint = base.rstrip("/") + "/api/v2/alerts"
    request_json(endpoint, [alert])
    if not record.get("resolved_at"):
        query = urlencode({"filter": 'event_id="' + record["event_id"] + '"'})
        active = request_json(endpoint + "?" + query)
        if not isinstance(active, list) or not any(
            item.get("labels") == alert["labels"]
            and timestamp(item["startsAt"]) == timestamp(alert["startsAt"])
            and timestamp(item["endsAt"]) == timestamp(end)
            and {r["name"] for r in item.get("receivers", [])} == {RECEIVER}
            for item in active
        ):
            raise Refused("reviewed_email_route_readback_required")
    return {"status": "resolution_accepted" if record.get("resolved_at")
            else "active_email_route_verified", "accepted_at": now.isoformat(),
            "email_delivery_proven": False}


def load(path):
    private(path)
    if path.stat().st_size > MAX_BYTES:
        raise Refused("alert_record_too_large")
    with path.open() as stream:
        record = json.load(stream)
    if (record.get("schema") != "coulomb.backup-alert-event.v1"
            or str(uuid.UUID(record["event_id"])) + ".json" != path.name
            or record["condition"] not in CONDITIONS or type(record["test"]) is not bool):
        raise Refused("invalid_alert_record")
    timestamp(record["observed_at"])
    if record.get("resolved_at"):
        if timestamp(record["resolved_at"]) < timestamp(record["observed_at"]):
            raise Refused("resolution_before_event")
    return record


def drain(state, hub_base, alertmanager_base, *, event=None, resolve=None, now=None,
          observations=None, recovered_capture=None,
          hub_sender=deliver_hub, email_sender=deliver_alertmanager):
    """Serialize producers, checkpoint before sends, and attempt channels independently.

    event is {event_id, condition, test}; a repeated ID must match. resolve names
    an existing UUID. No automatic deletion: a full journal refuses new events
    but still allows replay of existing events. Archive disposition is explicit.
    """
    now = now or datetime.now(UTC)
    if now.tzinfo is None:
        raise Refused("timezone_required")
    if event:
        event = dict(event)
        event["event_id"] = str(uuid.UUID(event["event_id"]))
        if set(event) != {"event_id", "condition", "test"} or (
            event["condition"] not in CONDITIONS or type(event["test"]) is not bool
        ):
            raise Refused("invalid_event")
    if resolve:
        resolve = str(uuid.UUID(resolve))
    if observations is not None and (set(observations) != CONDITIONS
                                    or any(type(v) is not bool for v in observations.values())):
        raise Refused("complete_health_observations_required")
    if recovered_capture:
        recovered_capture = timestamp(recovered_capture)
        if observations is None or any(observations.values()) or recovered_capture > now:
            raise Refused("healthy_new_capture_required")
    state.mkdir(mode=0o700, parents=True, exist_ok=True)
    private(state, directory=True)
    if state.resolve() != state.absolute():
        raise Refused("symlinked_alert_state")
    fd = os.open(state / "notify.lock", os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
    with os.fdopen(fd, "w") as lock:
        private(state / "notify.lock")
        fcntl.flock(lock, fcntl.LOCK_EX)
        paths = sorted(state.glob("*.json"))
        records = {p.stem: (p, load(p)) for p in paths}
        if event:
            key = event["event_id"]
            if key in records:
                if any(records[key][1][k] != v for k, v in event.items()):
                    raise Refused("event_id_reused_with_different_content")
            else:
                if len(records) >= MAX_EVENTS:
                    raise Refused("alert_journal_full_requires_archival")
                record = {"schema": "coulomb.backup-alert-event.v1", **event,
                          "origin": "explicit",
                          "observed_at": now.isoformat(), "hub_message_id": None,
                          "alertmanager": None}
                path = state / (key + ".json")
                checkpoint(path, record)
                records[key] = (path, record)
        if observations is not None:
            for condition, firing in sorted(observations.items()):
                active = [(p, r) for p, r in records.values()
                          if r.get("origin") == "health" and r["condition"] == condition
                          and not r["test"] and not r.get("resolved_at")]
                if firing and not active:
                    if len(records) >= MAX_EVENTS:
                        raise Refused("alert_journal_full_requires_archival")
                    key = str(uuid.uuid4())
                    record = {"schema": "coulomb.backup-alert-event.v1", "event_id": key,
                              "condition": condition, "test": False, "origin": "health",
                              "observed_at": now.isoformat(), "hub_message_id": None,
                              "alertmanager": None}
                    path = state / (key + ".json")
                    checkpoint(path, record)
                    records[key] = (path, record)
                elif not firing:
                    for path, record in active:
                        record["resolved_at"] = now.isoformat()
                        checkpoint(path, record)
        if recovered_capture:
            for path, record in records.values():
                if (record.get("origin") == "explicit" and not record["test"]
                        and record["condition"] == "backup_failed"
                        and not record.get("resolved_at")
                        and timestamp(record["observed_at"]) < recovered_capture):
                    record["resolved_at"] = now.isoformat()
                    checkpoint(path, record)
        if resolve:
            path, record = records[resolve]
            if not record.get("resolved_at"):
                record["resolved_at"] = now.isoformat()
                checkpoint(path, record)
        failures = []
        # Rotate unfinished work across bounded systemd invocations: a slow or
        # unreachable channel must not permanently starve later events.
        ordered = sorted(records.items(), key=lambda item: item[1][1].get("last_attempt_at", ""))
        for key, (path, record) in ordered:
            if (record.get("hub_message_id")
                    and (record.get("alertmanager") or {}).get("status") == "resolution_accepted"):
                continue
            record["last_attempt_at"] = now.isoformat()
            checkpoint(path, record)
            if not record.get("hub_message_id"):
                try:
                    record["hub_message_id"] = hub_sender(hub_base, record)
                    checkpoint(path, record)
                except Exception:
                    failures.append({"event_id": key, "channel": "state_hub"})
            accepted = record.get("alertmanager") or {}
            if accepted.get("status") == "resolution_accepted":
                continue
            try:
                record["alertmanager"] = email_sender(alertmanager_base, record, now)
                checkpoint(path, record)
            except Exception:
                failures.append({"event_id": key, "channel": "email_handoff"})
        return {"status": "pending_channel_retry" if failures else "channel_handoffs_verified",
                "failures": failures, "events": len(records), "email_delivery_proven": False,
                "automatic_monitoring_active": False}