#!/usr/bin/env python3 """Observe backup health, retain alert episodes, and refresh both delivery channels.""" import argparse import importlib.util import json from datetime import UTC, datetime from pathlib import Path from backup_alerts import MAX_EVENTS, drain spec = importlib.util.spec_from_file_location( "backup_health", Path(__file__).with_name("check-paired-backup-health.py")) health = importlib.util.module_from_spec(spec) spec.loader.exec_module(health) PENDING = {"multiple_pending_captures", "pending_requires_review", "backup_stalled"} def observations(report): reasons = set(report["reasons"]) return {"pending_backup": bool(reasons & PENDING), "backup_stale": "backup_stale" in reasons, "backup_failed": bool(reasons - PENDING - {"backup_stale"})} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--execute", action="store_true") parser.add_argument("--state-dir", type=Path, required=True) parser.add_argument("--mirror-receipt", type=Path, required=True) parser.add_argument("--alert-state-dir", type=Path, required=True) parser.add_argument("--minimum-free-bytes", type=int, required=True) parser.add_argument("--maximum-state-bytes", type=int, required=True) parser.add_argument("--hub-base", required=True) parser.add_argument("--alertmanager-base", required=True) args = parser.parse_args() now = datetime.now(UTC) report = health.check(args.state_dir, args.mirror_receipt, now, min_free_bytes=args.minimum_free_bytes, max_state_bytes=args.maximum_state_bytes) if sum(1 for _ in args.alert_state_dir.glob("*.json")) >= MAX_EVENTS * 3 // 4: report["reasons"] = sorted(set(report["reasons"]) | {"alert_journal_near_capacity"}) report["status"] = "unhealthy" if not args.execute: print(json.dumps(report)) return int(report["status"] != "healthy") try: delivery = drain(args.alert_state_dir, args.hub_base, args.alertmanager_base, now=now, observations=observations(report), recovered_capture=report.get("captured_at") if report["status"] == "healthy" else None) except Exception as error: delivery = {"status": "alert_delivery_requires_review", "failure_type": type(error).__name__} print(json.dumps({"health": report, "delivery": delivery})) # The unit measures successful monitoring, not backup health. A delivered # unhealthy observation must not trigger a second generic failure every hour. return int(delivery["status"] != "channel_handoffs_verified") if __name__ == "__main__": raise SystemExit(main())