tegwick
267d502d93
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m16s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
61 lines
2.7 KiB
Python
61 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Observe backup health, retain alert episodes, and refresh both delivery channels."""
|
|
|
|
import argparse
|
|
import importlib.util
|
|
import json
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
from backup_alerts import MAX_EVENTS, drain
|
|
|
|
spec = importlib.util.spec_from_file_location(
|
|
"backup_health", Path(__file__).with_name("check-paired-backup-health.py"))
|
|
health = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(health)
|
|
PENDING = {"multiple_pending_captures", "pending_requires_review", "backup_stalled"}
|
|
|
|
|
|
def observations(report):
|
|
reasons = set(report["reasons"])
|
|
return {"pending_backup": bool(reasons & PENDING), "backup_stale": "backup_stale" in reasons,
|
|
"backup_failed": bool(reasons - PENDING - {"backup_stale"})}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument("--state-dir", type=Path, required=True)
|
|
parser.add_argument("--mirror-receipt", type=Path, required=True)
|
|
parser.add_argument("--alert-state-dir", type=Path, required=True)
|
|
parser.add_argument("--minimum-free-bytes", type=int, required=True)
|
|
parser.add_argument("--maximum-state-bytes", type=int, required=True)
|
|
parser.add_argument("--hub-base", required=True)
|
|
parser.add_argument("--alertmanager-base", required=True)
|
|
args = parser.parse_args()
|
|
now = datetime.now(UTC)
|
|
report = health.check(args.state_dir, args.mirror_receipt, now,
|
|
min_free_bytes=args.minimum_free_bytes,
|
|
max_state_bytes=args.maximum_state_bytes)
|
|
if sum(1 for _ in args.alert_state_dir.glob("*.json")) >= MAX_EVENTS * 3 // 4:
|
|
report["reasons"] = sorted(set(report["reasons"]) | {"alert_journal_near_capacity"})
|
|
report["status"] = "unhealthy"
|
|
if not args.execute:
|
|
print(json.dumps(report))
|
|
return int(report["status"] != "healthy")
|
|
try:
|
|
delivery = drain(args.alert_state_dir, args.hub_base, args.alertmanager_base,
|
|
now=now, observations=observations(report),
|
|
recovered_capture=report.get("captured_at")
|
|
if report["status"] == "healthy" else None)
|
|
except Exception as error:
|
|
delivery = {"status": "alert_delivery_requires_review",
|
|
"failure_type": type(error).__name__}
|
|
print(json.dumps({"health": report, "delivery": delivery}))
|
|
# The unit measures successful monitoring, not backup health. A delivered
|
|
# unhealthy observation must not trigger a second generic failure every hour.
|
|
return int(delivery["status"] != "channel_handoffs_verified")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|