tegwick
267d502d93
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m16s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
225 lines
10 KiB
Python
225 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Read-only backup operating check. No provider calls, sends, retries or cleanup.
|
|
|
|
Recorded readback evidence is checked; current provider availability and operator
|
|
reception require separate native proofs. Timers remain subject to admission.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import stat
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
PREFIX = "platform-pg/application-archives/coulomb-social/"
|
|
PROJECT = "06d107bc-9d84-4672-b2a0-95701cc0c07f"
|
|
MIRROR_BUCKET = "railiance-coulomb-social-archives"
|
|
NAME = r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age"
|
|
BINDING = (
|
|
"application", "captured_at", "object_key", "version_id", "ciphertext_sha256",
|
|
"ciphertext_bytes", "age_recipient", "image", "database_sha256", "content_sha256",
|
|
)
|
|
|
|
|
|
class Refused(ValueError):
|
|
pass
|
|
|
|
|
|
def regular(path):
|
|
info = path.lstat()
|
|
if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid()
|
|
or info.st_mode & 0o077 or info.st_nlink != 1):
|
|
raise Refused("unsafe_state_entry")
|
|
return info
|
|
|
|
|
|
def read_json(path):
|
|
regular(path)
|
|
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
with os.fdopen(fd, "rb") as stream:
|
|
data = stream.read(1024 * 1024 + 1)
|
|
if len(data) > 1024 * 1024:
|
|
raise Refused("oversized_receipt")
|
|
value = json.loads(data)
|
|
if not isinstance(value, dict):
|
|
raise Refused("invalid_receipt")
|
|
return value
|
|
|
|
|
|
def timestamp(value):
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
raise Refused("naive_capture_timestamp")
|
|
return parsed.astimezone(UTC)
|
|
|
|
|
|
def validate_primary(receipt):
|
|
captured = timestamp(receipt["captured_at"])
|
|
name = receipt["local_archive"]
|
|
if (receipt.get("schema") != "platform.application-primary-archive.v1"
|
|
or receipt.get("application") != "coulomb-social"
|
|
or receipt.get("status") != "offsite_fetched_verified"
|
|
or receipt.get("immutable_create") is not True
|
|
or receipt.get("remote_retrieval_verified") is not True
|
|
or not isinstance(name, str) or not re.fullmatch(NAME, name)
|
|
or not name.startswith(captured.strftime("%Y%m%dT%H%M%SZ-"))
|
|
or receipt["object_key"] != PREFIX + captured.strftime("%Y/%m/%d/") + name
|
|
or receipt.get("destination") != "s3://railiance-platform-pg-backup/"
|
|
+ receipt["object_key"]
|
|
or not isinstance(receipt.get("version_id"), str)
|
|
or not receipt["version_id"] or receipt["version_id"] == "null"
|
|
or type(receipt.get("ciphertext_bytes")) is not int
|
|
or receipt["ciphertext_bytes"] <= 0
|
|
or receipt.get("encrypted_bytes") != receipt["ciphertext_bytes"]
|
|
or receipt.get("encrypted_sha256") != receipt.get("ciphertext_sha256")
|
|
or not isinstance(receipt.get("age_recipient"), str)
|
|
or not re.fullmatch(r"age1[a-z0-9]{58}", receipt["age_recipient"])
|
|
or not isinstance(receipt.get("image"), str)
|
|
or not re.fullmatch(r"forgejo\.coulomb\.social/coulomb/coulomb-social@sha256:"
|
|
r"[a-f0-9]{64}", receipt["image"])):
|
|
raise Refused("invalid_primary_evidence")
|
|
for field in ("ciphertext_sha256", "database_sha256", "content_sha256"):
|
|
if (not isinstance(receipt.get(field), str)
|
|
or not re.fullmatch(r"[a-f0-9]{64}", receipt[field])):
|
|
raise Refused("invalid_primary_hash")
|
|
return captured
|
|
|
|
|
|
def validate_mirror(primary, mirrored, *, require_retained_receipt=False):
|
|
validate_primary(mirrored)
|
|
ref = mirrored["recovery_mirror"]
|
|
if (any(mirrored[k] != primary[k] for k in BINDING)
|
|
or ref.get("project_id") != PROJECT or ref.get("bucket") != MIRROR_BUCKET
|
|
or ref.get("object_key") != "archives/" + primary["object_key"]
|
|
or ref.get("ciphertext_sha256") != primary["ciphertext_sha256"]
|
|
or ref.get("ciphertext_bytes") != primary["ciphertext_bytes"]
|
|
or ref.get("hash_readback_verified") is not True
|
|
or not isinstance(ref.get("version_id"), str)
|
|
or not ref["version_id"] or ref["version_id"] == "null"):
|
|
raise Refused("mirror_binding_differs")
|
|
if require_retained_receipt:
|
|
digest = ref.get("receipt_sha256", "")
|
|
version = ref.get("receipt_version_id")
|
|
if (not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest)
|
|
or ref.get("receipt_object_key") != "archives/" + primary["object_key"]
|
|
+ "." + digest + ".receipt.json"
|
|
or not isinstance(version, str) or not version or version == "null"
|
|
or any(c in version for c in "\r\n")
|
|
or type(ref.get("receipt_bytes")) is not int
|
|
or not 0 < ref["receipt_bytes"] <= 65536
|
|
or ref.get("receipt_readback_verified") is not True):
|
|
raise Refused("retained_mirror_receipt_required")
|
|
|
|
|
|
def check(state, mirror_receipt, now, *, min_free_bytes, max_state_bytes,
|
|
disk_usage=shutil.disk_usage):
|
|
if (now.tzinfo is None or type(min_free_bytes) is not int or min_free_bytes < 0
|
|
or type(max_state_bytes) is not int or max_state_bytes <= 0):
|
|
raise ValueError("Explicit nonnegative reserve and positive state budget required")
|
|
report = {
|
|
"schema": "coulomb.paired-backup-health.v1", "status": "unhealthy", "reasons": [],
|
|
"observed_at": now.astimezone(UTC).isoformat(),
|
|
"provider_contacted": False, "recorded_readback_only": True,
|
|
"state_modified": False, "notification_sent": False, "operator_reception_proven": False,
|
|
"minimum_free_bytes": min_free_bytes, "maximum_state_bytes": max_state_bytes,
|
|
}
|
|
reasons = report["reasons"]
|
|
try:
|
|
info = state.lstat()
|
|
if (not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid()
|
|
or info.st_mode & 0o077 or state.resolve() != state.absolute()):
|
|
raise Refused("unsafe_state_directory")
|
|
total, pending = 0, []
|
|
def walk_error(error):
|
|
raise error
|
|
|
|
for directory, directories, files in os.walk(state, followlinks=False, onerror=walk_error):
|
|
for name in directories:
|
|
path = Path(directory) / name
|
|
entry = path.lstat()
|
|
if (not stat.S_ISDIR(entry.st_mode) or entry.st_uid != os.geteuid()
|
|
or entry.st_mode & 0o077):
|
|
raise Refused("unsafe_state_entry")
|
|
if Path(directory) == state and name.startswith("pending-"):
|
|
pending.append(entry)
|
|
for name in files:
|
|
total += regular(Path(directory) / name).st_size
|
|
report.update(state_bytes=total, free_bytes=disk_usage(state).free,
|
|
pending_count=len(pending))
|
|
if total > max_state_bytes:
|
|
reasons.append("state_budget_exceeded")
|
|
if report["free_bytes"] < min_free_bytes:
|
|
reasons.append("insufficient_free_space")
|
|
busy = False
|
|
lock_path = state / "backup.lock"
|
|
if lock_path.exists() or lock_path.is_symlink():
|
|
regular(lock_path)
|
|
fd = os.open(lock_path, os.O_RDONLY | os.O_NOFOLLOW)
|
|
try:
|
|
try:
|
|
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
busy = True
|
|
finally:
|
|
os.close(fd)
|
|
report["backup_process_holds_lock"] = busy
|
|
if pending:
|
|
ages = [now.timestamp() - p.st_mtime for p in pending]
|
|
if len(pending) != 1:
|
|
reasons.append("multiple_pending_captures")
|
|
elif not busy:
|
|
reasons.append("pending_requires_review")
|
|
elif min(ages) < 0 or max(ages) > 35 * 60:
|
|
reasons.append("backup_stalled")
|
|
receipt = read_json(state / "last-success.json")
|
|
captured = validate_primary(receipt)
|
|
age = (now - captured).total_seconds()
|
|
report["capture_age_seconds"] = round(age, 3)
|
|
report["captured_at"] = captured.isoformat()
|
|
if age < 0:
|
|
reasons.append("capture_timestamp_in_future")
|
|
elif age > 26 * 3600:
|
|
reasons.append("backup_stale")
|
|
archive = state / receipt["local_archive"]
|
|
archive_info = regular(archive)
|
|
with archive.open("rb") as stream:
|
|
digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
|
if (archive_info.st_size != receipt["ciphertext_bytes"]
|
|
or digest != receipt["ciphertext_sha256"]):
|
|
reasons.append("local_ciphertext_differs")
|
|
try:
|
|
validate_mirror(receipt, read_json(mirror_receipt), require_retained_receipt=True)
|
|
report["matching_mirror_readback_recorded"] = True
|
|
except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError):
|
|
reasons.append("matching_mirror_evidence_missing_or_invalid")
|
|
except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError):
|
|
# No paths, values, receipt bodies or raw OS/provider errors enter output.
|
|
reasons.append("state_evidence_missing_or_invalid")
|
|
report["reasons"] = sorted(set(reasons))
|
|
if not reasons:
|
|
report["status"] = "healthy"
|
|
return report
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/coulomb-social-backup"))
|
|
parser.add_argument("--mirror-receipt", type=Path, required=True)
|
|
parser.add_argument("--minimum-free-bytes", type=int, required=True)
|
|
parser.add_argument("--maximum-state-bytes", type=int, required=True)
|
|
args = parser.parse_args()
|
|
result = check(args.state_dir, args.mirror_receipt, datetime.now(UTC),
|
|
min_free_bytes=args.minimum_free_bytes, max_state_bytes=args.maximum_state_bytes)
|
|
print(json.dumps(result, sort_keys=True))
|
|
return int(result["status"] != "healthy")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|