tegwick
11cab7298b
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m27s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a17f16-d99c-7ea3-95f0-8212897fba6b
400 lines
15 KiB
Python
400 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Node-side scheduled paired backup using platform age/Scaleway custody.
|
|
|
|
Default: preflight only. Live execution requires approval of the schedule,
|
|
quiescence and offsite consumer. No escrow key is needed to make a backup.
|
|
Expiration belongs to its owner.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
import backup_capacity as capacity
|
|
|
|
|
|
class Refused(RuntimeError):
|
|
pass
|
|
|
|
|
|
def digest(path: Path) -> str:
|
|
with path.open("rb") as stream:
|
|
return hashlib.file_digest(stream, "sha256").hexdigest()
|
|
|
|
|
|
def validate_pair(directory: Path) -> dict:
|
|
receipt = json.loads((directory / "receipt.json").read_text())
|
|
if (
|
|
receipt.get("status") != "captured"
|
|
or not receipt.get("writers_quiesced")
|
|
or not receipt.get("app_resumed")
|
|
or receipt.get("database") != "coulomb_social_db"
|
|
):
|
|
raise Refused("pair_incomplete")
|
|
for filename, prefix in [("database.dump", "database"), ("content.tar.gz", "content")]:
|
|
path = directory / filename
|
|
if path.is_symlink() or not path.is_file():
|
|
raise Refused("pair_invalid")
|
|
if digest(path) != receipt[prefix + "_sha256"]:
|
|
raise Refused("pair_hash_mismatch")
|
|
if path.stat().st_size != receipt[prefix + "_bytes"]:
|
|
raise Refused("pair_size_mismatch")
|
|
return receipt
|
|
|
|
|
|
def command(argv: list[str], *, env=None, timeout=900) -> None:
|
|
# File-backed output avoids pipe deadlocks and keeps binary/private child
|
|
# output out of diagnostics. Termination gives the snapshot its resume window.
|
|
with tempfile.TemporaryFile() as output:
|
|
process = subprocess.Popen(argv, env=env, stdout=output, stderr=output)
|
|
try:
|
|
result = process.wait(timeout=timeout)
|
|
except BaseException:
|
|
process.terminate()
|
|
try:
|
|
process.wait(timeout=400)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
raise
|
|
if result:
|
|
raise Refused("backup_command_failed")
|
|
|
|
|
|
def publish(
|
|
encrypted: Path,
|
|
fetched: Path,
|
|
transfer_receipt: Path,
|
|
captured_at: str,
|
|
tool: Path,
|
|
python: Path,
|
|
) -> dict:
|
|
command(
|
|
[
|
|
str(python),
|
|
str(tool),
|
|
"transfer",
|
|
"--application",
|
|
"coulomb-social",
|
|
"--source",
|
|
str(encrypted),
|
|
"--output",
|
|
str(fetched),
|
|
"--receipt",
|
|
str(transfer_receipt),
|
|
"--captured-at",
|
|
captured_at,
|
|
"--execute",
|
|
]
|
|
)
|
|
receipt = json.loads(transfer_receipt.read_text())
|
|
if (
|
|
receipt.get("status") != "offsite_fetched_verified"
|
|
or receipt.get("application") != "coulomb-social"
|
|
or receipt.get("captured_at") != captured_at
|
|
or not receipt.get("immutable_create")
|
|
or not receipt.get("remote_retrieval_verified")
|
|
or not receipt.get("version_id")
|
|
or receipt.get("version_id") == "null"
|
|
or receipt.get("ciphertext_sha256") != digest(encrypted)
|
|
or receipt.get("ciphertext_sha256") != digest(fetched)
|
|
or receipt.get("ciphertext_bytes") != encrypted.stat().st_size
|
|
or fetched.is_symlink()
|
|
or not fetched.is_file()
|
|
):
|
|
raise Refused("primary_receipt_not_verified")
|
|
return receipt
|
|
|
|
|
|
def write_receipt(path: Path, receipt: dict | list) -> None:
|
|
fd, name = tempfile.mkstemp(prefix="receipt-", dir=path.parent)
|
|
temporary = Path(name)
|
|
with os.fdopen(fd, "w") as stream:
|
|
json.dump(receipt, stream, indent=2)
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
temporary.replace(path)
|
|
|
|
|
|
def archive_pair(directory: Path, archive: Path) -> None:
|
|
with archive.open("xb") as stream:
|
|
os.chmod(archive, 0o600)
|
|
with tarfile.open(fileobj=stream, mode="w") as bundle:
|
|
for name in ["database.dump", "content.tar.gz", "receipt.json"]:
|
|
bundle.add(directory / name, arcname=name, recursive=False)
|
|
|
|
|
|
def run_backup(
|
|
state: Path,
|
|
primary_tool: Path,
|
|
recipient: str,
|
|
now: datetime,
|
|
*,
|
|
primary_python=Path("/opt/railiance-platform/backup-venv/bin/python"),
|
|
resume=False,
|
|
) -> dict:
|
|
# One owner-only state directory, one process, and one recovery point.
|
|
state.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
info = state.lstat()
|
|
if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.geteuid() or info.st_mode & 0o077:
|
|
raise Refused("state_directory_not_private")
|
|
fd = os.open(state / "backup.lock", os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "w") as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
raise Refused("backup_already_running") from None
|
|
pending = list(state.glob("pending-*"))
|
|
if resume:
|
|
if (
|
|
len(pending) != 1
|
|
or pending[0].is_symlink()
|
|
or not pending[0].is_dir()
|
|
or pending[0].stat().st_mode & 0o077
|
|
):
|
|
raise Refused("one_pending_pair_required")
|
|
stage = pending[0]
|
|
else:
|
|
if pending:
|
|
raise Refused("pending_backup_requires_review")
|
|
# Hold the writer lock across capacity check and capture. A sealed retry
|
|
# needs no new dump/tar/encryption; all previous scratch still counts.
|
|
phase = "resume" if resume and (stage / "sealed.json").is_file() else "capture"
|
|
capacity.require_capacity(state, phase=phase)
|
|
if not resume:
|
|
stage = Path(tempfile.mkdtemp(prefix="pending-", dir=state))
|
|
pair = stage / "pair"
|
|
helper = Path(__file__).with_name("snapshot-content-and-db.py")
|
|
if not resume:
|
|
command([sys.executable, str(helper), "--execute", "--destination", str(pair)])
|
|
captured = validate_pair(pair)
|
|
encrypted = stage / "packet.tar.age"
|
|
sealed_path = stage / "sealed.json"
|
|
binding = {
|
|
key: captured[key]
|
|
for key in [
|
|
"database_sha256",
|
|
"content_sha256",
|
|
"started_at",
|
|
"image",
|
|
]
|
|
}
|
|
if sealed_path.exists():
|
|
sealed = json.loads(sealed_path.read_text())
|
|
if (
|
|
sealed.get("binding") != binding
|
|
or sealed.get("age_recipient") != recipient
|
|
or not encrypted.is_file()
|
|
or encrypted.is_symlink()
|
|
or sealed.get("ciphertext_sha256") != digest(encrypted)
|
|
):
|
|
raise Refused("sealed_packet_binding_mismatch")
|
|
else:
|
|
if encrypted.exists():
|
|
raise Refused("partial_encryption_requires_review")
|
|
archive = stage / "packet.tar"
|
|
archive_pair(pair, archive)
|
|
command(["age", "-r", recipient, "-o", str(encrypted), str(archive)])
|
|
if not encrypted.is_file() or encrypted.stat().st_size == 0:
|
|
raise Refused("encryption_missing")
|
|
os.chmod(encrypted, 0o600)
|
|
sealed = {"binding": binding, "ciphertext_sha256": digest(encrypted),
|
|
"age_recipient": recipient}
|
|
write_receipt(sealed_path, sealed)
|
|
# Keep ciphertext and the primary key stable across an uncertain outcome.
|
|
# Each readback attempt has a fresh file; partial files remain private.
|
|
fd, fetched_name = tempfile.mkstemp(prefix="readback-", dir=stage)
|
|
os.close(fd)
|
|
fetched = Path(fetched_name)
|
|
fetched.unlink()
|
|
transfer = publish(
|
|
encrypted,
|
|
fetched,
|
|
stage / "primary-transfer.json",
|
|
captured["started_at"],
|
|
primary_tool,
|
|
primary_python,
|
|
)
|
|
object_key = transfer["object_key"]
|
|
filename = object_key.rsplit("/", 1)[-1]
|
|
if not object_key.startswith("platform-pg/application-archives/coulomb-social/"):
|
|
raise Refused("invalid_archive_scope")
|
|
if not re.fullmatch(r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age", filename):
|
|
raise Refused("invalid_archive_name")
|
|
receipt = {
|
|
"schema": "platform.application-primary-archive.v1",
|
|
"status": "offsite_fetched_verified",
|
|
"application": "coulomb-social",
|
|
"age_recipient": recipient,
|
|
"captured_at": captured["started_at"],
|
|
"completed_at": datetime.now(UTC).isoformat(),
|
|
"image": captured["image"],
|
|
"destination": transfer["destination"],
|
|
"object_key": object_key,
|
|
"version_id": transfer["version_id"],
|
|
"local_archive": filename,
|
|
"encrypted_sha256": digest(encrypted),
|
|
"encrypted_bytes": encrypted.stat().st_size,
|
|
"ciphertext_sha256": digest(encrypted),
|
|
"ciphertext_bytes": encrypted.stat().st_size,
|
|
"immutable_create": True,
|
|
"database_sha256": captured["database_sha256"],
|
|
"content_sha256": captured["content_sha256"],
|
|
"pause_and_capture_seconds": captured["elapsed_seconds"],
|
|
"remote_retrieval_verified": True,
|
|
"application_restore_verified": False,
|
|
}
|
|
target = state / filename
|
|
if target.is_symlink():
|
|
raise Refused("local_archive_collision")
|
|
if target.exists():
|
|
if digest(target) != receipt["encrypted_sha256"]:
|
|
raise Refused("local_archive_collision")
|
|
else:
|
|
shutil.copyfile(fetched, target)
|
|
os.chmod(target, 0o600)
|
|
inventory_path = state / "inventory.json"
|
|
inventory = json.loads(inventory_path.read_text()) if inventory_path.exists() else []
|
|
if not any(item["object_key"] == object_key for item in inventory):
|
|
inventory.append(receipt)
|
|
write_receipt(inventory_path, inventory)
|
|
write_receipt(state / "last-success.json", receipt)
|
|
shutil.rmtree(stage)
|
|
return receipt
|
|
|
|
|
|
def check_freshness(state: Path, now: datetime, hours: int = 26) -> dict:
|
|
receipt = json.loads((state / "last-success.json").read_text())
|
|
captured = datetime.fromisoformat(receipt["captured_at"].replace("Z", "+00:00"))
|
|
age = (now - captured).total_seconds()
|
|
if age < 0 or age > hours * 3600:
|
|
raise Refused("backup_stale")
|
|
if receipt.get("status") != "offsite_fetched_verified" or not receipt.get(
|
|
"remote_retrieval_verified"
|
|
):
|
|
raise Refused("backup_not_uploaded")
|
|
name = receipt["local_archive"]
|
|
if not re.fullmatch(r"\d{8}T\d{6}Z-[a-f0-9]{24}\.tar\.age", name):
|
|
raise Refused("invalid_archive_name")
|
|
if (
|
|
not receipt.get("version_id")
|
|
or receipt["version_id"] == "null"
|
|
or (state / name).is_symlink()
|
|
or digest(state / name) != receipt["encrypted_sha256"]
|
|
):
|
|
raise Refused("local_encrypted_hash_mismatch")
|
|
return {
|
|
"status": "fresh",
|
|
"age_hours": round(age / 3600, 2),
|
|
"remote_retrieval_verified": receipt["remote_retrieval_verified"],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--execute", action="store_true")
|
|
parser.add_argument(
|
|
"--resume-pending",
|
|
action="store_true",
|
|
help="Publish the one reviewed pending pair without pausing the app again",
|
|
)
|
|
parser.add_argument("--check-freshness", action="store_true")
|
|
parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/coulomb-social-backup"))
|
|
parser.add_argument(
|
|
"--primary-tool",
|
|
type=Path,
|
|
default=Path("/opt/railiance-platform/scripts/primary_application_archive.py"),
|
|
)
|
|
parser.add_argument(
|
|
"--primary-python",
|
|
type=Path,
|
|
default=Path("/opt/railiance-platform/backup-venv/bin/python"),
|
|
)
|
|
parser.add_argument(
|
|
"--age-recipient", default=os.environ.get("COULOMB_SOCIAL_BACKUP_AGE_RECIPIENT"),
|
|
help="Explicit Coulomb recovery public recipient; no shared/default key",
|
|
)
|
|
args = parser.parse_args()
|
|
if args.resume_pending and not args.execute:
|
|
raise Refused("resume_requires_execute")
|
|
now = datetime.now(UTC)
|
|
if args.check_freshness:
|
|
print(json.dumps(check_freshness(args.state_dir, now)))
|
|
elif args.execute:
|
|
if not isinstance(args.age_recipient, str) or not re.fullmatch(
|
|
r"age1[a-z0-9]{58}", args.age_recipient
|
|
):
|
|
raise Refused("explicit_age_recipient_required")
|
|
if (
|
|
not args.state_dir.is_absolute()
|
|
or not args.primary_tool.is_file()
|
|
or not args.primary_python.is_file()
|
|
):
|
|
raise Refused("invalid_backup_configuration")
|
|
capacity.enforce_file_limit()
|
|
# Fail before pausing the app if encryption or credential custody is absent.
|
|
# Encrypt an empty input into the captured output sink to validate the
|
|
# actual recipient/checksum before any capture or application pause.
|
|
command(["age", "-r", args.age_recipient, "/dev/null"])
|
|
command(
|
|
[
|
|
str(args.primary_python),
|
|
str(args.primary_tool),
|
|
"check",
|
|
"--application",
|
|
"coulomb-social",
|
|
]
|
|
)
|
|
print(
|
|
json.dumps(
|
|
run_backup(
|
|
args.state_dir,
|
|
args.primary_tool,
|
|
args.age_recipient,
|
|
now,
|
|
primary_python=args.primary_python,
|
|
resume=args.resume_pending,
|
|
)
|
|
)
|
|
)
|
|
else:
|
|
helper = Path(__file__).with_name("snapshot-content-and-db.py")
|
|
command([sys.executable, str(helper), "--destination", "/tmp/csoc-backup-review"])
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "reviewed",
|
|
"live_changes": False,
|
|
"offsite_auth_checked": False,
|
|
"immutable_primary": True,
|
|
}
|
|
)
|
|
)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
|
|
def terminate(signum, frame):
|
|
raise Refused("backup_interrupted")
|
|
|
|
signal.signal(signal.SIGTERM, terminate)
|
|
sys.exit(main())
|
|
except Exception as exc:
|
|
print(json.dumps({"status": "failed", "failure_type": type(exc).__name__}))
|
|
sys.exit(1)
|