#!/usr/bin/env python3 """Plan or execute one collision-safe S1 off-site envelope upload.""" from __future__ import annotations import argparse import base64 import hashlib import json import os import re import sys import tarfile import tempfile import urllib.error import urllib.parse import urllib.request import uuid from datetime import datetime, timezone from pathlib import Path from typing import Any import yaml from s1_backup import _sha256 from s1_backup_contract import BackupContractError from s1_restore import RestoreError, validate_bundle ROOT = Path(__file__).resolve().parents[1] SHA256_RE = re.compile(r"^[0-9a-f]{64}$") EXPECTED_MEMBERS = { "manifest.json", "receipt.json", "os-config.tar.gz.age", "packages.txt.age", } class OffsiteError(RuntimeError): """The off-site contract, evidence, or upload failed closed.""" class _NoRedirect(urllib.request.HTTPRedirectHandler): """Keep a credential-bearing file-drop request on its reviewed origin.""" def redirect_request( self, req: Any, fp: Any, code: int, msg: str, headers: Any, newurl: str, ) -> None: return None def _read_yaml(path: Path, label: str) -> dict[str, Any]: try: payload = yaml.safe_load(path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError) as exc: raise OffsiteError(f"cannot read {label}") from exc if not isinstance(payload, dict): raise OffsiteError(f"{label} must be a YAML object") return payload def contract_sha256(path: Path) -> str: return hashlib.sha256(path.read_bytes()).hexdigest() def load_contract(path: Path, *, fixture_http: bool = False) -> dict[str, Any]: payload = _read_yaml(path, "off-site contract") errors: list[str] = [] if payload.get("schema_version") != "1.0": errors.append("schema_version must be 1.0") if payload.get("route_id") != "railiance-backup-offsite-lane": errors.append("route_id must be railiance-backup-offsite-lane") if payload.get("owner_repo") != "railiance-platform": errors.append("owner_repo must be railiance-platform") provider = payload.get("provider") if not isinstance(provider, dict): errors.append("provider must be an object") provider = {} if provider.get("kind") != "nextcloud-file-drop": errors.append("provider.kind must be nextcloud-file-drop") base_url = provider.get("base_url") if not isinstance(base_url, str): errors.append("provider.base_url must be a URL") else: parsed = urllib.parse.urlsplit(base_url) allowed_scheme = "http" if fixture_http else "https" if parsed.scheme != allowed_scheme or not parsed.netloc or parsed.query or parsed.fragment: errors.append(f"provider.base_url must be a bounded {allowed_scheme} URL") if "@" in parsed.netloc or parsed.username or parsed.password: errors.append("provider.base_url must contain no user information") prefix = provider.get("remote_prefix") if not isinstance(prefix, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{1,62}", prefix): errors.append("provider.remote_prefix must be a safe single path segment") transport = payload.get("transport") expected_transport = { "method": "PUT", "content_type": "application/x-tar", "if_none_match": "*", "success_codes": [201, 204], "authentication": "basic-token-empty-password", "secret_env": "RAILIANCE_BACKUP_NC_TOKEN", "response_body": "suppressed", } if transport != expected_transport: errors.append("transport must match the reviewed write-only PUT contract") envelope = payload.get("envelope") if not isinstance(envelope, dict): errors.append("envelope must be an object") envelope = {} if envelope.get("format") != "deterministic-ustar": errors.append("envelope.format must be deterministic-ustar") if set(envelope.get("members", [])) != EXPECTED_MEMBERS: errors.append("envelope.members must contain exactly the encrypted bundle and metadata") if envelope.get("object_name_template") != "railiance-infra__{bundle_id}__s1-envelope.tar": errors.append("envelope.object_name_template is not the reviewed immutable name") acceptance_file = payload.get("owner_acceptance_file") if acceptance_file != "spec/s1-offsite-owner-acceptance.yaml": errors.append("owner_acceptance_file must name the bounded repository acceptance record") receipt_root = payload.get("receipt_root") if not isinstance(receipt_root, str) or not receipt_root.startswith("/") or receipt_root in { "/", "/etc", "/home", "/root", "/var", }: errors.append("receipt_root must be a dedicated absolute directory") if errors: raise OffsiteError("off-site contract failed:\n- " + "\n- ".join(errors)) return payload def load_acceptance(contract_path: Path, contract: dict[str, Any]) -> dict[str, Any]: acceptance_path = ROOT / contract["owner_acceptance_file"] # Fixture contracts carry their acceptance beside their fixture repository root. if contract_path.resolve().parent.parent != ROOT: acceptance_path = contract_path.resolve().parent.parent / contract["owner_acceptance_file"] payload = _read_yaml(acceptance_path, "owner acceptance") expected_digest = contract_sha256(contract_path) if payload.get("schema_version") != "1.0" or payload.get("owner_repo") != contract["owner_repo"]: raise OffsiteError("owner acceptance identity is invalid") if payload.get("status") == "accepted": if payload.get("contract_sha256") != expected_digest: raise OffsiteError("owner acceptance does not bind the current contract digest") try: uuid.UUID(str(payload.get("decision_id"))) except (ValueError, TypeError) as exc: raise OffsiteError("accepted owner contract requires a decision UUID") from exc if not isinstance(payload.get("accepted_at"), str) or not payload["accepted_at"].endswith("Z"): raise OffsiteError("accepted owner contract requires accepted_at") elif payload.get("status") != "pending": raise OffsiteError("owner acceptance status must be pending or accepted") return payload def review_contract(contract_path: Path, *, fixture_http: bool = False) -> dict[str, Any]: contract = load_contract(contract_path, fixture_http=fixture_http) digest = contract_sha256(contract_path) acceptance = load_acceptance(contract_path, contract) return { "approval": f"APPROVE S1-OFFSITE-CONTRACT-{digest}", "contract_sha256": digest, "owner_repo": contract["owner_repo"], "owner_status": acceptance["status"], "route_id": contract["route_id"], "status": "review-required" if acceptance["status"] != "accepted" else "accepted", } def _bundle_members(validated: dict[str, Any]) -> list[Path]: bundle = validated["bundle"] names = ["manifest.json", "receipt.json"] + sorted( item["filename"] for item in validated["manifest"]["artifacts"] ) if set(names) != EXPECTED_MEMBERS: raise OffsiteError("bundle does not match the reviewed off-site envelope membership") return [bundle / name for name in names] def build_envelope(bundle: Path, backup_spec: Path, destination: Path) -> dict[str, Any]: validated = validate_bundle(bundle, backup_spec) with tarfile.open(destination, "w", format=tarfile.USTAR_FORMAT) as archive: for source in _bundle_members(validated): info = tarfile.TarInfo(source.name) info.size = source.stat().st_size info.mode = 0o600 info.mtime = 0 info.uid = 0 info.gid = 0 info.uname = "" info.gname = "" with source.open("rb") as handle: archive.addfile(info, handle) return { "bundle_id": validated["manifest"]["bundle_id"], "envelope_sha256": _sha256(destination), "manifest_sha256": _sha256(validated["bundle"] / "manifest.json"), "size_bytes": destination.stat().st_size, } def plan_upload( *, bundle: Path, backup_spec: Path, contract_path: Path, fixture_http: bool = False ) -> dict[str, Any]: contract = load_contract(contract_path, fixture_http=fixture_http) acceptance = load_acceptance(contract_path, contract) with tempfile.TemporaryDirectory(prefix="s1-offsite-plan-") as temporary: envelope = Path(temporary) / "envelope.tar" metadata = build_envelope(bundle, backup_spec, envelope) object_name = contract["envelope"]["object_name_template"].format( bundle_id=metadata["bundle_id"] ) material = "\n".join( ( contract_sha256(contract_path), metadata["bundle_id"], metadata["envelope_sha256"], str(metadata["size_bytes"]), contract["provider"]["remote_prefix"], object_name, ) ) exact = hashlib.sha256(material.encode()).hexdigest()[:20].upper() return { **metadata, "approval": f"UPLOAD-S1-OFFSITE-{exact}" if acceptance["status"] == "accepted" else None, "contract_sha256": contract_sha256(contract_path), "executable": acceptance["status"] == "accepted", "object_name": object_name, "owner_status": acceptance["status"], "remote_object_identity": f"{contract['provider']['remote_prefix']}/{object_name}", "route_id": contract["route_id"], "status": "approval-required" if acceptance["status"] == "accepted" else "owner-review-required", } def _write_receipt(receipt_dir: Path, payload: dict[str, Any]) -> Path: receipt_dir.mkdir(parents=True, exist_ok=True, mode=0o700) final = receipt_dir / f"{payload['bundle_id']}.offsite.json" if final.exists(): raise OffsiteError("an off-site receipt already exists for this bundle") descriptor, temporary = tempfile.mkstemp(prefix=".offsite-receipt-", dir=receipt_dir) try: with os.fdopen(descriptor, "w", encoding="utf-8") as handle: json.dump(payload, handle, indent=2, sort_keys=True) handle.write("\n") handle.flush() os.fsync(handle.fileno()) os.chmod(temporary, 0o600) os.replace(temporary, final) return final except Exception: try: os.unlink(temporary) except FileNotFoundError: pass raise def upload( *, bundle: Path, backup_spec: Path, contract_path: Path, approval: str, receipt_dir: Path | None = None, fixture_http: bool = False, ) -> Path: contract = load_contract(contract_path, fixture_http=fixture_http) acceptance = load_acceptance(contract_path, contract) if acceptance["status"] != "accepted": raise OffsiteError("railiance-platform has not accepted the current upload contract") plan = plan_upload( bundle=bundle, backup_spec=backup_spec, contract_path=contract_path, fixture_http=fixture_http, ) if approval != plan["approval"]: raise OffsiteError("upload approval does not match the current exact envelope") secret_env = contract["transport"]["secret_env"] token = os.environ.get(secret_env) if not token or any(character in token for character in "\r\n"): raise OffsiteError(f"{secret_env} must be injected into the contained process") with tempfile.TemporaryDirectory(prefix="s1-offsite-upload-") as temporary: envelope = Path(temporary) / plan["object_name"] observed = build_envelope(bundle, backup_spec, envelope) if observed["envelope_sha256"] != plan["envelope_sha256"]: raise OffsiteError("envelope changed after approval") segments = ( contract["provider"]["base_url"].rstrip("/"), urllib.parse.quote(token, safe=""), urllib.parse.quote(contract["provider"]["remote_prefix"], safe=""), urllib.parse.quote(plan["object_name"], safe=""), ) destination = "/".join(segments) authorization = base64.b64encode(f"{token}:".encode()).decode() request = urllib.request.Request(destination, data=envelope.read_bytes(), method="PUT") request.add_header("Authorization", f"Basic {authorization}") request.add_header("Content-Type", contract["transport"]["content_type"]) request.add_header("If-None-Match", contract["transport"]["if_none_match"]) try: opener = urllib.request.build_opener(_NoRedirect) with opener.open(request, timeout=600) as response: status = response.status etag = response.headers.get("ETag") except urllib.error.HTTPError as exc: if exc.code in {409, 412}: raise OffsiteError("remote object collision; no success receipt was written") from exc raise OffsiteError(f"off-site PUT failed with HTTP {exc.code}; response suppressed") from exc except urllib.error.URLError as exc: raise OffsiteError("off-site PUT failed; endpoint and response suppressed") from exc if status not in contract["transport"]["success_codes"]: raise OffsiteError(f"off-site PUT returned unexpected HTTP {status}") payload = { "schema_version": "1.0", "receipt_id": str(uuid.uuid4()), "event_type": "offsite-upload", "status": "pass", "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "bundle_id": plan["bundle_id"], "manifest_sha256": plan["manifest_sha256"], "envelope_sha256": plan["envelope_sha256"], "size_bytes": plan["size_bytes"], "remote_object_identity": plan["remote_object_identity"], "route_id": contract["route_id"], "contract_sha256": plan["contract_sha256"], "owner_decision_id": acceptance["decision_id"], "http_status": status, "etag_sha256": hashlib.sha256(etag.encode()).hexdigest() if etag else None, } root = Path(receipt_dir or contract["receipt_root"]) return _write_receipt(root, payload) def validate_prune_receipt(bundle: Path, backup_spec: Path, receipt_path: Path) -> dict[str, Any]: validated = validate_bundle(bundle, backup_spec) try: payload = json.loads(receipt_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as exc: raise OffsiteError("off-site receipt is unreadable") from exc required = { "bundle_id", "contract_sha256", "created_at", "envelope_sha256", "etag_sha256", "event_type", "http_status", "manifest_sha256", "owner_decision_id", "receipt_id", "remote_object_identity", "route_id", "schema_version", "size_bytes", "status", } if not isinstance(payload, dict) or set(payload) != required: raise OffsiteError("off-site receipt fields are invalid") if payload.get("schema_version") != "1.0" or payload.get("event_type") != "offsite-upload": raise OffsiteError("off-site receipt identity is invalid") if payload.get("status") != "pass" or payload.get("bundle_id") != bundle.name: raise OffsiteError("off-site receipt is not passing for this bundle") if payload.get("route_id") != "railiance-backup-offsite-lane": raise OffsiteError("off-site receipt uses the wrong governed route") if payload.get("manifest_sha256") != _sha256(bundle / "manifest.json"): raise OffsiteError("off-site receipt does not bind the current manifest") if not SHA256_RE.fullmatch(str(payload.get("envelope_sha256"))): raise OffsiteError("off-site envelope digest is invalid") if payload.get("http_status") not in {201, 204}: raise OffsiteError("off-site receipt has no accepted PUT status") expected_identity = ( f"railiance-infra/railiance-infra__{bundle.name}__s1-envelope.tar" ) if payload.get("remote_object_identity") != expected_identity or token_shaped(expected_identity): raise OffsiteError("remote object identity is unsafe to retain") if not SHA256_RE.fullmatch(str(payload.get("contract_sha256"))): raise OffsiteError("off-site contract digest is invalid") etag_digest = payload.get("etag_sha256") if etag_digest is not None and not SHA256_RE.fullmatch(str(etag_digest)): raise OffsiteError("off-site ETag digest is invalid") if not isinstance(payload.get("size_bytes"), int) or not 0 < payload["size_bytes"] <= 536870912: raise OffsiteError("off-site envelope size is invalid") try: datetime.strptime(payload["created_at"], "%Y-%m-%dT%H:%M:%SZ") except (KeyError, TypeError, ValueError) as exc: raise OffsiteError("off-site receipt timestamp is invalid") from exc try: uuid.UUID(str(payload.get("receipt_id"))) uuid.UUID(str(payload.get("owner_decision_id"))) except (TypeError, ValueError) as exc: raise OffsiteError("off-site receipt decision identities are invalid") from exc if payload["manifest_sha256"] != _sha256(validated["bundle"] / "manifest.json"): raise OffsiteError("off-site receipt manifest validation failed") return payload def token_shaped(value: str) -> bool: lowered = value.lower() return any(marker in lowered for marker in ("token=", "password=", "authorization", "filesdrop/")) def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--contract", type=Path, default=ROOT / "spec/s1-offsite.yaml") parser.add_argument("--backup-spec", type=Path, default=ROOT / "spec/s1-backup.yaml") subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("review", help="render the exact owner contract approval") plan_parser = subparsers.add_parser("plan", help="plan one immutable envelope without credentials") plan_parser.add_argument("bundle", type=Path) upload_parser = subparsers.add_parser("upload", help="upload one approved immutable envelope") upload_parser.add_argument("bundle", type=Path) upload_parser.add_argument("--approval", required=True) upload_parser.add_argument("--receipt-dir", type=Path) args = parser.parse_args() try: if args.command == "review": result = review_contract(args.contract) elif args.command == "plan": result = plan_upload( bundle=args.bundle, backup_spec=args.backup_spec, contract_path=args.contract, ) else: receipt = upload( bundle=args.bundle, backup_spec=args.backup_spec, contract_path=args.contract, approval=args.approval, receipt_dir=args.receipt_dir, ) result = {"receipt": str(receipt), "status": "pass"} print(json.dumps(result, sort_keys=True)) return 0 except (BackupContractError, RestoreError, OffsiteError, OSError, tarfile.TarError) as exc: print(f"S1 off-site operation failed closed: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())