Prepare governed S1 offsite upload

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02994-7685-7940-bf34-3555b8256018

scripts/s1_backup.py

@@ -250,11 +250,33 @@ def plan_prune(spec_path: Path, output_dir: Path | None) -> dict[str, Any]:
         retention["keep_complete_bundles"],
         retention["max_total_bytes"],
     )
+    missing_evidence: list[str] = []
+    if declaration["offsite"]["required_before_prune"]:
+        from s1_offsite import OffsiteError, validate_prune_receipt
+
+        receipt_root = Path(declaration["offsite"]["receipt_root"])
+        for candidate in candidates:
+            try:
+                validate_prune_receipt(
+                    candidate,
+                    spec_path,
+                    receipt_root / f"{candidate.name}.offsite.json",
+                )
+            except (OffsiteError, OSError):
+                missing_evidence.append(candidate.name)
+    blocked = bool(missing_evidence)
     return {
-        "approval": _prune_approval(root, candidates) if candidates else None,
+        "approval": _prune_approval(root, candidates) if candidates and not blocked else None,
         "backup_root": str(root),
         "candidates": [path.name for path in candidates],
-        "status": "approval-required" if candidates else "nothing-to-prune",
+        "missing_offsite_evidence": missing_evidence,
+        "status": (
+            "blocked-missing-offsite-evidence"
+            if blocked
+            else "approval-required"
+            if candidates
+            else "nothing-to-prune"
+        ),
     }
 
 
@@ -262,6 +284,8 @@ def apply_prune(spec_path: Path, output_dir: Path | None, approval: str) -> dict
     plan = plan_prune(spec_path, output_dir)
     if not plan["candidates"]:
         return plan
+    if plan["missing_offsite_evidence"]:
+        raise BackupError("prune requires a passing off-site receipt for every candidate")
     if approval != plan["approval"]:
         raise BackupError("prune approval does not match the current exact candidate set")
     root = Path(plan["backup_root"])

scripts/s1_backup_contract.py

@@ -95,6 +95,19 @@ def validate_spec(payload: Any, *, policy_path: Path) -> dict[str, Any]:
             if not isinstance(value, int) or isinstance(value, bool) or not low <= value <= high:
                 errors.append(f"retention.{key} must be an integer from {low} to {high}")
 
+    offsite = payload.get("offsite")
+    if not isinstance(offsite, dict) or not isinstance(
+        offsite.get("required_before_prune"), bool
+    ):
+        errors.append("offsite must declare boolean required_before_prune and receipt_root")
+    else:
+        try:
+            receipt_root = _absolute_path(offsite.get("receipt_root"), "offsite.receipt_root")
+            if backup_root and receipt_root != backup_root + "/offsite-receipts":
+                errors.append("offsite.receipt_root must be the bounded backup-root receipt directory")
+        except BackupContractError as exc:
+            errors.append(str(exc))
+
     exclusions = payload.get("excluded_prefixes")
     normalized_exclusions: list[str] = []
     if not isinstance(exclusions, list):

scripts/s1_offsite.py

@@ -0,0 +1,456 @@
+#!/usr/bin/env python3
+"""Plan or execute one collision-safe S1 off-site envelope upload."""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import json
+import os
+import re
+import sys
+import tarfile
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from s1_backup import _sha256
+from s1_backup_contract import BackupContractError
+from s1_restore import RestoreError, validate_bundle
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+EXPECTED_MEMBERS = {
+    "manifest.json",
+    "receipt.json",
+    "os-config.tar.gz.age",
+    "packages.txt.age",
+}
+
+
+class OffsiteError(RuntimeError):
+    """The off-site contract, evidence, or upload failed closed."""
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    """Keep a credential-bearing file-drop request on its reviewed origin."""
+
+    def redirect_request(
+        self,
+        req: Any,
+        fp: Any,
+        code: int,
+        msg: str,
+        headers: Any,
+        newurl: str,
+    ) -> None:
+        return None
+
+
+def _read_yaml(path: Path, label: str) -> dict[str, Any]:
+    try:
+        payload = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except (OSError, yaml.YAMLError) as exc:
+        raise OffsiteError(f"cannot read {label}") from exc
+    if not isinstance(payload, dict):
+        raise OffsiteError(f"{label} must be a YAML object")
+    return payload
+
+
+def contract_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def load_contract(path: Path, *, fixture_http: bool = False) -> dict[str, Any]:
+    payload = _read_yaml(path, "off-site contract")
+    errors: list[str] = []
+    if payload.get("schema_version") != "1.0":
+        errors.append("schema_version must be 1.0")
+    if payload.get("route_id") != "railiance-backup-offsite-lane":
+        errors.append("route_id must be railiance-backup-offsite-lane")
+    if payload.get("owner_repo") != "railiance-platform":
+        errors.append("owner_repo must be railiance-platform")
+
+    provider = payload.get("provider")
+    if not isinstance(provider, dict):
+        errors.append("provider must be an object")
+        provider = {}
+    if provider.get("kind") != "nextcloud-file-drop":
+        errors.append("provider.kind must be nextcloud-file-drop")
+    base_url = provider.get("base_url")
+    if not isinstance(base_url, str):
+        errors.append("provider.base_url must be a URL")
+    else:
+        parsed = urllib.parse.urlsplit(base_url)
+        allowed_scheme = "http" if fixture_http else "https"
+        if parsed.scheme != allowed_scheme or not parsed.netloc or parsed.query or parsed.fragment:
+            errors.append(f"provider.base_url must be a bounded {allowed_scheme} URL")
+        if "@" in parsed.netloc or parsed.username or parsed.password:
+            errors.append("provider.base_url must contain no user information")
+    prefix = provider.get("remote_prefix")
+    if not isinstance(prefix, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{1,62}", prefix):
+        errors.append("provider.remote_prefix must be a safe single path segment")
+
+    transport = payload.get("transport")
+    expected_transport = {
+        "method": "PUT",
+        "content_type": "application/x-tar",
+        "if_none_match": "*",
+        "success_codes": [201, 204],
+        "authentication": "basic-token-empty-password",
+        "secret_env": "RAILIANCE_BACKUP_NC_TOKEN",
+        "response_body": "suppressed",
+    }
+    if transport != expected_transport:
+        errors.append("transport must match the reviewed write-only PUT contract")
+
+    envelope = payload.get("envelope")
+    if not isinstance(envelope, dict):
+        errors.append("envelope must be an object")
+        envelope = {}
+    if envelope.get("format") != "deterministic-ustar":
+        errors.append("envelope.format must be deterministic-ustar")
+    if set(envelope.get("members", [])) != EXPECTED_MEMBERS:
+        errors.append("envelope.members must contain exactly the encrypted bundle and metadata")
+    if envelope.get("object_name_template") != "railiance-infra__{bundle_id}__s1-envelope.tar":
+        errors.append("envelope.object_name_template is not the reviewed immutable name")
+
+    acceptance_file = payload.get("owner_acceptance_file")
+    if acceptance_file != "spec/s1-offsite-owner-acceptance.yaml":
+        errors.append("owner_acceptance_file must name the bounded repository acceptance record")
+    receipt_root = payload.get("receipt_root")
+    if not isinstance(receipt_root, str) or not receipt_root.startswith("/") or receipt_root in {
+        "/",
+        "/etc",
+        "/home",
+        "/root",
+        "/var",
+    }:
+        errors.append("receipt_root must be a dedicated absolute directory")
+    if errors:
+        raise OffsiteError("off-site contract failed:\n- " + "\n- ".join(errors))
+    return payload
+
+
+def load_acceptance(contract_path: Path, contract: dict[str, Any]) -> dict[str, Any]:
+    acceptance_path = ROOT / contract["owner_acceptance_file"]
+    # Fixture contracts carry their acceptance beside their fixture repository root.
+    if contract_path.resolve().parent.parent != ROOT:
+        acceptance_path = contract_path.resolve().parent.parent / contract["owner_acceptance_file"]
+    payload = _read_yaml(acceptance_path, "owner acceptance")
+    expected_digest = contract_sha256(contract_path)
+    if payload.get("schema_version") != "1.0" or payload.get("owner_repo") != contract["owner_repo"]:
+        raise OffsiteError("owner acceptance identity is invalid")
+    if payload.get("status") == "accepted":
+        if payload.get("contract_sha256") != expected_digest:
+            raise OffsiteError("owner acceptance does not bind the current contract digest")
+        try:
+            uuid.UUID(str(payload.get("decision_id")))
+        except (ValueError, TypeError) as exc:
+            raise OffsiteError("accepted owner contract requires a decision UUID") from exc
+        if not isinstance(payload.get("accepted_at"), str) or not payload["accepted_at"].endswith("Z"):
+            raise OffsiteError("accepted owner contract requires accepted_at")
+    elif payload.get("status") != "pending":
+        raise OffsiteError("owner acceptance status must be pending or accepted")
+    return payload
+
+
+def review_contract(contract_path: Path, *, fixture_http: bool = False) -> dict[str, Any]:
+    contract = load_contract(contract_path, fixture_http=fixture_http)
+    digest = contract_sha256(contract_path)
+    acceptance = load_acceptance(contract_path, contract)
+    return {
+        "approval": f"APPROVE S1-OFFSITE-CONTRACT-{digest}",
+        "contract_sha256": digest,
+        "owner_repo": contract["owner_repo"],
+        "owner_status": acceptance["status"],
+        "route_id": contract["route_id"],
+        "status": "review-required" if acceptance["status"] != "accepted" else "accepted",
+    }
+
+
+def _bundle_members(validated: dict[str, Any]) -> list[Path]:
+    bundle = validated["bundle"]
+    names = ["manifest.json", "receipt.json"] + sorted(
+        item["filename"] for item in validated["manifest"]["artifacts"]
+    )
+    if set(names) != EXPECTED_MEMBERS:
+        raise OffsiteError("bundle does not match the reviewed off-site envelope membership")
+    return [bundle / name for name in names]
+
+
+def build_envelope(bundle: Path, backup_spec: Path, destination: Path) -> dict[str, Any]:
+    validated = validate_bundle(bundle, backup_spec)
+    with tarfile.open(destination, "w", format=tarfile.USTAR_FORMAT) as archive:
+        for source in _bundle_members(validated):
+            info = tarfile.TarInfo(source.name)
+            info.size = source.stat().st_size
+            info.mode = 0o600
+            info.mtime = 0
+            info.uid = 0
+            info.gid = 0
+            info.uname = ""
+            info.gname = ""
+            with source.open("rb") as handle:
+                archive.addfile(info, handle)
+    return {
+        "bundle_id": validated["manifest"]["bundle_id"],
+        "envelope_sha256": _sha256(destination),
+        "manifest_sha256": _sha256(validated["bundle"] / "manifest.json"),
+        "size_bytes": destination.stat().st_size,
+    }
+
+
+def plan_upload(
+    *, bundle: Path, backup_spec: Path, contract_path: Path, fixture_http: bool = False
+) -> dict[str, Any]:
+    contract = load_contract(contract_path, fixture_http=fixture_http)
+    acceptance = load_acceptance(contract_path, contract)
+    with tempfile.TemporaryDirectory(prefix="s1-offsite-plan-") as temporary:
+        envelope = Path(temporary) / "envelope.tar"
+        metadata = build_envelope(bundle, backup_spec, envelope)
+    object_name = contract["envelope"]["object_name_template"].format(
+        bundle_id=metadata["bundle_id"]
+    )
+    material = "\n".join(
+        (
+            contract_sha256(contract_path),
+            metadata["bundle_id"],
+            metadata["envelope_sha256"],
+            str(metadata["size_bytes"]),
+            contract["provider"]["remote_prefix"],
+            object_name,
+        )
+    )
+    exact = hashlib.sha256(material.encode()).hexdigest()[:20].upper()
+    return {
+        **metadata,
+        "approval": f"UPLOAD-S1-OFFSITE-{exact}" if acceptance["status"] == "accepted" else None,
+        "contract_sha256": contract_sha256(contract_path),
+        "executable": acceptance["status"] == "accepted",
+        "object_name": object_name,
+        "owner_status": acceptance["status"],
+        "remote_object_identity": f"{contract['provider']['remote_prefix']}/{object_name}",
+        "route_id": contract["route_id"],
+        "status": "approval-required" if acceptance["status"] == "accepted" else "owner-review-required",
+    }
+
+
+def _write_receipt(receipt_dir: Path, payload: dict[str, Any]) -> Path:
+    receipt_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
+    final = receipt_dir / f"{payload['bundle_id']}.offsite.json"
+    if final.exists():
+        raise OffsiteError("an off-site receipt already exists for this bundle")
+    descriptor, temporary = tempfile.mkstemp(prefix=".offsite-receipt-", dir=receipt_dir)
+    try:
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            json.dump(payload, handle, indent=2, sort_keys=True)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.chmod(temporary, 0o600)
+        os.replace(temporary, final)
+        return final
+    except Exception:
+        try:
+            os.unlink(temporary)
+        except FileNotFoundError:
+            pass
+        raise
+
+
+def upload(
+    *,
+    bundle: Path,
+    backup_spec: Path,
+    contract_path: Path,
+    approval: str,
+    receipt_dir: Path | None = None,
+    fixture_http: bool = False,
+) -> Path:
+    contract = load_contract(contract_path, fixture_http=fixture_http)
+    acceptance = load_acceptance(contract_path, contract)
+    if acceptance["status"] != "accepted":
+        raise OffsiteError("railiance-platform has not accepted the current upload contract")
+    plan = plan_upload(
+        bundle=bundle,
+        backup_spec=backup_spec,
+        contract_path=contract_path,
+        fixture_http=fixture_http,
+    )
+    if approval != plan["approval"]:
+        raise OffsiteError("upload approval does not match the current exact envelope")
+    secret_env = contract["transport"]["secret_env"]
+    token = os.environ.get(secret_env)
+    if not token or any(character in token for character in "\r\n"):
+        raise OffsiteError(f"{secret_env} must be injected into the contained process")
+
+    with tempfile.TemporaryDirectory(prefix="s1-offsite-upload-") as temporary:
+        envelope = Path(temporary) / plan["object_name"]
+        observed = build_envelope(bundle, backup_spec, envelope)
+        if observed["envelope_sha256"] != plan["envelope_sha256"]:
+            raise OffsiteError("envelope changed after approval")
+        segments = (
+            contract["provider"]["base_url"].rstrip("/"),
+            urllib.parse.quote(token, safe=""),
+            urllib.parse.quote(contract["provider"]["remote_prefix"], safe=""),
+            urllib.parse.quote(plan["object_name"], safe=""),
+        )
+        destination = "/".join(segments)
+        authorization = base64.b64encode(f"{token}:".encode()).decode()
+        request = urllib.request.Request(destination, data=envelope.read_bytes(), method="PUT")
+        request.add_header("Authorization", f"Basic {authorization}")
+        request.add_header("Content-Type", contract["transport"]["content_type"])
+        request.add_header("If-None-Match", contract["transport"]["if_none_match"])
+        try:
+            opener = urllib.request.build_opener(_NoRedirect)
+            with opener.open(request, timeout=600) as response:
+                status = response.status
+                etag = response.headers.get("ETag")
+        except urllib.error.HTTPError as exc:
+            if exc.code in {409, 412}:
+                raise OffsiteError("remote object collision; no success receipt was written") from exc
+            raise OffsiteError(f"off-site PUT failed with HTTP {exc.code}; response suppressed") from exc
+        except urllib.error.URLError as exc:
+            raise OffsiteError("off-site PUT failed; endpoint and response suppressed") from exc
+        if status not in contract["transport"]["success_codes"]:
+            raise OffsiteError(f"off-site PUT returned unexpected HTTP {status}")
+
+    payload = {
+        "schema_version": "1.0",
+        "receipt_id": str(uuid.uuid4()),
+        "event_type": "offsite-upload",
+        "status": "pass",
+        "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "bundle_id": plan["bundle_id"],
+        "manifest_sha256": plan["manifest_sha256"],
+        "envelope_sha256": plan["envelope_sha256"],
+        "size_bytes": plan["size_bytes"],
+        "remote_object_identity": plan["remote_object_identity"],
+        "route_id": contract["route_id"],
+        "contract_sha256": plan["contract_sha256"],
+        "owner_decision_id": acceptance["decision_id"],
+        "http_status": status,
+        "etag_sha256": hashlib.sha256(etag.encode()).hexdigest() if etag else None,
+    }
+    root = Path(receipt_dir or contract["receipt_root"])
+    return _write_receipt(root, payload)
+
+
+def validate_prune_receipt(bundle: Path, backup_spec: Path, receipt_path: Path) -> dict[str, Any]:
+    validated = validate_bundle(bundle, backup_spec)
+    try:
+        payload = json.loads(receipt_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as exc:
+        raise OffsiteError("off-site receipt is unreadable") from exc
+    required = {
+        "bundle_id",
+        "contract_sha256",
+        "created_at",
+        "envelope_sha256",
+        "etag_sha256",
+        "event_type",
+        "http_status",
+        "manifest_sha256",
+        "owner_decision_id",
+        "receipt_id",
+        "remote_object_identity",
+        "route_id",
+        "schema_version",
+        "size_bytes",
+        "status",
+    }
+    if not isinstance(payload, dict) or set(payload) != required:
+        raise OffsiteError("off-site receipt fields are invalid")
+    if payload.get("schema_version") != "1.0" or payload.get("event_type") != "offsite-upload":
+        raise OffsiteError("off-site receipt identity is invalid")
+    if payload.get("status") != "pass" or payload.get("bundle_id") != bundle.name:
+        raise OffsiteError("off-site receipt is not passing for this bundle")
+    if payload.get("route_id") != "railiance-backup-offsite-lane":
+        raise OffsiteError("off-site receipt uses the wrong governed route")
+    if payload.get("manifest_sha256") != _sha256(bundle / "manifest.json"):
+        raise OffsiteError("off-site receipt does not bind the current manifest")
+    if not SHA256_RE.fullmatch(str(payload.get("envelope_sha256"))):
+        raise OffsiteError("off-site envelope digest is invalid")
+    if payload.get("http_status") not in {201, 204}:
+        raise OffsiteError("off-site receipt has no accepted PUT status")
+    expected_identity = (
+        f"railiance-infra/railiance-infra__{bundle.name}__s1-envelope.tar"
+    )
+    if payload.get("remote_object_identity") != expected_identity or token_shaped(expected_identity):
+        raise OffsiteError("remote object identity is unsafe to retain")
+    if not SHA256_RE.fullmatch(str(payload.get("contract_sha256"))):
+        raise OffsiteError("off-site contract digest is invalid")
+    etag_digest = payload.get("etag_sha256")
+    if etag_digest is not None and not SHA256_RE.fullmatch(str(etag_digest)):
+        raise OffsiteError("off-site ETag digest is invalid")
+    if not isinstance(payload.get("size_bytes"), int) or not 0 < payload["size_bytes"] <= 536870912:
+        raise OffsiteError("off-site envelope size is invalid")
+    try:
+        datetime.strptime(payload["created_at"], "%Y-%m-%dT%H:%M:%SZ")
+    except (KeyError, TypeError, ValueError) as exc:
+        raise OffsiteError("off-site receipt timestamp is invalid") from exc
+    try:
+        uuid.UUID(str(payload.get("receipt_id")))
+        uuid.UUID(str(payload.get("owner_decision_id")))
+    except (TypeError, ValueError) as exc:
+        raise OffsiteError("off-site receipt decision identities are invalid") from exc
+    if payload["manifest_sha256"] != _sha256(validated["bundle"] / "manifest.json"):
+        raise OffsiteError("off-site receipt manifest validation failed")
+    return payload
+
+
+def token_shaped(value: str) -> bool:
+    lowered = value.lower()
+    return any(marker in lowered for marker in ("token=", "password=", "authorization", "filesdrop/"))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--contract", type=Path, default=ROOT / "spec/s1-offsite.yaml")
+    parser.add_argument("--backup-spec", type=Path, default=ROOT / "spec/s1-backup.yaml")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("review", help="render the exact owner contract approval")
+    plan_parser = subparsers.add_parser("plan", help="plan one immutable envelope without credentials")
+    plan_parser.add_argument("bundle", type=Path)
+    upload_parser = subparsers.add_parser("upload", help="upload one approved immutable envelope")
+    upload_parser.add_argument("bundle", type=Path)
+    upload_parser.add_argument("--approval", required=True)
+    upload_parser.add_argument("--receipt-dir", type=Path)
+    args = parser.parse_args()
+    try:
+        if args.command == "review":
+            result = review_contract(args.contract)
+        elif args.command == "plan":
+            result = plan_upload(
+                bundle=args.bundle,
+                backup_spec=args.backup_spec,
+                contract_path=args.contract,
+            )
+        else:
+            receipt = upload(
+                bundle=args.bundle,
+                backup_spec=args.backup_spec,
+                contract_path=args.contract,
+                approval=args.approval,
+                receipt_dir=args.receipt_dir,
+            )
+            result = {"receipt": str(receipt), "status": "pass"}
+        print(json.dumps(result, sort_keys=True))
+        return 0
+    except (BackupContractError, RestoreError, OffsiteError, OSError, tarfile.TarError) as exc:
+        print(f"S1 off-site operation failed closed: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())