diff --git a/Makefile b/Makefile index e9f84c1..00ffe4d 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,18 @@ s1-restore-isolated: ## Decrypt only into explicit empty staging: BUNDLE=... DES @test -n "$(BUNDLE)" && test -n "$(DEST)" && test -n "$(IDENTITY)" || (echo "Usage: make s1-restore-isolated BUNDLE=... DEST=/tmp/... IDENTITY=/path/to/age-identity"; exit 1) python3 scripts/s1_restore.py "$(BUNDLE)" --extract-to "$(DEST)" --identity "$(IDENTITY)" +s1-offsite-review: ## Render the exact railiance-platform upload-contract approval + python3 scripts/s1_offsite.py review + +s1-offsite-plan: ## Build a credential-free exact upload plan: BUNDLE=/absolute/path + @test -n "$(BUNDLE)" || (echo "Usage: make s1-offsite-plan BUNDLE=/absolute/path/to/s1-backup-*"; exit 1) + python3 scripts/s1_offsite.py plan "$(BUNDLE)" + +s1-offsite-upload: ## Contained owner-routed upload: BUNDLE=... APPROVE_S1_OFFSITE_UPLOAD=UPLOAD-S1-OFFSITE-... + @test -n "$(BUNDLE)" && test -n "$(APPROVE_S1_OFFSITE_UPLOAD)" || (echo "Run make s1-offsite-plan, then pass BUNDLE and its exact approval"; exit 1) + warden access railiance-backup-offsite-lane --field RAILIANCE_BACKUP_NC_TOKEN --exec -- \ + python3 scripts/s1_offsite.py upload "$(BUNDLE)" --approval "$(APPROVE_S1_OFFSITE_UPLOAD)" + s1-backup-deploy: ## Deploy and enable timer: HOST=... APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER @test -n "$(HOST)" || (echo "Usage: make s1-backup-deploy HOST=Railiance01 APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER"; exit 1) @test "$(APPROVE_S1_BACKUP_DEPLOY)" = "DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER" || (echo "Refusing deployment: exact approval is absent"; exit 1) diff --git a/ansible/roles/s1_backup/tasks/main.yml b/ansible/roles/s1_backup/tasks/main.yml index bc4d1e1..d71cd96 100644 --- a/ansible/roles/s1_backup/tasks/main.yml +++ b/ansible/roles/s1_backup/tasks/main.yml @@ -29,6 +29,7 @@ loop: - s1_backup.py - s1_backup_contract.py + - s1_offsite.py - s1_restore.py - name: Install S1 backup declaration @@ -39,6 +40,17 @@ group: root mode: "0644" +- name: Install pending-owner S1 off-site contract metadata + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../spec/{{ item }}" + dest: "/usr/local/lib/railiance-infra/s1-backup/spec/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - s1-offsite.yaml + - s1-offsite-owner-acceptance.yaml + - name: Install public recipient policy metadata ansible.builtin.copy: src: "{{ playbook_dir }}/../../.sops.yaml" diff --git a/docs/s1-backup-recovery.md b/docs/s1-backup-recovery.md index 9c542e9..077cf2e 100644 --- a/docs/s1-backup-recovery.md +++ b/docs/s1-backup-recovery.md @@ -46,7 +46,50 @@ make s1-backup-prune APPROVE_S1_BACKUP_PRUNE=PRUNE-S1-BACKUPS-... If the set changes between those commands, pruning fails closed. Candidates are atomically moved beneath a quarantine directory inside the declared backup -root before removal; paths outside that root are never accepted. +root before removal; paths outside that root are never accepted. Production +pruning also refuses every candidate that lacks a passing off-site receipt +bound to the current local manifest. + +## Governed off-site owner interface + +The upload contract is source-controlled in `spec/s1-offsite.yaml`. It packages +the two encrypted artifacts and their metadata into one deterministic tar +envelope, avoiding a remotely visible partial multi-object backup. Upload uses +an immutable object name and `If-None-Match: *`; a collision fails without a +success receipt. Neither the endpoint, response body, nor credential is placed +in the receipt. + +The contract is intentionally pending `railiance-platform` acceptance. Render +the exact review phrase without credentials: + +```bash +make s1-offsite-review +``` + +After the owner records acceptance of that exact digest, a fresh bundle can be +planned without access to the lane: + +```bash +make s1-offsite-plan BUNDLE=/opt/backup/railiance/infra/s1-backup-... +``` + +Only the exact resulting envelope approval can enter the contained upload +path. `warden access` injects the primary token into the child environment; it +is never accepted in argv, files, State Hub, or command output: + +```bash +make s1-offsite-upload \ + BUNDLE=/opt/backup/railiance/infra/s1-backup-... \ + APPROVE_S1_OFFSITE_UPLOAD=UPLOAD-S1-OFFSITE-... +``` + +This command remains disabled while +`spec/s1-offsite-owner-acceptance.yaml` is pending. Owner acceptance must also +confirm that the secret-shaped default discovered in the owner repository is +retired or rotated; its value must never be copied into this repository or an +approval message. The current route still lacks an authoritative workload +identity, so the reviewed interface is attended and is not attached to the +systemd timer. ## Inspection and isolated restore @@ -90,6 +133,6 @@ checks the approval and pins the deployed wrapper to the reviewed 40-character source revision. This approval installs and enables the timer only; it does not authorize pruning, off-host upload, private-key access, or a restore. -Off-host transfer remains `RAIL-HO-WP-0012-T05` and belongs to the governed -`railiance-backup-offsite-lane`. An attended off-host isolated drill remains -`RAIL-HO-WP-0012-T06`; neither action is implemented or implied here. +Off-host execution remains pending under `RAIL-HO-WP-0012-T05` and belongs to +the governed `railiance-backup-offsite-lane`. An attended off-host isolated +drill remains `RAIL-HO-WP-0012-T06`; neither live action is implied here. diff --git a/scripts/s1_backup.py b/scripts/s1_backup.py index a44bb59..ff747ba 100755 --- a/scripts/s1_backup.py +++ b/scripts/s1_backup.py @@ -250,11 +250,33 @@ def plan_prune(spec_path: Path, output_dir: Path | None) -> dict[str, Any]: retention["keep_complete_bundles"], retention["max_total_bytes"], ) + missing_evidence: list[str] = [] + if declaration["offsite"]["required_before_prune"]: + from s1_offsite import OffsiteError, validate_prune_receipt + + receipt_root = Path(declaration["offsite"]["receipt_root"]) + for candidate in candidates: + try: + validate_prune_receipt( + candidate, + spec_path, + receipt_root / f"{candidate.name}.offsite.json", + ) + except (OffsiteError, OSError): + missing_evidence.append(candidate.name) + blocked = bool(missing_evidence) return { - "approval": _prune_approval(root, candidates) if candidates else None, + "approval": _prune_approval(root, candidates) if candidates and not blocked else None, "backup_root": str(root), "candidates": [path.name for path in candidates], - "status": "approval-required" if candidates else "nothing-to-prune", + "missing_offsite_evidence": missing_evidence, + "status": ( + "blocked-missing-offsite-evidence" + if blocked + else "approval-required" + if candidates + else "nothing-to-prune" + ), } @@ -262,6 +284,8 @@ def apply_prune(spec_path: Path, output_dir: Path | None, approval: str) -> dict plan = plan_prune(spec_path, output_dir) if not plan["candidates"]: return plan + if plan["missing_offsite_evidence"]: + raise BackupError("prune requires a passing off-site receipt for every candidate") if approval != plan["approval"]: raise BackupError("prune approval does not match the current exact candidate set") root = Path(plan["backup_root"]) diff --git a/scripts/s1_backup_contract.py b/scripts/s1_backup_contract.py index 7550374..f20b23f 100755 --- a/scripts/s1_backup_contract.py +++ b/scripts/s1_backup_contract.py @@ -95,6 +95,19 @@ def validate_spec(payload: Any, *, policy_path: Path) -> dict[str, Any]: if not isinstance(value, int) or isinstance(value, bool) or not low <= value <= high: errors.append(f"retention.{key} must be an integer from {low} to {high}") + offsite = payload.get("offsite") + if not isinstance(offsite, dict) or not isinstance( + offsite.get("required_before_prune"), bool + ): + errors.append("offsite must declare boolean required_before_prune and receipt_root") + else: + try: + receipt_root = _absolute_path(offsite.get("receipt_root"), "offsite.receipt_root") + if backup_root and receipt_root != backup_root + "/offsite-receipts": + errors.append("offsite.receipt_root must be the bounded backup-root receipt directory") + except BackupContractError as exc: + errors.append(str(exc)) + exclusions = payload.get("excluded_prefixes") normalized_exclusions: list[str] = [] if not isinstance(exclusions, list): diff --git a/scripts/s1_offsite.py b/scripts/s1_offsite.py new file mode 100755 index 0000000..0d7ec19 --- /dev/null +++ b/scripts/s1_offsite.py @@ -0,0 +1,456 @@ +#!/usr/bin/env python3 +"""Plan or execute one collision-safe S1 off-site envelope upload.""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import re +import sys +import tarfile +import tempfile +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +from s1_backup import _sha256 +from s1_backup_contract import BackupContractError +from s1_restore import RestoreError, validate_bundle + + +ROOT = Path(__file__).resolve().parents[1] +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +EXPECTED_MEMBERS = { + "manifest.json", + "receipt.json", + "os-config.tar.gz.age", + "packages.txt.age", +} + + +class OffsiteError(RuntimeError): + """The off-site contract, evidence, or upload failed closed.""" + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep a credential-bearing file-drop request on its reviewed origin.""" + + def redirect_request( + self, + req: Any, + fp: Any, + code: int, + msg: str, + headers: Any, + newurl: str, + ) -> None: + return None + + +def _read_yaml(path: Path, label: str) -> dict[str, Any]: + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise OffsiteError(f"cannot read {label}") from exc + if not isinstance(payload, dict): + raise OffsiteError(f"{label} must be a YAML object") + return payload + + +def contract_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def load_contract(path: Path, *, fixture_http: bool = False) -> dict[str, Any]: + payload = _read_yaml(path, "off-site contract") + errors: list[str] = [] + if payload.get("schema_version") != "1.0": + errors.append("schema_version must be 1.0") + if payload.get("route_id") != "railiance-backup-offsite-lane": + errors.append("route_id must be railiance-backup-offsite-lane") + if payload.get("owner_repo") != "railiance-platform": + errors.append("owner_repo must be railiance-platform") + + provider = payload.get("provider") + if not isinstance(provider, dict): + errors.append("provider must be an object") + provider = {} + if provider.get("kind") != "nextcloud-file-drop": + errors.append("provider.kind must be nextcloud-file-drop") + base_url = provider.get("base_url") + if not isinstance(base_url, str): + errors.append("provider.base_url must be a URL") + else: + parsed = urllib.parse.urlsplit(base_url) + allowed_scheme = "http" if fixture_http else "https" + if parsed.scheme != allowed_scheme or not parsed.netloc or parsed.query or parsed.fragment: + errors.append(f"provider.base_url must be a bounded {allowed_scheme} URL") + if "@" in parsed.netloc or parsed.username or parsed.password: + errors.append("provider.base_url must contain no user information") + prefix = provider.get("remote_prefix") + if not isinstance(prefix, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{1,62}", prefix): + errors.append("provider.remote_prefix must be a safe single path segment") + + transport = payload.get("transport") + expected_transport = { + "method": "PUT", + "content_type": "application/x-tar", + "if_none_match": "*", + "success_codes": [201, 204], + "authentication": "basic-token-empty-password", + "secret_env": "RAILIANCE_BACKUP_NC_TOKEN", + "response_body": "suppressed", + } + if transport != expected_transport: + errors.append("transport must match the reviewed write-only PUT contract") + + envelope = payload.get("envelope") + if not isinstance(envelope, dict): + errors.append("envelope must be an object") + envelope = {} + if envelope.get("format") != "deterministic-ustar": + errors.append("envelope.format must be deterministic-ustar") + if set(envelope.get("members", [])) != EXPECTED_MEMBERS: + errors.append("envelope.members must contain exactly the encrypted bundle and metadata") + if envelope.get("object_name_template") != "railiance-infra__{bundle_id}__s1-envelope.tar": + errors.append("envelope.object_name_template is not the reviewed immutable name") + + acceptance_file = payload.get("owner_acceptance_file") + if acceptance_file != "spec/s1-offsite-owner-acceptance.yaml": + errors.append("owner_acceptance_file must name the bounded repository acceptance record") + receipt_root = payload.get("receipt_root") + if not isinstance(receipt_root, str) or not receipt_root.startswith("/") or receipt_root in { + "/", + "/etc", + "/home", + "/root", + "/var", + }: + errors.append("receipt_root must be a dedicated absolute directory") + if errors: + raise OffsiteError("off-site contract failed:\n- " + "\n- ".join(errors)) + return payload + + +def load_acceptance(contract_path: Path, contract: dict[str, Any]) -> dict[str, Any]: + acceptance_path = ROOT / contract["owner_acceptance_file"] + # Fixture contracts carry their acceptance beside their fixture repository root. + if contract_path.resolve().parent.parent != ROOT: + acceptance_path = contract_path.resolve().parent.parent / contract["owner_acceptance_file"] + payload = _read_yaml(acceptance_path, "owner acceptance") + expected_digest = contract_sha256(contract_path) + if payload.get("schema_version") != "1.0" or payload.get("owner_repo") != contract["owner_repo"]: + raise OffsiteError("owner acceptance identity is invalid") + if payload.get("status") == "accepted": + if payload.get("contract_sha256") != expected_digest: + raise OffsiteError("owner acceptance does not bind the current contract digest") + try: + uuid.UUID(str(payload.get("decision_id"))) + except (ValueError, TypeError) as exc: + raise OffsiteError("accepted owner contract requires a decision UUID") from exc + if not isinstance(payload.get("accepted_at"), str) or not payload["accepted_at"].endswith("Z"): + raise OffsiteError("accepted owner contract requires accepted_at") + elif payload.get("status") != "pending": + raise OffsiteError("owner acceptance status must be pending or accepted") + return payload + + +def review_contract(contract_path: Path, *, fixture_http: bool = False) -> dict[str, Any]: + contract = load_contract(contract_path, fixture_http=fixture_http) + digest = contract_sha256(contract_path) + acceptance = load_acceptance(contract_path, contract) + return { + "approval": f"APPROVE S1-OFFSITE-CONTRACT-{digest}", + "contract_sha256": digest, + "owner_repo": contract["owner_repo"], + "owner_status": acceptance["status"], + "route_id": contract["route_id"], + "status": "review-required" if acceptance["status"] != "accepted" else "accepted", + } + + +def _bundle_members(validated: dict[str, Any]) -> list[Path]: + bundle = validated["bundle"] + names = ["manifest.json", "receipt.json"] + sorted( + item["filename"] for item in validated["manifest"]["artifacts"] + ) + if set(names) != EXPECTED_MEMBERS: + raise OffsiteError("bundle does not match the reviewed off-site envelope membership") + return [bundle / name for name in names] + + +def build_envelope(bundle: Path, backup_spec: Path, destination: Path) -> dict[str, Any]: + validated = validate_bundle(bundle, backup_spec) + with tarfile.open(destination, "w", format=tarfile.USTAR_FORMAT) as archive: + for source in _bundle_members(validated): + info = tarfile.TarInfo(source.name) + info.size = source.stat().st_size + info.mode = 0o600 + info.mtime = 0 + info.uid = 0 + info.gid = 0 + info.uname = "" + info.gname = "" + with source.open("rb") as handle: + archive.addfile(info, handle) + return { + "bundle_id": validated["manifest"]["bundle_id"], + "envelope_sha256": _sha256(destination), + "manifest_sha256": _sha256(validated["bundle"] / "manifest.json"), + "size_bytes": destination.stat().st_size, + } + + +def plan_upload( + *, bundle: Path, backup_spec: Path, contract_path: Path, fixture_http: bool = False +) -> dict[str, Any]: + contract = load_contract(contract_path, fixture_http=fixture_http) + acceptance = load_acceptance(contract_path, contract) + with tempfile.TemporaryDirectory(prefix="s1-offsite-plan-") as temporary: + envelope = Path(temporary) / "envelope.tar" + metadata = build_envelope(bundle, backup_spec, envelope) + object_name = contract["envelope"]["object_name_template"].format( + bundle_id=metadata["bundle_id"] + ) + material = "\n".join( + ( + contract_sha256(contract_path), + metadata["bundle_id"], + metadata["envelope_sha256"], + str(metadata["size_bytes"]), + contract["provider"]["remote_prefix"], + object_name, + ) + ) + exact = hashlib.sha256(material.encode()).hexdigest()[:20].upper() + return { + **metadata, + "approval": f"UPLOAD-S1-OFFSITE-{exact}" if acceptance["status"] == "accepted" else None, + "contract_sha256": contract_sha256(contract_path), + "executable": acceptance["status"] == "accepted", + "object_name": object_name, + "owner_status": acceptance["status"], + "remote_object_identity": f"{contract['provider']['remote_prefix']}/{object_name}", + "route_id": contract["route_id"], + "status": "approval-required" if acceptance["status"] == "accepted" else "owner-review-required", + } + + +def _write_receipt(receipt_dir: Path, payload: dict[str, Any]) -> Path: + receipt_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + final = receipt_dir / f"{payload['bundle_id']}.offsite.json" + if final.exists(): + raise OffsiteError("an off-site receipt already exists for this bundle") + descriptor, temporary = tempfile.mkstemp(prefix=".offsite-receipt-", dir=receipt_dir) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, 0o600) + os.replace(temporary, final) + return final + except Exception: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def upload( + *, + bundle: Path, + backup_spec: Path, + contract_path: Path, + approval: str, + receipt_dir: Path | None = None, + fixture_http: bool = False, +) -> Path: + contract = load_contract(contract_path, fixture_http=fixture_http) + acceptance = load_acceptance(contract_path, contract) + if acceptance["status"] != "accepted": + raise OffsiteError("railiance-platform has not accepted the current upload contract") + plan = plan_upload( + bundle=bundle, + backup_spec=backup_spec, + contract_path=contract_path, + fixture_http=fixture_http, + ) + if approval != plan["approval"]: + raise OffsiteError("upload approval does not match the current exact envelope") + secret_env = contract["transport"]["secret_env"] + token = os.environ.get(secret_env) + if not token or any(character in token for character in "\r\n"): + raise OffsiteError(f"{secret_env} must be injected into the contained process") + + with tempfile.TemporaryDirectory(prefix="s1-offsite-upload-") as temporary: + envelope = Path(temporary) / plan["object_name"] + observed = build_envelope(bundle, backup_spec, envelope) + if observed["envelope_sha256"] != plan["envelope_sha256"]: + raise OffsiteError("envelope changed after approval") + segments = ( + contract["provider"]["base_url"].rstrip("/"), + urllib.parse.quote(token, safe=""), + urllib.parse.quote(contract["provider"]["remote_prefix"], safe=""), + urllib.parse.quote(plan["object_name"], safe=""), + ) + destination = "/".join(segments) + authorization = base64.b64encode(f"{token}:".encode()).decode() + request = urllib.request.Request(destination, data=envelope.read_bytes(), method="PUT") + request.add_header("Authorization", f"Basic {authorization}") + request.add_header("Content-Type", contract["transport"]["content_type"]) + request.add_header("If-None-Match", contract["transport"]["if_none_match"]) + try: + opener = urllib.request.build_opener(_NoRedirect) + with opener.open(request, timeout=600) as response: + status = response.status + etag = response.headers.get("ETag") + except urllib.error.HTTPError as exc: + if exc.code in {409, 412}: + raise OffsiteError("remote object collision; no success receipt was written") from exc + raise OffsiteError(f"off-site PUT failed with HTTP {exc.code}; response suppressed") from exc + except urllib.error.URLError as exc: + raise OffsiteError("off-site PUT failed; endpoint and response suppressed") from exc + if status not in contract["transport"]["success_codes"]: + raise OffsiteError(f"off-site PUT returned unexpected HTTP {status}") + + payload = { + "schema_version": "1.0", + "receipt_id": str(uuid.uuid4()), + "event_type": "offsite-upload", + "status": "pass", + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "bundle_id": plan["bundle_id"], + "manifest_sha256": plan["manifest_sha256"], + "envelope_sha256": plan["envelope_sha256"], + "size_bytes": plan["size_bytes"], + "remote_object_identity": plan["remote_object_identity"], + "route_id": contract["route_id"], + "contract_sha256": plan["contract_sha256"], + "owner_decision_id": acceptance["decision_id"], + "http_status": status, + "etag_sha256": hashlib.sha256(etag.encode()).hexdigest() if etag else None, + } + root = Path(receipt_dir or contract["receipt_root"]) + return _write_receipt(root, payload) + + +def validate_prune_receipt(bundle: Path, backup_spec: Path, receipt_path: Path) -> dict[str, Any]: + validated = validate_bundle(bundle, backup_spec) + try: + payload = json.loads(receipt_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise OffsiteError("off-site receipt is unreadable") from exc + required = { + "bundle_id", + "contract_sha256", + "created_at", + "envelope_sha256", + "etag_sha256", + "event_type", + "http_status", + "manifest_sha256", + "owner_decision_id", + "receipt_id", + "remote_object_identity", + "route_id", + "schema_version", + "size_bytes", + "status", + } + if not isinstance(payload, dict) or set(payload) != required: + raise OffsiteError("off-site receipt fields are invalid") + if payload.get("schema_version") != "1.0" or payload.get("event_type") != "offsite-upload": + raise OffsiteError("off-site receipt identity is invalid") + if payload.get("status") != "pass" or payload.get("bundle_id") != bundle.name: + raise OffsiteError("off-site receipt is not passing for this bundle") + if payload.get("route_id") != "railiance-backup-offsite-lane": + raise OffsiteError("off-site receipt uses the wrong governed route") + if payload.get("manifest_sha256") != _sha256(bundle / "manifest.json"): + raise OffsiteError("off-site receipt does not bind the current manifest") + if not SHA256_RE.fullmatch(str(payload.get("envelope_sha256"))): + raise OffsiteError("off-site envelope digest is invalid") + if payload.get("http_status") not in {201, 204}: + raise OffsiteError("off-site receipt has no accepted PUT status") + expected_identity = ( + f"railiance-infra/railiance-infra__{bundle.name}__s1-envelope.tar" + ) + if payload.get("remote_object_identity") != expected_identity or token_shaped(expected_identity): + raise OffsiteError("remote object identity is unsafe to retain") + if not SHA256_RE.fullmatch(str(payload.get("contract_sha256"))): + raise OffsiteError("off-site contract digest is invalid") + etag_digest = payload.get("etag_sha256") + if etag_digest is not None and not SHA256_RE.fullmatch(str(etag_digest)): + raise OffsiteError("off-site ETag digest is invalid") + if not isinstance(payload.get("size_bytes"), int) or not 0 < payload["size_bytes"] <= 536870912: + raise OffsiteError("off-site envelope size is invalid") + try: + datetime.strptime(payload["created_at"], "%Y-%m-%dT%H:%M:%SZ") + except (KeyError, TypeError, ValueError) as exc: + raise OffsiteError("off-site receipt timestamp is invalid") from exc + try: + uuid.UUID(str(payload.get("receipt_id"))) + uuid.UUID(str(payload.get("owner_decision_id"))) + except (TypeError, ValueError) as exc: + raise OffsiteError("off-site receipt decision identities are invalid") from exc + if payload["manifest_sha256"] != _sha256(validated["bundle"] / "manifest.json"): + raise OffsiteError("off-site receipt manifest validation failed") + return payload + + +def token_shaped(value: str) -> bool: + lowered = value.lower() + return any(marker in lowered for marker in ("token=", "password=", "authorization", "filesdrop/")) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--contract", type=Path, default=ROOT / "spec/s1-offsite.yaml") + parser.add_argument("--backup-spec", type=Path, default=ROOT / "spec/s1-backup.yaml") + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("review", help="render the exact owner contract approval") + plan_parser = subparsers.add_parser("plan", help="plan one immutable envelope without credentials") + plan_parser.add_argument("bundle", type=Path) + upload_parser = subparsers.add_parser("upload", help="upload one approved immutable envelope") + upload_parser.add_argument("bundle", type=Path) + upload_parser.add_argument("--approval", required=True) + upload_parser.add_argument("--receipt-dir", type=Path) + args = parser.parse_args() + try: + if args.command == "review": + result = review_contract(args.contract) + elif args.command == "plan": + result = plan_upload( + bundle=args.bundle, + backup_spec=args.backup_spec, + contract_path=args.contract, + ) + else: + receipt = upload( + bundle=args.bundle, + backup_spec=args.backup_spec, + contract_path=args.contract, + approval=args.approval, + receipt_dir=args.receipt_dir, + ) + result = {"receipt": str(receipt), "status": "pass"} + print(json.dumps(result, sort_keys=True)) + return 0 + except (BackupContractError, RestoreError, OffsiteError, OSError, tarfile.TarError) as exc: + print(f"S1 off-site operation failed closed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spec/s1-backup.yaml b/spec/s1-backup.yaml index f2c54f1..9d0a21e 100644 --- a/spec/s1-backup.yaml +++ b/spec/s1-backup.yaml @@ -10,6 +10,10 @@ retention: max_total_bytes: 268435456 freshness_hours: 26 +offsite: + required_before_prune: true + receipt_root: /opt/backup/railiance/infra/offsite-receipts + artifacts: os-config: format: tar.gz diff --git a/spec/s1-offsite-owner-acceptance.yaml b/spec/s1-offsite-owner-acceptance.yaml new file mode 100644 index 0000000..8df2429 --- /dev/null +++ b/spec/s1-offsite-owner-acceptance.yaml @@ -0,0 +1,6 @@ +schema_version: "1.0" +contract_sha256: pending +status: pending +owner_repo: railiance-platform +decision_id: null +accepted_at: null diff --git a/spec/s1-offsite.yaml b/spec/s1-offsite.yaml new file mode 100644 index 0000000..8878a8e --- /dev/null +++ b/spec/s1-offsite.yaml @@ -0,0 +1,30 @@ +schema_version: "1.0" + +route_id: railiance-backup-offsite-lane +owner_repo: railiance-platform +owner_acceptance_file: spec/s1-offsite-owner-acceptance.yaml + +provider: + kind: nextcloud-file-drop + base_url: https://nx4069.your-storageshare.de/public.php/dav/filesdrop + remote_prefix: railiance-infra + +transport: + method: PUT + content_type: application/x-tar + if_none_match: "*" + success_codes: [201, 204] + authentication: basic-token-empty-password + secret_env: RAILIANCE_BACKUP_NC_TOKEN + response_body: suppressed + +envelope: + format: deterministic-ustar + members: + - manifest.json + - receipt.json + - os-config.tar.gz.age + - packages.txt.age + object_name_template: "railiance-infra__{bundle_id}__s1-envelope.tar" + +receipt_root: /opt/backup/railiance/infra/offsite-receipts diff --git a/tests/test_s1_backup_recovery.py b/tests/test_s1_backup_recovery.py index 5e50cca..fedbdb8 100644 --- a/tests/test_s1_backup_recovery.py +++ b/tests/test_s1_backup_recovery.py @@ -1,13 +1,18 @@ from __future__ import annotations import copy +import base64 +import http.server import json +import os import shutil import subprocess import sys import tarfile import tempfile +import threading import unittest +import uuid from datetime import datetime, timedelta, timezone from pathlib import Path from unittest import mock @@ -26,6 +31,21 @@ from s1_backup import ( # noqa: E402 ) from s1_backup_contract import BackupContractError, load_spec, validate_spec # noqa: E402 from s1_restore import RestoreError, _safe_member, extract_bundle, validate_bundle # noqa: E402 +from s1_offsite import ( # noqa: E402 + OffsiteError, + contract_sha256, + plan_upload, + review_contract, + upload, + validate_prune_receipt, +) + +EXPECTED_OFFSITE_MEMBERS = { + "manifest.json", + "receipt.json", + "os-config.tar.gz.age", + "packages.txt.age", +} @unittest.skipUnless(shutil.which("age") and shutil.which("age-keygen"), "age tools required") @@ -207,6 +227,7 @@ class S1BackupRecoveryTests(unittest.TestCase): def test_prune_requires_current_exact_approval(self) -> None: declaration = yaml.safe_load(self.spec.read_text()) declaration["retention"]["keep_complete_bundles"] = 2 + declaration["offsite"]["required_before_prune"] = False self.spec.write_text(yaml.safe_dump(declaration, sort_keys=False)) for index in range(3): self._create(timestamp=f"2026082{index + 1}T120000Z") @@ -219,6 +240,17 @@ class S1BackupRecoveryTests(unittest.TestCase): self.assertEqual("pruned", result["status"]) self.assertFalse((self.output / plan["candidates"][0]).exists()) + def test_prune_blocks_without_offsite_evidence(self) -> None: + declaration = yaml.safe_load(self.spec.read_text()) + declaration["retention"]["keep_complete_bundles"] = 2 + self.spec.write_text(yaml.safe_dump(declaration, sort_keys=False)) + for index in range(3): + self._create(timestamp=f"2026082{index + 1}T120000Z") + plan = plan_prune(self.spec, self.output) + self.assertEqual("blocked-missing-offsite-evidence", plan["status"]) + self.assertIsNone(plan["approval"]) + self.assertEqual(["s1-backup-20260821T120000Z"], plan["missing_offsite_evidence"]) + def test_status_rejects_tampered_passing_receipt(self) -> None: bundle = self._create() receipt_path = bundle / "receipt.json" @@ -251,6 +283,187 @@ class S1BackupRecoveryTests(unittest.TestCase): bootstrap = (ROOT / "ansible/playbooks/bootstrap.yaml").read_text() self.assertNotIn("s1_backup", bootstrap) + def _offsite_contract(self, base_url: str, *, accepted: bool) -> Path: + contract = yaml.safe_load((ROOT / "spec/s1-offsite.yaml").read_text()) + contract["provider"]["base_url"] = base_url + contract_path = self.spec_dir / "s1-offsite.yaml" + contract_path.write_text(yaml.safe_dump(contract, sort_keys=False)) + acceptance = { + "schema_version": "1.0", + "contract_sha256": contract_sha256(contract_path) if accepted else "pending", + "status": "accepted" if accepted else "pending", + "owner_repo": "railiance-platform", + "decision_id": str(uuid.uuid4()) if accepted else None, + "accepted_at": "2026-08-23T12:00:00Z" if accepted else None, + } + (self.spec_dir / "s1-offsite-owner-acceptance.yaml").write_text( + yaml.safe_dump(acceptance, sort_keys=False) + ) + return contract_path + + def test_offsite_contract_is_pending_owner_review(self) -> None: + review = review_contract(ROOT / "spec/s1-offsite.yaml") + self.assertEqual("review-required", review["status"]) + self.assertTrue(review["approval"].startswith("APPROVE S1-OFFSITE-CONTRACT-")) + + def test_deterministic_single_object_offsite_upload_and_collision(self) -> None: + token = "fixture-upload-token" + objects: dict[str, bytes] = {} + + class Handler(http.server.BaseHTTPRequestHandler): + def do_PUT(self) -> None: # noqa: N802 + expected_auth = "Basic " + base64.b64encode(f"{token}:".encode()).decode() + if self.headers.get("Authorization") != expected_auth: + self.send_response(401) + self.end_headers() + return + if self.headers.get("If-None-Match") != "*": + self.send_response(428) + self.end_headers() + return + if self.path in objects: + self.send_response(412) + self.end_headers() + return + length = int(self.headers["Content-Length"]) + objects[self.path] = self.rfile.read(length) + self.send_response(201) + self.send_header("ETag", '"fixture-etag"') + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + bundle = self._create() + contract = self._offsite_contract( + f"http://127.0.0.1:{server.server_port}/filesdrop", accepted=True + ) + plan = plan_upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + fixture_http=True, + ) + self.assertTrue(plan["executable"]) + second_plan = plan_upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + fixture_http=True, + ) + self.assertEqual(plan["envelope_sha256"], second_plan["envelope_sha256"]) + receipt_dir = self.root / "offsite-receipts" + with mock.patch.dict(os.environ, {"RAILIANCE_BACKUP_NC_TOKEN": token}): + with self.assertRaisesRegex(OffsiteError, "exact envelope"): + upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + approval="UPLOAD-S1-OFFSITE-WRONG", + receipt_dir=receipt_dir, + fixture_http=True, + ) + self.assertEqual({}, objects) + receipt_path = upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + approval=plan["approval"], + receipt_dir=receipt_dir, + fixture_http=True, + ) + with self.assertRaisesRegex(OffsiteError, "collision"): + upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + approval=plan["approval"], + receipt_dir=receipt_dir, + fixture_http=True, + ) + receipt = validate_prune_receipt(bundle, self.spec, receipt_path) + self.assertEqual(plan["remote_object_identity"], receipt["remote_object_identity"]) + self.assertNotIn(token, receipt_path.read_text()) + self.assertEqual(1, len(objects)) + envelope_path = self.root / "observed-envelope.tar" + envelope_path.write_bytes(next(iter(objects.values()))) + with tarfile.open(envelope_path, "r:") as archive: + self.assertEqual(EXPECTED_OFFSITE_MEMBERS, {item.name for item in archive}) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + + def test_offsite_upload_refuses_pending_owner_and_wrong_approval(self) -> None: + bundle = self._create() + contract = self._offsite_contract("http://127.0.0.1:9/filesdrop", accepted=False) + plan = plan_upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + fixture_http=True, + ) + self.assertFalse(plan["executable"]) + self.assertIsNone(plan["approval"]) + with self.assertRaisesRegex(OffsiteError, "has not accepted"): + upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + approval="UPLOAD-S1-OFFSITE-WRONG", + receipt_dir=self.root / "receipts", + fixture_http=True, + ) + + def test_offsite_upload_never_follows_redirect(self) -> None: + paths: list[str] = [] + + class RedirectHandler(http.server.BaseHTTPRequestHandler): + def do_PUT(self) -> None: # noqa: N802 + paths.append(self.path) + self.send_response(307) + self.send_header("Location", "/credential-capture") + self.end_headers() + + def log_message(self, format: str, *args: object) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), RedirectHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + bundle = self._create() + contract = self._offsite_contract( + f"http://127.0.0.1:{server.server_port}/filesdrop", accepted=True + ) + plan = plan_upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + fixture_http=True, + ) + with mock.patch.dict(os.environ, {"RAILIANCE_BACKUP_NC_TOKEN": "redirect-token"}): + with self.assertRaisesRegex(OffsiteError, "HTTP 307"): + upload( + bundle=bundle, + backup_spec=self.spec, + contract_path=contract, + approval=plan["approval"], + receipt_dir=self.root / "redirect-receipts", + fixture_http=True, + ) + self.assertEqual(1, len(paths)) + self.assertNotEqual("/credential-capture", paths[0]) + self.assertFalse((self.root / "redirect-receipts").exists()) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=2) + if __name__ == "__main__": unittest.main() diff --git a/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md b/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md index 52f5366..cd80fea 100644 --- a/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md +++ b/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md @@ -126,7 +126,7 @@ no timer is installed merely by running a verification command. ```task id: RAIL-HO-WP-0012-T05 -status: wait +status: progress priority: high ``` @@ -142,6 +142,25 @@ dry-run proves object naming and collision behavior, a controlled upload is visible through owner-provided metadata, and local/off-host retention cannot delete the only recoverable copy. +**Source preparation (2026-08-23):** `spec/s1-offsite.yaml` and +`scripts/s1_offsite.py` define a pending-owner contract that wraps the complete +encrypted bundle and metadata into one deterministic immutable object. The +upload uses `If-None-Match: *`, accepts only a freshly rendered exact envelope +approval, suppresses endpoint/response data, and emits a local metadata-only +receipt. Production pruning now refuses candidates without a receipt bound to +the current manifest. A disposable WebDAV fixture proves successful upload, +collision and redirect refusal, pending-owner refusal, and absence of the +injected token from retained evidence. + +Owner review remains mandatory. The exact review digest is rendered by +`make s1-offsite-review`; `spec/s1-offsite-owner-acceptance.yaml` remains +`pending`, so no live upload can execute. Review of the owner implementation +also found a tracked secret-shaped default in its Forgejo backup script. Its +value was not copied or recorded; `railiance-platform` must confirm it is +retired or rotated before accepting this contract. The route's missing +authoritative workload identity keeps this interface attended rather than +scheduled. + ## T06 — Perform an attended isolated restore drill ```task @@ -178,13 +197,14 @@ decrypted configuration. ## Source delivery record — 2026-08-23 -- `make validate-s1-backup` passes the declaration and 16 fixture recovery +- `make validate-s1-backup` passes the declaration and 21 fixture recovery tests using disposable age identities and storage. -- The full repository suite passes 46 tests. The timer calendar is accepted by +- The full repository suite passes 51 tests. The timer calendar is accepted by `systemd-analyze`; deployment YAML and separation from bootstrap are checked as source contracts. - Native `ansible-playbook --syntax-check` was unavailable on the development workstation. The deployment remains unexecuted and gated by `DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER` plus a clean committed revision. -- T05 and T06 remain `wait`; no off-host write, retained-artifact deletion, - private-key access, or live-host restore occurred. +- T05 is source-prepared and pending owner acceptance; T06 remains `wait`. No + off-host write, retained-artifact deletion, private-key access, or live-host + restore occurred.