From 295bf43d5447afe9294213d22b2ebc05f337e44d Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 23 Aug 2026 13:13:13 +0200 Subject: [PATCH] Implement governed S1 backup recovery loop Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02994-7685-7940-bf34-3555b8256018 --- .forgejo/workflows/ci-smoke.yaml | 3 +- Makefile | 31 + ansible/playbooks/s1-backup.yaml | 20 + ansible/roles/s1_backup/handlers/main.yml | 4 + ansible/roles/s1_backup/tasks/main.yml | 81 +++ .../templates/railiance-backup-s1.j2 | 4 + .../templates/railiance-backup-s1.service.j2 | 18 + .../templates/railiance-backup-s1.timer.j2 | 11 + docs/s1-backup-recovery.md | 95 ++++ scripts/s1_backup.py | 534 ++++++++++++++++++ scripts/s1_backup_contract.py | 208 +++++++ scripts/s1_restore.py | 284 ++++++++++ spec/s1-backup.yaml | 53 ++ tests/test_s1_backup_recovery.py | 244 ++++++++ tools/cmd/railiance-backup-s1 | 82 +-- ...RAIL-HO-WP-0012-s1-backup-recovery-loop.md | 46 +- 16 files changed, 1623 insertions(+), 95 deletions(-) create mode 100644 ansible/playbooks/s1-backup.yaml create mode 100644 ansible/roles/s1_backup/handlers/main.yml create mode 100644 ansible/roles/s1_backup/tasks/main.yml create mode 100644 ansible/roles/s1_backup/templates/railiance-backup-s1.j2 create mode 100644 ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2 create mode 100644 ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2 create mode 100644 docs/s1-backup-recovery.md create mode 100755 scripts/s1_backup.py create mode 100755 scripts/s1_backup_contract.py create mode 100755 scripts/s1_restore.py create mode 100644 spec/s1-backup.yaml create mode 100644 tests/test_s1_backup_recovery.py diff --git a/.forgejo/workflows/ci-smoke.yaml b/.forgejo/workflows/ci-smoke.yaml index ebaccc9..e63f319 100644 --- a/.forgejo/workflows/ci-smoke.yaml +++ b/.forgejo/workflows/ci-smoke.yaml @@ -21,7 +21,7 @@ jobs: run: | set -eu apt-get update - apt-get install -y --no-install-recommends make python3 python3-yaml + apt-get install -y --no-install-recommends age make python3 python3-yaml - name: Validate S1 source contracts run: | set -eu @@ -31,6 +31,7 @@ jobs: python3 scripts/handoff_contract.py python3 scripts/sops_rotation.py --check python3 scripts/s1_receipt.py docs/evidence/s1-receipts/*.json + python3 scripts/s1_backup.py check python3 -m unittest discover -s tests -v make check-secrets diff --git a/Makefile b/Makefile index 681aebe..e9f84c1 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ SHELL := /usr/bin/env bash GITEA ?= gitea.example.com OWNER ?= coulomb REPO ?= railiance-infra +SOURCE_REVISION ?= $(shell git rev-parse HEAD 2>/dev/null) # New-host defaults (can be overridden: make new-host NAME=... TYPE=...) TYPE ?= cpx11 @@ -146,6 +147,36 @@ tf-providers-plan: ## Plan after an upgrade (uses HCLOUD_TOKEN if set) backup: ## Backup S1 OS config to /opt/backup/railiance/infra/ (age-encrypted, root required) sudo tools/cmd/railiance-backup-s1 +validate-s1-backup: ## Validate S1 backup declaration and recovery implementation without host changes + python3 scripts/s1_backup.py check + python3 -m unittest tests.test_s1_backup_recovery -v + +s1-backup-status: ## Check newest local S1 backup integrity, freshness, count, and disk budget + python3 scripts/s1_backup.py status + +s1-backup-prune-plan: ## Print retained bundles and their exact one-use deletion approval + python3 scripts/s1_backup.py prune-plan + +s1-backup-prune: ## Apply reviewed prune plan: APPROVE_S1_BACKUP_PRUNE=PRUNE-S1-BACKUPS-... + @test -n "$(APPROVE_S1_BACKUP_PRUNE)" || (echo "Run make s1-backup-prune-plan, then pass its exact approval"; exit 1) + python3 scripts/s1_backup.py prune --approval "$(APPROVE_S1_BACKUP_PRUNE)" + +s1-restore-inspect: ## Verify encrypted bundle metadata without a private key: BUNDLE=/absolute/path + @test -n "$(BUNDLE)" || (echo "Usage: make s1-restore-inspect BUNDLE=/absolute/path/to/s1-backup-*"; exit 1) + python3 scripts/s1_restore.py "$(BUNDLE)" + +s1-restore-isolated: ## Decrypt only into explicit empty staging: BUNDLE=... DEST=/tmp/... IDENTITY=... + @test -n "$(BUNDLE)" && test -n "$(DEST)" && test -n "$(IDENTITY)" || (echo "Usage: make s1-restore-isolated BUNDLE=... DEST=/tmp/... IDENTITY=/path/to/age-identity"; exit 1) + python3 scripts/s1_restore.py "$(BUNDLE)" --extract-to "$(DEST)" --identity "$(IDENTITY)" + +s1-backup-deploy: ## Deploy and enable timer: HOST=... APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER + @test -n "$(HOST)" || (echo "Usage: make s1-backup-deploy HOST=Railiance01 APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER"; exit 1) + @test "$(APPROVE_S1_BACKUP_DEPLOY)" = "DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER" || (echo "Refusing deployment: exact approval is absent"; exit 1) + @git diff --quiet && git diff --cached --quiet || (echo "Refusing deployment: commit the exact source first"; exit 1) + cd ansible && ansible-playbook playbooks/s1-backup.yaml --limit "$(HOST)" \ + -e railiance_backup_deploy_approval="$(APPROVE_S1_BACKUP_DEPLOY)" \ + -e railiance_backup_source_revision="$(SOURCE_REVISION)" + # ---- Ansible ---- ansible-bootstrap: ## Run base bootstrap play (users, ssh, ufw, sops-agent, custodian-agent) cd ansible && ansible-playbook playbooks/bootstrap.yaml $(ANSIBLE_USER_FLAG) diff --git a/ansible/playbooks/s1-backup.yaml b/ansible/playbooks/s1-backup.yaml new file mode 100644 index 0000000..74f8ba2 --- /dev/null +++ b/ansible/playbooks/s1-backup.yaml @@ -0,0 +1,20 @@ +--- +# Mutating deployment surface for RAIL-HO-WP-0012-T04. This playbook is not +# included by bootstrap or verification and requires an exact operator token. +- hosts: all + become: true + vars_files: + - ../inventory/group_vars/all.yaml + pre_tasks: + - name: Require exact approval for S1 backup timer deployment + ansible.builtin.assert: + that: + - railiance_backup_deploy_approval | default('') == 'DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER' + - railiance_backup_source_revision | default('') is match('^[0-9a-f]{40}$') + fail_msg: >- + Refusing host mutation. Set + railiance_backup_deploy_approval=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER + and a 40-character railiance_backup_source_revision after reviewing + this play and its rendered units. + roles: + - role: s1_backup diff --git a/ansible/roles/s1_backup/handlers/main.yml b/ansible/roles/s1_backup/handlers/main.yml new file mode 100644 index 0000000..9b1265d --- /dev/null +++ b/ansible/roles/s1_backup/handlers/main.yml @@ -0,0 +1,4 @@ +--- +- name: Reload systemd for S1 backup + ansible.builtin.systemd: + daemon_reload: true diff --git a/ansible/roles/s1_backup/tasks/main.yml b/ansible/roles/s1_backup/tasks/main.yml new file mode 100644 index 0000000..bc4d1e1 --- /dev/null +++ b/ansible/roles/s1_backup/tasks/main.yml @@ -0,0 +1,81 @@ +--- +- name: Install S1 backup runtime packages + ansible.builtin.package: + name: + - age + - python3 + - python3-yaml + state: present + +- name: Create bounded S1 backup directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: root + group: root + mode: "{{ item.mode }}" + loop: + - {path: /usr/local/lib/railiance-infra/s1-backup/scripts, mode: "0755"} + - {path: /usr/local/lib/railiance-infra/s1-backup/spec, mode: "0755"} + - {path: /opt/backup/railiance/infra, mode: "0700"} + +- name: Install S1 backup Python implementation + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../scripts/{{ item }}" + dest: "/usr/local/lib/railiance-infra/s1-backup/scripts/{{ item }}" + owner: root + group: root + mode: "0755" + loop: + - s1_backup.py + - s1_backup_contract.py + - s1_restore.py + +- name: Install S1 backup declaration + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../spec/s1-backup.yaml" + dest: /usr/local/lib/railiance-infra/s1-backup/spec/s1-backup.yaml + owner: root + group: root + mode: "0644" + +- name: Install public recipient policy metadata + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../.sops.yaml" + dest: /usr/local/lib/railiance-infra/s1-backup/.sops.yaml + owner: root + group: root + mode: "0644" + +- name: Install S1 backup command wrapper + ansible.builtin.template: + src: railiance-backup-s1.j2 + dest: /usr/local/sbin/railiance-backup-s1 + owner: root + group: root + mode: "0755" + +- name: Install S1 backup systemd service + ansible.builtin.template: + src: railiance-backup-s1.service.j2 + dest: /etc/systemd/system/railiance-backup-s1.service + owner: root + group: root + mode: "0644" + notify: Reload systemd for S1 backup + +- name: Install S1 backup systemd timer + ansible.builtin.template: + src: railiance-backup-s1.timer.j2 + dest: /etc/systemd/system/railiance-backup-s1.timer + owner: root + group: root + mode: "0644" + notify: Reload systemd for S1 backup + +- name: Enable daily S1 backup timer + ansible.builtin.systemd: + name: railiance-backup-s1.timer + enabled: true + state: started + daemon_reload: true diff --git a/ansible/roles/s1_backup/templates/railiance-backup-s1.j2 b/ansible/roles/s1_backup/templates/railiance-backup-s1.j2 new file mode 100644 index 0000000..542ceb9 --- /dev/null +++ b/ansible/roles/s1_backup/templates/railiance-backup-s1.j2 @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +exec python3 /usr/local/lib/railiance-infra/s1-backup/scripts/s1_backup.py create \ + --source-revision "{{ railiance_backup_source_revision }}" "$@" diff --git a/ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2 b/ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2 new file mode 100644 index 0000000..f4e3155 --- /dev/null +++ b/ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Encrypted Railiance S1 operating-system backup +Documentation=file:///usr/local/lib/railiance-infra/s1-backup/spec/s1-backup.yaml +After=local-fs.target +ConditionPathIsReadWrite=/opt/backup/railiance/infra + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/railiance-backup-s1 +User=root +Group=root +UMask=0077 +Nice=10 +PrivateTmp=true +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/opt/backup/railiance/infra diff --git a/ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2 b/ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2 new file mode 100644 index 0000000..a5f4d8b --- /dev/null +++ b/ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Daily encrypted Railiance S1 backup (RAIL-HO-WP-0012-T04) + +[Timer] +OnCalendar=*-*-* 02:15:00 UTC +RandomizedDelaySec=30min +Persistent=true +Unit=railiance-backup-s1.service + +[Install] +WantedBy=timers.target diff --git a/docs/s1-backup-recovery.md b/docs/s1-backup-recovery.md new file mode 100644 index 0000000..9c542e9 --- /dev/null +++ b/docs/s1-backup-recovery.md @@ -0,0 +1,95 @@ +# S1 encrypted backup and isolated recovery + +This repository can collect only the declared S1 operating-system +configuration and Debian package-selection evidence. It does not back up +Kubernetes state, databases, Forgejo, application data, user homes, secrets, +or private keys. The machine-readable boundary is +[`spec/s1-backup.yaml`](../spec/s1-backup.yaml). + +## Safe source gate + +Run this without root, host access, or an age private key: + +```bash +make validate-s1-backup +``` + +The declaration must agree exactly with the public age recipients in +`.sops.yaml`. Fixture tests prove encrypted atomic publication, failure +cleanup, receipt integrity, isolated restore, traversal rejection, freshness, +and exact retention approval. This command never installs or starts a timer. + +## Backup and status + +On a host, root can create one bundle: + +```bash +sudo tools/cmd/railiance-backup-s1 +make s1-backup-status +``` + +A final `s1-backup-YYYYMMDDTHHMMSSZ/` directory appears only after both +artifacts are encrypted and the manifest and receipt are durable. The receipt +contains hashes, sizes, host identity, source revision, declaration digest, +and the retention decision, but no decrypted content or secret material. +Status fails for missing, stale, over-budget, over-retained, or tampered newest +evidence. + +Retention is deliberately not an unattended delete. Render the current exact +candidate set, review it, and pass back the one-use approval derived from the +candidate receipt hashes: + +```bash +make s1-backup-prune-plan +make s1-backup-prune APPROVE_S1_BACKUP_PRUNE=PRUNE-S1-BACKUPS-... +``` + +If the set changes between those commands, pruning fails closed. Candidates +are atomically moved beneath a quarantine directory inside the declared backup +root before removal; paths outside that root are never accepted. + +## Inspection and isolated restore + +Inspection verifies receipt, manifest, declaration, encrypted hashes, sizes, +and membership without a private key: + +```bash +make s1-restore-inspect BUNDLE=/opt/backup/railiance/infra/s1-backup-... +``` + +An attended operator may then extract into an explicit absent or empty staging +directory outside this repository: + +```bash +make s1-restore-isolated \ + BUNDLE=/opt/backup/railiance/infra/s1-backup-... \ + DEST=/tmp/s1-restore-drill \ + IDENTITY=/operator-controlled/path/identity.age +``` + +The command suppresses decryption output, validates every archive member and +mode before extraction, rejects escaping links and traversal, and writes a +metadata-only restore receipt. It refuses `/`, `/etc`, `/home`, `/root`, +`/var`, repository paths, and non-empty destinations. There is no live restore +mode. + +## Timer deployment approval interface + +The timer runs daily at 02:15 UTC with up to 30 minutes of jitter. Its service +can write only the declared backup root under systemd's filesystem controls. +Deployment is a host mutation and is separate from bootstrap and verification: + +```bash +make s1-backup-deploy \ + HOST=Railiance01 \ + APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER +``` + +The Make target also refuses a dirty source tree. The playbook independently +checks the approval and pins the deployed wrapper to the reviewed 40-character +source revision. This approval installs and enables the timer only; it does +not authorize pruning, off-host upload, private-key access, or a restore. + +Off-host transfer remains `RAIL-HO-WP-0012-T05` and belongs to the governed +`railiance-backup-offsite-lane`. An attended off-host isolated drill remains +`RAIL-HO-WP-0012-T06`; neither action is implemented or implied here. diff --git a/scripts/s1_backup.py b/scripts/s1_backup.py new file mode 100755 index 0000000..a44bb59 --- /dev/null +++ b/scripts/s1_backup.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +"""Create and inspect atomic, age-encrypted Railiance S1 backup bundles.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import socket +import stat +import subprocess +import sys +import tarfile +import tempfile +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +from s1_backup_contract import ( + FORBIDDEN_PARTS, + BackupContractError, + declaration_sha256, + load_spec, +) + + +ROOT = Path(__file__).resolve().parents[1] +BUNDLE_RE = re.compile(r"^s1-backup-(\d{8}T\d{6}Z)$") +REVISION_RE = re.compile(r"^[0-9a-f]{7,40}$") + + +class BackupError(RuntimeError): + """A backup could not produce one complete, unambiguous bundle.""" + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _rfc3339(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _revision(explicit: str | None) -> str: + value = explicit + if value is None: + try: + if subprocess.check_output( + ["git", "status", "--porcelain"], + cwd=ROOT, + text=True, + stderr=subprocess.DEVNULL, + ).strip(): + raise BackupError("source checkout is dirty; commit the exact backup implementation") + value = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL + ).strip() + except (OSError, subprocess.CalledProcessError) as exc: + raise BackupError("source revision is unavailable; pass --source-revision") from exc + if not REVISION_RE.fullmatch(value): + raise BackupError("source revision must be a 7-40 character lowercase git revision") + return value + + +def _source_path(source_root: Path, declared: str) -> Path: + relative = PurePosixPath(declared).relative_to("/") + return source_root.joinpath(*relative.parts) + + +def _walk_members(source_root: Path, declared: str) -> list[Path]: + start = _source_path(source_root, declared) + if not start.exists() and not start.is_symlink(): + return [] + members = [start] + if start.is_dir() and not start.is_symlink(): + for current, directories, files in os.walk(start, followlinks=False): + directories.sort() + files.sort() + base = Path(current) + members.extend(base / name for name in directories) + members.extend(base / name for name in files) + return members + + +def _member_metadata(path: Path, source_root: Path) -> dict[str, Any]: + relative = path.relative_to(source_root) + lowered = {part.lower() for part in relative.parts} + filename = relative.name.lower() + if lowered & FORBIDDEN_PARTS or filename.endswith((".key", ".pem")) or any( + marker in filename for marker in ("credential", "private_key", "token") + ): + raise BackupError(f"runtime member is secret/private-key shaped: /{relative}") + info = path.lstat() + if stat.S_ISREG(info.st_mode): + kind = "file" + elif stat.S_ISDIR(info.st_mode): + kind = "directory" + elif stat.S_ISLNK(info.st_mode): + kind = "symlink" + target = Path(os.readlink(path)) + resolved = target if target.is_absolute() else path.parent / target + try: + resolved.resolve(strict=False).relative_to(source_root) + except ValueError as exc: + raise BackupError(f"runtime link escapes fixture/source root: /{relative}") from exc + else: + raise BackupError(f"unsupported filesystem object: /{path.relative_to(source_root)}") + return { + "kind": kind, + "mode": f"{stat.S_IMODE(info.st_mode):04o}", + "path": str(PurePosixPath(*relative.parts)), + "size_bytes": info.st_size if kind == "file" else 0, + } + + +def _build_os_archive( + *, source_root: Path, declaration: dict[str, Any], destination: Path +) -> list[dict[str, Any]]: + selected: list[Path] = [] + missing_required: list[str] = [] + for entry in declaration["artifacts"]["os-config"]["paths"]: + members = _walk_members(source_root, entry["path"]) + if not members: + if entry["required"]: + missing_required.append(entry["path"]) + continue + selected.extend(members) + if missing_required: + raise BackupError("required backup inputs are missing: " + ", ".join(missing_required)) + unique = sorted(set(selected), key=lambda path: str(path.relative_to(source_root))) + metadata = [_member_metadata(path, source_root) for path in unique] + by_path = {item["path"]: item for item in metadata} + for entry in declaration["artifacts"]["os-config"]["paths"]: + if not entry["required"]: + continue + relative = str(PurePosixPath(entry["path"]).relative_to("/")) + if by_path[relative]["mode"] != entry["expected_mode"]: + raise BackupError( + f"critical mode mismatch for {entry['path']}: " + f"expected {entry['expected_mode']}, observed {by_path[relative]['mode']}" + ) + with tarfile.open(destination, "w:gz", compresslevel=9) as archive: + for path in unique: + archive.add( + path, + arcname=str(PurePosixPath(*path.relative_to(source_root).parts)), + recursive=False, + ) + return metadata + + +def _write_packages( + destination: Path, declaration: dict[str, Any], packages_file: Path | None +) -> None: + if packages_file is not None: + if not packages_file.is_file(): + raise BackupError("--packages-file must identify a readable regular file") + shutil.copyfile(packages_file, destination) + return + command = declaration["artifacts"]["packages"]["command"] + try: + with destination.open("wb") as output: + subprocess.run(command, stdout=output, stderr=subprocess.DEVNULL, check=True) + except (OSError, subprocess.CalledProcessError) as exc: + raise BackupError("fixed package-selection command failed") from exc + + +def _age_binary(explicit: str | None) -> str: + if explicit: + return explicit + for candidate in (shutil.which("age"), "/usr/local/bin/age"): + if candidate and Path(candidate).is_file(): + return candidate + raise BackupError("age executable not found") + + +def _encrypt(age_binary: str, recipients: list[str], source: Path, destination: Path) -> None: + command = [age_binary] + for recipient in recipients: + command.extend(["-r", recipient]) + command.extend(["-o", str(destination), str(source)]) + try: + subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True) + except (OSError, subprocess.CalledProcessError) as exc: + raise BackupError("age encryption failed; no bundle was published") from exc + if not destination.is_file() or destination.stat().st_size == 0: + raise BackupError("age encryption produced no usable output") + + +def _bundle_size(path: Path) -> int: + return sum(item.stat().st_size for item in path.rglob("*") if item.is_file()) + + +def _complete_bundles(output_dir: Path) -> list[Path]: + bundles = [] + if not output_dir.is_dir(): + return bundles + for path in output_dir.iterdir(): + if not path.is_dir() or not BUNDLE_RE.fullmatch(path.name): + continue + receipt = path / "receipt.json" + try: + payload = json.loads(receipt.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if payload.get("event_type") == "backup" and payload.get("status") == "pass": + bundles.append(path) + return sorted(bundles, key=lambda item: item.name, reverse=True) + + +def _retention_candidates(output_dir: Path, keep: int, max_bytes: int) -> list[Path]: + bundles = _complete_bundles(output_dir) + retained = list(bundles) + candidates: list[Path] = [] + while len(retained) > keep: + candidates.append(retained.pop()) + while sum(_bundle_size(path) for path in retained) > max_bytes and len(retained) > 2: + candidates.append(retained.pop()) + return candidates + + +def _prune_approval(output_dir: Path, candidates: list[Path]) -> str: + material = "\n".join( + f"{path.name}:{_sha256(path / 'receipt.json')}" for path in sorted(candidates) + ) + digest = hashlib.sha256(material.encode()).hexdigest()[:16].upper() + return f"PRUNE-S1-BACKUPS-{digest}" + + +def plan_prune(spec_path: Path, output_dir: Path | None) -> dict[str, Any]: + declaration = load_spec(spec_path) + root = Path(output_dir or declaration["backup_root"]).resolve() + if root != Path(declaration["backup_root"]).resolve() and output_dir is None: + raise BackupError("retention root does not match the declaration") + retention = declaration["retention"] + candidates = _retention_candidates( + root, + retention["keep_complete_bundles"], + retention["max_total_bytes"], + ) + return { + "approval": _prune_approval(root, candidates) if candidates else None, + "backup_root": str(root), + "candidates": [path.name for path in candidates], + "status": "approval-required" if candidates else "nothing-to-prune", + } + + +def apply_prune(spec_path: Path, output_dir: Path | None, approval: str) -> dict[str, Any]: + plan = plan_prune(spec_path, output_dir) + if not plan["candidates"]: + return plan + if approval != plan["approval"]: + raise BackupError("prune approval does not match the current exact candidate set") + root = Path(plan["backup_root"]) + quarantine = root / f".prune-{uuid.uuid4()}" + quarantine.mkdir(mode=0o700) + moved: list[Path] = [] + try: + for name in plan["candidates"]: + source = root / name + if source.parent != root or not BUNDLE_RE.fullmatch(source.name): + raise BackupError("prune candidate escaped the declared backup root") + target = quarantine / source.name + os.replace(source, target) + moved.append(target) + directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except Exception: + for target in reversed(moved): + if target.exists(): + os.replace(target, root / target.name) + shutil.rmtree(quarantine, ignore_errors=True) + raise + shutil.rmtree(quarantine) + return { + "backup_root": str(root), + "deleted": plan["candidates"], + "status": "pruned", + } + + +def _retention_decision(output_dir: Path, keep: int, max_bytes: int, new_size: int) -> dict[str, Any]: + projected = [(path.name, _bundle_size(path)) for path in _complete_bundles(output_dir)] + projected.insert(0, ("new-bundle", new_size)) + candidates: list[str] = [] + total = sum(size for _, size in projected) + retained = list(projected) + while len(retained) > keep: + name, size = retained.pop() + candidates.append(name) + total -= size + while total > max_bytes and len(retained) > 2: + name, size = retained.pop() + candidates.append(name) + total -= size + return { + "keep_complete_bundles": keep, + "max_total_bytes": max_bytes, + "projected_total_bytes": total, + "prune_candidates": sorted(name for name in candidates if name != "new-bundle"), + "pruning_applied": False, + } + + +def create_bundle( + *, + spec_path: Path, + source_root: Path, + output_dir: Path | None, + packages_file: Path | None, + age_binary: str | None, + source_revision: str | None, + timestamp: str | None, + hostname: str | None, +) -> Path: + declaration = load_spec(spec_path) + source_root = source_root.resolve() + production = source_root == Path("/") + if production and os.geteuid() != 0: + raise BackupError("production backup requires root before creating the destination") + if packages_file is not None and production: + raise BackupError("--packages-file is allowed only with a fixture --source-root") + destination_root = Path(output_dir or declaration["backup_root"]) + if not destination_root.is_absolute(): + raise BackupError("output directory must be absolute") + if destination_root.resolve() in {Path("/"), Path("/etc"), Path("/home"), Path("/root")}: + raise BackupError("output directory must be bounded") + + observed = _utc_now() + stamp = timestamp or observed.strftime("%Y%m%dT%H%M%SZ") + if not re.fullmatch(r"\d{8}T\d{6}Z", stamp): + raise BackupError("timestamp must be YYYYMMDDTHHMMSSZ") + bundle_name = f"s1-backup-{stamp}" + final = destination_root / bundle_name + if final.exists(): + raise BackupError(f"refusing to overwrite existing bundle {bundle_name}") + + destination_root.mkdir(parents=True, exist_ok=True, mode=0o700) + stage = Path(tempfile.mkdtemp(prefix=f".{bundle_name}.", dir=destination_root)) + os.chmod(stage, 0o700) + try: + plain_os = stage / ".os-config.tar.gz" + plain_packages = stage / ".packages.txt" + members = _build_os_archive( + source_root=source_root, declaration=declaration, destination=plain_os + ) + _write_packages(plain_packages, declaration, packages_file) + executable = _age_binary(age_binary) + encrypted = { + "os-config": stage / "os-config.tar.gz.age", + "packages": stage / "packages.txt.age", + } + _encrypt(executable, declaration["recipients"], plain_os, encrypted["os-config"]) + _encrypt(executable, declaration["recipients"], plain_packages, encrypted["packages"]) + plain_os.unlink() + plain_packages.unlink() + + revision = _revision(source_revision) + declaration_digest = declaration_sha256(spec_path) + artifacts = [ + { + "artifact_class": artifact_class, + "encrypted_sha256": _sha256(path), + "filename": path.name, + "size_bytes": path.stat().st_size, + } + for artifact_class, path in encrypted.items() + ] + manifest = { + "schema_version": "1.0", + "bundle_id": bundle_name, + "created_at": _rfc3339(observed), + "host": hostname or socket.gethostname(), + "source_revision": revision, + "declaration_sha256": declaration_digest, + "recipient_sha256": [ + hashlib.sha256(value.encode()).hexdigest() for value in declaration["recipients"] + ], + "artifacts": artifacts, + "os_config_members": members, + "package_source": "fixture-file" if packages_file else "dpkg --get-selections", + } + manifest_path = stage / "manifest.json" + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + retention = declaration["retention"] + receipt = { + "schema_version": "1.0", + "receipt_id": str(uuid.uuid4()), + "event_type": "backup", + "status": "pass", + "created_at": _rfc3339(observed), + "bundle_id": bundle_name, + "host": manifest["host"], + "source_revision": revision, + "declaration_sha256": declaration_digest, + "manifest_sha256": _sha256(manifest_path), + "artifacts": artifacts, + "retention": _retention_decision( + destination_root, + retention["keep_complete_bundles"], + retention["max_total_bytes"], + _bundle_size(stage), + ), + } + (stage / "receipt.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + for path in stage.iterdir(): + if path.is_file(): + with path.open("rb") as handle: + os.fsync(handle.fileno()) + os.replace(stage, final) + directory_fd = os.open(destination_root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return final + except Exception: + shutil.rmtree(stage, ignore_errors=True) + raise + + +def inspect_status(spec_path: Path, output_dir: Path | None) -> dict[str, Any]: + declaration = load_spec(spec_path) + root = Path(output_dir or declaration["backup_root"]) + bundles = _complete_bundles(root) + if not bundles: + raise BackupError("no complete S1 backup bundle exists") + newest = bundles[0] + # Import lazily to keep the create path independent while making status + # prove the newest evidence, rather than trusting a pass-shaped JSON file. + from s1_restore import RestoreError, validate_bundle + + try: + receipt = validate_bundle(newest, spec_path)["receipt"] + created = datetime.fromisoformat(receipt["created_at"].replace("Z", "+00:00")) + except (RestoreError, KeyError, TypeError, ValueError) as exc: + raise BackupError(f"newest backup evidence is invalid: {exc}") from exc + fresh_until = created + timedelta(hours=declaration["retention"]["freshness_hours"]) + total = sum(_bundle_size(bundle) for bundle in bundles) + result = { + "bundle": newest.name, + "bundle_count": len(bundles), + "created_at": receipt["created_at"], + "fresh_until": _rfc3339(fresh_until), + "status": "pass", + "total_bytes": total, + } + if _utc_now() >= fresh_until: + result["status"] = "stale" + if len(bundles) > declaration["retention"]["keep_complete_bundles"]: + result["status"] = "over-retention" + if total > declaration["retention"]["max_total_bytes"]: + result["status"] = "over-budget" + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + check = subparsers.add_parser("check", help="validate source-only backup inputs") + check.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + create = subparsers.add_parser("create", help="create one complete encrypted bundle") + create.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + create.add_argument("--source-root", type=Path, default=Path("/")) + create.add_argument("--output-dir", type=Path) + create.add_argument("--packages-file", type=Path) + create.add_argument("--age-bin") + create.add_argument("--source-revision") + create.add_argument("--timestamp") + create.add_argument("--hostname") + status_parser = subparsers.add_parser("status", help="fail if backup evidence is absent or stale") + status_parser.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + status_parser.add_argument("--output-dir", type=Path) + prune_plan = subparsers.add_parser( + "prune-plan", help="print the exact retained bundles requiring deletion approval" + ) + prune_plan.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + prune_plan.add_argument("--output-dir", type=Path) + prune = subparsers.add_parser("prune", help="apply one exact, freshly rendered prune plan") + prune.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + prune.add_argument("--output-dir", type=Path) + prune.add_argument("--approval", required=True) + args = parser.parse_args() + try: + if args.command == "check": + declaration = load_spec(args.spec) + result = {"ok": True, "artifacts": sorted(declaration["artifacts"])} + elif args.command == "create": + bundle = create_bundle( + spec_path=args.spec, + source_root=args.source_root, + output_dir=args.output_dir, + packages_file=args.packages_file, + age_binary=args.age_bin, + source_revision=args.source_revision, + timestamp=args.timestamp, + hostname=args.hostname, + ) + result = {"bundle": str(bundle), "status": "pass"} + elif args.command == "status": + result = inspect_status(args.spec, args.output_dir) + elif args.command == "prune-plan": + result = plan_prune(args.spec, args.output_dir) + else: + result = apply_prune(args.spec, args.output_dir, args.approval) + print(json.dumps(result, sort_keys=True)) + return 0 if result.get("status", "pass") == "pass" else 1 + except (BackupContractError, BackupError, OSError, tarfile.TarError) as exc: + print(f"S1 backup failed closed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/s1_backup_contract.py b/scripts/s1_backup_contract.py new file mode 100755 index 0000000..7550374 --- /dev/null +++ b/scripts/s1_backup_contract.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Validate the declared S1 backup membership and public-recipient policy.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path, PurePosixPath +from typing import Any + +import yaml + + +SCHEMA_VERSION = "1.0" +AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{20,}$") +REQUIRED_EXCLUSIONS = { + "/etc/ssl/private", + "/home", + "/root", + "/var/lib/kubelet", + "/var/lib/rancher", +} +FORBIDDEN_PARTS = {"id_rsa", "id_ed25519", "keys.txt", "private", "secrets"} + + +class BackupContractError(ValueError): + """The backup declaration is ambiguous, unsafe, or inconsistent.""" + + +def _absolute_path(value: Any, label: str) -> str: + if not isinstance(value, str) or not value.startswith("/"): + raise BackupContractError(f"{label} must be an absolute path") + path = PurePosixPath(value) + if ".." in path.parts or str(path) != value.rstrip("/"): + raise BackupContractError(f"{label} must be normalized without traversal") + return str(path) + + +def _sops_recipients(path: Path) -> set[str]: + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise BackupContractError(f"cannot read recipient policy {path}: {exc}") from exc + recipients: set[str] = set() + for rule in payload.get("creation_rules", []) if isinstance(payload, dict) else []: + for group in rule.get("key_groups", []) if isinstance(rule, dict) else []: + age = group.get("age", []) if isinstance(group, dict) else [] + if isinstance(age, list): + recipients.update(item.strip() for item in age if isinstance(item, str)) + if not recipients: + raise BackupContractError("recipient policy contains no age recipients") + return recipients + + +def validate_spec(payload: Any, *, policy_path: Path) -> dict[str, Any]: + if not isinstance(payload, dict): + raise BackupContractError("backup declaration must be a YAML object") + errors: list[str] = [] + if str(payload.get("schema_version")) != SCHEMA_VERSION: + errors.append(f"schema_version must be {SCHEMA_VERSION}") + try: + backup_root = _absolute_path(payload.get("backup_root"), "backup_root") + except BackupContractError as exc: + errors.append(str(exc)) + backup_root = "" + if backup_root in {"", "/", "/etc", "/home", "/root", "/var"}: + errors.append("backup_root must be a dedicated bounded directory") + + recipients = payload.get("recipients") + if not isinstance(recipients, list) or not recipients or not all( + isinstance(item, str) and AGE_RECIPIENT_RE.fullmatch(item) for item in recipients + ): + errors.append("recipients must be a non-empty list of age public recipients") + recipients = [] + elif len(set(recipients)) != len(recipients): + errors.append("recipients contains duplicates") + try: + policy_recipients = _sops_recipients(policy_path) + if set(recipients) != policy_recipients: + errors.append("backup recipients must exactly match the SOPS public-recipient policy") + except BackupContractError as exc: + errors.append(str(exc)) + + retention = payload.get("retention") + if not isinstance(retention, dict): + errors.append("retention must be an object") + else: + for key, low, high in ( + ("keep_complete_bundles", 2, 90), + ("max_total_bytes", 1048576, 10737418240), + ("freshness_hours", 1, 168), + ): + value = retention.get(key) + if not isinstance(value, int) or isinstance(value, bool) or not low <= value <= high: + errors.append(f"retention.{key} must be an integer from {low} to {high}") + + exclusions = payload.get("excluded_prefixes") + normalized_exclusions: list[str] = [] + if not isinstance(exclusions, list): + errors.append("excluded_prefixes must be a list") + else: + for index, value in enumerate(exclusions): + try: + normalized_exclusions.append( + _absolute_path(value, f"excluded_prefixes[{index}]") + ) + except BackupContractError as exc: + errors.append(str(exc)) + missing = sorted(REQUIRED_EXCLUSIONS - set(normalized_exclusions)) + if missing: + errors.append(f"excluded_prefixes omits governed exclusions {', '.join(missing)}") + + artifacts = payload.get("artifacts") + if not isinstance(artifacts, dict) or set(artifacts) != {"os-config", "packages"}: + errors.append("artifacts must contain exactly os-config and packages") + artifacts = {} + os_config = artifacts.get("os-config", {}) + paths = os_config.get("paths") if isinstance(os_config, dict) else None + declared: list[str] = [] + if not isinstance(paths, list) or not paths: + errors.append("artifacts.os-config.paths must be a non-empty list") + else: + for index, item in enumerate(paths): + label = f"artifacts.os-config.paths[{index}]" + if not isinstance(item, dict) or not isinstance(item.get("required"), bool): + errors.append(f"{label} must contain path and boolean required") + continue + expected_mode = item.get("expected_mode") + if item["required"] and not ( + isinstance(expected_mode, str) and re.fullmatch(r"0[0-7]{3}", expected_mode) + ): + errors.append(f"{label} requires expected_mode as four octal digits") + if not item["required"] and expected_mode is not None: + errors.append(f"{label}.expected_mode is allowed only for required paths") + try: + path = _absolute_path(item.get("path"), f"{label}.path") + except BackupContractError as exc: + errors.append(str(exc)) + continue + declared.append(path) + lowered = {part.lower() for part in PurePosixPath(path).parts} + if lowered & FORBIDDEN_PARTS: + errors.append(f"{label}.path selects secret/private-key shaped content") + if any(path == prefix or path.startswith(prefix + "/") for prefix in normalized_exclusions): + errors.append(f"{label}.path falls under excluded prefix") + if len(set(declared)) != len(declared): + errors.append("artifacts.os-config.paths contains duplicates") + for left in declared: + for right in declared: + if left != right and right.startswith(left + "/"): + errors.append(f"declared paths overlap: {left} contains {right}") + if isinstance(os_config, dict) and os_config.get("format") != "tar.gz": + errors.append("artifacts.os-config.format must be tar.gz") + packages = artifacts.get("packages", {}) + if not isinstance(packages, dict) or packages.get("format") != "dpkg-selections": + errors.append("artifacts.packages.format must be dpkg-selections") + elif packages.get("command") != ["dpkg", "--get-selections"]: + errors.append("artifacts.packages.command must be the fixed dpkg selection argv") + + if errors: + raise BackupContractError("backup contract failed:\n- " + "\n- ".join(errors)) + return payload + + +def load_spec(path: Path) -> dict[str, Any]: + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise BackupContractError(f"cannot read {path}: {exc}") from exc + policy_value = payload.get("recipient_policy") if isinstance(payload, dict) else None + if not isinstance(policy_value, str) or not policy_value: + raise BackupContractError("recipient_policy must name a repository-relative file") + policy_relative = PurePosixPath(policy_value) + if policy_relative.is_absolute() or ".." in policy_relative.parts: + raise BackupContractError("recipient_policy must be a normalized repository-relative path") + root = path.resolve().parent.parent + return validate_spec(payload, policy_path=root / policy_relative) + + +def declaration_sha256(path: Path) -> str: + import hashlib + + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("spec", nargs="?", type=Path, default=Path("spec/s1-backup.yaml")) + args = parser.parse_args() + payload = load_spec(args.spec) + print( + json.dumps( + { + "artifacts": sorted(payload["artifacts"]), + "declaration_sha256": declaration_sha256(args.spec), + "ok": True, + "path_count": len(payload["artifacts"]["os-config"]["paths"]), + "recipients": len(payload["recipients"]), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/s1_restore.py b/scripts/s1_restore.py new file mode 100755 index 0000000..e4b4e6e --- /dev/null +++ b/scripts/s1_restore.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Inspect or extract an S1 backup into an isolated staging directory.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + +from s1_backup import BackupError, _age_binary, _sha256 +from s1_backup_contract import BackupContractError, declaration_sha256, load_spec + + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST_KEYS = { + "artifacts", + "bundle_id", + "created_at", + "declaration_sha256", + "host", + "os_config_members", + "package_source", + "recipient_sha256", + "schema_version", + "source_revision", +} +BACKUP_RECEIPT_KEYS = { + "artifacts", + "bundle_id", + "created_at", + "declaration_sha256", + "event_type", + "host", + "manifest_sha256", + "receipt_id", + "retention", + "schema_version", + "source_revision", + "status", +} + + +class RestoreError(RuntimeError): + """An artifact or restore destination is unsafe or inconsistent.""" + + +def _json(path: Path, label: str) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RestoreError(f"cannot read {label}") from exc + if not isinstance(payload, dict): + raise RestoreError(f"{label} must be a JSON object") + return payload + + +def validate_bundle(bundle: Path, spec_path: Path) -> dict[str, Any]: + bundle = bundle.resolve() + if not bundle.is_dir() or not bundle.name.startswith("s1-backup-"): + raise RestoreError("bundle must be one complete s1-backup-* directory") + declaration = load_spec(spec_path) + manifest_path = bundle / "manifest.json" + receipt_path = bundle / "receipt.json" + manifest = _json(manifest_path, "manifest") + receipt = _json(receipt_path, "receipt") + if set(manifest) != MANIFEST_KEYS: + raise RestoreError("manifest fields do not match the metadata-only contract") + if set(receipt) != BACKUP_RECEIPT_KEYS: + raise RestoreError("receipt fields do not match the metadata-only contract") + if manifest.get("schema_version") != "1.0" or receipt.get("schema_version") != "1.0": + raise RestoreError("unsupported backup evidence schema version") + if receipt.get("event_type") != "backup" or receipt.get("status") != "pass": + raise RestoreError("bundle has no passing backup receipt") + if manifest.get("bundle_id") != bundle.name or receipt.get("bundle_id") != bundle.name: + raise RestoreError("bundle identity does not match its directory") + for shared in ( + "artifacts", + "created_at", + "declaration_sha256", + "host", + "source_revision", + ): + if receipt.get(shared) != manifest.get(shared): + raise RestoreError(f"receipt and manifest {shared} differ") + if receipt.get("manifest_sha256") != _sha256(manifest_path): + raise RestoreError("manifest digest does not match the receipt") + expected_declaration = declaration_sha256(spec_path) + if manifest.get("declaration_sha256") != expected_declaration: + raise RestoreError("bundle was not produced from the supplied declaration") + if receipt.get("declaration_sha256") != expected_declaration: + raise RestoreError("receipt declaration digest is inconsistent") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list) or { + item.get("artifact_class") for item in artifacts if isinstance(item, dict) + } != {"os-config", "packages"}: + raise RestoreError("manifest must describe exactly two artifact classes") + for item in artifacts: + filename = item.get("filename") + if not isinstance(filename, str) or Path(filename).name != filename: + raise RestoreError("artifact filename must be a basename") + artifact = bundle / filename + if not artifact.is_file(): + raise RestoreError(f"artifact is missing: {filename}") + if item.get("encrypted_sha256") != _sha256(artifact): + raise RestoreError(f"artifact digest mismatch: {filename}") + if item.get("size_bytes") != artifact.stat().st_size: + raise RestoreError(f"artifact size mismatch: {filename}") + members = manifest.get("os_config_members") + if not isinstance(members, list) or not members: + raise RestoreError("manifest contains no OS configuration membership") + member_by_path = { + item.get("path"): item for item in members if isinstance(item, dict) + } + for entry in declaration["artifacts"]["os-config"]["paths"]: + if not entry["required"]: + continue + relative = str(PurePosixPath(entry["path"]).relative_to("/")) + member = member_by_path.get(relative) + if not member or member.get("mode") != entry["expected_mode"]: + raise RestoreError(f"required member or declared critical mode differs: {entry['path']}") + return {"bundle": bundle, "declaration": declaration, "manifest": manifest, "receipt": receipt} + + +def _safe_member(member: tarfile.TarInfo) -> dict[str, Any]: + path = PurePosixPath(member.name) + if path.is_absolute() or ".." in path.parts or str(path) != member.name.rstrip("/"): + raise RestoreError(f"unsafe archive member path: {member.name}") + if member.isdev() or member.isfifo(): + raise RestoreError(f"unsupported archive member type: {member.name}") + if not (member.isfile() or member.isdir() or member.issym() or member.islnk()): + raise RestoreError(f"unsupported archive member type: {member.name}") + if member.issym() or member.islnk(): + target = PurePosixPath(member.linkname) + combined = target if target.is_absolute() else path.parent / target + if target.is_absolute() or ".." in combined.parts: + raise RestoreError(f"archive link escapes staging root: {member.name}") + kind = "file" if member.isfile() else "directory" if member.isdir() else "symlink" + return { + "kind": kind, + "mode": f"{member.mode:04o}", + "path": str(path), + "size_bytes": member.size if member.isfile() else 0, + } + + +def _decrypt(age_binary: str, identity: Path, source: Path, destination: Path) -> None: + try: + subprocess.run( + [age_binary, "-d", "-i", str(identity), "-o", str(destination), str(source)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise RestoreError("artifact decryption failed with output suppressed") from exc + + +def _safe_destination(destination: Path) -> Path: + resolved = destination.resolve() + if not resolved.is_absolute() or resolved in { + Path("/"), + Path("/etc"), + Path("/home"), + Path("/root"), + Path("/var"), + }: + raise RestoreError("restore destination must be a bounded isolated directory") + try: + resolved.relative_to(ROOT) + except ValueError: + pass + else: + raise RestoreError("restore destination must not be inside the repository") + if not resolved.parent.is_dir(): + raise RestoreError("restore destination parent must already exist") + if resolved.exists() and (not resolved.is_dir() or any(resolved.iterdir())): + raise RestoreError("restore destination must be absent or an empty directory") + return resolved + + +def extract_bundle( + *, bundle: Path, spec_path: Path, destination: Path, identity: Path, age_binary: str | None +) -> Path: + validated = validate_bundle(bundle, spec_path) + if not identity.is_file(): + raise RestoreError("age identity must be supplied as a readable file") + final = _safe_destination(destination) + stage = Path(tempfile.mkdtemp(prefix=f".{final.name}.", dir=final.parent)) + os.chmod(stage, 0o700) + try: + artifacts = { + item["artifact_class"]: validated["bundle"] / item["filename"] + for item in validated["manifest"]["artifacts"] + } + executable = _age_binary(age_binary) + plain_archive = stage / ".os-config.tar.gz" + _decrypt(executable, identity, artifacts["os-config"], plain_archive) + with tarfile.open(plain_archive, "r:gz") as archive: + observed = [_safe_member(member) for member in archive.getmembers()] + expected = validated["manifest"]["os_config_members"] + if observed != expected: + raise RestoreError("decrypted archive membership or modes differ from manifest") + archive.extractall(path=stage, filter="data") + plain_archive.unlink() + packages = stage / "packages.txt" + _decrypt(executable, identity, artifacts["packages"], packages) + os.chmod(packages, 0o600) + if packages.stat().st_size == 0 or b"\x00" in packages.read_bytes(): + raise RestoreError("package-selection artifact is empty or not text") + + receipt = { + "schema_version": "1.0", + "receipt_id": str(uuid.uuid4()), + "event_type": "restore", + "status": "pass", + "created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "bundle_id": validated["manifest"]["bundle_id"], + "source_revision": validated["manifest"]["source_revision"], + "declaration_sha256": validated["manifest"]["declaration_sha256"], + "manifest_sha256": _sha256(validated["bundle"] / "manifest.json"), + "member_count": len(validated["manifest"]["os_config_members"]), + "isolated": True, + } + (stage / "restore-receipt.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + if final.exists(): + final.rmdir() + os.replace(stage, final) + return final + except Exception: + shutil.rmtree(stage, ignore_errors=True) + raise + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("bundle", type=Path) + parser.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml") + parser.add_argument("--extract-to", type=Path) + parser.add_argument("--identity", type=Path) + parser.add_argument("--age-bin") + args = parser.parse_args() + try: + if args.extract_to is None: + if args.identity is not None: + parser.error("--identity is used only with --extract-to") + validated = validate_bundle(args.bundle, args.spec) + result = { + "artifacts": len(validated["manifest"]["artifacts"]), + "bundle": validated["manifest"]["bundle_id"], + "member_count": len(validated["manifest"]["os_config_members"]), + "status": "pass", + } + else: + if args.identity is None: + parser.error("--extract-to requires --identity") + restored = extract_bundle( + bundle=args.bundle, + spec_path=args.spec, + destination=args.extract_to, + identity=args.identity, + age_binary=args.age_bin, + ) + result = {"restored_to": str(restored), "status": "pass"} + print(json.dumps(result, sort_keys=True)) + return 0 + except (BackupContractError, BackupError, RestoreError, OSError, tarfile.TarError) as exc: + print(f"S1 restore failed closed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/spec/s1-backup.yaml b/spec/s1-backup.yaml new file mode 100644 index 0000000..f2c54f1 --- /dev/null +++ b/spec/s1-backup.yaml @@ -0,0 +1,53 @@ +schema_version: "1.0" + +backup_root: /opt/backup/railiance/infra +recipient_policy: .sops.yaml +recipients: + - age1aq8twfd78wvpra0had8cezcnj96tj4q0068edrz5jez8d6xwmflqdepsh4 + +retention: + keep_complete_bundles: 7 + max_total_bytes: 268435456 + freshness_hours: 26 + +artifacts: + os-config: + format: tar.gz + paths: + - path: /etc/ssh/sshd_config + required: true + expected_mode: "0644" + - path: /etc/ssh/sshd_config.d + required: false + - path: /etc/ufw/ufw.conf + required: false + - path: /etc/ufw/user.rules + required: false + - path: /etc/ufw/user6.rules + required: false + - path: /etc/fail2ban/jail.local + required: false + - path: /etc/fail2ban/jail.d + required: false + - path: /etc/hosts + required: true + expected_mode: "0644" + - path: /etc/hostname + required: true + expected_mode: "0644" + - path: /etc/apt/sources.list.d + required: false + packages: + format: dpkg-selections + command: [dpkg, --get-selections] + +excluded_prefixes: + - /etc/age + - /etc/kubernetes + - /etc/rancher + - /etc/sops + - /etc/ssl/private + - /home + - /root + - /var/lib/kubelet + - /var/lib/rancher diff --git a/tests/test_s1_backup_recovery.py b/tests/test_s1_backup_recovery.py new file mode 100644 index 0000000..4ebcc05 --- /dev/null +++ b/tests/test_s1_backup_recovery.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import copy +import json +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest import mock + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from s1_backup import ( # noqa: E402 + BackupError, + apply_prune, + create_bundle, + inspect_status, + plan_prune, +) +from s1_backup_contract import BackupContractError, load_spec, validate_spec # noqa: E402 +from s1_restore import RestoreError, _safe_member, extract_bundle, validate_bundle # noqa: E402 + + +@unittest.skipUnless(shutil.which("age") and shutil.which("age-keygen"), "age tools required") +class S1BackupRecoveryTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.fixture = self.root / "fixture" + (self.fixture / "etc/ssh/sshd_config.d").mkdir(parents=True) + (self.fixture / "etc/ssh/sshd_config").write_text("PasswordAuthentication no\n") + (self.fixture / "etc/ssh/sshd_config.d/10-hardening.conf").write_text( + "PermitRootLogin no\n" + ) + (self.fixture / "etc/hosts").write_text("127.0.0.1 localhost\n") + (self.fixture / "etc/hostname").write_text("fixture-host\n") + self.packages = self.root / "packages.txt" + self.packages.write_text("curl\tinstall\ngit\tinstall\n") + self.identity = self.root / "identity.age" + subprocess.run( + ["age-keygen", "-o", str(self.identity)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + self.recipient = subprocess.check_output( + ["age-keygen", "-y", str(self.identity)], text=True + ).strip() + policy = { + "creation_rules": [{"key_groups": [{"age": [self.recipient]}]}] + } + (self.root / ".sops.yaml").write_text(yaml.safe_dump(policy)) + spec = yaml.safe_load((ROOT / "spec/s1-backup.yaml").read_text()) + spec["recipients"] = [self.recipient] + self.spec_dir = self.root / "spec" + self.spec_dir.mkdir() + self.spec = self.spec_dir / "s1-backup.yaml" + self.spec.write_text(yaml.safe_dump(spec, sort_keys=False)) + self.output = self.root / "bundles" + + def tearDown(self) -> None: + self.temp.cleanup() + + def _create(self, **overrides: object) -> Path: + arguments = { + "spec_path": self.spec, + "source_root": self.fixture, + "output_dir": self.output, + "packages_file": self.packages, + "age_binary": shutil.which("age"), + "source_revision": "a" * 40, + "timestamp": "20260823T120000Z", + "hostname": "fixture-host", + } + arguments.update(overrides) + return create_bundle(**arguments) + + def test_repository_contract_validates_without_private_key(self) -> None: + declaration = load_spec(ROOT / "spec/s1-backup.yaml") + self.assertEqual(10, len(declaration["artifacts"]["os-config"]["paths"])) + + def test_relative_and_excluded_paths_fail(self) -> None: + payload = yaml.safe_load(self.spec.read_text()) + bad = copy.deepcopy(payload) + bad["artifacts"]["os-config"]["paths"][0]["path"] = "etc/ssh/sshd_config" + with self.assertRaisesRegex(BackupContractError, "absolute"): + validate_spec(bad, policy_path=self.root / ".sops.yaml") + bad = copy.deepcopy(payload) + bad["artifacts"]["os-config"]["paths"][0]["path"] = "/root/id_ed25519" + with self.assertRaisesRegex(BackupContractError, "secret|excluded"): + validate_spec(bad, policy_path=self.root / ".sops.yaml") + + def test_atomic_encrypted_bundle_round_trip(self) -> None: + bundle = self._create() + self.assertEqual( + {"manifest.json", "os-config.tar.gz.age", "packages.txt.age", "receipt.json"}, + {path.name for path in bundle.iterdir()}, + ) + validated = validate_bundle(bundle, self.spec) + self.assertEqual("pass", validated["receipt"]["status"]) + restore = self.root / "restore" + extract_bundle( + bundle=bundle, + spec_path=self.spec, + destination=restore, + identity=self.identity, + age_binary=shutil.which("age"), + ) + self.assertEqual("fixture-host\n", (restore / "etc/hostname").read_text()) + self.assertIn("curl", (restore / "packages.txt").read_text()) + self.assertEqual("pass", json.loads((restore / "restore-receipt.json").read_text())["status"]) + + def test_encryption_failure_publishes_nothing(self) -> None: + with self.assertRaisesRegex(BackupError, "no bundle was published"): + self._create(age_binary="/bin/false") + self.assertEqual([], list(self.output.iterdir())) + + def test_missing_required_input_publishes_nothing(self) -> None: + (self.fixture / "etc/hostname").unlink() + with self.assertRaisesRegex(BackupError, "required backup inputs"): + self._create() + self.assertEqual([], list(self.output.iterdir())) + + def test_runtime_secret_shaped_member_publishes_nothing(self) -> None: + (self.fixture / "etc/ssh/sshd_config.d/operator-token").write_text("do-not-copy\n") + with self.assertRaisesRegex(BackupError, "secret/private-key shaped"): + self._create() + self.assertEqual([], list(self.output.iterdir())) + + def test_critical_mode_drift_publishes_nothing(self) -> None: + (self.fixture / "etc/ssh/sshd_config").chmod(0o600) + with self.assertRaisesRegex(BackupError, "critical mode mismatch"): + self._create() + self.assertEqual([], list(self.output.iterdir())) + + def test_root_check_precedes_destination_creation(self) -> None: + destination = self.root / "must-not-exist" + with mock.patch("s1_backup.os.geteuid", return_value=1000): + with self.assertRaisesRegex(BackupError, "requires root"): + create_bundle( + spec_path=self.spec, + source_root=Path("/"), + output_dir=destination, + packages_file=None, + age_binary=shutil.which("age"), + source_revision="a" * 40, + timestamp="20260823T120000Z", + hostname="fixture-host", + ) + self.assertFalse(destination.exists()) + + def test_tampered_artifact_fails_before_decryption(self) -> None: + bundle = self._create() + with (bundle / "packages.txt.age").open("ab") as handle: + handle.write(b"tamper") + with self.assertRaisesRegex(RestoreError, "digest mismatch"): + validate_bundle(bundle, self.spec) + + def test_archive_traversal_and_escaping_link_fail(self) -> None: + member = tarfile.TarInfo("../../etc/shadow") + with self.assertRaisesRegex(RestoreError, "unsafe archive member"): + _safe_member(member) + link = tarfile.TarInfo("etc/escape") + link.type = tarfile.SYMTYPE + link.linkname = "../../root/key" + with self.assertRaisesRegex(RestoreError, "escapes"): + _safe_member(link) + + def test_restore_destination_cannot_be_repository(self) -> None: + bundle = self._create() + with self.assertRaisesRegex(RestoreError, "repository"): + extract_bundle( + bundle=bundle, + spec_path=self.spec, + destination=ROOT / "restore-test", + identity=self.identity, + age_binary=shutil.which("age"), + ) + + def test_status_fails_when_receipt_is_stale(self) -> None: + bundle = self._create() + created = datetime.fromisoformat( + json.loads((bundle / "receipt.json").read_text())["created_at"].replace("Z", "+00:00") + ) + with mock.patch("s1_backup._utc_now", return_value=created + timedelta(hours=27)): + self.assertEqual("stale", inspect_status(self.spec, self.output)["status"]) + + def test_prune_requires_current_exact_approval(self) -> None: + declaration = yaml.safe_load(self.spec.read_text()) + declaration["retention"]["keep_complete_bundles"] = 2 + self.spec.write_text(yaml.safe_dump(declaration, sort_keys=False)) + for index in range(3): + self._create(timestamp=f"2026082{index + 1}T120000Z") + plan = plan_prune(self.spec, self.output) + self.assertEqual(["s1-backup-20260821T120000Z"], plan["candidates"]) + with self.assertRaisesRegex(BackupError, "exact candidate set"): + apply_prune(self.spec, self.output, "PRUNE-S1-BACKUPS-WRONG") + self.assertTrue((self.output / plan["candidates"][0]).exists()) + result = apply_prune(self.spec, self.output, plan["approval"]) + self.assertEqual("pruned", result["status"]) + self.assertFalse((self.output / plan["candidates"][0]).exists()) + + def test_status_rejects_tampered_passing_receipt(self) -> None: + bundle = self._create() + receipt_path = bundle / "receipt.json" + receipt = json.loads(receipt_path.read_text()) + receipt["created_at"] = "2020-01-01T00:00:00Z" + receipt_path.write_text(json.dumps(receipt)) + with self.assertRaisesRegex(BackupError, "invalid"): + inspect_status(self.spec, self.output) + + def test_timer_calendar_and_unit_are_source_renderable(self) -> None: + timer = ROOT / "ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2" + service = ROOT / "ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2" + timer_text = timer.read_text() + service_text = service.read_text() + self.assertIn("OnCalendar=*-*-* 02:15:00 UTC", timer_text) + self.assertIn("Persistent=true", timer_text) + self.assertIn("ReadWritePaths=/opt/backup/railiance/infra", service_text) + if shutil.which("systemd-analyze"): + result = subprocess.run( + ["systemd-analyze", "calendar", "*-*-* 02:15:00 UTC"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(0, result.returncode, result.stderr) + + playbook = yaml.safe_load((ROOT / "ansible/playbooks/s1-backup.yaml").read_text()) + self.assertEqual("s1_backup", playbook[0]["roles"][0]["role"]) + self.assertIn("DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER", str(playbook)) + bootstrap = (ROOT / "ansible/playbooks/bootstrap.yaml").read_text() + self.assertNotIn("s1_backup", bootstrap) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/cmd/railiance-backup-s1 b/tools/cmd/railiance-backup-s1 index 97fba45..d00a273 100755 --- a/tools/cmd/railiance-backup-s1 +++ b/tools/cmd/railiance-backup-s1 @@ -1,80 +1,8 @@ #!/usr/bin/env bash -# tools/cmd/railiance-backup-s1 — S1 OS & Provisioning backup -# Backs up: key OS config files applied by Ansible (sshd, ufw, fail2ban, hosts) -# Encryption: age (reuses SOPS key pair from .sops.yaml) -# Output: /opt/backup/railiance/infra/ -# No network required. Requires root to read /etc/. +# Repository entry point. The deployed systemd wrapper uses the same Python +# implementation from /usr/local/lib/railiance-infra. set -euo pipefail -# ── Configuration ────────────────────────────────────────────────────────────── -AGE_PUBLIC_KEY="age1aq8twfd78wvpra0had8cezcnj96tj4q0068edrz5jez8d6xwmflqdepsh4" -BACKUP_DIR="/opt/backup/railiance/infra" -KEEP=7 -TS="$(date -u +%Y%m%dT%H%M%SZ)" - -# Colour helpers (no external dependency) -ok() { printf " ✅ %-12s %s\n" "$1" "$2"; } -warn() { printf " ⚠️ %-12s %s\n" "$1" "$2"; } -bad() { printf " ❌ %-12s %s\n" "$1" "$2"; } - -mkdir -p "${BACKUP_DIR}" -printf "\nrailiance-infra (S1) backup — %s\n" "${TS}" -printf "%0.s-" $(seq 1 44); echo - -# ── Root check ───────────────────────────────────────────────────────────────── -if [[ $EUID -ne 0 ]]; then - bad "root" "this script requires root — run via: sudo make backup" - exit 1 -fi - -# ── OS config snapshot ───────────────────────────────────────────────────────── -# Captures the live state of files that Ansible manages. -# These may drift from the playbooks if manual changes were made. -ok "os-config" "snapshotting /etc config…" - -OS_FILES=( - /etc/ssh/sshd_config - /etc/ssh/sshd_config.d/ - /etc/ufw/ufw.conf - /etc/ufw/user.rules - /etc/ufw/user6.rules - /etc/fail2ban/jail.local - /etc/fail2ban/jail.d/ - /etc/hosts - /etc/hostname - /etc/apt/sources.list.d/ -) - -TMP_OS="$(mktemp -d)" -for item in "${OS_FILES[@]}"; do - [[ -e "${item}" ]] || continue - dest="${TMP_OS}$(dirname "${item}")" - mkdir -p "${dest}" - cp -a "${item}" "${dest}/" 2>/dev/null || true -done - -tar -czf - -C "${TMP_OS}" . \ - | age -r "${AGE_PUBLIC_KEY}" -o "${BACKUP_DIR}/os-config-${TS}.tar.gz.age" -rm -rf "${TMP_OS}" -ok "os-config" "encrypted → os-config-${TS}.tar.gz.age" - -# ── Installed packages list ──────────────────────────────────────────────────── -if command -v dpkg &>/dev/null; then - dpkg --get-selections \ - | age -r "${AGE_PUBLIC_KEY}" -o "${BACKUP_DIR}/packages-${TS}.txt.age" - ok "packages" "encrypted → packages-${TS}.txt.age" -fi - -# ── Prune local cache ────────────────────────────────────────────────────────── -for pattern in "os-config-*.tar.gz.age" "packages-*.txt.age"; do - find "${BACKUP_DIR}" -name "${pattern}" | sort -r | tail -n +$((KEEP + 1)) | xargs -r rm -f -done -ok "prune" "kept last ${KEEP} of each type" - -# ── Stamp ────────────────────────────────────────────────────────────────────── -echo "${TS}" > "${BACKUP_DIR}/.last-backup" - -echo -ok "done" "backup complete — ${TS}" -echo " Location: ${BACKUP_DIR}" -echo " Decrypt with: age -d -i ~/.config/sops/age/keys.txt " +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)" +exec python3 "${REPO_ROOT}/scripts/s1_backup.py" create "$@" diff --git a/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md b/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md index d14fdf2..c2dc227 100644 --- a/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md +++ b/workplans/RAIL-HO-WP-0012-s1-backup-recovery-loop.md @@ -4,7 +4,7 @@ type: workplan title: "Close the encrypted S1 backup and recovery loop" domain: financials repo: railiance-infra -status: proposed +status: active owner: codex topic_slug: railiance created: "2026-08-23" @@ -30,13 +30,12 @@ operator approval. ## Current State and Risks -`tools/cmd/railiance-backup-s1` currently creates age-encrypted OS-configuration -and package-selection files under `/opt/backup/railiance/infra` and retains the -latest seven of each. It is not scheduled, has no off-host transfer, produces -no manifest or receipt, has no restoration tool or drill, duplicates an age -recipient in executable source, suppresses individual copy failures, and can -leave ambiguous partial output. The output is therefore a useful local helper, -not current proof of recoverability. +The source-side local loop now has a declaration, atomic encrypted bundle, +metadata-only evidence, integrity/freshness status, exact-approval retention, +isolated restore, and separately approved timer deployment interface. No timer +has been deployed and no retained artifact has been deleted by this workplan. +Off-host transfer and an attended drill remain open, so the repository still +cannot claim operational recoverability from local source tests alone. ## Delivery Order @@ -48,7 +47,7 @@ tasks and an attended isolated drill. ```task id: RAIL-HO-WP-0012-T01 -status: todo +status: done priority: high ``` @@ -66,7 +65,7 @@ requires neither root nor a decryption key. ```task id: RAIL-HO-WP-0012-T02 -status: todo +status: done priority: high ``` @@ -89,7 +88,7 @@ tests leave no ambiguous success state. ```task id: RAIL-HO-WP-0012-T03 -status: todo +status: done priority: high ``` @@ -110,7 +109,7 @@ and no default invocation can overwrite a host file. ```task id: RAIL-HO-WP-0012-T04 -status: todo +status: done priority: medium ``` @@ -164,15 +163,28 @@ decrypted configuration. ## Acceptance -- [ ] S1 backup membership, exclusions, recipients, and retention are declared +- [x] S1 backup membership, exclusions, recipients, and retention are declared and validated from source. -- [ ] Backup publication is atomic, encrypted, and accompanied by a safe +- [x] Backup publication is atomic, encrypted, and accompanied by a safe manifest and receipt. -- [ ] Restore defaults to isolated inspection and rejects tampering, traversal, +- [x] Restore defaults to isolated inspection and rejects tampering, traversal, and accidental live overwrite. -- [ ] Scheduling and freshness checks are source-controlled but deployed only +- [x] Scheduling and freshness checks are source-controlled but deployed only through an explicit host-change approval. - [ ] A governed off-host copy and an attended isolated restore drill prove the selected S1 state is recoverable. -- [ ] No cluster, platform, tenant, secret-issuance, or private-key custody +- [x] No cluster, platform, tenant, secret-issuance, or private-key custody responsibility is absorbed into this repository. + +## Source delivery record — 2026-08-23 + +- `make validate-s1-backup` passes the declaration and 15 fixture recovery + tests using disposable age identities and storage. +- The full repository suite passes 45 tests. The timer calendar is accepted by + `systemd-analyze`; deployment YAML and separation from bootstrap are checked + as source contracts. +- Native `ansible-playbook --syntax-check` was unavailable on the development + workstation. The deployment remains unexecuted and gated by + `DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER` plus a clean committed revision. +- T05 and T06 remain `wait`; no off-host write, retained-artifact deletion, + private-key access, or live-host restore occurred.