Implement governed S1 backup recovery loop
Some checks failed
CI Smoke / source-contract (push) Failing after 9s
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a02994-7685-7940-bf34-3555b8256018
This commit is contained in:
codex 2026-08-23 13:13:13 +02:00
parent 40e295e3bd
commit 295bf43d54
16 changed files with 1623 additions and 95 deletions

View file

@ -21,7 +21,7 @@ jobs:
run: |
set -eu
apt-get update
apt-get install -y --no-install-recommends make python3 python3-yaml
apt-get install -y --no-install-recommends age make python3 python3-yaml
- name: Validate S1 source contracts
run: |
set -eu
@ -31,6 +31,7 @@ jobs:
python3 scripts/handoff_contract.py
python3 scripts/sops_rotation.py --check
python3 scripts/s1_receipt.py docs/evidence/s1-receipts/*.json
python3 scripts/s1_backup.py check
python3 -m unittest discover -s tests -v
make check-secrets

View file

@ -6,6 +6,7 @@ SHELL := /usr/bin/env bash
GITEA ?= gitea.example.com
OWNER ?= coulomb
REPO ?= railiance-infra
SOURCE_REVISION ?= $(shell git rev-parse HEAD 2>/dev/null)
# New-host defaults (can be overridden: make new-host NAME=... TYPE=...)
TYPE ?= cpx11
@ -146,6 +147,36 @@ tf-providers-plan: ## Plan after an upgrade (uses HCLOUD_TOKEN if set)
backup: ## Backup S1 OS config to /opt/backup/railiance/infra/ (age-encrypted, root required)
sudo tools/cmd/railiance-backup-s1
validate-s1-backup: ## Validate S1 backup declaration and recovery implementation without host changes
python3 scripts/s1_backup.py check
python3 -m unittest tests.test_s1_backup_recovery -v
s1-backup-status: ## Check newest local S1 backup integrity, freshness, count, and disk budget
python3 scripts/s1_backup.py status
s1-backup-prune-plan: ## Print retained bundles and their exact one-use deletion approval
python3 scripts/s1_backup.py prune-plan
s1-backup-prune: ## Apply reviewed prune plan: APPROVE_S1_BACKUP_PRUNE=PRUNE-S1-BACKUPS-...
@test -n "$(APPROVE_S1_BACKUP_PRUNE)" || (echo "Run make s1-backup-prune-plan, then pass its exact approval"; exit 1)
python3 scripts/s1_backup.py prune --approval "$(APPROVE_S1_BACKUP_PRUNE)"
s1-restore-inspect: ## Verify encrypted bundle metadata without a private key: BUNDLE=/absolute/path
@test -n "$(BUNDLE)" || (echo "Usage: make s1-restore-inspect BUNDLE=/absolute/path/to/s1-backup-*"; exit 1)
python3 scripts/s1_restore.py "$(BUNDLE)"
s1-restore-isolated: ## Decrypt only into explicit empty staging: BUNDLE=... DEST=/tmp/... IDENTITY=...
@test -n "$(BUNDLE)" && test -n "$(DEST)" && test -n "$(IDENTITY)" || (echo "Usage: make s1-restore-isolated BUNDLE=... DEST=/tmp/... IDENTITY=/path/to/age-identity"; exit 1)
python3 scripts/s1_restore.py "$(BUNDLE)" --extract-to "$(DEST)" --identity "$(IDENTITY)"
s1-backup-deploy: ## Deploy and enable timer: HOST=... APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER
@test -n "$(HOST)" || (echo "Usage: make s1-backup-deploy HOST=Railiance01 APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER"; exit 1)
@test "$(APPROVE_S1_BACKUP_DEPLOY)" = "DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER" || (echo "Refusing deployment: exact approval is absent"; exit 1)
@git diff --quiet && git diff --cached --quiet || (echo "Refusing deployment: commit the exact source first"; exit 1)
cd ansible && ansible-playbook playbooks/s1-backup.yaml --limit "$(HOST)" \
-e railiance_backup_deploy_approval="$(APPROVE_S1_BACKUP_DEPLOY)" \
-e railiance_backup_source_revision="$(SOURCE_REVISION)"
# ---- Ansible ----
ansible-bootstrap: ## Run base bootstrap play (users, ssh, ufw, sops-agent, custodian-agent)
cd ansible && ansible-playbook playbooks/bootstrap.yaml $(ANSIBLE_USER_FLAG)

View file

@ -0,0 +1,20 @@
---
# Mutating deployment surface for RAIL-HO-WP-0012-T04. This playbook is not
# included by bootstrap or verification and requires an exact operator token.
- hosts: all
become: true
vars_files:
- ../inventory/group_vars/all.yaml
pre_tasks:
- name: Require exact approval for S1 backup timer deployment
ansible.builtin.assert:
that:
- railiance_backup_deploy_approval | default('') == 'DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER'
- railiance_backup_source_revision | default('') is match('^[0-9a-f]{40}$')
fail_msg: >-
Refusing host mutation. Set
railiance_backup_deploy_approval=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER
and a 40-character railiance_backup_source_revision after reviewing
this play and its rendered units.
roles:
- role: s1_backup

View file

@ -0,0 +1,4 @@
---
- name: Reload systemd for S1 backup
ansible.builtin.systemd:
daemon_reload: true

View file

@ -0,0 +1,81 @@
---
- name: Install S1 backup runtime packages
ansible.builtin.package:
name:
- age
- python3
- python3-yaml
state: present
- name: Create bounded S1 backup directories
ansible.builtin.file:
path: "{{ item.path }}"
state: directory
owner: root
group: root
mode: "{{ item.mode }}"
loop:
- {path: /usr/local/lib/railiance-infra/s1-backup/scripts, mode: "0755"}
- {path: /usr/local/lib/railiance-infra/s1-backup/spec, mode: "0755"}
- {path: /opt/backup/railiance/infra, mode: "0700"}
- name: Install S1 backup Python implementation
ansible.builtin.copy:
src: "{{ playbook_dir }}/../../scripts/{{ item }}"
dest: "/usr/local/lib/railiance-infra/s1-backup/scripts/{{ item }}"
owner: root
group: root
mode: "0755"
loop:
- s1_backup.py
- s1_backup_contract.py
- s1_restore.py
- name: Install S1 backup declaration
ansible.builtin.copy:
src: "{{ playbook_dir }}/../../spec/s1-backup.yaml"
dest: /usr/local/lib/railiance-infra/s1-backup/spec/s1-backup.yaml
owner: root
group: root
mode: "0644"
- name: Install public recipient policy metadata
ansible.builtin.copy:
src: "{{ playbook_dir }}/../../.sops.yaml"
dest: /usr/local/lib/railiance-infra/s1-backup/.sops.yaml
owner: root
group: root
mode: "0644"
- name: Install S1 backup command wrapper
ansible.builtin.template:
src: railiance-backup-s1.j2
dest: /usr/local/sbin/railiance-backup-s1
owner: root
group: root
mode: "0755"
- name: Install S1 backup systemd service
ansible.builtin.template:
src: railiance-backup-s1.service.j2
dest: /etc/systemd/system/railiance-backup-s1.service
owner: root
group: root
mode: "0644"
notify: Reload systemd for S1 backup
- name: Install S1 backup systemd timer
ansible.builtin.template:
src: railiance-backup-s1.timer.j2
dest: /etc/systemd/system/railiance-backup-s1.timer
owner: root
group: root
mode: "0644"
notify: Reload systemd for S1 backup
- name: Enable daily S1 backup timer
ansible.builtin.systemd:
name: railiance-backup-s1.timer
enabled: true
state: started
daemon_reload: true

View file

@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
exec python3 /usr/local/lib/railiance-infra/s1-backup/scripts/s1_backup.py create \
--source-revision "{{ railiance_backup_source_revision }}" "$@"

View file

@ -0,0 +1,18 @@
[Unit]
Description=Encrypted Railiance S1 operating-system backup
Documentation=file:///usr/local/lib/railiance-infra/s1-backup/spec/s1-backup.yaml
After=local-fs.target
ConditionPathIsReadWrite=/opt/backup/railiance/infra
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/railiance-backup-s1
User=root
Group=root
UMask=0077
Nice=10
PrivateTmp=true
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/opt/backup/railiance/infra

View file

@ -0,0 +1,11 @@
[Unit]
Description=Daily encrypted Railiance S1 backup (RAIL-HO-WP-0012-T04)
[Timer]
OnCalendar=*-*-* 02:15:00 UTC
RandomizedDelaySec=30min
Persistent=true
Unit=railiance-backup-s1.service
[Install]
WantedBy=timers.target

View file

@ -0,0 +1,95 @@
# S1 encrypted backup and isolated recovery
This repository can collect only the declared S1 operating-system
configuration and Debian package-selection evidence. It does not back up
Kubernetes state, databases, Forgejo, application data, user homes, secrets,
or private keys. The machine-readable boundary is
[`spec/s1-backup.yaml`](../spec/s1-backup.yaml).
## Safe source gate
Run this without root, host access, or an age private key:
```bash
make validate-s1-backup
```
The declaration must agree exactly with the public age recipients in
`.sops.yaml`. Fixture tests prove encrypted atomic publication, failure
cleanup, receipt integrity, isolated restore, traversal rejection, freshness,
and exact retention approval. This command never installs or starts a timer.
## Backup and status
On a host, root can create one bundle:
```bash
sudo tools/cmd/railiance-backup-s1
make s1-backup-status
```
A final `s1-backup-YYYYMMDDTHHMMSSZ/` directory appears only after both
artifacts are encrypted and the manifest and receipt are durable. The receipt
contains hashes, sizes, host identity, source revision, declaration digest,
and the retention decision, but no decrypted content or secret material.
Status fails for missing, stale, over-budget, over-retained, or tampered newest
evidence.
Retention is deliberately not an unattended delete. Render the current exact
candidate set, review it, and pass back the one-use approval derived from the
candidate receipt hashes:
```bash
make s1-backup-prune-plan
make s1-backup-prune APPROVE_S1_BACKUP_PRUNE=PRUNE-S1-BACKUPS-...
```
If the set changes between those commands, pruning fails closed. Candidates
are atomically moved beneath a quarantine directory inside the declared backup
root before removal; paths outside that root are never accepted.
## Inspection and isolated restore
Inspection verifies receipt, manifest, declaration, encrypted hashes, sizes,
and membership without a private key:
```bash
make s1-restore-inspect BUNDLE=/opt/backup/railiance/infra/s1-backup-...
```
An attended operator may then extract into an explicit absent or empty staging
directory outside this repository:
```bash
make s1-restore-isolated \
BUNDLE=/opt/backup/railiance/infra/s1-backup-... \
DEST=/tmp/s1-restore-drill \
IDENTITY=/operator-controlled/path/identity.age
```
The command suppresses decryption output, validates every archive member and
mode before extraction, rejects escaping links and traversal, and writes a
metadata-only restore receipt. It refuses `/`, `/etc`, `/home`, `/root`,
`/var`, repository paths, and non-empty destinations. There is no live restore
mode.
## Timer deployment approval interface
The timer runs daily at 02:15 UTC with up to 30 minutes of jitter. Its service
can write only the declared backup root under systemd's filesystem controls.
Deployment is a host mutation and is separate from bootstrap and verification:
```bash
make s1-backup-deploy \
HOST=Railiance01 \
APPROVE_S1_BACKUP_DEPLOY=DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER
```
The Make target also refuses a dirty source tree. The playbook independently
checks the approval and pins the deployed wrapper to the reviewed 40-character
source revision. This approval installs and enables the timer only; it does
not authorize pruning, off-host upload, private-key access, or a restore.
Off-host transfer remains `RAIL-HO-WP-0012-T05` and belongs to the governed
`railiance-backup-offsite-lane`. An attended off-host isolated drill remains
`RAIL-HO-WP-0012-T06`; neither action is implemented or implied here.

534
scripts/s1_backup.py Executable file
View file

@ -0,0 +1,534 @@
#!/usr/bin/env python3
"""Create and inspect atomic, age-encrypted Railiance S1 backup bundles."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import re
import shutil
import socket
import stat
import subprocess
import sys
import tarfile
import tempfile
import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path, PurePosixPath
from typing import Any
from s1_backup_contract import (
FORBIDDEN_PARTS,
BackupContractError,
declaration_sha256,
load_spec,
)
ROOT = Path(__file__).resolve().parents[1]
BUNDLE_RE = re.compile(r"^s1-backup-(\d{8}T\d{6}Z)$")
REVISION_RE = re.compile(r"^[0-9a-f]{7,40}$")
class BackupError(RuntimeError):
"""A backup could not produce one complete, unambiguous bundle."""
def _sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _utc_now() -> datetime:
return datetime.now(timezone.utc)
def _rfc3339(value: datetime) -> str:
return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
def _revision(explicit: str | None) -> str:
value = explicit
if value is None:
try:
if subprocess.check_output(
["git", "status", "--porcelain"],
cwd=ROOT,
text=True,
stderr=subprocess.DEVNULL,
).strip():
raise BackupError("source checkout is dirty; commit the exact backup implementation")
value = subprocess.check_output(
["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, stderr=subprocess.DEVNULL
).strip()
except (OSError, subprocess.CalledProcessError) as exc:
raise BackupError("source revision is unavailable; pass --source-revision") from exc
if not REVISION_RE.fullmatch(value):
raise BackupError("source revision must be a 7-40 character lowercase git revision")
return value
def _source_path(source_root: Path, declared: str) -> Path:
relative = PurePosixPath(declared).relative_to("/")
return source_root.joinpath(*relative.parts)
def _walk_members(source_root: Path, declared: str) -> list[Path]:
start = _source_path(source_root, declared)
if not start.exists() and not start.is_symlink():
return []
members = [start]
if start.is_dir() and not start.is_symlink():
for current, directories, files in os.walk(start, followlinks=False):
directories.sort()
files.sort()
base = Path(current)
members.extend(base / name for name in directories)
members.extend(base / name for name in files)
return members
def _member_metadata(path: Path, source_root: Path) -> dict[str, Any]:
relative = path.relative_to(source_root)
lowered = {part.lower() for part in relative.parts}
filename = relative.name.lower()
if lowered & FORBIDDEN_PARTS or filename.endswith((".key", ".pem")) or any(
marker in filename for marker in ("credential", "private_key", "token")
):
raise BackupError(f"runtime member is secret/private-key shaped: /{relative}")
info = path.lstat()
if stat.S_ISREG(info.st_mode):
kind = "file"
elif stat.S_ISDIR(info.st_mode):
kind = "directory"
elif stat.S_ISLNK(info.st_mode):
kind = "symlink"
target = Path(os.readlink(path))
resolved = target if target.is_absolute() else path.parent / target
try:
resolved.resolve(strict=False).relative_to(source_root)
except ValueError as exc:
raise BackupError(f"runtime link escapes fixture/source root: /{relative}") from exc
else:
raise BackupError(f"unsupported filesystem object: /{path.relative_to(source_root)}")
return {
"kind": kind,
"mode": f"{stat.S_IMODE(info.st_mode):04o}",
"path": str(PurePosixPath(*relative.parts)),
"size_bytes": info.st_size if kind == "file" else 0,
}
def _build_os_archive(
*, source_root: Path, declaration: dict[str, Any], destination: Path
) -> list[dict[str, Any]]:
selected: list[Path] = []
missing_required: list[str] = []
for entry in declaration["artifacts"]["os-config"]["paths"]:
members = _walk_members(source_root, entry["path"])
if not members:
if entry["required"]:
missing_required.append(entry["path"])
continue
selected.extend(members)
if missing_required:
raise BackupError("required backup inputs are missing: " + ", ".join(missing_required))
unique = sorted(set(selected), key=lambda path: str(path.relative_to(source_root)))
metadata = [_member_metadata(path, source_root) for path in unique]
by_path = {item["path"]: item for item in metadata}
for entry in declaration["artifacts"]["os-config"]["paths"]:
if not entry["required"]:
continue
relative = str(PurePosixPath(entry["path"]).relative_to("/"))
if by_path[relative]["mode"] != entry["expected_mode"]:
raise BackupError(
f"critical mode mismatch for {entry['path']}: "
f"expected {entry['expected_mode']}, observed {by_path[relative]['mode']}"
)
with tarfile.open(destination, "w:gz", compresslevel=9) as archive:
for path in unique:
archive.add(
path,
arcname=str(PurePosixPath(*path.relative_to(source_root).parts)),
recursive=False,
)
return metadata
def _write_packages(
destination: Path, declaration: dict[str, Any], packages_file: Path | None
) -> None:
if packages_file is not None:
if not packages_file.is_file():
raise BackupError("--packages-file must identify a readable regular file")
shutil.copyfile(packages_file, destination)
return
command = declaration["artifacts"]["packages"]["command"]
try:
with destination.open("wb") as output:
subprocess.run(command, stdout=output, stderr=subprocess.DEVNULL, check=True)
except (OSError, subprocess.CalledProcessError) as exc:
raise BackupError("fixed package-selection command failed") from exc
def _age_binary(explicit: str | None) -> str:
if explicit:
return explicit
for candidate in (shutil.which("age"), "/usr/local/bin/age"):
if candidate and Path(candidate).is_file():
return candidate
raise BackupError("age executable not found")
def _encrypt(age_binary: str, recipients: list[str], source: Path, destination: Path) -> None:
command = [age_binary]
for recipient in recipients:
command.extend(["-r", recipient])
command.extend(["-o", str(destination), str(source)])
try:
subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
except (OSError, subprocess.CalledProcessError) as exc:
raise BackupError("age encryption failed; no bundle was published") from exc
if not destination.is_file() or destination.stat().st_size == 0:
raise BackupError("age encryption produced no usable output")
def _bundle_size(path: Path) -> int:
return sum(item.stat().st_size for item in path.rglob("*") if item.is_file())
def _complete_bundles(output_dir: Path) -> list[Path]:
bundles = []
if not output_dir.is_dir():
return bundles
for path in output_dir.iterdir():
if not path.is_dir() or not BUNDLE_RE.fullmatch(path.name):
continue
receipt = path / "receipt.json"
try:
payload = json.loads(receipt.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
continue
if payload.get("event_type") == "backup" and payload.get("status") == "pass":
bundles.append(path)
return sorted(bundles, key=lambda item: item.name, reverse=True)
def _retention_candidates(output_dir: Path, keep: int, max_bytes: int) -> list[Path]:
bundles = _complete_bundles(output_dir)
retained = list(bundles)
candidates: list[Path] = []
while len(retained) > keep:
candidates.append(retained.pop())
while sum(_bundle_size(path) for path in retained) > max_bytes and len(retained) > 2:
candidates.append(retained.pop())
return candidates
def _prune_approval(output_dir: Path, candidates: list[Path]) -> str:
material = "\n".join(
f"{path.name}:{_sha256(path / 'receipt.json')}" for path in sorted(candidates)
)
digest = hashlib.sha256(material.encode()).hexdigest()[:16].upper()
return f"PRUNE-S1-BACKUPS-{digest}"
def plan_prune(spec_path: Path, output_dir: Path | None) -> dict[str, Any]:
declaration = load_spec(spec_path)
root = Path(output_dir or declaration["backup_root"]).resolve()
if root != Path(declaration["backup_root"]).resolve() and output_dir is None:
raise BackupError("retention root does not match the declaration")
retention = declaration["retention"]
candidates = _retention_candidates(
root,
retention["keep_complete_bundles"],
retention["max_total_bytes"],
)
return {
"approval": _prune_approval(root, candidates) if candidates else None,
"backup_root": str(root),
"candidates": [path.name for path in candidates],
"status": "approval-required" if candidates else "nothing-to-prune",
}
def apply_prune(spec_path: Path, output_dir: Path | None, approval: str) -> dict[str, Any]:
plan = plan_prune(spec_path, output_dir)
if not plan["candidates"]:
return plan
if approval != plan["approval"]:
raise BackupError("prune approval does not match the current exact candidate set")
root = Path(plan["backup_root"])
quarantine = root / f".prune-{uuid.uuid4()}"
quarantine.mkdir(mode=0o700)
moved: list[Path] = []
try:
for name in plan["candidates"]:
source = root / name
if source.parent != root or not BUNDLE_RE.fullmatch(source.name):
raise BackupError("prune candidate escaped the declared backup root")
target = quarantine / source.name
os.replace(source, target)
moved.append(target)
directory_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
except Exception:
for target in reversed(moved):
if target.exists():
os.replace(target, root / target.name)
shutil.rmtree(quarantine, ignore_errors=True)
raise
shutil.rmtree(quarantine)
return {
"backup_root": str(root),
"deleted": plan["candidates"],
"status": "pruned",
}
def _retention_decision(output_dir: Path, keep: int, max_bytes: int, new_size: int) -> dict[str, Any]:
projected = [(path.name, _bundle_size(path)) for path in _complete_bundles(output_dir)]
projected.insert(0, ("new-bundle", new_size))
candidates: list[str] = []
total = sum(size for _, size in projected)
retained = list(projected)
while len(retained) > keep:
name, size = retained.pop()
candidates.append(name)
total -= size
while total > max_bytes and len(retained) > 2:
name, size = retained.pop()
candidates.append(name)
total -= size
return {
"keep_complete_bundles": keep,
"max_total_bytes": max_bytes,
"projected_total_bytes": total,
"prune_candidates": sorted(name for name in candidates if name != "new-bundle"),
"pruning_applied": False,
}
def create_bundle(
*,
spec_path: Path,
source_root: Path,
output_dir: Path | None,
packages_file: Path | None,
age_binary: str | None,
source_revision: str | None,
timestamp: str | None,
hostname: str | None,
) -> Path:
declaration = load_spec(spec_path)
source_root = source_root.resolve()
production = source_root == Path("/")
if production and os.geteuid() != 0:
raise BackupError("production backup requires root before creating the destination")
if packages_file is not None and production:
raise BackupError("--packages-file is allowed only with a fixture --source-root")
destination_root = Path(output_dir or declaration["backup_root"])
if not destination_root.is_absolute():
raise BackupError("output directory must be absolute")
if destination_root.resolve() in {Path("/"), Path("/etc"), Path("/home"), Path("/root")}:
raise BackupError("output directory must be bounded")
observed = _utc_now()
stamp = timestamp or observed.strftime("%Y%m%dT%H%M%SZ")
if not re.fullmatch(r"\d{8}T\d{6}Z", stamp):
raise BackupError("timestamp must be YYYYMMDDTHHMMSSZ")
bundle_name = f"s1-backup-{stamp}"
final = destination_root / bundle_name
if final.exists():
raise BackupError(f"refusing to overwrite existing bundle {bundle_name}")
destination_root.mkdir(parents=True, exist_ok=True, mode=0o700)
stage = Path(tempfile.mkdtemp(prefix=f".{bundle_name}.", dir=destination_root))
os.chmod(stage, 0o700)
try:
plain_os = stage / ".os-config.tar.gz"
plain_packages = stage / ".packages.txt"
members = _build_os_archive(
source_root=source_root, declaration=declaration, destination=plain_os
)
_write_packages(plain_packages, declaration, packages_file)
executable = _age_binary(age_binary)
encrypted = {
"os-config": stage / "os-config.tar.gz.age",
"packages": stage / "packages.txt.age",
}
_encrypt(executable, declaration["recipients"], plain_os, encrypted["os-config"])
_encrypt(executable, declaration["recipients"], plain_packages, encrypted["packages"])
plain_os.unlink()
plain_packages.unlink()
revision = _revision(source_revision)
declaration_digest = declaration_sha256(spec_path)
artifacts = [
{
"artifact_class": artifact_class,
"encrypted_sha256": _sha256(path),
"filename": path.name,
"size_bytes": path.stat().st_size,
}
for artifact_class, path in encrypted.items()
]
manifest = {
"schema_version": "1.0",
"bundle_id": bundle_name,
"created_at": _rfc3339(observed),
"host": hostname or socket.gethostname(),
"source_revision": revision,
"declaration_sha256": declaration_digest,
"recipient_sha256": [
hashlib.sha256(value.encode()).hexdigest() for value in declaration["recipients"]
],
"artifacts": artifacts,
"os_config_members": members,
"package_source": "fixture-file" if packages_file else "dpkg --get-selections",
}
manifest_path = stage / "manifest.json"
manifest_path.write_text(
json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
retention = declaration["retention"]
receipt = {
"schema_version": "1.0",
"receipt_id": str(uuid.uuid4()),
"event_type": "backup",
"status": "pass",
"created_at": _rfc3339(observed),
"bundle_id": bundle_name,
"host": manifest["host"],
"source_revision": revision,
"declaration_sha256": declaration_digest,
"manifest_sha256": _sha256(manifest_path),
"artifacts": artifacts,
"retention": _retention_decision(
destination_root,
retention["keep_complete_bundles"],
retention["max_total_bytes"],
_bundle_size(stage),
),
}
(stage / "receipt.json").write_text(
json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
for path in stage.iterdir():
if path.is_file():
with path.open("rb") as handle:
os.fsync(handle.fileno())
os.replace(stage, final)
directory_fd = os.open(destination_root, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
return final
except Exception:
shutil.rmtree(stage, ignore_errors=True)
raise
def inspect_status(spec_path: Path, output_dir: Path | None) -> dict[str, Any]:
declaration = load_spec(spec_path)
root = Path(output_dir or declaration["backup_root"])
bundles = _complete_bundles(root)
if not bundles:
raise BackupError("no complete S1 backup bundle exists")
newest = bundles[0]
# Import lazily to keep the create path independent while making status
# prove the newest evidence, rather than trusting a pass-shaped JSON file.
from s1_restore import RestoreError, validate_bundle
try:
receipt = validate_bundle(newest, spec_path)["receipt"]
created = datetime.fromisoformat(receipt["created_at"].replace("Z", "+00:00"))
except (RestoreError, KeyError, TypeError, ValueError) as exc:
raise BackupError(f"newest backup evidence is invalid: {exc}") from exc
fresh_until = created + timedelta(hours=declaration["retention"]["freshness_hours"])
total = sum(_bundle_size(bundle) for bundle in bundles)
result = {
"bundle": newest.name,
"bundle_count": len(bundles),
"created_at": receipt["created_at"],
"fresh_until": _rfc3339(fresh_until),
"status": "pass",
"total_bytes": total,
}
if _utc_now() >= fresh_until:
result["status"] = "stale"
if len(bundles) > declaration["retention"]["keep_complete_bundles"]:
result["status"] = "over-retention"
if total > declaration["retention"]["max_total_bytes"]:
result["status"] = "over-budget"
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)
check = subparsers.add_parser("check", help="validate source-only backup inputs")
check.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
create = subparsers.add_parser("create", help="create one complete encrypted bundle")
create.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
create.add_argument("--source-root", type=Path, default=Path("/"))
create.add_argument("--output-dir", type=Path)
create.add_argument("--packages-file", type=Path)
create.add_argument("--age-bin")
create.add_argument("--source-revision")
create.add_argument("--timestamp")
create.add_argument("--hostname")
status_parser = subparsers.add_parser("status", help="fail if backup evidence is absent or stale")
status_parser.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
status_parser.add_argument("--output-dir", type=Path)
prune_plan = subparsers.add_parser(
"prune-plan", help="print the exact retained bundles requiring deletion approval"
)
prune_plan.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
prune_plan.add_argument("--output-dir", type=Path)
prune = subparsers.add_parser("prune", help="apply one exact, freshly rendered prune plan")
prune.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
prune.add_argument("--output-dir", type=Path)
prune.add_argument("--approval", required=True)
args = parser.parse_args()
try:
if args.command == "check":
declaration = load_spec(args.spec)
result = {"ok": True, "artifacts": sorted(declaration["artifacts"])}
elif args.command == "create":
bundle = create_bundle(
spec_path=args.spec,
source_root=args.source_root,
output_dir=args.output_dir,
packages_file=args.packages_file,
age_binary=args.age_bin,
source_revision=args.source_revision,
timestamp=args.timestamp,
hostname=args.hostname,
)
result = {"bundle": str(bundle), "status": "pass"}
elif args.command == "status":
result = inspect_status(args.spec, args.output_dir)
elif args.command == "prune-plan":
result = plan_prune(args.spec, args.output_dir)
else:
result = apply_prune(args.spec, args.output_dir, args.approval)
print(json.dumps(result, sort_keys=True))
return 0 if result.get("status", "pass") == "pass" else 1
except (BackupContractError, BackupError, OSError, tarfile.TarError) as exc:
print(f"S1 backup failed closed: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())

208
scripts/s1_backup_contract.py Executable file
View file

@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""Validate the declared S1 backup membership and public-recipient policy."""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path, PurePosixPath
from typing import Any
import yaml
SCHEMA_VERSION = "1.0"
AGE_RECIPIENT_RE = re.compile(r"^age1[023456789acdefghjklmnpqrstuvwxyz]{20,}$")
REQUIRED_EXCLUSIONS = {
"/etc/ssl/private",
"/home",
"/root",
"/var/lib/kubelet",
"/var/lib/rancher",
}
FORBIDDEN_PARTS = {"id_rsa", "id_ed25519", "keys.txt", "private", "secrets"}
class BackupContractError(ValueError):
"""The backup declaration is ambiguous, unsafe, or inconsistent."""
def _absolute_path(value: Any, label: str) -> str:
if not isinstance(value, str) or not value.startswith("/"):
raise BackupContractError(f"{label} must be an absolute path")
path = PurePosixPath(value)
if ".." in path.parts or str(path) != value.rstrip("/"):
raise BackupContractError(f"{label} must be normalized without traversal")
return str(path)
def _sops_recipients(path: Path) -> set[str]:
try:
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
except (OSError, yaml.YAMLError) as exc:
raise BackupContractError(f"cannot read recipient policy {path}: {exc}") from exc
recipients: set[str] = set()
for rule in payload.get("creation_rules", []) if isinstance(payload, dict) else []:
for group in rule.get("key_groups", []) if isinstance(rule, dict) else []:
age = group.get("age", []) if isinstance(group, dict) else []
if isinstance(age, list):
recipients.update(item.strip() for item in age if isinstance(item, str))
if not recipients:
raise BackupContractError("recipient policy contains no age recipients")
return recipients
def validate_spec(payload: Any, *, policy_path: Path) -> dict[str, Any]:
if not isinstance(payload, dict):
raise BackupContractError("backup declaration must be a YAML object")
errors: list[str] = []
if str(payload.get("schema_version")) != SCHEMA_VERSION:
errors.append(f"schema_version must be {SCHEMA_VERSION}")
try:
backup_root = _absolute_path(payload.get("backup_root"), "backup_root")
except BackupContractError as exc:
errors.append(str(exc))
backup_root = ""
if backup_root in {"", "/", "/etc", "/home", "/root", "/var"}:
errors.append("backup_root must be a dedicated bounded directory")
recipients = payload.get("recipients")
if not isinstance(recipients, list) or not recipients or not all(
isinstance(item, str) and AGE_RECIPIENT_RE.fullmatch(item) for item in recipients
):
errors.append("recipients must be a non-empty list of age public recipients")
recipients = []
elif len(set(recipients)) != len(recipients):
errors.append("recipients contains duplicates")
try:
policy_recipients = _sops_recipients(policy_path)
if set(recipients) != policy_recipients:
errors.append("backup recipients must exactly match the SOPS public-recipient policy")
except BackupContractError as exc:
errors.append(str(exc))
retention = payload.get("retention")
if not isinstance(retention, dict):
errors.append("retention must be an object")
else:
for key, low, high in (
("keep_complete_bundles", 2, 90),
("max_total_bytes", 1048576, 10737418240),
("freshness_hours", 1, 168),
):
value = retention.get(key)
if not isinstance(value, int) or isinstance(value, bool) or not low <= value <= high:
errors.append(f"retention.{key} must be an integer from {low} to {high}")
exclusions = payload.get("excluded_prefixes")
normalized_exclusions: list[str] = []
if not isinstance(exclusions, list):
errors.append("excluded_prefixes must be a list")
else:
for index, value in enumerate(exclusions):
try:
normalized_exclusions.append(
_absolute_path(value, f"excluded_prefixes[{index}]")
)
except BackupContractError as exc:
errors.append(str(exc))
missing = sorted(REQUIRED_EXCLUSIONS - set(normalized_exclusions))
if missing:
errors.append(f"excluded_prefixes omits governed exclusions {', '.join(missing)}")
artifacts = payload.get("artifacts")
if not isinstance(artifacts, dict) or set(artifacts) != {"os-config", "packages"}:
errors.append("artifacts must contain exactly os-config and packages")
artifacts = {}
os_config = artifacts.get("os-config", {})
paths = os_config.get("paths") if isinstance(os_config, dict) else None
declared: list[str] = []
if not isinstance(paths, list) or not paths:
errors.append("artifacts.os-config.paths must be a non-empty list")
else:
for index, item in enumerate(paths):
label = f"artifacts.os-config.paths[{index}]"
if not isinstance(item, dict) or not isinstance(item.get("required"), bool):
errors.append(f"{label} must contain path and boolean required")
continue
expected_mode = item.get("expected_mode")
if item["required"] and not (
isinstance(expected_mode, str) and re.fullmatch(r"0[0-7]{3}", expected_mode)
):
errors.append(f"{label} requires expected_mode as four octal digits")
if not item["required"] and expected_mode is not None:
errors.append(f"{label}.expected_mode is allowed only for required paths")
try:
path = _absolute_path(item.get("path"), f"{label}.path")
except BackupContractError as exc:
errors.append(str(exc))
continue
declared.append(path)
lowered = {part.lower() for part in PurePosixPath(path).parts}
if lowered & FORBIDDEN_PARTS:
errors.append(f"{label}.path selects secret/private-key shaped content")
if any(path == prefix or path.startswith(prefix + "/") for prefix in normalized_exclusions):
errors.append(f"{label}.path falls under excluded prefix")
if len(set(declared)) != len(declared):
errors.append("artifacts.os-config.paths contains duplicates")
for left in declared:
for right in declared:
if left != right and right.startswith(left + "/"):
errors.append(f"declared paths overlap: {left} contains {right}")
if isinstance(os_config, dict) and os_config.get("format") != "tar.gz":
errors.append("artifacts.os-config.format must be tar.gz")
packages = artifacts.get("packages", {})
if not isinstance(packages, dict) or packages.get("format") != "dpkg-selections":
errors.append("artifacts.packages.format must be dpkg-selections")
elif packages.get("command") != ["dpkg", "--get-selections"]:
errors.append("artifacts.packages.command must be the fixed dpkg selection argv")
if errors:
raise BackupContractError("backup contract failed:\n- " + "\n- ".join(errors))
return payload
def load_spec(path: Path) -> dict[str, Any]:
try:
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
except (OSError, yaml.YAMLError) as exc:
raise BackupContractError(f"cannot read {path}: {exc}") from exc
policy_value = payload.get("recipient_policy") if isinstance(payload, dict) else None
if not isinstance(policy_value, str) or not policy_value:
raise BackupContractError("recipient_policy must name a repository-relative file")
policy_relative = PurePosixPath(policy_value)
if policy_relative.is_absolute() or ".." in policy_relative.parts:
raise BackupContractError("recipient_policy must be a normalized repository-relative path")
root = path.resolve().parent.parent
return validate_spec(payload, policy_path=root / policy_relative)
def declaration_sha256(path: Path) -> str:
import hashlib
return hashlib.sha256(path.read_bytes()).hexdigest()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("spec", nargs="?", type=Path, default=Path("spec/s1-backup.yaml"))
args = parser.parse_args()
payload = load_spec(args.spec)
print(
json.dumps(
{
"artifacts": sorted(payload["artifacts"]),
"declaration_sha256": declaration_sha256(args.spec),
"ok": True,
"path_count": len(payload["artifacts"]["os-config"]["paths"]),
"recipients": len(payload["recipients"]),
},
sort_keys=True,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())

284
scripts/s1_restore.py Executable file
View file

@ -0,0 +1,284 @@
#!/usr/bin/env python3
"""Inspect or extract an S1 backup into an isolated staging directory."""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
import uuid
from datetime import datetime, timezone
from pathlib import Path, PurePosixPath
from typing import Any
from s1_backup import BackupError, _age_binary, _sha256
from s1_backup_contract import BackupContractError, declaration_sha256, load_spec
ROOT = Path(__file__).resolve().parents[1]
MANIFEST_KEYS = {
"artifacts",
"bundle_id",
"created_at",
"declaration_sha256",
"host",
"os_config_members",
"package_source",
"recipient_sha256",
"schema_version",
"source_revision",
}
BACKUP_RECEIPT_KEYS = {
"artifacts",
"bundle_id",
"created_at",
"declaration_sha256",
"event_type",
"host",
"manifest_sha256",
"receipt_id",
"retention",
"schema_version",
"source_revision",
"status",
}
class RestoreError(RuntimeError):
"""An artifact or restore destination is unsafe or inconsistent."""
def _json(path: Path, label: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RestoreError(f"cannot read {label}") from exc
if not isinstance(payload, dict):
raise RestoreError(f"{label} must be a JSON object")
return payload
def validate_bundle(bundle: Path, spec_path: Path) -> dict[str, Any]:
bundle = bundle.resolve()
if not bundle.is_dir() or not bundle.name.startswith("s1-backup-"):
raise RestoreError("bundle must be one complete s1-backup-* directory")
declaration = load_spec(spec_path)
manifest_path = bundle / "manifest.json"
receipt_path = bundle / "receipt.json"
manifest = _json(manifest_path, "manifest")
receipt = _json(receipt_path, "receipt")
if set(manifest) != MANIFEST_KEYS:
raise RestoreError("manifest fields do not match the metadata-only contract")
if set(receipt) != BACKUP_RECEIPT_KEYS:
raise RestoreError("receipt fields do not match the metadata-only contract")
if manifest.get("schema_version") != "1.0" or receipt.get("schema_version") != "1.0":
raise RestoreError("unsupported backup evidence schema version")
if receipt.get("event_type") != "backup" or receipt.get("status") != "pass":
raise RestoreError("bundle has no passing backup receipt")
if manifest.get("bundle_id") != bundle.name or receipt.get("bundle_id") != bundle.name:
raise RestoreError("bundle identity does not match its directory")
for shared in (
"artifacts",
"created_at",
"declaration_sha256",
"host",
"source_revision",
):
if receipt.get(shared) != manifest.get(shared):
raise RestoreError(f"receipt and manifest {shared} differ")
if receipt.get("manifest_sha256") != _sha256(manifest_path):
raise RestoreError("manifest digest does not match the receipt")
expected_declaration = declaration_sha256(spec_path)
if manifest.get("declaration_sha256") != expected_declaration:
raise RestoreError("bundle was not produced from the supplied declaration")
if receipt.get("declaration_sha256") != expected_declaration:
raise RestoreError("receipt declaration digest is inconsistent")
artifacts = manifest.get("artifacts")
if not isinstance(artifacts, list) or {
item.get("artifact_class") for item in artifacts if isinstance(item, dict)
} != {"os-config", "packages"}:
raise RestoreError("manifest must describe exactly two artifact classes")
for item in artifacts:
filename = item.get("filename")
if not isinstance(filename, str) or Path(filename).name != filename:
raise RestoreError("artifact filename must be a basename")
artifact = bundle / filename
if not artifact.is_file():
raise RestoreError(f"artifact is missing: {filename}")
if item.get("encrypted_sha256") != _sha256(artifact):
raise RestoreError(f"artifact digest mismatch: {filename}")
if item.get("size_bytes") != artifact.stat().st_size:
raise RestoreError(f"artifact size mismatch: {filename}")
members = manifest.get("os_config_members")
if not isinstance(members, list) or not members:
raise RestoreError("manifest contains no OS configuration membership")
member_by_path = {
item.get("path"): item for item in members if isinstance(item, dict)
}
for entry in declaration["artifacts"]["os-config"]["paths"]:
if not entry["required"]:
continue
relative = str(PurePosixPath(entry["path"]).relative_to("/"))
member = member_by_path.get(relative)
if not member or member.get("mode") != entry["expected_mode"]:
raise RestoreError(f"required member or declared critical mode differs: {entry['path']}")
return {"bundle": bundle, "declaration": declaration, "manifest": manifest, "receipt": receipt}
def _safe_member(member: tarfile.TarInfo) -> dict[str, Any]:
path = PurePosixPath(member.name)
if path.is_absolute() or ".." in path.parts or str(path) != member.name.rstrip("/"):
raise RestoreError(f"unsafe archive member path: {member.name}")
if member.isdev() or member.isfifo():
raise RestoreError(f"unsupported archive member type: {member.name}")
if not (member.isfile() or member.isdir() or member.issym() or member.islnk()):
raise RestoreError(f"unsupported archive member type: {member.name}")
if member.issym() or member.islnk():
target = PurePosixPath(member.linkname)
combined = target if target.is_absolute() else path.parent / target
if target.is_absolute() or ".." in combined.parts:
raise RestoreError(f"archive link escapes staging root: {member.name}")
kind = "file" if member.isfile() else "directory" if member.isdir() else "symlink"
return {
"kind": kind,
"mode": f"{member.mode:04o}",
"path": str(path),
"size_bytes": member.size if member.isfile() else 0,
}
def _decrypt(age_binary: str, identity: Path, source: Path, destination: Path) -> None:
try:
subprocess.run(
[age_binary, "-d", "-i", str(identity), "-o", str(destination), str(source)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=True,
)
except (OSError, subprocess.CalledProcessError) as exc:
raise RestoreError("artifact decryption failed with output suppressed") from exc
def _safe_destination(destination: Path) -> Path:
resolved = destination.resolve()
if not resolved.is_absolute() or resolved in {
Path("/"),
Path("/etc"),
Path("/home"),
Path("/root"),
Path("/var"),
}:
raise RestoreError("restore destination must be a bounded isolated directory")
try:
resolved.relative_to(ROOT)
except ValueError:
pass
else:
raise RestoreError("restore destination must not be inside the repository")
if not resolved.parent.is_dir():
raise RestoreError("restore destination parent must already exist")
if resolved.exists() and (not resolved.is_dir() or any(resolved.iterdir())):
raise RestoreError("restore destination must be absent or an empty directory")
return resolved
def extract_bundle(
*, bundle: Path, spec_path: Path, destination: Path, identity: Path, age_binary: str | None
) -> Path:
validated = validate_bundle(bundle, spec_path)
if not identity.is_file():
raise RestoreError("age identity must be supplied as a readable file")
final = _safe_destination(destination)
stage = Path(tempfile.mkdtemp(prefix=f".{final.name}.", dir=final.parent))
os.chmod(stage, 0o700)
try:
artifacts = {
item["artifact_class"]: validated["bundle"] / item["filename"]
for item in validated["manifest"]["artifacts"]
}
executable = _age_binary(age_binary)
plain_archive = stage / ".os-config.tar.gz"
_decrypt(executable, identity, artifacts["os-config"], plain_archive)
with tarfile.open(plain_archive, "r:gz") as archive:
observed = [_safe_member(member) for member in archive.getmembers()]
expected = validated["manifest"]["os_config_members"]
if observed != expected:
raise RestoreError("decrypted archive membership or modes differ from manifest")
archive.extractall(path=stage, filter="data")
plain_archive.unlink()
packages = stage / "packages.txt"
_decrypt(executable, identity, artifacts["packages"], packages)
os.chmod(packages, 0o600)
if packages.stat().st_size == 0 or b"\x00" in packages.read_bytes():
raise RestoreError("package-selection artifact is empty or not text")
receipt = {
"schema_version": "1.0",
"receipt_id": str(uuid.uuid4()),
"event_type": "restore",
"status": "pass",
"created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"bundle_id": validated["manifest"]["bundle_id"],
"source_revision": validated["manifest"]["source_revision"],
"declaration_sha256": validated["manifest"]["declaration_sha256"],
"manifest_sha256": _sha256(validated["bundle"] / "manifest.json"),
"member_count": len(validated["manifest"]["os_config_members"]),
"isolated": True,
}
(stage / "restore-receipt.json").write_text(
json.dumps(receipt, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
if final.exists():
final.rmdir()
os.replace(stage, final)
return final
except Exception:
shutil.rmtree(stage, ignore_errors=True)
raise
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("bundle", type=Path)
parser.add_argument("--spec", type=Path, default=ROOT / "spec" / "s1-backup.yaml")
parser.add_argument("--extract-to", type=Path)
parser.add_argument("--identity", type=Path)
parser.add_argument("--age-bin")
args = parser.parse_args()
try:
if args.extract_to is None:
if args.identity is not None:
parser.error("--identity is used only with --extract-to")
validated = validate_bundle(args.bundle, args.spec)
result = {
"artifacts": len(validated["manifest"]["artifacts"]),
"bundle": validated["manifest"]["bundle_id"],
"member_count": len(validated["manifest"]["os_config_members"]),
"status": "pass",
}
else:
if args.identity is None:
parser.error("--extract-to requires --identity")
restored = extract_bundle(
bundle=args.bundle,
spec_path=args.spec,
destination=args.extract_to,
identity=args.identity,
age_binary=args.age_bin,
)
result = {"restored_to": str(restored), "status": "pass"}
print(json.dumps(result, sort_keys=True))
return 0
except (BackupContractError, BackupError, RestoreError, OSError, tarfile.TarError) as exc:
print(f"S1 restore failed closed: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())

53
spec/s1-backup.yaml Normal file
View file

@ -0,0 +1,53 @@
schema_version: "1.0"
backup_root: /opt/backup/railiance/infra
recipient_policy: .sops.yaml
recipients:
- age1aq8twfd78wvpra0had8cezcnj96tj4q0068edrz5jez8d6xwmflqdepsh4
retention:
keep_complete_bundles: 7
max_total_bytes: 268435456
freshness_hours: 26
artifacts:
os-config:
format: tar.gz
paths:
- path: /etc/ssh/sshd_config
required: true
expected_mode: "0644"
- path: /etc/ssh/sshd_config.d
required: false
- path: /etc/ufw/ufw.conf
required: false
- path: /etc/ufw/user.rules
required: false
- path: /etc/ufw/user6.rules
required: false
- path: /etc/fail2ban/jail.local
required: false
- path: /etc/fail2ban/jail.d
required: false
- path: /etc/hosts
required: true
expected_mode: "0644"
- path: /etc/hostname
required: true
expected_mode: "0644"
- path: /etc/apt/sources.list.d
required: false
packages:
format: dpkg-selections
command: [dpkg, --get-selections]
excluded_prefixes:
- /etc/age
- /etc/kubernetes
- /etc/rancher
- /etc/sops
- /etc/ssl/private
- /home
- /root
- /var/lib/kubelet
- /var/lib/rancher

View file

@ -0,0 +1,244 @@
from __future__ import annotations
import copy
import json
import shutil
import subprocess
import sys
import tarfile
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
from unittest import mock
import yaml
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
from s1_backup import ( # noqa: E402
BackupError,
apply_prune,
create_bundle,
inspect_status,
plan_prune,
)
from s1_backup_contract import BackupContractError, load_spec, validate_spec # noqa: E402
from s1_restore import RestoreError, _safe_member, extract_bundle, validate_bundle # noqa: E402
@unittest.skipUnless(shutil.which("age") and shutil.which("age-keygen"), "age tools required")
class S1BackupRecoveryTests(unittest.TestCase):
def setUp(self) -> None:
self.temp = tempfile.TemporaryDirectory()
self.root = Path(self.temp.name)
self.fixture = self.root / "fixture"
(self.fixture / "etc/ssh/sshd_config.d").mkdir(parents=True)
(self.fixture / "etc/ssh/sshd_config").write_text("PasswordAuthentication no\n")
(self.fixture / "etc/ssh/sshd_config.d/10-hardening.conf").write_text(
"PermitRootLogin no\n"
)
(self.fixture / "etc/hosts").write_text("127.0.0.1 localhost\n")
(self.fixture / "etc/hostname").write_text("fixture-host\n")
self.packages = self.root / "packages.txt"
self.packages.write_text("curl\tinstall\ngit\tinstall\n")
self.identity = self.root / "identity.age"
subprocess.run(
["age-keygen", "-o", str(self.identity)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=True,
)
self.recipient = subprocess.check_output(
["age-keygen", "-y", str(self.identity)], text=True
).strip()
policy = {
"creation_rules": [{"key_groups": [{"age": [self.recipient]}]}]
}
(self.root / ".sops.yaml").write_text(yaml.safe_dump(policy))
spec = yaml.safe_load((ROOT / "spec/s1-backup.yaml").read_text())
spec["recipients"] = [self.recipient]
self.spec_dir = self.root / "spec"
self.spec_dir.mkdir()
self.spec = self.spec_dir / "s1-backup.yaml"
self.spec.write_text(yaml.safe_dump(spec, sort_keys=False))
self.output = self.root / "bundles"
def tearDown(self) -> None:
self.temp.cleanup()
def _create(self, **overrides: object) -> Path:
arguments = {
"spec_path": self.spec,
"source_root": self.fixture,
"output_dir": self.output,
"packages_file": self.packages,
"age_binary": shutil.which("age"),
"source_revision": "a" * 40,
"timestamp": "20260823T120000Z",
"hostname": "fixture-host",
}
arguments.update(overrides)
return create_bundle(**arguments)
def test_repository_contract_validates_without_private_key(self) -> None:
declaration = load_spec(ROOT / "spec/s1-backup.yaml")
self.assertEqual(10, len(declaration["artifacts"]["os-config"]["paths"]))
def test_relative_and_excluded_paths_fail(self) -> None:
payload = yaml.safe_load(self.spec.read_text())
bad = copy.deepcopy(payload)
bad["artifacts"]["os-config"]["paths"][0]["path"] = "etc/ssh/sshd_config"
with self.assertRaisesRegex(BackupContractError, "absolute"):
validate_spec(bad, policy_path=self.root / ".sops.yaml")
bad = copy.deepcopy(payload)
bad["artifacts"]["os-config"]["paths"][0]["path"] = "/root/id_ed25519"
with self.assertRaisesRegex(BackupContractError, "secret|excluded"):
validate_spec(bad, policy_path=self.root / ".sops.yaml")
def test_atomic_encrypted_bundle_round_trip(self) -> None:
bundle = self._create()
self.assertEqual(
{"manifest.json", "os-config.tar.gz.age", "packages.txt.age", "receipt.json"},
{path.name for path in bundle.iterdir()},
)
validated = validate_bundle(bundle, self.spec)
self.assertEqual("pass", validated["receipt"]["status"])
restore = self.root / "restore"
extract_bundle(
bundle=bundle,
spec_path=self.spec,
destination=restore,
identity=self.identity,
age_binary=shutil.which("age"),
)
self.assertEqual("fixture-host\n", (restore / "etc/hostname").read_text())
self.assertIn("curl", (restore / "packages.txt").read_text())
self.assertEqual("pass", json.loads((restore / "restore-receipt.json").read_text())["status"])
def test_encryption_failure_publishes_nothing(self) -> None:
with self.assertRaisesRegex(BackupError, "no bundle was published"):
self._create(age_binary="/bin/false")
self.assertEqual([], list(self.output.iterdir()))
def test_missing_required_input_publishes_nothing(self) -> None:
(self.fixture / "etc/hostname").unlink()
with self.assertRaisesRegex(BackupError, "required backup inputs"):
self._create()
self.assertEqual([], list(self.output.iterdir()))
def test_runtime_secret_shaped_member_publishes_nothing(self) -> None:
(self.fixture / "etc/ssh/sshd_config.d/operator-token").write_text("do-not-copy\n")
with self.assertRaisesRegex(BackupError, "secret/private-key shaped"):
self._create()
self.assertEqual([], list(self.output.iterdir()))
def test_critical_mode_drift_publishes_nothing(self) -> None:
(self.fixture / "etc/ssh/sshd_config").chmod(0o600)
with self.assertRaisesRegex(BackupError, "critical mode mismatch"):
self._create()
self.assertEqual([], list(self.output.iterdir()))
def test_root_check_precedes_destination_creation(self) -> None:
destination = self.root / "must-not-exist"
with mock.patch("s1_backup.os.geteuid", return_value=1000):
with self.assertRaisesRegex(BackupError, "requires root"):
create_bundle(
spec_path=self.spec,
source_root=Path("/"),
output_dir=destination,
packages_file=None,
age_binary=shutil.which("age"),
source_revision="a" * 40,
timestamp="20260823T120000Z",
hostname="fixture-host",
)
self.assertFalse(destination.exists())
def test_tampered_artifact_fails_before_decryption(self) -> None:
bundle = self._create()
with (bundle / "packages.txt.age").open("ab") as handle:
handle.write(b"tamper")
with self.assertRaisesRegex(RestoreError, "digest mismatch"):
validate_bundle(bundle, self.spec)
def test_archive_traversal_and_escaping_link_fail(self) -> None:
member = tarfile.TarInfo("../../etc/shadow")
with self.assertRaisesRegex(RestoreError, "unsafe archive member"):
_safe_member(member)
link = tarfile.TarInfo("etc/escape")
link.type = tarfile.SYMTYPE
link.linkname = "../../root/key"
with self.assertRaisesRegex(RestoreError, "escapes"):
_safe_member(link)
def test_restore_destination_cannot_be_repository(self) -> None:
bundle = self._create()
with self.assertRaisesRegex(RestoreError, "repository"):
extract_bundle(
bundle=bundle,
spec_path=self.spec,
destination=ROOT / "restore-test",
identity=self.identity,
age_binary=shutil.which("age"),
)
def test_status_fails_when_receipt_is_stale(self) -> None:
bundle = self._create()
created = datetime.fromisoformat(
json.loads((bundle / "receipt.json").read_text())["created_at"].replace("Z", "+00:00")
)
with mock.patch("s1_backup._utc_now", return_value=created + timedelta(hours=27)):
self.assertEqual("stale", inspect_status(self.spec, self.output)["status"])
def test_prune_requires_current_exact_approval(self) -> None:
declaration = yaml.safe_load(self.spec.read_text())
declaration["retention"]["keep_complete_bundles"] = 2
self.spec.write_text(yaml.safe_dump(declaration, sort_keys=False))
for index in range(3):
self._create(timestamp=f"2026082{index + 1}T120000Z")
plan = plan_prune(self.spec, self.output)
self.assertEqual(["s1-backup-20260821T120000Z"], plan["candidates"])
with self.assertRaisesRegex(BackupError, "exact candidate set"):
apply_prune(self.spec, self.output, "PRUNE-S1-BACKUPS-WRONG")
self.assertTrue((self.output / plan["candidates"][0]).exists())
result = apply_prune(self.spec, self.output, plan["approval"])
self.assertEqual("pruned", result["status"])
self.assertFalse((self.output / plan["candidates"][0]).exists())
def test_status_rejects_tampered_passing_receipt(self) -> None:
bundle = self._create()
receipt_path = bundle / "receipt.json"
receipt = json.loads(receipt_path.read_text())
receipt["created_at"] = "2020-01-01T00:00:00Z"
receipt_path.write_text(json.dumps(receipt))
with self.assertRaisesRegex(BackupError, "invalid"):
inspect_status(self.spec, self.output)
def test_timer_calendar_and_unit_are_source_renderable(self) -> None:
timer = ROOT / "ansible/roles/s1_backup/templates/railiance-backup-s1.timer.j2"
service = ROOT / "ansible/roles/s1_backup/templates/railiance-backup-s1.service.j2"
timer_text = timer.read_text()
service_text = service.read_text()
self.assertIn("OnCalendar=*-*-* 02:15:00 UTC", timer_text)
self.assertIn("Persistent=true", timer_text)
self.assertIn("ReadWritePaths=/opt/backup/railiance/infra", service_text)
if shutil.which("systemd-analyze"):
result = subprocess.run(
["systemd-analyze", "calendar", "*-*-* 02:15:00 UTC"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.assertEqual(0, result.returncode, result.stderr)
playbook = yaml.safe_load((ROOT / "ansible/playbooks/s1-backup.yaml").read_text())
self.assertEqual("s1_backup", playbook[0]["roles"][0]["role"])
self.assertIn("DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER", str(playbook))
bootstrap = (ROOT / "ansible/playbooks/bootstrap.yaml").read_text()
self.assertNotIn("s1_backup", bootstrap)
if __name__ == "__main__":
unittest.main()

View file

@ -1,80 +1,8 @@
#!/usr/bin/env bash
# tools/cmd/railiance-backup-s1 — S1 OS & Provisioning backup
# Backs up: key OS config files applied by Ansible (sshd, ufw, fail2ban, hosts)
# Encryption: age (reuses SOPS key pair from .sops.yaml)
# Output: /opt/backup/railiance/infra/
# No network required. Requires root to read /etc/.
# Repository entry point. The deployed systemd wrapper uses the same Python
# implementation from /usr/local/lib/railiance-infra.
set -euo pipefail
# ── Configuration ──────────────────────────────────────────────────────────────
AGE_PUBLIC_KEY="age1aq8twfd78wvpra0had8cezcnj96tj4q0068edrz5jez8d6xwmflqdepsh4"
BACKUP_DIR="/opt/backup/railiance/infra"
KEEP=7
TS="$(date -u +%Y%m%dT%H%M%SZ)"
# Colour helpers (no external dependency)
ok() { printf " ✅ %-12s %s\n" "$1" "$2"; }
warn() { printf " ⚠️ %-12s %s\n" "$1" "$2"; }
bad() { printf " ❌ %-12s %s\n" "$1" "$2"; }
mkdir -p "${BACKUP_DIR}"
printf "\nrailiance-infra (S1) backup — %s\n" "${TS}"
printf "%0.s-" $(seq 1 44); echo
# ── Root check ─────────────────────────────────────────────────────────────────
if [[ $EUID -ne 0 ]]; then
bad "root" "this script requires root — run via: sudo make backup"
exit 1
fi
# ── OS config snapshot ─────────────────────────────────────────────────────────
# Captures the live state of files that Ansible manages.
# These may drift from the playbooks if manual changes were made.
ok "os-config" "snapshotting /etc config…"
OS_FILES=(
/etc/ssh/sshd_config
/etc/ssh/sshd_config.d/
/etc/ufw/ufw.conf
/etc/ufw/user.rules
/etc/ufw/user6.rules
/etc/fail2ban/jail.local
/etc/fail2ban/jail.d/
/etc/hosts
/etc/hostname
/etc/apt/sources.list.d/
)
TMP_OS="$(mktemp -d)"
for item in "${OS_FILES[@]}"; do
[[ -e "${item}" ]] || continue
dest="${TMP_OS}$(dirname "${item}")"
mkdir -p "${dest}"
cp -a "${item}" "${dest}/" 2>/dev/null || true
done
tar -czf - -C "${TMP_OS}" . \
| age -r "${AGE_PUBLIC_KEY}" -o "${BACKUP_DIR}/os-config-${TS}.tar.gz.age"
rm -rf "${TMP_OS}"
ok "os-config" "encrypted → os-config-${TS}.tar.gz.age"
# ── Installed packages list ────────────────────────────────────────────────────
if command -v dpkg &>/dev/null; then
dpkg --get-selections \
| age -r "${AGE_PUBLIC_KEY}" -o "${BACKUP_DIR}/packages-${TS}.txt.age"
ok "packages" "encrypted → packages-${TS}.txt.age"
fi
# ── Prune local cache ──────────────────────────────────────────────────────────
for pattern in "os-config-*.tar.gz.age" "packages-*.txt.age"; do
find "${BACKUP_DIR}" -name "${pattern}" | sort -r | tail -n +$((KEEP + 1)) | xargs -r rm -f
done
ok "prune" "kept last ${KEEP} of each type"
# ── Stamp ──────────────────────────────────────────────────────────────────────
echo "${TS}" > "${BACKUP_DIR}/.last-backup"
echo
ok "done" "backup complete — ${TS}"
echo " Location: ${BACKUP_DIR}"
echo " Decrypt with: age -d -i ~/.config/sops/age/keys.txt <file>"
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
exec python3 "${REPO_ROOT}/scripts/s1_backup.py" create "$@"

View file

@ -4,7 +4,7 @@ type: workplan
title: "Close the encrypted S1 backup and recovery loop"
domain: financials
repo: railiance-infra
status: proposed
status: active
owner: codex
topic_slug: railiance
created: "2026-08-23"
@ -30,13 +30,12 @@ operator approval.
## Current State and Risks
`tools/cmd/railiance-backup-s1` currently creates age-encrypted OS-configuration
and package-selection files under `/opt/backup/railiance/infra` and retains the
latest seven of each. It is not scheduled, has no off-host transfer, produces
no manifest or receipt, has no restoration tool or drill, duplicates an age
recipient in executable source, suppresses individual copy failures, and can
leave ambiguous partial output. The output is therefore a useful local helper,
not current proof of recoverability.
The source-side local loop now has a declaration, atomic encrypted bundle,
metadata-only evidence, integrity/freshness status, exact-approval retention,
isolated restore, and separately approved timer deployment interface. No timer
has been deployed and no retained artifact has been deleted by this workplan.
Off-host transfer and an attended drill remain open, so the repository still
cannot claim operational recoverability from local source tests alone.
## Delivery Order
@ -48,7 +47,7 @@ tasks and an attended isolated drill.
```task
id: RAIL-HO-WP-0012-T01
status: todo
status: done
priority: high
```
@ -66,7 +65,7 @@ requires neither root nor a decryption key.
```task
id: RAIL-HO-WP-0012-T02
status: todo
status: done
priority: high
```
@ -89,7 +88,7 @@ tests leave no ambiguous success state.
```task
id: RAIL-HO-WP-0012-T03
status: todo
status: done
priority: high
```
@ -110,7 +109,7 @@ and no default invocation can overwrite a host file.
```task
id: RAIL-HO-WP-0012-T04
status: todo
status: done
priority: medium
```
@ -164,15 +163,28 @@ decrypted configuration.
## Acceptance
- [ ] S1 backup membership, exclusions, recipients, and retention are declared
- [x] S1 backup membership, exclusions, recipients, and retention are declared
and validated from source.
- [ ] Backup publication is atomic, encrypted, and accompanied by a safe
- [x] Backup publication is atomic, encrypted, and accompanied by a safe
manifest and receipt.
- [ ] Restore defaults to isolated inspection and rejects tampering, traversal,
- [x] Restore defaults to isolated inspection and rejects tampering, traversal,
and accidental live overwrite.
- [ ] Scheduling and freshness checks are source-controlled but deployed only
- [x] Scheduling and freshness checks are source-controlled but deployed only
through an explicit host-change approval.
- [ ] A governed off-host copy and an attended isolated restore drill prove the
selected S1 state is recoverable.
- [ ] No cluster, platform, tenant, secret-issuance, or private-key custody
- [x] No cluster, platform, tenant, secret-issuance, or private-key custody
responsibility is absorbed into this repository.
## Source delivery record — 2026-08-23
- `make validate-s1-backup` passes the declaration and 15 fixture recovery
tests using disposable age identities and storage.
- The full repository suite passes 45 tests. The timer calendar is accepted by
`systemd-analyze`; deployment YAML and separation from bootstrap are checked
as source contracts.
- Native `ansible-playbook --syntax-check` was unavailable on the development
workstation. The deployment remains unexecuted and gated by
`DEPLOY-RAIL-HO-WP-0012-S1-BACKUP-TIMER` plus a clean committed revision.
- T05 and T06 remain `wait`; no off-host write, retained-artifact deletion,
private-key access, or live-host restore occurred.