d0a5a27926
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02994-7685-7940-bf34-3555b8256018
456 lines
19 KiB
Python
Executable file
456 lines
19 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Plan or execute one collision-safe S1 off-site envelope upload."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import yaml
|
|
|
|
from s1_backup import _sha256
|
|
from s1_backup_contract import BackupContractError
|
|
from s1_restore import RestoreError, validate_bundle
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
EXPECTED_MEMBERS = {
|
|
"manifest.json",
|
|
"receipt.json",
|
|
"os-config.tar.gz.age",
|
|
"packages.txt.age",
|
|
}
|
|
|
|
|
|
class OffsiteError(RuntimeError):
|
|
"""The off-site contract, evidence, or upload failed closed."""
|
|
|
|
|
|
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
"""Keep a credential-bearing file-drop request on its reviewed origin."""
|
|
|
|
def redirect_request(
|
|
self,
|
|
req: Any,
|
|
fp: Any,
|
|
code: int,
|
|
msg: str,
|
|
headers: Any,
|
|
newurl: str,
|
|
) -> None:
|
|
return None
|
|
|
|
|
|
def _read_yaml(path: Path, label: str) -> dict[str, Any]:
|
|
try:
|
|
payload = yaml.safe_load(path.read_text(encoding="utf-8"))
|
|
except (OSError, yaml.YAMLError) as exc:
|
|
raise OffsiteError(f"cannot read {label}") from exc
|
|
if not isinstance(payload, dict):
|
|
raise OffsiteError(f"{label} must be a YAML object")
|
|
return payload
|
|
|
|
|
|
def contract_sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def load_contract(path: Path, *, fixture_http: bool = False) -> dict[str, Any]:
|
|
payload = _read_yaml(path, "off-site contract")
|
|
errors: list[str] = []
|
|
if payload.get("schema_version") != "1.0":
|
|
errors.append("schema_version must be 1.0")
|
|
if payload.get("route_id") != "railiance-backup-offsite-lane":
|
|
errors.append("route_id must be railiance-backup-offsite-lane")
|
|
if payload.get("owner_repo") != "railiance-platform":
|
|
errors.append("owner_repo must be railiance-platform")
|
|
|
|
provider = payload.get("provider")
|
|
if not isinstance(provider, dict):
|
|
errors.append("provider must be an object")
|
|
provider = {}
|
|
if provider.get("kind") != "nextcloud-file-drop":
|
|
errors.append("provider.kind must be nextcloud-file-drop")
|
|
base_url = provider.get("base_url")
|
|
if not isinstance(base_url, str):
|
|
errors.append("provider.base_url must be a URL")
|
|
else:
|
|
parsed = urllib.parse.urlsplit(base_url)
|
|
allowed_scheme = "http" if fixture_http else "https"
|
|
if parsed.scheme != allowed_scheme or not parsed.netloc or parsed.query or parsed.fragment:
|
|
errors.append(f"provider.base_url must be a bounded {allowed_scheme} URL")
|
|
if "@" in parsed.netloc or parsed.username or parsed.password:
|
|
errors.append("provider.base_url must contain no user information")
|
|
prefix = provider.get("remote_prefix")
|
|
if not isinstance(prefix, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]{1,62}", prefix):
|
|
errors.append("provider.remote_prefix must be a safe single path segment")
|
|
|
|
transport = payload.get("transport")
|
|
expected_transport = {
|
|
"method": "PUT",
|
|
"content_type": "application/x-tar",
|
|
"if_none_match": "*",
|
|
"success_codes": [201, 204],
|
|
"authentication": "basic-token-empty-password",
|
|
"secret_env": "RAILIANCE_BACKUP_NC_TOKEN",
|
|
"response_body": "suppressed",
|
|
}
|
|
if transport != expected_transport:
|
|
errors.append("transport must match the reviewed write-only PUT contract")
|
|
|
|
envelope = payload.get("envelope")
|
|
if not isinstance(envelope, dict):
|
|
errors.append("envelope must be an object")
|
|
envelope = {}
|
|
if envelope.get("format") != "deterministic-ustar":
|
|
errors.append("envelope.format must be deterministic-ustar")
|
|
if set(envelope.get("members", [])) != EXPECTED_MEMBERS:
|
|
errors.append("envelope.members must contain exactly the encrypted bundle and metadata")
|
|
if envelope.get("object_name_template") != "railiance-infra__{bundle_id}__s1-envelope.tar":
|
|
errors.append("envelope.object_name_template is not the reviewed immutable name")
|
|
|
|
acceptance_file = payload.get("owner_acceptance_file")
|
|
if acceptance_file != "spec/s1-offsite-owner-acceptance.yaml":
|
|
errors.append("owner_acceptance_file must name the bounded repository acceptance record")
|
|
receipt_root = payload.get("receipt_root")
|
|
if not isinstance(receipt_root, str) or not receipt_root.startswith("/") or receipt_root in {
|
|
"/",
|
|
"/etc",
|
|
"/home",
|
|
"/root",
|
|
"/var",
|
|
}:
|
|
errors.append("receipt_root must be a dedicated absolute directory")
|
|
if errors:
|
|
raise OffsiteError("off-site contract failed:\n- " + "\n- ".join(errors))
|
|
return payload
|
|
|
|
|
|
def load_acceptance(contract_path: Path, contract: dict[str, Any]) -> dict[str, Any]:
|
|
acceptance_path = ROOT / contract["owner_acceptance_file"]
|
|
# Fixture contracts carry their acceptance beside their fixture repository root.
|
|
if contract_path.resolve().parent.parent != ROOT:
|
|
acceptance_path = contract_path.resolve().parent.parent / contract["owner_acceptance_file"]
|
|
payload = _read_yaml(acceptance_path, "owner acceptance")
|
|
expected_digest = contract_sha256(contract_path)
|
|
if payload.get("schema_version") != "1.0" or payload.get("owner_repo") != contract["owner_repo"]:
|
|
raise OffsiteError("owner acceptance identity is invalid")
|
|
if payload.get("status") == "accepted":
|
|
if payload.get("contract_sha256") != expected_digest:
|
|
raise OffsiteError("owner acceptance does not bind the current contract digest")
|
|
try:
|
|
uuid.UUID(str(payload.get("decision_id")))
|
|
except (ValueError, TypeError) as exc:
|
|
raise OffsiteError("accepted owner contract requires a decision UUID") from exc
|
|
if not isinstance(payload.get("accepted_at"), str) or not payload["accepted_at"].endswith("Z"):
|
|
raise OffsiteError("accepted owner contract requires accepted_at")
|
|
elif payload.get("status") != "pending":
|
|
raise OffsiteError("owner acceptance status must be pending or accepted")
|
|
return payload
|
|
|
|
|
|
def review_contract(contract_path: Path, *, fixture_http: bool = False) -> dict[str, Any]:
|
|
contract = load_contract(contract_path, fixture_http=fixture_http)
|
|
digest = contract_sha256(contract_path)
|
|
acceptance = load_acceptance(contract_path, contract)
|
|
return {
|
|
"approval": f"APPROVE S1-OFFSITE-CONTRACT-{digest}",
|
|
"contract_sha256": digest,
|
|
"owner_repo": contract["owner_repo"],
|
|
"owner_status": acceptance["status"],
|
|
"route_id": contract["route_id"],
|
|
"status": "review-required" if acceptance["status"] != "accepted" else "accepted",
|
|
}
|
|
|
|
|
|
def _bundle_members(validated: dict[str, Any]) -> list[Path]:
|
|
bundle = validated["bundle"]
|
|
names = ["manifest.json", "receipt.json"] + sorted(
|
|
item["filename"] for item in validated["manifest"]["artifacts"]
|
|
)
|
|
if set(names) != EXPECTED_MEMBERS:
|
|
raise OffsiteError("bundle does not match the reviewed off-site envelope membership")
|
|
return [bundle / name for name in names]
|
|
|
|
|
|
def build_envelope(bundle: Path, backup_spec: Path, destination: Path) -> dict[str, Any]:
|
|
validated = validate_bundle(bundle, backup_spec)
|
|
with tarfile.open(destination, "w", format=tarfile.USTAR_FORMAT) as archive:
|
|
for source in _bundle_members(validated):
|
|
info = tarfile.TarInfo(source.name)
|
|
info.size = source.stat().st_size
|
|
info.mode = 0o600
|
|
info.mtime = 0
|
|
info.uid = 0
|
|
info.gid = 0
|
|
info.uname = ""
|
|
info.gname = ""
|
|
with source.open("rb") as handle:
|
|
archive.addfile(info, handle)
|
|
return {
|
|
"bundle_id": validated["manifest"]["bundle_id"],
|
|
"envelope_sha256": _sha256(destination),
|
|
"manifest_sha256": _sha256(validated["bundle"] / "manifest.json"),
|
|
"size_bytes": destination.stat().st_size,
|
|
}
|
|
|
|
|
|
def plan_upload(
|
|
*, bundle: Path, backup_spec: Path, contract_path: Path, fixture_http: bool = False
|
|
) -> dict[str, Any]:
|
|
contract = load_contract(contract_path, fixture_http=fixture_http)
|
|
acceptance = load_acceptance(contract_path, contract)
|
|
with tempfile.TemporaryDirectory(prefix="s1-offsite-plan-") as temporary:
|
|
envelope = Path(temporary) / "envelope.tar"
|
|
metadata = build_envelope(bundle, backup_spec, envelope)
|
|
object_name = contract["envelope"]["object_name_template"].format(
|
|
bundle_id=metadata["bundle_id"]
|
|
)
|
|
material = "\n".join(
|
|
(
|
|
contract_sha256(contract_path),
|
|
metadata["bundle_id"],
|
|
metadata["envelope_sha256"],
|
|
str(metadata["size_bytes"]),
|
|
contract["provider"]["remote_prefix"],
|
|
object_name,
|
|
)
|
|
)
|
|
exact = hashlib.sha256(material.encode()).hexdigest()[:20].upper()
|
|
return {
|
|
**metadata,
|
|
"approval": f"UPLOAD-S1-OFFSITE-{exact}" if acceptance["status"] == "accepted" else None,
|
|
"contract_sha256": contract_sha256(contract_path),
|
|
"executable": acceptance["status"] == "accepted",
|
|
"object_name": object_name,
|
|
"owner_status": acceptance["status"],
|
|
"remote_object_identity": f"{contract['provider']['remote_prefix']}/{object_name}",
|
|
"route_id": contract["route_id"],
|
|
"status": "approval-required" if acceptance["status"] == "accepted" else "owner-review-required",
|
|
}
|
|
|
|
|
|
def _write_receipt(receipt_dir: Path, payload: dict[str, Any]) -> Path:
|
|
receipt_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
final = receipt_dir / f"{payload['bundle_id']}.offsite.json"
|
|
if final.exists():
|
|
raise OffsiteError("an off-site receipt already exists for this bundle")
|
|
descriptor, temporary = tempfile.mkstemp(prefix=".offsite-receipt-", dir=receipt_dir)
|
|
try:
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, indent=2, sort_keys=True)
|
|
handle.write("\n")
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.chmod(temporary, 0o600)
|
|
os.replace(temporary, final)
|
|
return final
|
|
except Exception:
|
|
try:
|
|
os.unlink(temporary)
|
|
except FileNotFoundError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def upload(
|
|
*,
|
|
bundle: Path,
|
|
backup_spec: Path,
|
|
contract_path: Path,
|
|
approval: str,
|
|
receipt_dir: Path | None = None,
|
|
fixture_http: bool = False,
|
|
) -> Path:
|
|
contract = load_contract(contract_path, fixture_http=fixture_http)
|
|
acceptance = load_acceptance(contract_path, contract)
|
|
if acceptance["status"] != "accepted":
|
|
raise OffsiteError("railiance-platform has not accepted the current upload contract")
|
|
plan = plan_upload(
|
|
bundle=bundle,
|
|
backup_spec=backup_spec,
|
|
contract_path=contract_path,
|
|
fixture_http=fixture_http,
|
|
)
|
|
if approval != plan["approval"]:
|
|
raise OffsiteError("upload approval does not match the current exact envelope")
|
|
secret_env = contract["transport"]["secret_env"]
|
|
token = os.environ.get(secret_env)
|
|
if not token or any(character in token for character in "\r\n"):
|
|
raise OffsiteError(f"{secret_env} must be injected into the contained process")
|
|
|
|
with tempfile.TemporaryDirectory(prefix="s1-offsite-upload-") as temporary:
|
|
envelope = Path(temporary) / plan["object_name"]
|
|
observed = build_envelope(bundle, backup_spec, envelope)
|
|
if observed["envelope_sha256"] != plan["envelope_sha256"]:
|
|
raise OffsiteError("envelope changed after approval")
|
|
segments = (
|
|
contract["provider"]["base_url"].rstrip("/"),
|
|
urllib.parse.quote(token, safe=""),
|
|
urllib.parse.quote(contract["provider"]["remote_prefix"], safe=""),
|
|
urllib.parse.quote(plan["object_name"], safe=""),
|
|
)
|
|
destination = "/".join(segments)
|
|
authorization = base64.b64encode(f"{token}:".encode()).decode()
|
|
request = urllib.request.Request(destination, data=envelope.read_bytes(), method="PUT")
|
|
request.add_header("Authorization", f"Basic {authorization}")
|
|
request.add_header("Content-Type", contract["transport"]["content_type"])
|
|
request.add_header("If-None-Match", contract["transport"]["if_none_match"])
|
|
try:
|
|
opener = urllib.request.build_opener(_NoRedirect)
|
|
with opener.open(request, timeout=600) as response:
|
|
status = response.status
|
|
etag = response.headers.get("ETag")
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code in {409, 412}:
|
|
raise OffsiteError("remote object collision; no success receipt was written") from exc
|
|
raise OffsiteError(f"off-site PUT failed with HTTP {exc.code}; response suppressed") from exc
|
|
except urllib.error.URLError as exc:
|
|
raise OffsiteError("off-site PUT failed; endpoint and response suppressed") from exc
|
|
if status not in contract["transport"]["success_codes"]:
|
|
raise OffsiteError(f"off-site PUT returned unexpected HTTP {status}")
|
|
|
|
payload = {
|
|
"schema_version": "1.0",
|
|
"receipt_id": str(uuid.uuid4()),
|
|
"event_type": "offsite-upload",
|
|
"status": "pass",
|
|
"created_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"bundle_id": plan["bundle_id"],
|
|
"manifest_sha256": plan["manifest_sha256"],
|
|
"envelope_sha256": plan["envelope_sha256"],
|
|
"size_bytes": plan["size_bytes"],
|
|
"remote_object_identity": plan["remote_object_identity"],
|
|
"route_id": contract["route_id"],
|
|
"contract_sha256": plan["contract_sha256"],
|
|
"owner_decision_id": acceptance["decision_id"],
|
|
"http_status": status,
|
|
"etag_sha256": hashlib.sha256(etag.encode()).hexdigest() if etag else None,
|
|
}
|
|
root = Path(receipt_dir or contract["receipt_root"])
|
|
return _write_receipt(root, payload)
|
|
|
|
|
|
def validate_prune_receipt(bundle: Path, backup_spec: Path, receipt_path: Path) -> dict[str, Any]:
|
|
validated = validate_bundle(bundle, backup_spec)
|
|
try:
|
|
payload = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as exc:
|
|
raise OffsiteError("off-site receipt is unreadable") from exc
|
|
required = {
|
|
"bundle_id",
|
|
"contract_sha256",
|
|
"created_at",
|
|
"envelope_sha256",
|
|
"etag_sha256",
|
|
"event_type",
|
|
"http_status",
|
|
"manifest_sha256",
|
|
"owner_decision_id",
|
|
"receipt_id",
|
|
"remote_object_identity",
|
|
"route_id",
|
|
"schema_version",
|
|
"size_bytes",
|
|
"status",
|
|
}
|
|
if not isinstance(payload, dict) or set(payload) != required:
|
|
raise OffsiteError("off-site receipt fields are invalid")
|
|
if payload.get("schema_version") != "1.0" or payload.get("event_type") != "offsite-upload":
|
|
raise OffsiteError("off-site receipt identity is invalid")
|
|
if payload.get("status") != "pass" or payload.get("bundle_id") != bundle.name:
|
|
raise OffsiteError("off-site receipt is not passing for this bundle")
|
|
if payload.get("route_id") != "railiance-backup-offsite-lane":
|
|
raise OffsiteError("off-site receipt uses the wrong governed route")
|
|
if payload.get("manifest_sha256") != _sha256(bundle / "manifest.json"):
|
|
raise OffsiteError("off-site receipt does not bind the current manifest")
|
|
if not SHA256_RE.fullmatch(str(payload.get("envelope_sha256"))):
|
|
raise OffsiteError("off-site envelope digest is invalid")
|
|
if payload.get("http_status") not in {201, 204}:
|
|
raise OffsiteError("off-site receipt has no accepted PUT status")
|
|
expected_identity = (
|
|
f"railiance-infra/railiance-infra__{bundle.name}__s1-envelope.tar"
|
|
)
|
|
if payload.get("remote_object_identity") != expected_identity or token_shaped(expected_identity):
|
|
raise OffsiteError("remote object identity is unsafe to retain")
|
|
if not SHA256_RE.fullmatch(str(payload.get("contract_sha256"))):
|
|
raise OffsiteError("off-site contract digest is invalid")
|
|
etag_digest = payload.get("etag_sha256")
|
|
if etag_digest is not None and not SHA256_RE.fullmatch(str(etag_digest)):
|
|
raise OffsiteError("off-site ETag digest is invalid")
|
|
if not isinstance(payload.get("size_bytes"), int) or not 0 < payload["size_bytes"] <= 536870912:
|
|
raise OffsiteError("off-site envelope size is invalid")
|
|
try:
|
|
datetime.strptime(payload["created_at"], "%Y-%m-%dT%H:%M:%SZ")
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise OffsiteError("off-site receipt timestamp is invalid") from exc
|
|
try:
|
|
uuid.UUID(str(payload.get("receipt_id")))
|
|
uuid.UUID(str(payload.get("owner_decision_id")))
|
|
except (TypeError, ValueError) as exc:
|
|
raise OffsiteError("off-site receipt decision identities are invalid") from exc
|
|
if payload["manifest_sha256"] != _sha256(validated["bundle"] / "manifest.json"):
|
|
raise OffsiteError("off-site receipt manifest validation failed")
|
|
return payload
|
|
|
|
|
|
def token_shaped(value: str) -> bool:
|
|
lowered = value.lower()
|
|
return any(marker in lowered for marker in ("token=", "password=", "authorization", "filesdrop/"))
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--contract", type=Path, default=ROOT / "spec/s1-offsite.yaml")
|
|
parser.add_argument("--backup-spec", type=Path, default=ROOT / "spec/s1-backup.yaml")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
subparsers.add_parser("review", help="render the exact owner contract approval")
|
|
plan_parser = subparsers.add_parser("plan", help="plan one immutable envelope without credentials")
|
|
plan_parser.add_argument("bundle", type=Path)
|
|
upload_parser = subparsers.add_parser("upload", help="upload one approved immutable envelope")
|
|
upload_parser.add_argument("bundle", type=Path)
|
|
upload_parser.add_argument("--approval", required=True)
|
|
upload_parser.add_argument("--receipt-dir", type=Path)
|
|
args = parser.parse_args()
|
|
try:
|
|
if args.command == "review":
|
|
result = review_contract(args.contract)
|
|
elif args.command == "plan":
|
|
result = plan_upload(
|
|
bundle=args.bundle,
|
|
backup_spec=args.backup_spec,
|
|
contract_path=args.contract,
|
|
)
|
|
else:
|
|
receipt = upload(
|
|
bundle=args.bundle,
|
|
backup_spec=args.backup_spec,
|
|
contract_path=args.contract,
|
|
approval=args.approval,
|
|
receipt_dir=args.receipt_dir,
|
|
)
|
|
result = {"receipt": str(receipt), "status": "pass"}
|
|
print(json.dumps(result, sort_keys=True))
|
|
return 0
|
|
except (BackupContractError, RestoreError, OffsiteError, OSError, tarfile.TarError) as exc:
|
|
print(f"S1 off-site operation failed closed: {exc}", file=sys.stderr)
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|