coulomb/railiance-platform
5
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

RPF-WP-0015: T02 synthetic sender lane, T03 checklist revision, ops-warden re-acceptance

Browse source 
Addresses round-1 owner review requests from audit-core, rapp-postgres,
railiance-cluster and railiance-infra; refreshes the review contract.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 680842@bnt-lap001
Assistant-Session: 54eb9e8e-7d6b-4f0c-b7a6-a6db62075883
This commit is contained in:
codex 2026-10-14 12:49:40 +02:00
parent aee6fe79d7
commit ce8286dff9
13 changed files with 2032 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

6
scripts/audit-core-database-lease-recovery.py
View file

@ -100,6 +100,12 @@ def load_approval(path: Path, now: datetime, *, require_open_window: bool) -> di
load = document.get("synthetic_load") or {}
if not load.get("contract_id") or not load.get("driver_revision"):
raise ProcedureError("approval receipt lacks the synthetic-load contract")
sender = document.get("synthetic_sender")
# Strict naming/scope/expiry rules live in audit_core_synthetic_sender_lane.py.
if not isinstance(sender, dict) or set(sender) != {"identity", "source", "tenant", "expires_at"} or not all(
isinstance(value, str) and value for value in sender.values()
):
raise ProcedureError("approval receipt lacks the projected synthetic sender identity")
return document

525
scripts/audit-core-recovery-preflight.py
View file

@ -104,6 +104,23 @@ def latest_completed_backup(backups: dict[str, Any], now: datetime) -> dict[str,
}
def running_pod_image(pod: dict[str, Any]) -> dict[str, Any]:
"""Image actually running in the serving pod, not only the Deployment spec.
``status.containerStatuses[].imageID`` carries the digest the kubelet
resolved; a spec that matches while an older ReplicaSet pod still runs must
fail closed.
"""
containers = pod.get("spec", {}).get("containers", [])
statuses = pod.get("status", {}).get("containerStatuses", [])
name = containers[0].get("name") if containers else None
spec_image = containers[0].get("image") if containers else None
status = next((item for item in statuses if item.get("name") == name), None)
image_id = (status or {}).get("imageID") or ""
match = re.search(r"@(sha256:[0-9a-f]{64})$", image_id)
return {"image": spec_image, "digest": match.group(1) if match else None}
def resource_condition(resource: dict[str, Any]) -> dict[str, Any]:
conditions = resource.get("status", {}).get("conditions", [])
current = conditions[0] if conditions else {}
@ -247,6 +264,7 @@ def common_state(remote: Remote, now: datetime) -> dict[str, Any]:
except json.JSONDecodeError as exc:
raise PreflightError("OpenBao seal state returned invalid JSON") from exc
image = deployment["spec"]["template"]["spec"]["containers"][0]["image"]
running = running_pod_image(pod)
state = {
"captured_at": now.replace(microsecond=0).isoformat().replace("+00:00", "Z"),
"node": {
@ -280,6 +298,12 @@ def common_state(remote: Remote, now: datetime) -> dict[str, Any]:
"audit_core": {
"image": image,
"image_matches_reviewed_digest": image == EXPECTED_IMAGE,
"running_pod_image": running["image"],
"running_pod_image_digest": running["digest"],
"running_pod_image_matches_reviewed_digest": (
running["image"] == EXPECTED_IMAGE
and running["digest"] == EXPECTED_IMAGE.rsplit("@", 1)[1]
),
"ready_replicas": deployment.get("status", {}).get("readyReplicas", 0),
"replicas": deployment.get("spec", {}).get("replicas", 0),
"pod_uid": pod["metadata"]["uid"],
@ -303,6 +327,7 @@ def automated_common_pass(state: dict[str, Any]) -> bool:
all(item["ready"] for item in state["external_secret_stores"].values()),
all(item["ready"] for item in state["external_secrets"].values()),
state["audit_core"]["image_matches_reviewed_digest"],
state["audit_core"]["running_pod_image_matches_reviewed_digest"],
state["audit_core"]["ready_replicas"] == state["audit_core"]["replicas"] == 1,
)
)
@ -366,14 +391,431 @@ def database_lease_preflight(remote: Remote, args: argparse.Namespace) -> dict[s
}
# --- node-reboot specific checks -------------------------------------------
# Named core platform workloads (namespace, kind, name) that must be fully
# Ready. Every other Deployment/StatefulSet in these namespaces must be Ready
# as well, so a renamed or newly added operator component cannot hide.
CORE_PLATFORM_WORKLOADS = (
("kube-system", "Deployment", "coredns"),
("kube-system", "Deployment", "local-path-provisioner"),
("kube-system", "Deployment", "traefik"),
("kube-system", "Deployment", "metrics-server"),
("cert-manager", "Deployment", "cert-manager"),
("cnpg-system", "Deployment", "cnpg-controller-manager"),
("external-secrets", "Deployment", "external-secrets"),
("knative-serving", "Deployment", "activator"),
("knative-serving", "Deployment", "controller"),
("knative-serving", "Deployment", "net-kourier-controller"),
("kourier-system", "Deployment", "3scale-kourier-gateway"),
("argocd", "Deployment", "argocd-repo-server"),
("argocd", "StatefulSet", "argocd-application-controller"),
("openbao", "StatefulSet", "openbao"),
)
CORE_PLATFORM_NAMESPACES = frozenset(ns for ns, _, _ in CORE_PLATFORM_WORKLOADS)
# CNPG clusters observed on 2026-10-14. The live list is discovered; this set
# only makes a vanished cluster fail closed instead of silently shrinking scope.
EXPECTED_CNPG_CLUSTERS = frozenset({
"databases/apps-pg", "databases/forgejo-db", "databases/net-kingdom-pg",
"databases/platform-pg", "databases/platform-pg-2", "databases/state-hub-db",
"target-revenue/target-revenue-pg", "user-engine/user-engine-pg",
})
FRESH_BACKUP_REQUIRED = frozenset({"databases/platform-pg"})
FRESH_BACKUP_RECOMMENDED = frozenset({"databases/state-hub-db", "user-engine/user-engine-pg"})
# Declared host firewall (railiance-infra RAIL-HO-WP-0009 / Goss).
UFW_ALLOWED = frozenset({"OpenSSH", "2224/tcp"})
UFW_FORBIDDEN_PORTS = ("6443", "8472")
K3S_API_TUNNEL = "k3s-api-railiance01"
OPS_WARDEN_RECEIPT = "reviews/RPF-WP-0017-ops-warden-reacceptance-2026-10-14.json"
CONSUMED_DECISION_PREFIXES = ("449a697a", "85724c0c")
CONSUMED_SCENARIO_IDS = frozenset({"WARDEN-WP-0027-T02-DRILL-20260822-01"})
SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{5,127}$")
FORBIDDEN_RECEIPT_MARKERS = (
"BEGIN PRIVATE KEY", "BEGIN OPENSSH PRIVATE KEY", "AGE-SECRET-KEY-1",
"OPENBAO_ROOT_TOKEN", "VAULT_TOKEN", "hvs.", "password=", "://",
)
def workload_ready(resource: dict[str, Any]) -> bool:
wanted = resource.get("spec", {}).get("replicas", 1)
return wanted >= 1 and resource.get("status", {}).get("readyReplicas", 0) == wanted
def core_platform_workloads(workloads: dict[str, Any]) -> dict[str, Any]:
"""Readiness of named core workloads plus every workload in core namespaces."""
observed: dict[tuple[str, str, str], bool] = {}
for item in workloads.get("items", []):
meta = item.get("metadata", {})
if meta.get("namespace") in CORE_PLATFORM_NAMESPACES:
key = (meta["namespace"], item.get("kind"), meta.get("name"))
observed[key] = workload_ready(item)
missing = sorted("/".join(key) for key in CORE_PLATFORM_WORKLOADS if key not in observed)
unready = sorted("/".join(key) for key, ready in observed.items() if not ready)
return {
"required": ["/".join(key) for key in CORE_PLATFORM_WORKLOADS],
"missing": missing,
"unready": unready,
"ready": not missing and not unready,
}
def failing_pods(pod_list: dict[str, Any]) -> list[str]:
"""``namespace/name`` of every pod that is neither Succeeded nor Running+Ready."""
failing = []
for pod in pod_list.get("items", []):
status = pod.get("status", {})
phase = status.get("phase")
if phase == "Succeeded":
continue
statuses = status.get("containerStatuses", [])
if phase == "Running" and statuses and all(item.get("ready") for item in statuses):
continue
meta = pod.get("metadata", {})
failing.append(f"{meta.get('namespace')}/{meta.get('name')}")
return sorted(failing)
def failing_pod_diff(baseline: list[str], current: list[str]) -> dict[str, Any]:
new = sorted(set(current) - set(baseline))
return {
"baseline_count": len(baseline),
"current_count": len(current),
"new_failing": new,
"resolved": sorted(set(baseline) - set(current)),
"no_new_failures": not new,
}
def system_identifier(controldata: str) -> str | None:
match = re.search(r"^Database system identifier:\s+(\d+)\s*$", controldata, re.MULTILINE)
return match.group(1) if match else None
def cnpg_identity(cluster: dict[str, Any], pvc_uids: dict[str, str]) -> dict[str, Any]:
"""Value-safe identity/health of one CNPG cluster (no credentials, no DSNs)."""
meta = cluster.get("metadata", {})
status = cluster.get("status", {})
primary = status.get("currentPrimary")
backup_configured = bool(cluster.get("spec", {}).get("backup"))
archiving = condition(cluster, "ContinuousArchiving")
return {
"uid": meta.get("uid"),
"instances": cluster.get("spec", {}).get("instances", 0),
"ready_instances": status.get("readyInstances", 0),
"phase": status.get("phase"),
"current_primary": primary,
"primary_pvc_uid": pvc_uids.get(primary) if primary else None,
"healthy_pvc": sorted(status.get("healthyPVC", [])),
"backup_configured": backup_configured,
"continuous_archiving": archiving == "True" if backup_configured else None,
"system_id": None,
}
def cnpg_healthy(identity: dict[str, Any]) -> bool:
return (
identity["ready_instances"] == identity["instances"] == 1
and bool(identity["uid"])
and bool(identity["primary_pvc_uid"])
and bool(identity["system_id"])
and identity["current_primary"] in identity["healthy_pvc"]
and identity["continuous_archiving"] is not False
)
def latest_backups_by_cluster(backups: dict[str, Any], now: datetime) -> dict[str, Any]:
grouped: dict[str, list[dict[str, Any]]] = {}
for item in backups.get("items", []):
meta = item.get("metadata", {})
cluster = item.get("spec", {}).get("cluster", {}).get("name")
if cluster:
grouped.setdefault(f"{meta.get('namespace')}/{cluster}", []).append(item)
result = {}
for key, items in grouped.items():
try:
result[key] = latest_completed_backup({"items": items}, now)
except PreflightError:
result[key] = None
return result
def eso_inventory(stores: dict[str, Any], external_secrets: dict[str, Any]) -> dict[str, Any]:
"""Every ClusterSecretStore Valid and every ExternalSecret SecretSynced."""
def bad(items: list[dict[str, Any]], reason: str, namespaced: bool) -> list[str]:
out = []
for item in items:
ready = next(
(c for c in item.get("status", {}).get("conditions", []) if c.get("type") == "Ready"),
{},
)
if ready.get("status") != "True" or ready.get("reason") != reason:
meta = item.get("metadata", {})
out.append(f"{meta.get('namespace')}/{meta.get('name')}" if namespaced else meta.get("name"))
return sorted(out)
store_items = stores.get("items", [])
es_items = external_secrets.get("items", [])
invalid_stores = bad(store_items, "Valid", False)
unsynced = bad(es_items, "SecretSynced", True)
return {
"cluster_store_count": len(store_items),
"invalid_cluster_stores": invalid_stores,
"external_secret_count": len(es_items),
"unsynced_external_secrets": unsynced,
"ready": bool(store_items) and bool(es_items) and not invalid_stores and not unsynced,
}
def ufw_state(status_text: str) -> dict[str, Any]:
"""Compare ``ufw status`` with the declared OpenSSH + Nydus 2224 allow set."""
active = bool(re.search(r"^Status:\s+active\s*$", status_text, re.MULTILINE))
targets: set[str] = set()
in_rules = False
for line in status_text.splitlines():
if line.startswith("--"):
in_rules = True
continue
if in_rules and line.strip():
target = re.split(r"\s{2,}", line.strip())[0].replace(" (v6)", "")
targets.add(target)
forbidden = sorted(
target for target in targets
if any(re.match(rf"^{port}(/|:|$)", target) for port in UFW_FORBIDDEN_PORTS)
)
return {
"active": active,
"allowed_targets": sorted(targets),
"forbidden_ports_open": forbidden,
"matches_declaration": active and targets == UFW_ALLOWED and not forbidden,
}
def boots_since(list_boots_json: str, since: datetime) -> int:
"""Count boots whose first entry is after ``since`` (journalctl --list-boots -o json)."""
try:
boots = json.loads(list_boots_json)
except json.JSONDecodeError as exc:
raise PreflightError("boot list returned invalid JSON") from exc
count = 0
for boot in boots:
first = boot.get("first_entry")
if isinstance(first, int) and datetime.fromtimestamp(first / 1_000_000, UTC) > since:
count += 1
return count
def validate_ops_warden_receipt(path: Path) -> dict[str, Any]:
"""RPF-WP-0017 re-acceptance of the ops-warden revision for the attended login."""
try:
raw = path.read_text(encoding="utf-8")
receipt = json.loads(raw)
except (OSError, json.JSONDecodeError) as exc:
raise PreflightError("ops-warden acceptance receipt is unavailable or invalid JSON") from exc
if not isinstance(receipt, dict) or receipt.get("workplan_id", "RPF-WP-0017") != "RPF-WP-0017" or "RPF-WP-0017" not in raw:
raise PreflightError("ops-warden acceptance receipt is not an RPF-WP-0017 receipt")
if any(marker in raw for marker in FORBIDDEN_RECEIPT_MARKERS[:-1]):
raise PreflightError("ops-warden acceptance receipt contains forbidden material")
if receipt.get("secret_values_observed") is True or receipt.get("sensitive_material_recorded") is True:
raise PreflightError("ops-warden acceptance receipt records sensitive material")
disposition = next(
(str(receipt[key]).lower() for key in ("current_decision", "disposition", "decision", "status", "result") if key in receipt),
"",
)
if disposition not in {"accept", "accepted", "approve", "approved"}:
raise PreflightError("ops-warden acceptance receipt does not record acceptance")
revision = next(
(
receipt[key] for key in ("current_reviewed_revision", "reviewed_revision", "accepted_revision", "ops_warden_revision", "revision")
if isinstance(receipt.get(key), str) and re.fullmatch(r"[0-9a-f]{40}", receipt[key])
),
None,
)
if not revision:
raise PreflightError("ops-warden acceptance receipt names no exact 40-hex revision")
return {"path": str(path), "accepted_revision": revision, "verified": True}
CONSOLE_RECEIPT_ID = re.compile(r"^RAIL01-CONSOLE-(\d{8})T\d{6}Z$")
def validate_console_receipt(path: Path, *, window_id: str | None, now: datetime) -> dict[str, Any]:
"""railiance-infra console attestation (docs/railiance01-console-access.md).
Accept only a same-day receipt for this window that proves the console
shows the host and is independent of the guest network. Role-only fields;
any credential-looking or placeholder content is denied.
"""
try:
import yaml # type: ignore[import-untyped]
raw = path.read_text(encoding="utf-8")
receipt = yaml.safe_load(raw)
except Exception as exc: # noqa: BLE001 - any parse/import failure is fail-closed
raise PreflightError("provider-console receipt is unavailable or invalid YAML") from exc
if not isinstance(receipt, dict):
raise PreflightError("provider-console receipt has an unsupported shape")
if any(marker in raw for marker in FORBIDDEN_RECEIPT_MARKERS) or "<" in raw:
raise PreflightError("provider-console receipt contains forbidden material/placeholder")
match = CONSOLE_RECEIPT_ID.fullmatch(str(receipt.get("receipt_id", "")))
if not match or path.stem != receipt["receipt_id"]:
raise PreflightError("provider-console receipt id is not RAIL01-CONSOLE-<YYYYMMDDTHHMMSSZ>")
try:
attested = rfc3339(str(receipt["attested_at"]))
except (KeyError, ValueError) as exc:
raise PreflightError("provider-console receipt has invalid attested_at") from exc
today = now.astimezone(UTC).strftime("%Y%m%d")
if match.group(1) != today or attested.strftime("%Y%m%d") != today or attested > now:
raise PreflightError("provider-console receipt is not from the window day")
if receipt.get("host") != "railiance01" or receipt.get("panel_login") != "ok":
raise PreflightError("provider-console receipt does not prove panel access to railiance01")
if receipt.get("console_shows_host") is not True or receipt.get("independent_of_guest_network") is not True:
raise PreflightError("provider-console receipt does not prove an independent host console")
if not window_id or str(receipt.get("reboot_window_id")) != window_id:
raise PreflightError("provider-console receipt does not name the approved window id")
return {
"receipt_id": receipt["receipt_id"],
"attested_at": attested.isoformat().replace("+00:00", "Z"),
"attested_by_role": receipt.get("attested_by_role"),
"console_shows_host": True,
"independent_of_guest_network": True,
"window_id_matches": True,
"verified": True,
}
def fresh_decision_gates(scenario_id: str | None, decision_id: str | None) -> dict[str, bool]:
"""Consumed drill scenario/decisions can never authorize this window."""
return {
"new_scenario_id_recorded": bool(scenario_id)
and bool(SAFE_ID.fullmatch(scenario_id))
and scenario_id not in CONSUMED_SCENARIO_IDS,
"fresh_human_decision_recorded": bool(decision_id)
and bool(SAFE_ID.fullmatch(decision_id))
and not decision_id.lower().startswith(CONSUMED_DECISION_PREFIXES),
}
def reboot_comparison(pre: dict[str, Any], post: dict[str, Any], boots: int) -> dict[str, Any]:
"""Post-reboot proof: exactly one reboot, same database identities, no new failures."""
pre_host, post_host = pre.get("host", {}), post["host"]
identity_changes = []
for key, before in pre.get("cnpg_clusters", {}).items():
after = post["cnpg_clusters"].get(key)
for field in ("uid", "primary_pvc_uid", "system_id"):
if after is None or after.get(field) != before.get(field):
identity_changes.append(f"{key}:{field}")
pod_diff = failing_pod_diff(pre.get("failing_pods", []), post["failing_pods"])
return {
"boot_id_changed": bool(pre_host.get("boot_id"))
and pre_host.get("boot_id") != post_host["boot_id"],
"boots_since_pre_capture": boots,
"exactly_one_reboot": boots == 1 and pre_host.get("boot_id") != post_host["boot_id"],
"database_identity_changes": sorted(identity_changes),
"failing_pod_diff": pod_diff,
"passed": boots == 1
and pre_host.get("boot_id") != post_host["boot_id"]
and not identity_changes
and pod_diff["no_new_failures"],
}
def tunnel_answers(name: str) -> bool:
try:
completed = subprocess.run(
["bridge", "check", name], text=True, capture_output=True, check=False, timeout=30
)
except (OSError, subprocess.TimeoutExpired):
return False
rows = [line.split() for line in completed.stdout.splitlines() if line.startswith(name)]
return completed.returncode == 0 and bool(rows) and rows[0][-1] == "yes"
def reboot_state(remote: Remote, now: datetime) -> dict[str, Any]:
workloads = remote.kubectl_json(
["get", "deployments,statefulsets", "-A"], label="read core workload state"
)
pods = remote.kubectl_json(["get", "pods", "-A"], label="read all-namespace pod phases")
clusters = remote.kubectl_json(
["get", "clusters.postgresql.cnpg.io", "-A"], label="discover CNPG clusters"
)
pvcs = remote.kubectl_json(["get", "pvc", "-A"], label="read PVC identities")
pvc_uids = {
f"{item['metadata']['namespace']}/{item['metadata']['name']}": item["metadata"]["uid"]
for item in pvcs.get("items", [])
}
terminating_pvcs = sorted(
f"{item['metadata']['namespace']}/{item['metadata']['name']}"
for item in pvcs.get("items", [])
if item["metadata"].get("deletionTimestamp")
)
cnpg: dict[str, Any] = {}
for cluster in clusters.get("items", []):
ns, name = cluster["metadata"]["namespace"], cluster["metadata"]["name"]
ns_pvcs = {
key.split("/", 1)[1]: uid for key, uid in pvc_uids.items() if key.startswith(f"{ns}/")
}
identity = cnpg_identity(cluster, ns_pvcs)
if identity["current_primary"]:
controldata = remote.kubectl(
[
"-n", ns, "exec", identity["current_primary"], "-c", "postgres", "--",
"pg_controldata", "/var/lib/postgresql/data/pgdata",
],
label=f"read {ns}/{name} system identifier",
)
identity["system_id"] = system_identifier(controldata)
cnpg[f"{ns}/{name}"] = identity
backups = remote.kubectl_json(["get", "backups.postgresql.cnpg.io", "-A"], label="read CNPG backup metadata")
stores = remote.kubectl_json(["get", "clustersecretstores"], label="read ESO store conditions")
projections = remote.kubectl_json(["get", "externalsecrets", "-A"], label="read ESO projection conditions")
ufw = remote.run(["sudo", "-n", "ufw", "status"], label="read host firewall state")
ntp = remote.run(
["timedatectl", "show", "-p", "NTPSynchronized", "--value"], label="read time sync state"
)
return {
"host": {
"boot_id": remote.run(["cat", "/proc/sys/kernel/random/boot_id"], label="read boot id"),
"ntp_synchronized": ntp.strip() == "yes",
"firewall": ufw_state(ufw),
},
"k3s_api_tunnel_answers": tunnel_answers(K3S_API_TUNNEL),
"core_platform_workloads": core_platform_workloads(workloads),
"failing_pods": failing_pods(pods),
"terminating_pvcs": terminating_pvcs,
"cnpg_clusters": cnpg,
"missing_expected_cnpg_clusters": sorted(EXPECTED_CNPG_CLUSTERS - set(cnpg)),
"cnpg_backups": latest_backups_by_cluster(backups, now),
"eso": eso_inventory(stores, projections),
}
def reboot_automated_pass(extra: dict[str, Any], max_backup_age_hours: float) -> dict[str, bool]:
backups = extra["cnpg_backups"]
return {
"ntp_synchronized": extra["host"]["ntp_synchronized"],
"firewall_matches_declaration": extra["host"]["firewall"]["matches_declaration"],
"k3s_api_tunnel_answers": extra["k3s_api_tunnel_answers"],
"core_platform_workloads_ready": extra["core_platform_workloads"]["ready"],
"all_cnpg_clusters_healthy": not extra["missing_expected_cnpg_clusters"]
and all(cnpg_healthy(item) for item in extra["cnpg_clusters"].values()),
"required_backups_fresh": all(
backups.get(key) is not None and backups[key]["age_hours"] <= max_backup_age_hours
for key in FRESH_BACKUP_REQUIRED
),
"all_eso_stores_valid_and_synced": extra["eso"]["ready"],
}
def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]:
now = datetime.now(UTC)
state = common_state(remote, now)
backups = remote.kubectl_json(
["-n", "databases", "get", "backup", "-l", "cnpg.io/cluster=platform-pg"],
label="read platform-pg backup metadata",
)
latest = latest_completed_backup(backups, now)
extra = reboot_state(remote, now)
latest = extra["cnpg_backups"].get("databases/platform-pg")
if latest is None:
raise PreflightError("no completed platform-pg backup is visible")
snapshot_evidence = Path(args.openbao_snapshot_evidence).resolve() if args.openbao_snapshot_evidence else None
snapshot_receipt = None
if snapshot_evidence:
@ -383,25 +825,75 @@ def reboot_preflight(remote: Remote, args: argparse.Namespace) -> dict[str, Any]
now=now,
max_age_hours=args.max_snapshot_age_hours,
)
# Gate failures stay value-safe gate results; they never abort evidence capture.
receipt_errors: dict[str, str] = {}
warden_receipt = None
try:
warden_receipt = validate_ops_warden_receipt(Path(args.ops_warden_acceptance_receipt))
except PreflightError as exc:
receipt_errors["ops_warden_acceptance_receipt"] = str(exc)
console_receipt = None
if args.provider_console_receipt:
try:
console_receipt = validate_console_receipt(
Path(args.provider_console_receipt), window_id=args.approved_window_id, now=now
)
except PreflightError as exc:
receipt_errors["provider_console_receipt"] = str(exc)
else:
receipt_errors["provider_console_receipt"] = "not supplied (RAIL-HO-WP-0011-T09 pending)"
checks = reboot_automated_pass(extra, args.max_backup_age_hours)
comparison = None
if args.pre_reboot_evidence:
try:
pre = json.loads(Path(args.pre_reboot_evidence).read_text(encoding="utf-8"))["reboot_evidence"]
except (OSError, json.JSONDecodeError, KeyError, TypeError) as exc:
raise PreflightError("pre-reboot evidence is unavailable or has no reboot_evidence") from exc
boot_list = remote.run(
["sudo", "-n", "journalctl", "--list-boots", "-o", "json", "--no-pager"],
label="read boot list",
)
comparison = reboot_comparison(pre, extra, boots_since(boot_list, rfc3339(pre["captured_at"])))
checks["post_reboot_comparison_passed"] = comparison["passed"]
else:
# Pre-reboot: stuck Terminating PVCs must be resolved or explicitly listed.
listed = set(args.accept_terminating_pvc or [])
checks["terminating_pvcs_resolved_or_listed"] = set(extra["terminating_pvcs"]) <= listed
decision = fresh_decision_gates(args.scenario_id, args.human_decision_id)
gates = {
"approved_window_id_recorded": bool(args.approved_window_id),
**decision,
"ops_warden_revision_accepted_for_attended_login": bool(warden_receipt),
"openbao_snapshot_evidence_present": bool(snapshot_receipt),
"unseal_quorum_attested": args.unseal_quorum_attested,
"provider_console_access_attested": args.provider_console_attested,
"provider_console_receipt_verified": bool(console_receipt),
"host_cluster_platform_database_audit_owners_acknowledged": args.all_owners_ack,
"railiance_infra_independent_abort_owner_acknowledged": args.infra_abort_owner_ack,
"attended_abort_operator_named": bool(args.abort_operator),
}
automated = automated_common_pass(state) and latest["age_hours"] <= args.max_backup_age_hours
automated = (
automated_common_pass(state)
and latest["age_hours"] <= args.max_backup_age_hours
and all(checks.values())
)
return {
"procedure": "railiance01-coordinated-reboot",
"phase": "post-reboot" if comparison else "pre-reboot",
"preflight_only": True,
"automated_checks_passed": automated,
"ready_for_live_execution": automated and all(gates.values()),
"automated_checks": checks,
"ready_for_live_execution": automated and all(gates.values()) and not comparison,
"operator_gates": gates,
"latest_platform_pg_backup": latest,
"max_backup_age_hours": args.max_backup_age_hours,
"fresh_backup_recommended_for": sorted(FRESH_BACKUP_RECOMMENDED),
"openbao_snapshot_receipt": snapshot_receipt,
"ops_warden_acceptance_receipt": warden_receipt,
"provider_console_receipt": console_receipt,
"receipt_gate_failures": receipt_errors,
"max_snapshot_age_hours": args.max_snapshot_age_hours,
"reboot_evidence": {"captured_at": state["captured_at"], **extra},
"reboot_comparison": comparison,
"state": state,
"secret_values_observed": False,
}
@ -418,7 +910,22 @@ def main() -> int:
parser.add_argument("--synthetic-load-id")
parser.add_argument("--openbao-snapshot-evidence")
parser.add_argument("--unseal-quorum-attested", action="store_true")
parser.add_argument("--provider-console-attested", action="store_true")
parser.add_argument(
"--provider-console-receipt",
help="railiance-infra RAIL01-CONSOLE-<ts>.yaml receipt for this window",
)
parser.add_argument("--infra-abort-owner-ack", action="store_true")
parser.add_argument("--scenario-id", help="new scenario id; consumed drill ids are denied")
parser.add_argument("--human-decision-id", help="fresh human GO decision id")
parser.add_argument("--ops-warden-acceptance-receipt", default=OPS_WARDEN_RECEIPT)
parser.add_argument(
"--accept-terminating-pvc", action="append",
help="namespace/name of a pre-existing Terminating PVC accepted for the window",
)
parser.add_argument(
"--pre-reboot-evidence",
help="pre-reboot preflight JSON; switches node-reboot to post-reboot comparison",
)
parser.add_argument("--all-owners-ack", action="store_true")
parser.add_argument("--max-backup-age-hours", type=float, default=26.0)
parser.add_argument("--max-snapshot-age-hours", type=float, default=24.0)

439
scripts/audit_core_synthetic_sender_lane.py Normal file
View file

@ -0,0 +1,439 @@
#!/usr/bin/env python3
"""Attended, value-safe synthetic sender lane for audit-core recovery exercises.
One temporary audit-core sender identity is added to the registry at
``platform/workloads/audit-core/senders`` for exactly one approved window and
removed again afterwards. The identity is parameterized by the approved window
receipt (``synthetic_sender`` block), not by constants in this file, so the
dated Whitehat ``-01``/``-02`` procedures are never copied.
Ordering contract (RAILIANCE-WP-0024-T02):
project -> audit-core restart -> T02 exercise -> evidence -> cleanup -> restart
* ``preflight`` is read-only. It never reads the sender registry values unless
``--inspect-registry`` is given (attended; values stay in memory only).
* ``project`` is time-gated to the opening minutes of the approved window and
writes the bearer only to a new mode-0600 file outside any Git work tree.
* ``cleanup`` is idempotent and refuses to run until the T02 exercise evidence
is recorded, unless an explicit abort reason is given.
No command prints a bearer, the registry, OpenBao bodies, or Kubernetes Secret
data.
"""
from __future__ import annotations
import argparse
import importlib.util
import json
import os
import re
import secrets
import stat
import sys
import time
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
LANE = "audit-core-synthetic-sender"
REGISTRY_PATH = "platform/workloads/audit-core/senders"
DRIVER_INTERFACE = ROOT / "interfaces" / "RAILIANCE-WP-0024-T02-synthetic-load-driver.json"
PROJECT_CUTOFF = timedelta(minutes=3)
MIN_REMAINING = timedelta(minutes=10)
NAME_RE = re.compile(r"^[a-z][a-z0-9-]{2,62}$")
TENANT_RE = re.compile(r"^tenant:trial:[a-z0-9][a-z0-9-]{1,62}$")
RFC3339_Z_RE = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
class LaneError(RuntimeError):
pass
def _load(name: str, filename: str) -> Any:
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename)
if not spec or not spec.loader:
raise LaneError(f"cannot load {filename}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
# Reuse, do not copy: the live-proven registry/reload helpers from the -02 lane
# (Operator, safe_run, secure_unlink, force_sender_sync_and_restart) and the
# reviewed T02 receipt/attest-chain/preflight gates from the lease harness.
CREDS = _load("audit_core_whitehat_e2_credentials_02", "audit-core-whitehat-e2-credentials-02.py")
LEASE = _load("audit_core_database_lease_recovery", "audit-core-database-lease-recovery.py")
PREFLIGHT = LEASE.PREFLIGHT
ERRORS = (OSError, IndexError, KeyError, LaneError, CREDS.ProcedureError, LEASE.ProcedureError, PREFLIGHT.PreflightError)
def utc_now() -> datetime:
return datetime.now(UTC)
def stamp(value: datetime) -> str:
return value.astimezone(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def confirm_token(approval: dict[str, Any]) -> str:
return f"{LANE}:{approval['approval_id']}:attended"
def driver_contract_id() -> str:
try:
return json.loads(DRIVER_INTERFACE.read_text(encoding="utf-8"))["contract_id"]
except (OSError, json.JSONDecodeError, KeyError) as exc:
raise LaneError("synthetic-load driver interface is unavailable") from exc
def sender_spec(approval: dict[str, Any]) -> dict[str, Any]:
"""Validate the receipt's single synthetic identity; return it normalized."""
block = approval.get("synthetic_sender")
if not isinstance(block, dict):
raise LaneError("approval receipt has no synthetic_sender block")
if set(block) != {"identity", "source", "tenant", "expires_at"}:
raise LaneError("synthetic_sender must name exactly identity, source, tenant and expires_at")
name, source, tenant, expires_text = (
block["identity"], block["source"], block["tenant"], block["expires_at"]
)
if not isinstance(name, str) or not NAME_RE.match(name) or "synthetic" not in name:
raise LaneError("synthetic_sender.identity must be a lowercase name containing 'synthetic'")
if not isinstance(source, str) or not NAME_RE.match(source) or "synthetic" not in source:
raise LaneError("synthetic_sender.source must be a named synthetic source, never '*'")
if not isinstance(tenant, str) or not TENANT_RE.match(tenant):
raise LaneError("synthetic_sender.tenant must be exactly one tenant:trial:<name>")
if not isinstance(expires_text, str) or not RFC3339_Z_RE.match(expires_text):
raise LaneError("synthetic_sender.expires_at must be RFC3339 UTC (YYYY-MM-DDTHH:MM:SSZ)")
expires = LEASE.parse_time(expires_text)
window_end = LEASE.parse_time(approval["window"]["end"])
if expires > window_end:
raise LaneError("synthetic_sender.expires_at must not be later than the window end")
load = approval.get("synthetic_load") or {}
if load.get("contract_id") != driver_contract_id():
raise LaneError("synthetic_load.contract_id does not match the registered driver interface")
return {"name": name, "source": source, "tenant": tenant, "expires_at": expires_text, "expires": expires}
def load_receipt(path: Path, now: datetime, *, require_open_window: bool) -> tuple[dict[str, Any], dict[str, Any]]:
approval = LEASE.load_approval(path, now, require_open_window=require_open_window)
return approval, sender_spec(approval)
def assert_projection_time(approval: dict[str, Any], spec: dict[str, Any], now: datetime) -> None:
start = LEASE.parse_time(approval["window"]["start"])
if not start <= now <= start + PROJECT_CUTOFF:
raise LaneError(
f"projection permitted only {stamp(start)} through {stamp(start + PROJECT_CUTOFF)}; now {stamp(now)}"
)
if spec["expires"] - now < MIN_REMAINING:
raise LaneError("identity would expire before a full T02 exercise could complete")
def registry_entry(spec: dict[str, Any], token: str) -> dict[str, Any]:
return {
"name": spec["name"],
"tokens": [token],
"sources": [spec["source"]],
"tenants": [spec["tenant"]],
"may_write": True,
"may_read": True,
"expires_at": spec["expires_at"],
}
def registry_conflicts(registry: list[Any], spec: dict[str, Any]) -> list[str]:
"""Name, source, or tenant overlap with any existing identity is a conflict."""
conflicts: list[str] = []
for entry in registry:
if not isinstance(entry, dict):
continue
label = str(entry.get("name"))
if entry.get("name") == spec["name"]:
conflicts.append(f"identity {label} already exists")
if spec["source"] in (entry.get("sources") or []):
conflicts.append(f"source is already granted to {label}")
if spec["tenant"] in (entry.get("tenants") or []):
conflicts.append(f"tenant is already granted to {label}")
return conflicts
def add_identity(registry: list[Any], spec: dict[str, Any], token: str) -> list[Any]:
conflicts = registry_conflicts(registry, spec)
if conflicts:
raise LaneError("registry conflict: " + "; ".join(conflicts))
return [*registry, registry_entry(spec, token)]
def remove_identity(registry: list[Any], spec: dict[str, Any]) -> tuple[list[Any], int]:
"""Remove only the exact named identity, and only if it is ours."""
updated: list[Any] = []
removed = 0
for entry in registry:
if isinstance(entry, dict) and entry.get("name") == spec["name"]:
if entry.get("sources") != [spec["source"]] or entry.get("tenants") != [spec["tenant"]]:
raise LaneError("named identity exists with a different scope; refusing to remove it")
removed += 1
continue
updated.append(entry)
return updated, removed
def _inside_git_tree(path: Path) -> bool:
return any((parent / ".git").exists() for parent in path.parents)
def check_bearer_path(path: Path, *, must_exist: bool) -> None:
if not path.is_absolute():
raise LaneError("bearer file must be an absolute path")
parent = path.parent.resolve()
target = parent / path.name
if target == ROOT or ROOT in target.parents or _inside_git_tree(target):
raise LaneError("bearer file must be outside every Git work tree")
info = parent.stat()
if info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) & 0o077:
raise LaneError("bearer file directory must be owned by the operator with mode 0700")
if must_exist:
details = target.lstat()
if not stat.S_ISREG(details.st_mode) or stat.S_IMODE(details.st_mode) != 0o600:
raise LaneError("bearer file must be a regular mode-0600 file")
elif target.exists() or target.is_symlink():
raise LaneError("bearer file already exists; run cleanup first")
def write_bearer(path: Path, token: str) -> None:
check_bearer_path(path, must_exist=False)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "w", encoding="utf-8") as handle:
handle.write(token)
handle.flush()
os.fsync(handle.fileno())
os.chmod(path, 0o600)
def assert_evidence(path: Path | None, abort_reason: str | None, approval: dict[str, Any]) -> dict[str, Any]:
"""Cleanup restarts audit-core, so it must not precede the recorded evidence."""
if abort_reason:
return {"mode": "abort", "reason": abort_reason.strip()[:200]}
if path is None:
raise LaneError("cleanup requires --exercise-evidence (recorded T02 result) or --abort-reason")
try:
evidence = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise LaneError("exercise evidence is unavailable or invalid JSON") from exc
if (
not isinstance(evidence, dict)
or evidence.get("procedure") != LEASE.PROCEDURE
or evidence.get("approval_id") != approval["approval_id"]
or evidence.get("secret_values_observed") is not False
or not evidence.get("completed_at")
):
raise LaneError("exercise evidence does not belong to this approved T02 window")
return {"mode": "evidence-recorded", "completed_at": evidence["completed_at"]}
def cluster_state(remote: Any) -> dict[str, Any]:
state = PREFLIGHT.common_state(remote, utc_now())
return {
"automated_checks_passed": PREFLIGHT.automated_common_pass(state),
"openbao_unsealed": state["openbao"]["initialized"] and not state["openbao"]["sealed"],
"senders_external_secret_ready": state["external_secrets"]["audit-core-senders"]["ready"],
"image_matches_reviewed_digest": state["audit_core"]["image_matches_reviewed_digest"],
"ready": f"{state['audit_core']['ready_replicas']}/{state['audit_core']['replicas']}",
"pod_uid": state["audit_core"]["pod_uid"],
}
def settled_state(remote: Any, *, previous_uid: str | None, timeout: float = 90) -> dict[str, Any]:
"""Wait until exactly one Ready serving pod remains (old pod terminated)."""
deadline = time.monotonic() + timeout
while True:
try:
state = cluster_state(remote)
if state["ready"] == "1/1" and state["pod_uid"] != previous_uid:
return state
except PREFLIGHT.PreflightError:
pass
if time.monotonic() >= deadline:
raise LaneError("audit-core did not settle on one Ready restarted pod")
time.sleep(3)
def registry_metadata(operator: Any) -> dict[str, Any]:
raw = operator.bao(["kv", "metadata", "get", "-format=json", REGISTRY_PATH], label="read sender registry metadata").stdout
try:
data = json.loads(raw)["data"]
return {"current_version": data["current_version"], "updated_time": data["updated_time"]}
except (json.JSONDecodeError, KeyError, TypeError) as exc:
raise LaneError("sender registry metadata is incomplete") from exc
def time_state(approval: dict[str, Any], now: datetime) -> str:
start = LEASE.parse_time(approval["window"]["start"])
end = LEASE.parse_time(approval["window"]["end"])
if now < start:
return "before-window"
if now <= start + PROJECT_CUTOFF:
return "projection-window-open"
return "projection-cutoff-passed" if now <= end else "window-closed"
def identity_view(spec: dict[str, Any]) -> dict[str, Any]:
return {key: spec[key] for key in ("name", "source", "tenant", "expires_at")}
def preflight(args: argparse.Namespace) -> dict[str, Any]:
now = utc_now()
approval, spec = load_receipt(args.approval, now, require_open_window=False)
result: dict[str, Any] = {
"lane": LANE,
"approval_id": approval["approval_id"],
"time_state": time_state(approval, now),
"identity": identity_view(spec),
"confirm": confirm_token(approval),
"registry_inspected": False,
"secret_values_observed": False,
}
if args.bearer_file is not None:
check_bearer_path(args.bearer_file, must_exist=False)
result["bearer_file_path_ok"] = True
if args.offline:
result["live_checks"] = "skipped"
return result
remote = PREFLIGHT.Remote(args.remote)
result["audit_core"] = cluster_state(remote)
operator = CREDS.Operator(args.remote, args.token_file)
result["sender_registry_metadata"] = registry_metadata(operator)
if args.inspect_registry:
# Attended only: values are held in memory and never emitted.
registry = operator.registry()
result["registry_inspected"] = True
result["identity_present"] = any(
isinstance(entry, dict) and entry.get("name") == spec["name"] for entry in registry
)
result["registry_conflicts"] = registry_conflicts(registry, spec)
registry.clear()
result["automated_checks_passed"] = result["audit_core"]["automated_checks_passed"]
return result
def _cleanup_registry(operator: Any, spec: dict[str, Any]) -> int:
updated, removed = remove_identity(operator.registry(), spec)
if removed:
operator.write_registry(updated)
CREDS.force_sender_sync_and_restart(operator)
updated.clear()
return removed
def project(args: argparse.Namespace) -> dict[str, Any]:
now = utc_now()
approval, spec = load_receipt(args.approval, now, require_open_window=True)
if args.confirm != confirm_token(approval):
raise LaneError(f"project requires --confirm {confirm_token(approval)}")
assert_projection_time(approval, spec, now)
if args.bearer_file is None:
raise LaneError("project requires --bearer-file outside Git")
check_bearer_path(args.bearer_file, must_exist=False)
remote = PREFLIGHT.Remote(args.remote)
before = cluster_state(remote)
if not before["automated_checks_passed"]:
raise LaneError("audit-core recovery preflight is not green; refusing to project")
LEASE.attest_chain_state(remote)
operator = CREDS.Operator(args.remote, args.token_file)
registry = operator.registry()
conflicts = registry_conflicts(registry, spec)
if conflicts:
raise LaneError("registry conflict: " + "; ".join(conflicts))
token = secrets.token_urlsafe(48)
bearer_written = registry_written = False
try:
write_bearer(args.bearer_file, token)
bearer_written = True
operator.write_registry(add_identity(registry, spec, token))
registry_written = True
CREDS.force_sender_sync_and_restart(operator)
after = settled_state(remote, previous_uid=before["pod_uid"])
except BaseException:
if registry_written:
_cleanup_registry(operator, spec)
if bearer_written:
CREDS.secure_unlink(args.bearer_file)
raise
finally:
token = ""
registry.clear()
return {
"lane": LANE,
"approval_id": approval["approval_id"],
"projected_at": stamp(utc_now()),
"identity": identity_view(spec),
"bearer_file": str(args.bearer_file),
"bearer_file_mode": "0600",
"audit_core_restarted": True,
"audit_core_pod_uid_after_projection": after["pod_uid"],
"next_step": "run the T02 exercise; its baseline must capture this pod UID",
"secret_values_observed": False,
}
def cleanup(args: argparse.Namespace) -> dict[str, Any]:
approval, spec = load_receipt(args.approval, utc_now(), require_open_window=False)
if args.confirm != confirm_token(approval):
raise LaneError(f"cleanup requires --confirm {confirm_token(approval)}")
gate = assert_evidence(args.exercise_evidence, args.abort_reason, approval)
operator = CREDS.Operator(args.remote, args.token_file)
removed = _cleanup_registry(operator, spec)
remaining = any(
isinstance(entry, dict) and entry.get("name") == spec["name"] for entry in operator.registry()
)
if remaining:
raise LaneError("temporary identity remains after cleanup")
bearer_removed = False
if args.bearer_file is not None and (args.bearer_file.exists() or args.bearer_file.is_symlink()):
check_bearer_path(args.bearer_file, must_exist=True)
CREDS.secure_unlink(args.bearer_file)
bearer_removed = True
after = settled_state(PREFLIGHT.Remote(args.remote), previous_uid=None)
return {
"lane": LANE,
"approval_id": approval["approval_id"],
"cleanup_gate": gate,
"cleanup_completed_at": stamp(utc_now()),
"identity": spec["name"],
"identities_removed": removed,
"audit_core_restarted": bool(removed),
"bearer_file_removed": bearer_removed,
"audit_core_ready": after["ready"],
"secret_values_observed": False,
}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("command", choices=["preflight", "project", "cleanup"])
parser.add_argument("--approval", type=Path, required=True, help="approved T02 window receipt")
parser.add_argument("--bearer-file", type=Path, help="absolute path outside Git (mode-0700 directory)")
parser.add_argument("--remote", default="railiance01")
parser.add_argument("--token-file", type=Path, default=Path.home() / ".local/openbao/platform-admin.token")
parser.add_argument("--confirm")
parser.add_argument("--offline", action="store_true", help="preflight: validate receipt/path only")
parser.add_argument("--inspect-registry", action="store_true", help="preflight: attended registry conflict check")
parser.add_argument("--exercise-evidence", type=Path, help="cleanup: recorded T02 exercise result JSON")
parser.add_argument("--abort-reason", help="cleanup: abort path when no exercise evidence exists")
args = parser.parse_args(argv)
try:
handler = {"preflight": preflight, "project": project, "cleanup": cleanup}[args.command]
result = handler(args)
except ERRORS as exc:
print(f"synthetic sender lane failed: {exc}", file=sys.stderr)
return 1
print(json.dumps(result, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())