From a55b05bf7d20dfd689656580d17c38171ea24f36 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 3 Oct 2026 21:48:51 +0200 Subject: [PATCH] Inspect backup client delivery and record gated refresh requirements Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1029d-2c93-7500-a0e0-e3e089668b29 --- scripts/inspect_backup_delivery.py | 84 ++++++++++ ...form-backup-restore-evidence-scheduling.md | 143 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 scripts/inspect_backup_delivery.py diff --git a/scripts/inspect_backup_delivery.py b/scripts/inspect_backup_delivery.py new file mode 100644 index 0000000..708b656 --- /dev/null +++ b/scripts/inspect_backup_delivery.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Read-only backup delivery inspection; never reads a Kubernetes Secret.""" +import argparse +import hashlib +import json +import re +import subprocess +from datetime import datetime, timezone +from pathlib import Path + + +def capture(args): + result = subprocess.run(args, capture_output=True, timeout=30, check=False) + if result.returncode: + raise RuntimeError("inspection_command_failed") + return result.stdout + + +def inspect(platform_root, command=capture): + def resource(kind, name=None, namespace=None): + args = ["kubectl"] + if namespace: + args += ["-n", namespace] + args += ["get", kind] + if name: + args += [name] + return json.loads(command(args + ["-o", "json"])) + nodes = resource("nodes") + if not any(a.get("address") == "92.205.62.239" for n in nodes["items"] + for a in n.get("status", {}).get("addresses", [])): + raise RuntimeError("wrong_cluster") + es = resource("externalsecret", "actcore-backup-offsite", "activity-core") + deployment = resource("deployment", "actcore-worker", "activity-core") + spec = deployment["spec"]["template"]["spec"] + workers = [c for c in spec["containers"] if any( + m["mountPath"] == "/opt/railiance-platform" for m in c.get("volumeMounts", []))] + if len(workers) != 1: + raise RuntimeError("ambiguous_worker") + volumes = [v for v in spec["volumes"] if v["name"] == "backup-verified"] + if len(volumes) != 1: + raise RuntimeError("missing_bundle") + name = volumes[0]["configMap"]["name"] + if not re.fullmatch(r"backup-verified-[a-f0-9]{12}", name): + raise RuntimeError("unexpected_bundle_name") + cm = resource("configmap", name, "activity-core") + common = platform_root / "lib/railiance-backup-common.sh" + digest = hashlib.sha256(common.read_bytes()).hexdigest() + bundle_digest = hashlib.sha256(cm["data"]["lib__railiance-backup-common.sh"].encode()).hexdigest() + mounted = {} + for lane, path in [("forgejo", "/opt/railiance-backup-verified/lib/railiance-backup-common.sh"), + ("cnpg", "/opt/railiance-platform/lib/railiance-backup-common.sh")]: + output = command(["kubectl", "-n", "activity-core", "exec", "deployment/actcore-worker", + "-c", workers[0]["name"], "--", "sha256sum", path]) + actual = output.split()[0].decode() + if not re.fullmatch(r"[a-f0-9]{64}", actual): + raise RuntimeError("invalid_digest_output") + mounted[lane] = actual == digest + return {"read_only": True, "secret_objects_read": False, "inspected_at": datetime.now(timezone.utc).isoformat(), + "source_common_sha256": digest, "source_public_dav_header_present": b"X-Requested-With" in common.read_bytes(), + "bundle": name, "bundle_common_matches_source": bundle_digest == digest, + "mounted_common_matches_source": mounted, + "externalsecret_ready": any(c.get("type") == "Ready" and c.get("status") == "True" + for c in es.get("status", {}).get("conditions", [])), + "externalsecret_refresh_time": es.get("status", {}).get("refreshTime"), + "refresh_interval": es["spec"].get("refreshInterval"), + "worker_secret_env_requires_new_process": any(e.get("secretRef", {}).get("name") == "actcore-runtime-secret" + for e in workers[0].get("envFrom", [])), + "credential_currentness_proved": False, "provider_roundtrip_proved": False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--platform-root", type=Path, required=True) + args = parser.parse_args() + try: + receipt = inspect(args.platform_root) + except Exception: + receipt = {"read_only": True, "ok": False, "error_code": "backup_delivery_inspection_failed"} + print(json.dumps(receipt, sort_keys=True)) + return 0 if receipt.get("ok", True) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/workplans/ACTIVITY-WP-0042-platform-backup-restore-evidence-scheduling.md b/workplans/ACTIVITY-WP-0042-platform-backup-restore-evidence-scheduling.md index 121af45..bb5ece3 100644 --- a/workplans/ACTIVITY-WP-0042-platform-backup-restore-evidence-scheduling.md +++ b/workplans/ACTIVITY-WP-0042-platform-backup-restore-evidence-scheduling.md @@ -228,3 +228,146 @@ T05 still requires platform acceptance, not another local candidate implementati No new workplan/task, identity claim, credential fetch, scheduled restore or owner acceptance was invented. The source-only backup hardening remains committed; this review does not claim it was rolled out to production. + + +## Inspect downstream shared backup-lane delivery + +```task +id: ACTIVITY-WP-0042-T07 +status: done +priority: high +``` + +2026-10-03 NK-WP-0045 recovery follow-up: inspect actual backup consumer delivery +without reading Secret objects. `scripts/inspect_backup_delivery.py` checks the +railiance01 node, ESO readiness/refresh metadata, fixed source/library digests +and mounted Forgejo/CNPG library digests. Captures and discards raw command +failures; receipts contain only safe metadata, digests and booleans. + +Forgejo uses immutable ConfigMap `backup-verified-0220ca56520c`; CNPG uses the +railiance01 `/home/tegwick/railiance-platform` hostPath. They are separate source +delivery paths. ExternalSecret `actcore-backup-offsite` refreshes each hour into +`actcore-runtime-secret` with Merge/Retain; worker envFrom credentials remain +in existing processes after a Secret refresh. Ready/SecretSynced is delivery +health, not valid provider credentials or backup proof. Platform common source +`50da37cf50016b03a7e6a537e87772a72979aba1` now includes public DAV +`X-Requested-With: XMLHttpRequest`; vendored/mounted versions need verification. + +Knowledge orientation: package 0.2.4 digest +`025aab4f38246219c924bc94954e70ea0e6e7f72bcaf8d4344cd599602d345c3`, +start withheld because informed-decision source changed; reviewed that README +and current custodian environment orientation directly. No orientation claim +establishes runtime authority. + +## Refresh the existing backup consumer after the owner repairs the lane + +```task +id: ACTIVITY-WP-0042-T08 +status: wait +priority: high +blocking_reason: "Platform must supply an authenticated replacement-lane/CAS and encrypted history upload/readback receipt; then a reviewed scoped worker bundle/hostPath and env refresh is required." +``` + +The reviewed-folder inspection for NK-WP-0045 found the current token and +expected labelled share absent; upload and authenticated route probes return +401. Lane KV v3 being synced does not resolve this provider mismatch. + +Owner handoff: railiance-platform must first publish its non-secret receipt +identifying the replacement lane KV version, exact two updated field names +(`NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`), preserved AGE recipient, owner-reviewed +share binding, and encrypted retained-history upload/readback proof. No token +or URL goes into this repo or receipts. Root custody recovery can finish +independently of this larger recurring backup consumer acceptance. + +Prepare the bounded worker change only after that owner receipt: refresh ESO +and verify status/resource metadata; replace the Forgejo immutable bundle +using the platform-owned reviewed bundle builder; synchronize the CNPG hostPath +source separately through repo-manager; reconcile the new bundle reference in +`k8s/gitops/runtime.yaml` before ArgoCD synchronization. Preserve unrelated +live mounts/environment. Do not use the old direct deployment patch helper +against this adopted ArgoCD application. Perform one reviewed worker rollout +to refresh envFrom only after source and ESO prerequisites; compare new pod +start to ESO refresh and run this inspector again. Secret object bodies are +never diagnostic output. + +Require safe non-secret post-rollout source digests and readiness plus an +owner-approved existing bounded backup run/provider receipt before claiming +activity delivery acceptance. No automatic scheduled backup invocation, new +operation admission, restore capability or customer recovery readiness is +claimed. T02/T03/T05 keep their separate existing dependencies. + + +## Inspect downstream shared backup-lane delivery + +```task +id: ACTIVITY-WP-0042-T07 +status: done +priority: high +``` + +2026-10-03 NK-WP-0045 recovery follow-up: inspect actual backup consumer delivery +without reading Secret objects. `scripts/inspect_backup_delivery.py` checks the +railiance01 node, ESO readiness/refresh metadata, fixed source/library digests +and mounted Forgejo/CNPG library digests. Captures and discards raw command +failures; receipts contain only safe metadata, digests and booleans. + +Forgejo uses immutable ConfigMap `backup-verified-0220ca56520c`; CNPG uses the +railiance01 `/home/tegwick/railiance-platform` hostPath. They are separate source +delivery paths. ExternalSecret `actcore-backup-offsite` refreshes each hour into +`actcore-runtime-secret` with Merge/Retain; worker envFrom credentials remain +in existing processes after a Secret refresh. Ready/SecretSynced is delivery +health, not valid provider credentials or backup proof. Platform common source +`50da37cf50016b03a7e6a537e87772a72979aba1` now includes public DAV +`X-Requested-With: XMLHttpRequest`; vendored/mounted versions need verification. + +Knowledge orientation: package 0.2.4 digest +`025aab4f38246219c924bc94954e70ea0e6e7f72bcaf8d4344cd599602d345c3`, +start withheld because informed-decision source changed; reviewed that README +and current custodian environment orientation directly. No orientation claim +establishes runtime authority. + +## Refresh the existing backup consumer after the owner repairs the lane + +```task +id: ACTIVITY-WP-0042-T08 +status: wait +priority: high +blocking_reason: "Platform must supply an authenticated replacement-lane/CAS and encrypted history upload/readback receipt; then a reviewed scoped worker bundle/hostPath and env refresh is required." +``` + +The reviewed-folder inspection for NK-WP-0045 found the current token and +expected labelled share absent; upload and authenticated route probes return +401. Lane KV v3 being synced does not resolve this provider mismatch. + +Owner handoff: railiance-platform must first publish its non-secret receipt +identifying the replacement lane KV version, exact two updated field names +(`NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`), preserved AGE recipient, owner-reviewed +share binding, and encrypted retained-history upload/readback proof. No token +or URL goes into this repo or receipts. Root custody recovery can finish +independently of this larger recurring backup consumer acceptance. + +Prepare the bounded worker change only after that owner receipt: refresh ESO +and verify status/resource metadata; replace the Forgejo immutable bundle +using the platform-owned reviewed bundle builder; synchronize the CNPG hostPath +source separately through repo-manager; reconcile the new bundle reference in +`k8s/gitops/runtime.yaml` before ArgoCD synchronization. Preserve unrelated +live mounts/environment. Do not use the old direct deployment patch helper +against this adopted ArgoCD application. Perform one reviewed worker rollout +to refresh envFrom only after source and ESO prerequisites; compare new pod +start to ESO refresh and run this inspector again. Secret object bodies are +never diagnostic output. + +Require safe non-secret post-rollout source digests and readiness plus an +owner-approved existing bounded backup run/provider receipt before claiming +activity delivery acceptance. No automatic scheduled backup invocation, new +operation admission, restore capability or customer recovery readiness is +claimed. T02/T03/T05 keep their separate existing dependencies. + + +Read-only inspection receipt (2026-10-03T19:47:12Z): ESO Ready, last refresh +19:29:02Z. Source common SHA256 +`09f349ae815339732db505d6896fb1ca384be7c66b40574750a3ba30d4d922e0`; +immutable bundle common and both mounted Forgejo/CNPG libraries differ. No +secret object read, provider probe or live mutation performed. Three injected +inspection checks passed: current/stale source, delivery versus credential +proof, and wrong-cluster refusal. `git diff --check` passed.