diff --git a/api/services/historical_projection.py b/api/services/historical_projection.py index 37c1d9a..1a8a79c 100644 --- a/api/services/historical_projection.py +++ b/api/services/historical_projection.py @@ -72,6 +72,111 @@ BINDINGS = json.loads(r'''[ "record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED", "uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e", "parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED" + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9", + "record_id": "RAIL-BS-WP-0002", + "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d", + "parent_record_id": "RAIL-BS-WP-0002", + "tasks": [ + { + "label": "T01", + "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40", + "status": "done" + }, + { + "label": "T02", + "uuid": "e8510646-46ed-4697-a345-f3d3009eea78", + "status": "done" + }, + { + "label": "T03", + "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5", + "status": "done" + }, + { + "label": "T04", + "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f", + "status": "done" + }, + { + "label": "T05", + "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a", + "record_id": "RAIL-BS-WP-0003", + "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b", + "parent_record_id": "RAIL-BS-WP-0003", + "tasks": [ + { + "label": "T01", + "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc", + "status": "done" + }, + { + "label": "T02", + "uuid": "ab166073-30a7-4702-a037-4091e8706e20", + "status": "done" + }, + { + "label": "T03", + "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01", + "status": "done" + }, + { + "label": "T04", + "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3", + "record_id": "RAIL-BS-WP-0004", + "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625", + "parent_record_id": "RAIL-BS-WP-0004", + "tasks": [ + { + "label": "T01", + "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b", + "status": "done" + }, + { + "label": "T02", + "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880", + "status": "done" + }, + { + "label": "T03", + "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", + "status": "done" + }, + { + "label": "T04", + "uuid": "08233868-d522-4117-bc4e-6c0f52545665", + "status": "done" + }, + { + "label": "T05", + "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", + "status": "done" + }, + { + "label": "T06", + "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07", + "status": "done" + } + ] } ]''') diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md new file mode 100644 index 0000000..595cd5c --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md @@ -0,0 +1,177 @@ +--- +id: RAIL-BS-WP-0002 +type: workplan +title: "k3s and Kubernetes Platform Baseline" +domain: financials +repo: railiance-cluster +status: finished +owner: railiance +topic_slug: railiance +repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8" +state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d" +created: "2026-03-09" +updated: "2026-03-10" +completed: "2026-03-10" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# k3s and Kubernetes Platform Baseline + +## Goal + +Install k3s, Helm, and the baseline Kubernetes services on the converged +HostEurope node. This workplan picks up exactly where `railiance-hosts` +leaves off: a hardened, verified OS node that is ready for Kubernetes. + +## Pre-condition + +`railiance-infra` converge + Goss verify must pass before any task here +is executed: + +```bash +# In railiance-infra/ +make converge +make verify # must exit 0 +``` + +## Boundary + +This repo owns everything from k3s upward. It must not re-configure items +defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003: +`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`. + +**Out of scope here:** platform services (PostgreSQL, storage, identity) +→ `railiance-platform`. Application deployments (Gitea, coulomb services) +→ `railiance-apps`. + +--- + +## Tasks + +### T01 — Ansible playbook: install k3s (server mode) + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40" +completed: "2026-03-10" +``` + +Harden `ansible/bootstrap.yml` to a production-ready k3s install: + +- Use the official k3s install script pinned to a specific version + (`INSTALL_K3S_VERSION=v1.35.1+k3s1`) +- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"` + (cluster-init enables embedded etcd for future HA expansion) +- Wait for node `Ready` before proceeding: + ```bash + k3s kubectl wait node --all --for=condition=Ready --timeout=120s + ``` +- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope` + +**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server +and the control node (via kubeconfig). + +--- + +### T02 — Helm installation + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78" +completed: "2026-03-10" +``` + +Add a task (or a role `roles/helm/`) that: + +1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm` +2. Verifies the checksum +3. Confirms `helm version` succeeds + +**Done when:** `helm version` succeeds on the HostEurope node. + +--- + +### T03 — Smoke test: k3s + Helm + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5" +completed: "2026-03-10" +``` + +Extend `tests/smoke_kube.sh` to assert: + +- `k3s kubectl get nodes` → node in Ready state +- `helm version` exits 0 +- CoreDNS pod running in `kube-system` +- Traefik ingress controller pod running (default in k3s) + +Run via: +```bash +ansible-playbook -i ansible/hosts.ini ansible/smoke.yml +``` +or directly over SSH if the kubeconfig is available locally. + +**Done when:** all assertions pass and the script exits 0. + +--- + +### T04 — Commit kubeconfig management notes + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f" +completed: "2026-03-10" +``` + +Document in `docs/kubeconfig.md`: + +- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`) +- How to merge it into `~/.kube/config` +- How to switch context: `kubectl config use-context default` +- Security note: kubeconfig is gitignored (contains cluster CA + client cert) + +**Done when:** doc written and committed. + +--- + +### T05 — Add `make k3s-install` and `make smoke` targets + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f" +completed: "2026-03-10" +``` + +Add to Makefile (create one if none exists): + +```makefile +k3s-install: ## Install k3s and Helm on all inventory hosts + ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml + +smoke: ## Run Kubernetes smoke tests + bash tests/smoke_kube.sh +``` + +**Done when:** both targets work and are listed in `make help`. + +--- + +## References + +- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline) +- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure) +- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09 +- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md` +- k3s releases: https://github.com/k3s-io/k3s/releases diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md new file mode 100644 index 0000000..fe7b21d --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md @@ -0,0 +1,196 @@ +--- +id: RAIL-BS-WP-0003 +type: workplan +title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +created: "2026-03-10" +updated: "2026-03-10" +state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover + +## Summary + +On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to +restart. pgpool — the connection pooler between Gitea and PostgreSQL — then +entered CrashLoopBackOff and produced no logs. As a result Gitea's login +and all write operations hung indefinitely. The root page was still served +(from Valkey cache) which masked the failure. + +The fix was to patch a missing key in a Kubernetes secret. The root cause is +that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not +populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql` +secret, even though the pgpool pod requires it at startup. + +--- + +## Timeline + +| Time (UTC) | Event | +|---|---| +| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) | +| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff | +| ~11:00 | User noticed Gitea login hanging; home page still loading | +| ~13:00 | Root cause identified: missing `pgpool-password` secret key | +| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly | +| ~13:15 | Gitea fully operational | + +--- + +## Root Cause + +The Bitnami `pgpool` container startup script reads the file +`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the +`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume +mount. That secret key was never created by the Helm chart, so the file did +not exist. The container exited immediately with no logs. + +The pod had been running for 20 days without a restart, so this gap was +never discovered during initial deployment. + +--- + +## Evidence + +```bash +# Secret was missing the pgpool-password key +sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml +# data: keys were password, postgres-password, repmgr-password only +# pgpool-password was absent + +# pgpool pod describe showed 824 back-off restarts over 173 minutes +# No logs in either current or --previous output +sudo k3s kubectl logs -n default --previous +# (empty) + +# Gitea process had zero TCP connections to PostgreSQL port 5432 +# but many connections to Valkey port 6379 +cat /proc//net/tcp | grep 1538 # 1538 = 5432 hex — no results +``` + +--- + +## Immediate Fix Applied + +```bash +# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0) +sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \ + --type='json' \ + -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]' + +# Restart pgpool +sudo k3s kubectl delete pod -n default +``` + +--- + +## Risk: Fix Will Be Lost on helm upgrade + +The patched secret is managed by Helm (annotation: +`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the +secret from the chart template, which does not include `pgpool-password`, +and the bug will recur. + +--- + +## Tasks + +### T01 — Add pgpool-password to Helm values + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" +``` + +Create or update `helm/gitea-values.yaml` (or equivalent) to permanently +include the pgpool-password so it survives `helm upgrade`: + +```yaml +postgresql-ha: + postgresql: + pgpoolPassword: +``` + +**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly +without manual secret patching. + +--- + +### T02 — Add pgpool health check to smoke test + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20" +``` + +Extend `tests/smoke_kube.sh` to assert: + +```bash +# All postgresql-ha pods Running +kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1 + +# pgpool specifically not in CrashLoopBackOff +kubectl get pod -n default -l app.kubernetes.io/component=pgpool \ + -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash +``` + +**Done when:** the smoke test catches a pgpool failure within 5 minutes. + +--- + +### T03 — Add HA failover test + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01" +``` + +Create `tests/test_ha_failover.sh` that: + +1. Records Gitea login response time (baseline) +2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default` +3. Waits for repmgr to promote a replica (max 60s) +4. Asserts Gitea login POST still succeeds within 10s +5. Asserts pgpool pod is Running (not CrashLoopBackOff) +6. Asserts all postgresql pods return to Running + +This test must pass before any PostgreSQL HA deployment is considered done. + +**Done when:** script exits 0 against a live cluster. + +--- + +### T04 — Document the incident in docs/ + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" +``` + +Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full +timeline, root cause, and fix, so future operators understand what happened +and how to recover. + +**Done when:** doc committed and linked from `docs/README.md`. + +--- + +## References + +- Bitnami postgresql-ha chart v16.2.2 +- Gitea Helm chart v12.2.0 +- Related decision: D3 (HA testing policy) in `DECISIONS.md` diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md new file mode 100644 index 0000000..89d14ff --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md @@ -0,0 +1,275 @@ +--- +id: RAIL-BS-WP-0004 +type: workplan +title: "Integrated Backup — S2 Kubernetes Runtime Layer" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +topic_slug: railiance +state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625" +created: "2026-02-25" +updated: "2026-03-26" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Integrated Backup — S2 Kubernetes Runtime Layer + +## Goal + +Implement the Q3 (Operability & Resilience) integrated backup for +railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state — +encrypted with age, written to a local directory on the server. No external +dependencies required. + +## Architecture (Decision D4) + +Each railiance repo implements its own backup for what it owns. No central +backup service. See `DECISIONS.md` D4 for full rationale. + +**Standard interface every railiance repo must provide:** + +```bash +make backup # encrypt + write to /opt/backup/railiance// +make restore # restore from most recent local backup +``` + +Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key). +Output: `/opt/backup/railiance/cluster/` on the server. + +## What S2 (railiance-cluster) owns and must back up + +| Asset | Why it matters | +|---|---| +| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets | +| Helm release values | Runtime values not in git (any manually applied overrides) | +| kubeconfig | Admin access to the cluster | + +**Not S2's responsibility:** +- Custodian State Hub DB → the-custodian owns this +- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern +- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this +- PostgreSQL data volumes → S3 (railiance-platform) owns this + +## Encryption + +Reuse the age public key from `.sops.yaml`: + +```bash +AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}') +tar -czf - | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age +``` + +Decryption requires the private key at `~/.config/sops/age/keys.txt` +(same key used for `sops -d`). No additional key management needed. + +## Extension Point EP-RAIL-005 + +Once all five OAS layers implement this interface, the custodian can +orchestrate a full-stack backup with: + +```bash +for repo in railiance-infra railiance-cluster railiance-platform \ + railiance-enablement railiance-apps; do + make -C ~/$repo backup +done +``` + +No special protocol needed — just the standard interface. + +--- + +## Tasks + +### T01 — Define backup directory and encryption wrapper + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b" +``` + +Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`): + +- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`) +- Encrypt each artifact with age using public key from `.sops.yaml` +- Write timestamp-named files: `etcd-.snap.age`, `helm-values-.tar.gz.age`, `kubeconfig-.yaml.age` +- Keep last 7 of each type +- Write `.last-backup` stamp +- Exit 0 on success, non-zero on any failure +- No network required + +Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based +custodian DB — wrong scope, not applicable to this server). + +**Done when:** `make backup` runs on COULOMBCORE without error and files +appear in `/opt/backup/railiance/cluster/`. + +--- + +### T02 — Back up k3s state (SQLite hot backup) + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880" +``` + +k3s has built-in etcd snapshot support: + +```bash +sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ) +# Default location: /var/lib/rancher/k3s/server/db/snapshots/ +``` + +Add to the backup script: take a fresh snapshot, encrypt with age, +copy to `/opt/backup/railiance/cluster/`. + +> **Note — verify etcd is in use before implementing:** +> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`. +> Without it, k3s uses SQLite and this command will fail. +> Verify first: `sudo k3s etcd-snapshot ls 2>&1` + +> **Note — sudo required:** etcd snapshot requires root. See T06 for how +> this is resolved (backup runs under root's crontab). + +**Done when:** backup includes a current etcd snapshot. + +--- + +### T03 — Back up Helm release values + +```task +id: T03 +status: done +priority: medium +state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3" +``` + +Capture current runtime Helm values for all releases: + +```bash +KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \ + jq -r '.[].name + " " + .namespace' | \ + while read name ns; do + KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml + done +``` + +Tar and age-encrypt into `helm-values-.tar.gz.age`. + +> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable +> only by default. The backup script must either run as root (see T06) or k3s +> must be configured with `--write-kubeconfig-mode=644`. Running as root +> (via root crontab) is the chosen approach — no config change needed. + +**Done when:** backup includes a snapshot of all Helm release values. + +--- + +### T04 — Back up kubeconfig + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665" +``` + +Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`) +into `kubeconfig-.yaml.age` in the backup directory. + +**Done when:** backup includes the encrypted kubeconfig. + +--- + +### T05 — make restore target + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8" +``` + +Add `tools/cmd/railiance-restore-s2` that decrypts and lists available +backups, with guided restore for the etcd snapshot case. + +Restore of etcd from snapshot: +```bash +sudo k3s server --cluster-reset \ + --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/ +``` + +**Done when:** `make restore` prints available backups and a restore guide. + +--- + +### T06 — Install cron job and run restore drill + +```task +id: T06 +status: done +priority: medium +state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07" +``` + +#### Solving the sudo problem + +The backup script needs root for two reasons: +- `k3s etcd-snapshot save` requires root +- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only + +**Solution: run the cron under root's crontab.** + +This is the correct pattern for system-level backup jobs. It avoids a +proliferating sudoers whitelist (one entry per command, brittle to maintain) +and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in +production. The backup writes to `/opt/backup/` which is root-owned anyway. + +Install the cron as root: + +```bash +sudo crontab -e +# Add: +0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1 +``` + +Note: use the absolute path to the repo — `~` does not expand reliably in +root's crontab unless HOME is set. + +Verify it is installed: +```bash +sudo crontab -l | grep railiance +``` + +#### Restore drill + +Once T01–T04 are done, run a decrypt-and-verify drill: + +```bash +# Decrypt the etcd snapshot and verify it is a valid snapshot file +sudo age -d -i ~/.config/sops/age/keys.txt \ + /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \ + | file - + +# Record the drill +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \ + >> /opt/backup/railiance/cluster/restore-drill.log +``` + +**Done when:** cron installed under root, drill completes without error, +log entry written. + +--- + +## References + +- Decision D4: Integrated backup per capability (`DECISIONS.md`) +- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement) +- OAS Q3: Operability & Resilience +- Extension point EP-RAIL-005: Custodian full-stack backup orchestration +- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore diff --git a/tests/test_cluster_historical_projection.py b/tests/test_cluster_historical_projection.py new file mode 100644 index 0000000..36db2df --- /dev/null +++ b/tests/test_cluster_historical_projection.py @@ -0,0 +1,67 @@ +import shutil +import uuid +from pathlib import Path +import pytest +from api.services import forge_projection as fp +from api.services import historical_projection as hp +from tests.test_forge_projection import _FakeSession,_Repo,_Row,_TaskRow + +FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster' +@pytest.fixture +def bound(tmp_path): + root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root) + derived=fp.derive_from_checkout(root,'railiance-cluster','candidate') + assert len(derived.workplans)==3 + rows=[];tasks=[] + for parent in derived.workplans: + row=_Row(parent.record_id.lower(),'finished',parent.relative_path);row.id=uuid.UUID(parent.uuid);rows.append(row) + for source in parent.tasks: + task=_TaskRow(source.record_id,status='done',workplan_id=row.id);task.id=uuid.UUID(source.uuid);tasks.append(task) + assert len(tasks)==15 + return root,derived,rows,tasks + +def test_exact_three_sources_derive_only_frozen_15_task_uuids(bound): + root,derived,rows,tasks=bound + assert all(parent.historical_preservation for parent in derived.workplans) + assert all(parent.uuid!=fp.derived_record_uuid(parent.record_id) for parent in derived.workplans) + expected={item['uuid'] for entry in hp.BINDINGS if entry['repo']=='railiance-cluster' for item in entry['tasks']} + assert {str(t.id) for t in tasks}==expected + +@pytest.mark.asyncio +async def test_existing_rows_are_entirely_unchanged_on_normal_receiving(bound): + root,derived,rows,tasks=bound;before=[dict(x.__dict__) for x in rows+tasks] + session=_FakeSession(_Repo(),rows,task_rows=tasks) + outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived) + assert outcome.status=='noop' and not outcome.refused + assert [dict(x.__dict__) for x in rows+tasks]==before + assert not session.added and not session.deleted + +@pytest.mark.parametrize('defect',['missing-parent','missing-task','foreign-holder','nonterminal-task','nonterminal-parent','duplicate-current-label']) +@pytest.mark.asyncio +async def test_unsupported_actual_binding_refuses_before_any_mutation(bound,defect): + root,derived,rows,tasks=bound + if defect=='missing-parent':rows.pop() + elif defect=='missing-task':tasks.pop() + elif defect=='foreign-holder':tasks[0].workplan_id=uuid.uuid4() + elif defect=='nonterminal-task':tasks[0].status='progress' + elif defect=='duplicate-current-label':tasks.append(_TaskRow(tasks[0].record_id,status='done',workplan_id=tasks[0].workplan_id)) + else:rows[0].status='active' + before=[dict(x.__dict__) for x in rows+tasks];session=_FakeSession(_Repo(),rows,task_rows=tasks) + outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived) + assert outcome.status=='refused';assert [dict(x.__dict__) for x in rows+tasks]==before + assert not session.added and not session.deleted + +@pytest.mark.parametrize('change',['hash-drift','missing','symlink','live-source','missing-source-uuid']) +def test_frozen_source_drift_refuses_derivation(bound,change): + root,*_=bound;p=next((root/'workplans/archived').glob('*.md'));raw=p.read_bytes() + if change=='missing':p.unlink() + elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target) + elif change=='live-source':p.write_bytes(raw.replace(b'status: finished',b'status: active',1)) + elif change=='missing-source-uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1)) + else:p.write_bytes(raw+b'\nunreviewed\n') + with pytest.raises(fp.ForgeDeriveError,match='Historical source binding'):fp.derive_from_checkout(root,'railiance-cluster','candidate') + +def test_unknown_repository_never_trusts_frozen_source_uuids(bound): + root,derived,*_=bound;other=fp.derive_from_checkout(root,'different-repo','candidate') + assert not any(parent.historical_preservation for parent in other.workplans) + assert not {parent.uuid for parent in derived.workplans}&{parent.uuid for parent in other.workplans} diff --git a/workplans/STATE-WP-0095-historical-source-receiving.md b/workplans/STATE-WP-0095-historical-source-receiving.md index a48c8c4..384a2f7 100644 --- a/workplans/STATE-WP-0095-historical-source-receiving.md +++ b/workplans/STATE-WP-0095-historical-source-receiving.md @@ -4,10 +4,10 @@ type: workplan title: "Preserve exact closed historical source bindings in Forge receiving" domain: infotech repo: state-hub -status: finished +status: active owner: codex created: "2026-10-04" -updated: "2026-10-06" +updated: "2026-10-17" flavor: implementation depends_on: [] state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058" @@ -167,3 +167,73 @@ had 54 workplans and 288 tasks before and after. Evidence: [activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json). Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and ACTIVITY-WP-0045-T02 remain with their owners. + + +## Cluster exact historical receiving successor, 2026-10-17 + +This scoped extension retains the completed Activity/Custodian outcomes above. +Three frozen cluster archives contain three existing parent UUIDs and 15 existing +closed task UUIDs. Extend the existing exact historical receiving path before +registrar admission; never derive replacement identities or modify those +sources. Cross-owner observation attaches to existing SHR-WP-0001, with +RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly. +The local derived case is waiting; no live worker injection is authorized. + +## Prepare exact cluster source preservation and refusal cases + +```task +id: STATE-WP-0095-T05 +status: done +priority: high +state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b" +``` + +Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager. +Candidate receiving tests pass 26: existing rows remain entirely unchanged; +missing/foreign/nonterminal rows and unknown, changed, missing or symlinked +source refuse. Read-only public metadata confirms all 18 existing rows have the +expected ownership and terminal status. These are source and metadata checks, +not delivered receiving or a committed native trial. + +## Deliver reviewed cluster receiving successor before registrar admission + +```task +id: STATE-WP-0095-T06 +status: wait +priority: high +depends_on: [STATE-WP-0095-T05] +needs_human: true +intervention_note: "Review exact committed source, CI image and owner deployment proposal before rollout." +state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4" +``` + +Use the normal CI image and Helm owner path after exact revision/image pin, +fresh headroom and reviewed dry-run/diff. API/MCP currently run main-7d99282; +Helm revision75 is the observed rollback baseline, requiring freshness before +execution. No deployment assent exists for this successor. Verify delivered +module hash, health, both readiness checks and expected migrated schema +c4d5e6f7a8b9; verify exact +three parents/15 tasks and rejection behavior without committed probe changes. +Only the verified return unblocks RMGR-WP-0019-T05. The full current image also carries inbox-feed migration c4d5e6f7a8b9, +although this receiver patch adds none. Helm rollback leaves that migration in +place; unrelated migration acceptance and old-image compatibility need normal +owner evidence before rollout approval. Do not run destructive default local +DB tests; use only exact disposable DB containment. Costs and monetary risk are +unknown pending exact rollout proposal; no zero-cost or autonomy grant is inferred. + +## Verify ordinary cluster receiving and immutable historical rows + +```task +id: STATE-WP-0095-T07 +status: wait +priority: high +depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06] +state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d" +``` + +Capture normal cluster owner publication/receiving receipts and compare entire +existing row snapshots and exact source bytes. No creation, movement, relabel, +unretirement or retirement is authorized for these histories. A rollback first +fences cluster sync and restores registrar refusal before removing receiving +support; it never rewrites archived IDs or UUIDs. This work does not establish +original TLS recovery or owner-native admission.