diff --git a/src/repo_manager/historical_sources.py b/src/repo_manager/historical_sources.py index 6eb2072..5ff0fa3 100644 --- a/src/repo_manager/historical_sources.py +++ b/src/repo_manager/historical_sources.py @@ -70,6 +70,111 @@ BINDINGS = json.loads(r'''[ "record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED", "uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e", "parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED" + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md", + "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9", + "record_id": "RAIL-BS-WP-0002", + "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d", + "parent_record_id": "RAIL-BS-WP-0002", + "tasks": [ + { + "label": "T01", + "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40", + "status": "done" + }, + { + "label": "T02", + "uuid": "e8510646-46ed-4697-a345-f3d3009eea78", + "status": "done" + }, + { + "label": "T03", + "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5", + "status": "done" + }, + { + "label": "T04", + "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f", + "status": "done" + }, + { + "label": "T05", + "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md", + "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a", + "record_id": "RAIL-BS-WP-0003", + "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b", + "parent_record_id": "RAIL-BS-WP-0003", + "tasks": [ + { + "label": "T01", + "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc", + "status": "done" + }, + { + "label": "T02", + "uuid": "ab166073-30a7-4702-a037-4091e8706e20", + "status": "done" + }, + { + "label": "T03", + "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01", + "status": "done" + }, + { + "label": "T04", + "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d", + "status": "done" + } + ] + }, + { + "repo": "railiance-cluster", + "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md", + "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3", + "record_id": "RAIL-BS-WP-0004", + "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625", + "parent_record_id": "RAIL-BS-WP-0004", + "tasks": [ + { + "label": "T01", + "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b", + "status": "done" + }, + { + "label": "T02", + "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880", + "status": "done" + }, + { + "label": "T03", + "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3", + "status": "done" + }, + { + "label": "T04", + "uuid": "08233868-d522-4117-bc4e-6c0f52545665", + "status": "done" + }, + { + "label": "T05", + "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8", + "status": "done" + }, + { + "label": "T06", + "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07", + "status": "done" + } + ] } ]''') @@ -83,3 +188,24 @@ def historical_source_paths(repo_root): raise ValueError("Historical source binding requires owner review: " + entry["path"]) paths.add(entry["path"]) return paths + + +def historical_source_occurrences(repo_root): + """Exact frozen closed task occurrences; never a general local-ID alias.""" + from repo_manager.parse.workplan import parse_workplan_file + from uuid import UUID + paths=historical_source_paths(repo_root) + occurrences=set(paths) + for entry in BINDINGS: + if entry['repo']!=repo_root.name or 'tasks' not in entry:continue + parsed=parse_workplan_file(repo_root/entry['path'],repo_root=repo_root) + actual=[{'label':task.id,'uuid':task.state_hub_task_id,'status':task.status} for task in parsed.tasks] + if (parsed.id!=entry['record_id'] or parsed.state_hub_workstream_id!=entry['uuid'] + or parsed.status not in {'finished','archived','completed'} or actual!=entry['tasks'] + or any(task['status'] not in {'done','cancel'} for task in actual)): + raise ValueError('Historical task binding requires owner review: '+entry['path']) + UUID(entry['uuid']) + for number,task in enumerate(actual,1): + UUID(task['uuid']) + occurrences.add(entry['path']+'#task-block-'+str(number)) + return occurrences diff --git a/src/repo_manager/identifiers.py b/src/repo_manager/identifiers.py index 186854d..7dbbdd0 100644 --- a/src/repo_manager/identifiers.py +++ b/src/repo_manager/identifiers.py @@ -854,8 +854,9 @@ def ensure_missing_work_record_identifiers( # allocation candidates. Keep conformance reporting strict while allowing # ordinary registration to preserve those bindings. Local aliases still # require the explicit parent allowlist in the kind registry. - from repo_manager.historical_sources import historical_source_paths - historical_sources: set[str] = historical_source_paths(repo_root) + from repo_manager.historical_sources import historical_source_occurrences + frozen_sources = historical_source_occurrences(repo_root) + historical_sources: set[str] = set(frozen_sources) for path in iter_workplan_files(repo_root): parsed = parse_workplan_file(path, repo_root=repo_root) if parsed.status not in {"finished", "archived", "completed"} or not parsed.id: @@ -889,11 +890,18 @@ def ensure_missing_work_record_identifiers( item for item in identity["invalid_identifiers"] if item["source"] in historical_sources ] - if invalid or identity["identity_collisions"]: + # Strict scans keep legacy nonconformance. Only collisions wholly contained + # in exact hash-bound closed task occurrences can be preserved by receiving; + # any unknown/live source or qualified/workplan collision remains fatal. + historical_collisions=[item for item in identity['identity_collisions'] + if item['kind']=='task' and re.fullmatch(r'T[0-9]{2}',item['id']) + and all(source in frozen_sources for source in item['sources'])] + collisions=[item for item in identity['identity_collisions'] if item not in historical_collisions] + if invalid or collisions: raise ValueError( "work-record identities are not safe to derive: " f"invalid={invalid!r}, " - f"collisions={identity['identity_collisions']!r}" + f"collisions={collisions!r}" ) rendered: dict[Path, str] = {} @@ -966,6 +974,7 @@ def ensure_missing_work_record_identifiers( "namespace": namespace, "assignments": assignments, "historical_invalid_identifiers": historical, + "historical_identity_collisions": historical_collisions, "files_changed": sorted(str(path.relative_to(repo_root)) for path in rendered), } diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md new file mode 100644 index 0000000..595cd5c --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md @@ -0,0 +1,177 @@ +--- +id: RAIL-BS-WP-0002 +type: workplan +title: "k3s and Kubernetes Platform Baseline" +domain: financials +repo: railiance-cluster +status: finished +owner: railiance +topic_slug: railiance +repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8" +state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d" +created: "2026-03-09" +updated: "2026-03-10" +completed: "2026-03-10" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# k3s and Kubernetes Platform Baseline + +## Goal + +Install k3s, Helm, and the baseline Kubernetes services on the converged +HostEurope node. This workplan picks up exactly where `railiance-hosts` +leaves off: a hardened, verified OS node that is ready for Kubernetes. + +## Pre-condition + +`railiance-infra` converge + Goss verify must pass before any task here +is executed: + +```bash +# In railiance-infra/ +make converge +make verify # must exit 0 +``` + +## Boundary + +This repo owns everything from k3s upward. It must not re-configure items +defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003: +`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`. + +**Out of scope here:** platform services (PostgreSQL, storage, identity) +→ `railiance-platform`. Application deployments (Gitea, coulomb services) +→ `railiance-apps`. + +--- + +## Tasks + +### T01 — Ansible playbook: install k3s (server mode) + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40" +completed: "2026-03-10" +``` + +Harden `ansible/bootstrap.yml` to a production-ready k3s install: + +- Use the official k3s install script pinned to a specific version + (`INSTALL_K3S_VERSION=v1.35.1+k3s1`) +- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"` + (cluster-init enables embedded etcd for future HA expansion) +- Wait for node `Ready` before proceeding: + ```bash + k3s kubectl wait node --all --for=condition=Ready --timeout=120s + ``` +- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope` + +**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server +and the control node (via kubeconfig). + +--- + +### T02 — Helm installation + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78" +completed: "2026-03-10" +``` + +Add a task (or a role `roles/helm/`) that: + +1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm` +2. Verifies the checksum +3. Confirms `helm version` succeeds + +**Done when:** `helm version` succeeds on the HostEurope node. + +--- + +### T03 — Smoke test: k3s + Helm + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5" +completed: "2026-03-10" +``` + +Extend `tests/smoke_kube.sh` to assert: + +- `k3s kubectl get nodes` → node in Ready state +- `helm version` exits 0 +- CoreDNS pod running in `kube-system` +- Traefik ingress controller pod running (default in k3s) + +Run via: +```bash +ansible-playbook -i ansible/hosts.ini ansible/smoke.yml +``` +or directly over SSH if the kubeconfig is available locally. + +**Done when:** all assertions pass and the script exits 0. + +--- + +### T04 — Commit kubeconfig management notes + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f" +completed: "2026-03-10" +``` + +Document in `docs/kubeconfig.md`: + +- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`) +- How to merge it into `~/.kube/config` +- How to switch context: `kubectl config use-context default` +- Security note: kubeconfig is gitignored (contains cluster CA + client cert) + +**Done when:** doc written and committed. + +--- + +### T05 — Add `make k3s-install` and `make smoke` targets + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f" +completed: "2026-03-10" +``` + +Add to Makefile (create one if none exists): + +```makefile +k3s-install: ## Install k3s and Helm on all inventory hosts + ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml + +smoke: ## Run Kubernetes smoke tests + bash tests/smoke_kube.sh +``` + +**Done when:** both targets work and are listed in `make help`. + +--- + +## References + +- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline) +- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure) +- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09 +- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md` +- k3s releases: https://github.com/k3s-io/k3s/releases diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md new file mode 100644 index 0000000..fe7b21d --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md @@ -0,0 +1,196 @@ +--- +id: RAIL-BS-WP-0003 +type: workplan +title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +created: "2026-03-10" +updated: "2026-03-10" +state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover + +## Summary + +On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to +restart. pgpool — the connection pooler between Gitea and PostgreSQL — then +entered CrashLoopBackOff and produced no logs. As a result Gitea's login +and all write operations hung indefinitely. The root page was still served +(from Valkey cache) which masked the failure. + +The fix was to patch a missing key in a Kubernetes secret. The root cause is +that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not +populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql` +secret, even though the pgpool pod requires it at startup. + +--- + +## Timeline + +| Time (UTC) | Event | +|---|---| +| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) | +| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff | +| ~11:00 | User noticed Gitea login hanging; home page still loading | +| ~13:00 | Root cause identified: missing `pgpool-password` secret key | +| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly | +| ~13:15 | Gitea fully operational | + +--- + +## Root Cause + +The Bitnami `pgpool` container startup script reads the file +`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the +`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume +mount. That secret key was never created by the Helm chart, so the file did +not exist. The container exited immediately with no logs. + +The pod had been running for 20 days without a restart, so this gap was +never discovered during initial deployment. + +--- + +## Evidence + +```bash +# Secret was missing the pgpool-password key +sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml +# data: keys were password, postgres-password, repmgr-password only +# pgpool-password was absent + +# pgpool pod describe showed 824 back-off restarts over 173 minutes +# No logs in either current or --previous output +sudo k3s kubectl logs -n default --previous +# (empty) + +# Gitea process had zero TCP connections to PostgreSQL port 5432 +# but many connections to Valkey port 6379 +cat /proc//net/tcp | grep 1538 # 1538 = 5432 hex — no results +``` + +--- + +## Immediate Fix Applied + +```bash +# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0) +sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \ + --type='json' \ + -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]' + +# Restart pgpool +sudo k3s kubectl delete pod -n default +``` + +--- + +## Risk: Fix Will Be Lost on helm upgrade + +The patched secret is managed by Helm (annotation: +`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the +secret from the chart template, which does not include `pgpool-password`, +and the bug will recur. + +--- + +## Tasks + +### T01 — Add pgpool-password to Helm values + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc" +``` + +Create or update `helm/gitea-values.yaml` (or equivalent) to permanently +include the pgpool-password so it survives `helm upgrade`: + +```yaml +postgresql-ha: + postgresql: + pgpoolPassword: +``` + +**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly +without manual secret patching. + +--- + +### T02 — Add pgpool health check to smoke test + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20" +``` + +Extend `tests/smoke_kube.sh` to assert: + +```bash +# All postgresql-ha pods Running +kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1 + +# pgpool specifically not in CrashLoopBackOff +kubectl get pod -n default -l app.kubernetes.io/component=pgpool \ + -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash +``` + +**Done when:** the smoke test catches a pgpool failure within 5 minutes. + +--- + +### T03 — Add HA failover test + +```task +id: T03 +status: done +priority: high +state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01" +``` + +Create `tests/test_ha_failover.sh` that: + +1. Records Gitea login response time (baseline) +2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default` +3. Waits for repmgr to promote a replica (max 60s) +4. Asserts Gitea login POST still succeeds within 10s +5. Asserts pgpool pod is Running (not CrashLoopBackOff) +6. Asserts all postgresql pods return to Running + +This test must pass before any PostgreSQL HA deployment is considered done. + +**Done when:** script exits 0 against a live cluster. + +--- + +### T04 — Document the incident in docs/ + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d" +``` + +Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full +timeline, root cause, and fix, so future operators understand what happened +and how to recover. + +**Done when:** doc committed and linked from `docs/README.md`. + +--- + +## References + +- Bitnami postgresql-ha chart v16.2.2 +- Gitea Helm chart v12.2.0 +- Related decision: D3 (HA testing policy) in `DECISIONS.md` diff --git a/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md new file mode 100644 index 0000000..89d14ff --- /dev/null +++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md @@ -0,0 +1,275 @@ +--- +id: RAIL-BS-WP-0004 +type: workplan +title: "Integrated Backup — S2 Kubernetes Runtime Layer" +domain: financials +repo: railiance-cluster +status: finished +owner: tegwick +topic_slug: railiance +state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625" +created: "2026-02-25" +updated: "2026-03-26" +--- + +> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`). + +# Integrated Backup — S2 Kubernetes Runtime Layer + +## Goal + +Implement the Q3 (Operability & Resilience) integrated backup for +railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state — +encrypted with age, written to a local directory on the server. No external +dependencies required. + +## Architecture (Decision D4) + +Each railiance repo implements its own backup for what it owns. No central +backup service. See `DECISIONS.md` D4 for full rationale. + +**Standard interface every railiance repo must provide:** + +```bash +make backup # encrypt + write to /opt/backup/railiance// +make restore # restore from most recent local backup +``` + +Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key). +Output: `/opt/backup/railiance/cluster/` on the server. + +## What S2 (railiance-cluster) owns and must back up + +| Asset | Why it matters | +|---|---| +| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets | +| Helm release values | Runtime values not in git (any manually applied overrides) | +| kubeconfig | Admin access to the cluster | + +**Not S2's responsibility:** +- Custodian State Hub DB → the-custodian owns this +- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern +- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this +- PostgreSQL data volumes → S3 (railiance-platform) owns this + +## Encryption + +Reuse the age public key from `.sops.yaml`: + +```bash +AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}') +tar -czf - | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age +``` + +Decryption requires the private key at `~/.config/sops/age/keys.txt` +(same key used for `sops -d`). No additional key management needed. + +## Extension Point EP-RAIL-005 + +Once all five OAS layers implement this interface, the custodian can +orchestrate a full-stack backup with: + +```bash +for repo in railiance-infra railiance-cluster railiance-platform \ + railiance-enablement railiance-apps; do + make -C ~/$repo backup +done +``` + +No special protocol needed — just the standard interface. + +--- + +## Tasks + +### T01 — Define backup directory and encryption wrapper + +```task +id: T01 +status: done +priority: high +state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b" +``` + +Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`): + +- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`) +- Encrypt each artifact with age using public key from `.sops.yaml` +- Write timestamp-named files: `etcd-.snap.age`, `helm-values-.tar.gz.age`, `kubeconfig-.yaml.age` +- Keep last 7 of each type +- Write `.last-backup` stamp +- Exit 0 on success, non-zero on any failure +- No network required + +Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based +custodian DB — wrong scope, not applicable to this server). + +**Done when:** `make backup` runs on COULOMBCORE without error and files +appear in `/opt/backup/railiance/cluster/`. + +--- + +### T02 — Back up k3s state (SQLite hot backup) + +```task +id: T02 +status: done +priority: high +state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880" +``` + +k3s has built-in etcd snapshot support: + +```bash +sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ) +# Default location: /var/lib/rancher/k3s/server/db/snapshots/ +``` + +Add to the backup script: take a fresh snapshot, encrypt with age, +copy to `/opt/backup/railiance/cluster/`. + +> **Note — verify etcd is in use before implementing:** +> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`. +> Without it, k3s uses SQLite and this command will fail. +> Verify first: `sudo k3s etcd-snapshot ls 2>&1` + +> **Note — sudo required:** etcd snapshot requires root. See T06 for how +> this is resolved (backup runs under root's crontab). + +**Done when:** backup includes a current etcd snapshot. + +--- + +### T03 — Back up Helm release values + +```task +id: T03 +status: done +priority: medium +state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3" +``` + +Capture current runtime Helm values for all releases: + +```bash +KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \ + jq -r '.[].name + " " + .namespace' | \ + while read name ns; do + KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml + done +``` + +Tar and age-encrypt into `helm-values-.tar.gz.age`. + +> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable +> only by default. The backup script must either run as root (see T06) or k3s +> must be configured with `--write-kubeconfig-mode=644`. Running as root +> (via root crontab) is the chosen approach — no config change needed. + +**Done when:** backup includes a snapshot of all Helm release values. + +--- + +### T04 — Back up kubeconfig + +```task +id: T04 +status: done +priority: medium +state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665" +``` + +Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`) +into `kubeconfig-.yaml.age` in the backup directory. + +**Done when:** backup includes the encrypted kubeconfig. + +--- + +### T05 — make restore target + +```task +id: T05 +status: done +priority: medium +state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8" +``` + +Add `tools/cmd/railiance-restore-s2` that decrypts and lists available +backups, with guided restore for the etcd snapshot case. + +Restore of etcd from snapshot: +```bash +sudo k3s server --cluster-reset \ + --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/ +``` + +**Done when:** `make restore` prints available backups and a restore guide. + +--- + +### T06 — Install cron job and run restore drill + +```task +id: T06 +status: done +priority: medium +state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07" +``` + +#### Solving the sudo problem + +The backup script needs root for two reasons: +- `k3s etcd-snapshot save` requires root +- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only + +**Solution: run the cron under root's crontab.** + +This is the correct pattern for system-level backup jobs. It avoids a +proliferating sudoers whitelist (one entry per command, brittle to maintain) +and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in +production. The backup writes to `/opt/backup/` which is root-owned anyway. + +Install the cron as root: + +```bash +sudo crontab -e +# Add: +0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1 +``` + +Note: use the absolute path to the repo — `~` does not expand reliably in +root's crontab unless HOME is set. + +Verify it is installed: +```bash +sudo crontab -l | grep railiance +``` + +#### Restore drill + +Once T01–T04 are done, run a decrypt-and-verify drill: + +```bash +# Decrypt the etcd snapshot and verify it is a valid snapshot file +sudo age -d -i ~/.config/sops/age/keys.txt \ + /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \ + | file - + +# Record the drill +echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \ + >> /opt/backup/railiance/cluster/restore-drill.log +``` + +**Done when:** cron installed under root, drill completes without error, +log entry written. + +--- + +## References + +- Decision D4: Integrated backup per capability (`DECISIONS.md`) +- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement) +- OAS Q3: Operability & Resilience +- Extension point EP-RAIL-005: Custodian full-stack backup orchestration +- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore diff --git a/tests/test_cluster_frozen_history.py b/tests/test_cluster_frozen_history.py new file mode 100644 index 0000000..b5d9498 --- /dev/null +++ b/tests/test_cluster_frozen_history.py @@ -0,0 +1,52 @@ +import shutil +from pathlib import Path +import pytest +from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure +from repo_manager.record_identity import scan_record_identities + +FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster' +@pytest.fixture +def repo(tmp_path): + root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root) + live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n') + return root,live + +def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo): + root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root) + assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5 + preview=ensure(root,execute=False);assert len(preview['assignments'])==2 + assert preview['historical_invalid_identifiers']==before['invalid_identifiers'] + assert preview['historical_identity_collisions']==before['identity_collisions'] + result=ensure(root,execute=True);assert len(result['assignments'])==2 + assert all(p.read_bytes()==raw for p,raw in archive.items()) + assert not scan_record_identities(root)['ok'] + assert ensure(root,execute=False)['assignments']==[] + +@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid']) +def test_unknown_history_change_refuses_before_any_allocation(repo,change): + root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes() + raw=p.read_bytes() + if change=='missing':p.unlink() + elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target) + elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1)) + elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1)) + elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1)) + else:p.write_bytes(raw+b'\nunreviewed change\n') + with pytest.raises(ValueError):ensure(root) + assert live.read_bytes()==before + +def test_any_unknown_or_live_source_in_collision_still_refuses(repo): + root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md' + other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') + with pytest.raises(ValueError,match='identities are not safe'):ensure(root) + assert live.read_bytes()==before + +def test_other_repo_does_not_inherit_exact_frozen_exception(repo): + root,live=repo;other=root.with_name('other');root.rename(other) + with pytest.raises(ValueError,match='identities are not safe'):ensure(other) + +def test_qualified_collision_stays_global(repo): + root,live=repo;before=live.read_bytes() + (root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n') + with pytest.raises(ValueError,match='collisions='):ensure(root) + assert live.read_bytes()==before diff --git a/workplans/RMGR-WP-0019-historical-source-admission.md b/workplans/RMGR-WP-0019-historical-source-admission.md index fcdb3dc..5bc6b7d 100644 --- a/workplans/RMGR-WP-0019-historical-source-admission.md +++ b/workplans/RMGR-WP-0019-historical-source-admission.md @@ -4,10 +4,10 @@ type: workplan title: "Admit exact historical source views after receiving preservation support" domain: infotech repo: repo-manager -status: finished +status: active owner: codex created: "2026-10-04" -updated: "2026-10-06" +updated: "2026-10-17" flavor: implementation depends_on: [] state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab" @@ -146,3 +146,96 @@ e22516cc finished) holding 64 tasks (24+12+7+21). Receipt: Activity sync at e809 primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired / 0 refused, 0 task changes, historical UUIDs preserved. Evidence: state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json. + + +## Cluster exact historical source successor, 2026-10-17 + +This scoped extension preserves the completed Activity/Custodian work above. +The cluster publisher now refuses three closed RAIL-BS source files containing +15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs, +and UUIDs. No new alias or general syntax exemption is introduced. Evidence and +exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian +NetKingdom change packet. The existing SHR-WP-0001 project driver owns the +Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the +blocked cluster publication. No worker or human gate is woken here. + +## Prepare exact cluster registrar successor + +```task +id: RMGR-WP-0019-T04 +status: done +priority: high +state_hub_task_id: "1ccb06c6-ad30-5847-8704-e5628ba81f35" +``` + +Prepared full-file SHA256 bindings for three archives and 15 exact task +occurrences; strict scans still return 15 invalid labels and five collisions. +Unknown/live sources, changed/missing/symlinked histories, UUID drift and +qualified/global collisions refuse before allocation. Candidate tests pass 47; +canonical application remains a separate reviewed step. The read-only current +cluster preview proposes seven ordinary UUID bindings in two files and changes +no bytes. One pre-existing ADHOC field is included explicitly: task +RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives +3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only. + +## Admit cluster registrar successor only after receiving verification + +```task +id: RMGR-WP-0019-T05 +status: done +priority: high +depends_on: [STATE-WP-0095-T06] +state_hub_task_id: "36f47e21-42d0-5aef-b065-4a5dd3af0726" +``` + +Retain the current guard until exact reviewed receiving source has been delivered +and verified through State Hub's normal owner deployment. Match both manifests, +then integrate reviewed registrar source and verify ordinary allocator preview. +No direct API patch, history rename or bypass substitutes for this gate. + +## Run ordinary cluster owner publication and verify preservation + +```task +id: RMGR-WP-0019-T06 +status: wait +priority: high +depends_on: [RMGR-WP-0019-T05] +state_hub_task_id: "4d1675b6-f49f-5b01-8d76-bbf2030d1b34" +``` + +Only then run normal owner consistency/publication with the reviewed expected +file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary +generated indexes. Preserve all three archived files and all 18 existing +parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07. +Do not infer TLS recovery, credential custody or native acceptance from sync. + + +## Receiver-first registrar source follow-through, 2026-10-17 + +The exact receiving return is retained in Custodian +`docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/verified-receiving-return.json`, +bound to the actual native rollout/preservation/refusal receipts. It verifies the +reviewed receiver source/image/module, API/MCP readiness, expected additive schema, +three existing historical parents/15 tasks and rolled-back safe refusal probes. +Only after that return was the exact registrar patch +`232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc` +admitted in the Repo Manager owner source. T04 retains its source-preparation +outcome; T05 records this receiving-gated registrar admission, not a substitute +for receiving. Canonical focused registrar/identity checks passed 47 cases in 5.84 seconds. +The actual bounded read-only current Cluster allocator preview agreed with the +reviewed source: seven ordinary missing UUID bindings in exactly the new WP0018 and +one-field existing ADHOC backfill, with 15 invalid labels/five collisions still +reported by the strict conformance scan. The three historical file bytes, IDs, +UUIDs and ownership are unchanged. No broad syntax relaxation or direct Hub UUID +patch is introduced. Exact source application/test/preview receipts travel with +the same packet; source publication/projection is verified separately. + +T06 remains wait until actual ordinary Cluster owner publication and receiving +preservation are verified. Registrar source readiness alone does not mark that +publication complete. Preserve the 18 existing rows and all three archives; the +root coordinator owns the normal seven-field managed allocation and dirty-source +preservation. Own Repo Manager projection must show exactly T01–T05 done, T06 +wait and active workplan status, including the managed new T04/T05/T06 UUIDs and +unchanged earlier task UUIDs. The separate scanner RMGR-WP-0021-T05 completion +and unrelated owner work remain untouched. No native TLS recovery, custody, +current registration/private admission or NetKingdom target acceptance follows.