Admit exact frozen Cluster histories after verified receiving
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1468b-63fa-7710-85dc-8f2e16b3ebfa
This commit is contained in:
parent
9ebde0a9bd
commit
70cd754996
7 changed files with 934 additions and 6 deletions
|
|
@ -70,6 +70,111 @@ BINDINGS = json.loads(r'''[
|
|||
"record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED",
|
||||
"uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e",
|
||||
"parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED"
|
||||
},
|
||||
{
|
||||
"repo": "railiance-cluster",
|
||||
"path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
|
||||
"sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9",
|
||||
"record_id": "RAIL-BS-WP-0002",
|
||||
"uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d",
|
||||
"parent_record_id": "RAIL-BS-WP-0002",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "T01",
|
||||
"uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T02",
|
||||
"uuid": "e8510646-46ed-4697-a345-f3d3009eea78",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T03",
|
||||
"uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T04",
|
||||
"uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T05",
|
||||
"uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f",
|
||||
"status": "done"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"repo": "railiance-cluster",
|
||||
"path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
|
||||
"sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a",
|
||||
"record_id": "RAIL-BS-WP-0003",
|
||||
"uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b",
|
||||
"parent_record_id": "RAIL-BS-WP-0003",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "T01",
|
||||
"uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T02",
|
||||
"uuid": "ab166073-30a7-4702-a037-4091e8706e20",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T03",
|
||||
"uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T04",
|
||||
"uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d",
|
||||
"status": "done"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"repo": "railiance-cluster",
|
||||
"path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
|
||||
"sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3",
|
||||
"record_id": "RAIL-BS-WP-0004",
|
||||
"uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625",
|
||||
"parent_record_id": "RAIL-BS-WP-0004",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "T01",
|
||||
"uuid": "4526a842-ea31-4874-9231-92ab556cfe7b",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T02",
|
||||
"uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T03",
|
||||
"uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T04",
|
||||
"uuid": "08233868-d522-4117-bc4e-6c0f52545665",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T05",
|
||||
"uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8",
|
||||
"status": "done"
|
||||
},
|
||||
{
|
||||
"label": "T06",
|
||||
"uuid": "f8e4a094-c367-40eb-b895-da17bc144b07",
|
||||
"status": "done"
|
||||
}
|
||||
]
|
||||
}
|
||||
]''')
|
||||
|
||||
|
|
@ -83,3 +188,24 @@ def historical_source_paths(repo_root):
|
|||
raise ValueError("Historical source binding requires owner review: " + entry["path"])
|
||||
paths.add(entry["path"])
|
||||
return paths
|
||||
|
||||
|
||||
def historical_source_occurrences(repo_root):
|
||||
"""Exact frozen closed task occurrences; never a general local-ID alias."""
|
||||
from repo_manager.parse.workplan import parse_workplan_file
|
||||
from uuid import UUID
|
||||
paths=historical_source_paths(repo_root)
|
||||
occurrences=set(paths)
|
||||
for entry in BINDINGS:
|
||||
if entry['repo']!=repo_root.name or 'tasks' not in entry:continue
|
||||
parsed=parse_workplan_file(repo_root/entry['path'],repo_root=repo_root)
|
||||
actual=[{'label':task.id,'uuid':task.state_hub_task_id,'status':task.status} for task in parsed.tasks]
|
||||
if (parsed.id!=entry['record_id'] or parsed.state_hub_workstream_id!=entry['uuid']
|
||||
or parsed.status not in {'finished','archived','completed'} or actual!=entry['tasks']
|
||||
or any(task['status'] not in {'done','cancel'} for task in actual)):
|
||||
raise ValueError('Historical task binding requires owner review: '+entry['path'])
|
||||
UUID(entry['uuid'])
|
||||
for number,task in enumerate(actual,1):
|
||||
UUID(task['uuid'])
|
||||
occurrences.add(entry['path']+'#task-block-'+str(number))
|
||||
return occurrences
|
||||
|
|
|
|||
|
|
@ -854,8 +854,9 @@ def ensure_missing_work_record_identifiers(
|
|||
# allocation candidates. Keep conformance reporting strict while allowing
|
||||
# ordinary registration to preserve those bindings. Local aliases still
|
||||
# require the explicit parent allowlist in the kind registry.
|
||||
from repo_manager.historical_sources import historical_source_paths
|
||||
historical_sources: set[str] = historical_source_paths(repo_root)
|
||||
from repo_manager.historical_sources import historical_source_occurrences
|
||||
frozen_sources = historical_source_occurrences(repo_root)
|
||||
historical_sources: set[str] = set(frozen_sources)
|
||||
for path in iter_workplan_files(repo_root):
|
||||
parsed = parse_workplan_file(path, repo_root=repo_root)
|
||||
if parsed.status not in {"finished", "archived", "completed"} or not parsed.id:
|
||||
|
|
@ -889,11 +890,18 @@ def ensure_missing_work_record_identifiers(
|
|||
item for item in identity["invalid_identifiers"]
|
||||
if item["source"] in historical_sources
|
||||
]
|
||||
if invalid or identity["identity_collisions"]:
|
||||
# Strict scans keep legacy nonconformance. Only collisions wholly contained
|
||||
# in exact hash-bound closed task occurrences can be preserved by receiving;
|
||||
# any unknown/live source or qualified/workplan collision remains fatal.
|
||||
historical_collisions=[item for item in identity['identity_collisions']
|
||||
if item['kind']=='task' and re.fullmatch(r'T[0-9]{2}',item['id'])
|
||||
and all(source in frozen_sources for source in item['sources'])]
|
||||
collisions=[item for item in identity['identity_collisions'] if item not in historical_collisions]
|
||||
if invalid or collisions:
|
||||
raise ValueError(
|
||||
"work-record identities are not safe to derive: "
|
||||
f"invalid={invalid!r}, "
|
||||
f"collisions={identity['identity_collisions']!r}"
|
||||
f"collisions={collisions!r}"
|
||||
)
|
||||
|
||||
rendered: dict[Path, str] = {}
|
||||
|
|
@ -966,6 +974,7 @@ def ensure_missing_work_record_identifiers(
|
|||
"namespace": namespace,
|
||||
"assignments": assignments,
|
||||
"historical_invalid_identifiers": historical,
|
||||
"historical_identity_collisions": historical_collisions,
|
||||
"files_changed": sorted(str(path.relative_to(repo_root)) for path in rendered),
|
||||
}
|
||||
|
||||
|
|
|
|||
177
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md
vendored
Normal file
177
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md
vendored
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
---
|
||||
id: RAIL-BS-WP-0002
|
||||
type: workplan
|
||||
title: "k3s and Kubernetes Platform Baseline"
|
||||
domain: financials
|
||||
repo: railiance-cluster
|
||||
status: finished
|
||||
owner: railiance
|
||||
topic_slug: railiance
|
||||
repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
|
||||
state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
|
||||
created: "2026-03-09"
|
||||
updated: "2026-03-10"
|
||||
completed: "2026-03-10"
|
||||
---
|
||||
|
||||
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
|
||||
|
||||
# k3s and Kubernetes Platform Baseline
|
||||
|
||||
## Goal
|
||||
|
||||
Install k3s, Helm, and the baseline Kubernetes services on the converged
|
||||
HostEurope node. This workplan picks up exactly where `railiance-hosts`
|
||||
leaves off: a hardened, verified OS node that is ready for Kubernetes.
|
||||
|
||||
## Pre-condition
|
||||
|
||||
`railiance-infra` converge + Goss verify must pass before any task here
|
||||
is executed:
|
||||
|
||||
```bash
|
||||
# In railiance-infra/
|
||||
make converge
|
||||
make verify # must exit 0
|
||||
```
|
||||
|
||||
## Boundary
|
||||
|
||||
This repo owns everything from k3s upward. It must not re-configure items
|
||||
defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
|
||||
`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
|
||||
|
||||
**Out of scope here:** platform services (PostgreSQL, storage, identity)
|
||||
→ `railiance-platform`. Application deployments (Gitea, coulomb services)
|
||||
→ `railiance-apps`.
|
||||
|
||||
---
|
||||
|
||||
## Tasks
|
||||
|
||||
### T01 — Ansible playbook: install k3s (server mode)
|
||||
|
||||
```task
|
||||
id: T01
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
|
||||
completed: "2026-03-10"
|
||||
```
|
||||
|
||||
Harden `ansible/bootstrap.yml` to a production-ready k3s install:
|
||||
|
||||
- Use the official k3s install script pinned to a specific version
|
||||
(`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
|
||||
- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
|
||||
(cluster-init enables embedded etcd for future HA expansion)
|
||||
- Wait for node `Ready` before proceeding:
|
||||
```bash
|
||||
k3s kubectl wait node --all --for=condition=Ready --timeout=120s
|
||||
```
|
||||
- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
|
||||
|
||||
**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
|
||||
and the control node (via kubeconfig).
|
||||
|
||||
---
|
||||
|
||||
### T02 — Helm installation
|
||||
|
||||
```task
|
||||
id: T02
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
|
||||
completed: "2026-03-10"
|
||||
```
|
||||
|
||||
Add a task (or a role `roles/helm/`) that:
|
||||
|
||||
1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
|
||||
2. Verifies the checksum
|
||||
3. Confirms `helm version` succeeds
|
||||
|
||||
**Done when:** `helm version` succeeds on the HostEurope node.
|
||||
|
||||
---
|
||||
|
||||
### T03 — Smoke test: k3s + Helm
|
||||
|
||||
```task
|
||||
id: T03
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
|
||||
completed: "2026-03-10"
|
||||
```
|
||||
|
||||
Extend `tests/smoke_kube.sh` to assert:
|
||||
|
||||
- `k3s kubectl get nodes` → node in Ready state
|
||||
- `helm version` exits 0
|
||||
- CoreDNS pod running in `kube-system`
|
||||
- Traefik ingress controller pod running (default in k3s)
|
||||
|
||||
Run via:
|
||||
```bash
|
||||
ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
|
||||
```
|
||||
or directly over SSH if the kubeconfig is available locally.
|
||||
|
||||
**Done when:** all assertions pass and the script exits 0.
|
||||
|
||||
---
|
||||
|
||||
### T04 — Commit kubeconfig management notes
|
||||
|
||||
```task
|
||||
id: T04
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
|
||||
completed: "2026-03-10"
|
||||
```
|
||||
|
||||
Document in `docs/kubeconfig.md`:
|
||||
|
||||
- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
|
||||
- How to merge it into `~/.kube/config`
|
||||
- How to switch context: `kubectl config use-context default`
|
||||
- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
|
||||
|
||||
**Done when:** doc written and committed.
|
||||
|
||||
---
|
||||
|
||||
### T05 — Add `make k3s-install` and `make smoke` targets
|
||||
|
||||
```task
|
||||
id: T05
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
|
||||
completed: "2026-03-10"
|
||||
```
|
||||
|
||||
Add to Makefile (create one if none exists):
|
||||
|
||||
```makefile
|
||||
k3s-install: ## Install k3s and Helm on all inventory hosts
|
||||
ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
|
||||
|
||||
smoke: ## Run Kubernetes smoke tests
|
||||
bash tests/smoke_kube.sh
|
||||
```
|
||||
|
||||
**Done when:** both targets work and are listed in `make help`.
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
|
||||
- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
|
||||
- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
|
||||
- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
|
||||
- k3s releases: https://github.com/k3s-io/k3s/releases
|
||||
196
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md
vendored
Normal file
196
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md
vendored
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
---
|
||||
id: RAIL-BS-WP-0003
|
||||
type: workplan
|
||||
title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
|
||||
domain: financials
|
||||
repo: railiance-cluster
|
||||
status: finished
|
||||
owner: tegwick
|
||||
created: "2026-03-10"
|
||||
updated: "2026-03-10"
|
||||
state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
|
||||
---
|
||||
|
||||
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
|
||||
|
||||
# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
|
||||
|
||||
## Summary
|
||||
|
||||
On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
|
||||
restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
|
||||
entered CrashLoopBackOff and produced no logs. As a result Gitea's login
|
||||
and all write operations hung indefinitely. The root page was still served
|
||||
(from Valkey cache) which masked the failure.
|
||||
|
||||
The fix was to patch a missing key in a Kubernetes secret. The root cause is
|
||||
that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
|
||||
populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
|
||||
secret, even though the pgpool pod requires it at startup.
|
||||
|
||||
---
|
||||
|
||||
## Timeline
|
||||
|
||||
| Time (UTC) | Event |
|
||||
|---|---|
|
||||
| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
|
||||
| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
|
||||
| ~11:00 | User noticed Gitea login hanging; home page still loading |
|
||||
| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
|
||||
| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
|
||||
| ~13:15 | Gitea fully operational |
|
||||
|
||||
---
|
||||
|
||||
## Root Cause
|
||||
|
||||
The Bitnami `pgpool` container startup script reads the file
|
||||
`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
|
||||
`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
|
||||
mount. That secret key was never created by the Helm chart, so the file did
|
||||
not exist. The container exited immediately with no logs.
|
||||
|
||||
The pod had been running for 20 days without a restart, so this gap was
|
||||
never discovered during initial deployment.
|
||||
|
||||
---
|
||||
|
||||
## Evidence
|
||||
|
||||
```bash
|
||||
# Secret was missing the pgpool-password key
|
||||
sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
|
||||
# data: keys were password, postgres-password, repmgr-password only
|
||||
# pgpool-password was absent
|
||||
|
||||
# pgpool pod describe showed 824 back-off restarts over 173 minutes
|
||||
# No logs in either current or --previous output
|
||||
sudo k3s kubectl logs -n default <pgpool-pod> --previous
|
||||
# (empty)
|
||||
|
||||
# Gitea process had zero TCP connections to PostgreSQL port 5432
|
||||
# but many connections to Valkey port 6379
|
||||
cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Immediate Fix Applied
|
||||
|
||||
```bash
|
||||
# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
|
||||
sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
|
||||
--type='json' \
|
||||
-p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
|
||||
|
||||
# Restart pgpool
|
||||
sudo k3s kubectl delete pod -n default <pgpool-pod-name>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Risk: Fix Will Be Lost on helm upgrade
|
||||
|
||||
The patched secret is managed by Helm (annotation:
|
||||
`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
|
||||
secret from the chart template, which does not include `pgpool-password`,
|
||||
and the bug will recur.
|
||||
|
||||
---
|
||||
|
||||
## Tasks
|
||||
|
||||
### T01 — Add pgpool-password to Helm values
|
||||
|
||||
```task
|
||||
id: T01
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
|
||||
```
|
||||
|
||||
Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
|
||||
include the pgpool-password so it survives `helm upgrade`:
|
||||
|
||||
```yaml
|
||||
postgresql-ha:
|
||||
postgresql:
|
||||
pgpoolPassword: <value matching sr-check-password>
|
||||
```
|
||||
|
||||
**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
|
||||
without manual secret patching.
|
||||
|
||||
---
|
||||
|
||||
### T02 — Add pgpool health check to smoke test
|
||||
|
||||
```task
|
||||
id: T02
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
|
||||
```
|
||||
|
||||
Extend `tests/smoke_kube.sh` to assert:
|
||||
|
||||
```bash
|
||||
# All postgresql-ha pods Running
|
||||
kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
|
||||
|
||||
# pgpool specifically not in CrashLoopBackOff
|
||||
kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
|
||||
-o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
|
||||
```
|
||||
|
||||
**Done when:** the smoke test catches a pgpool failure within 5 minutes.
|
||||
|
||||
---
|
||||
|
||||
### T03 — Add HA failover test
|
||||
|
||||
```task
|
||||
id: T03
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
|
||||
```
|
||||
|
||||
Create `tests/test_ha_failover.sh` that:
|
||||
|
||||
1. Records Gitea login response time (baseline)
|
||||
2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
|
||||
3. Waits for repmgr to promote a replica (max 60s)
|
||||
4. Asserts Gitea login POST still succeeds within 10s
|
||||
5. Asserts pgpool pod is Running (not CrashLoopBackOff)
|
||||
6. Asserts all postgresql pods return to Running
|
||||
|
||||
This test must pass before any PostgreSQL HA deployment is considered done.
|
||||
|
||||
**Done when:** script exits 0 against a live cluster.
|
||||
|
||||
---
|
||||
|
||||
### T04 — Document the incident in docs/
|
||||
|
||||
```task
|
||||
id: T04
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
|
||||
```
|
||||
|
||||
Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
|
||||
timeline, root cause, and fix, so future operators understand what happened
|
||||
and how to recover.
|
||||
|
||||
**Done when:** doc committed and linked from `docs/README.md`.
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- Bitnami postgresql-ha chart v16.2.2
|
||||
- Gitea Helm chart v12.2.0
|
||||
- Related decision: D3 (HA testing policy) in `DECISIONS.md`
|
||||
275
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md
vendored
Normal file
275
tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md
vendored
Normal file
|
|
@ -0,0 +1,275 @@
|
|||
---
|
||||
id: RAIL-BS-WP-0004
|
||||
type: workplan
|
||||
title: "Integrated Backup — S2 Kubernetes Runtime Layer"
|
||||
domain: financials
|
||||
repo: railiance-cluster
|
||||
status: finished
|
||||
owner: tegwick
|
||||
topic_slug: railiance
|
||||
state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
|
||||
created: "2026-02-25"
|
||||
updated: "2026-03-26"
|
||||
---
|
||||
|
||||
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
|
||||
|
||||
# Integrated Backup — S2 Kubernetes Runtime Layer
|
||||
|
||||
## Goal
|
||||
|
||||
Implement the Q3 (Operability & Resilience) integrated backup for
|
||||
railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
|
||||
encrypted with age, written to a local directory on the server. No external
|
||||
dependencies required.
|
||||
|
||||
## Architecture (Decision D4)
|
||||
|
||||
Each railiance repo implements its own backup for what it owns. No central
|
||||
backup service. See `DECISIONS.md` D4 for full rationale.
|
||||
|
||||
**Standard interface every railiance repo must provide:**
|
||||
|
||||
```bash
|
||||
make backup # encrypt + write to /opt/backup/railiance/<layer>/
|
||||
make restore # restore from most recent local backup
|
||||
```
|
||||
|
||||
Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
|
||||
Output: `/opt/backup/railiance/cluster/` on the server.
|
||||
|
||||
## What S2 (railiance-cluster) owns and must back up
|
||||
|
||||
| Asset | Why it matters |
|
||||
|---|---|
|
||||
| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
|
||||
| Helm release values | Runtime values not in git (any manually applied overrides) |
|
||||
| kubeconfig | Admin access to the cluster |
|
||||
|
||||
**Not S2's responsibility:**
|
||||
- Custodian State Hub DB → the-custodian owns this
|
||||
- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
|
||||
- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
|
||||
- PostgreSQL data volumes → S3 (railiance-platform) owns this
|
||||
|
||||
## Encryption
|
||||
|
||||
Reuse the age public key from `.sops.yaml`:
|
||||
|
||||
```bash
|
||||
AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
|
||||
tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
|
||||
```
|
||||
|
||||
Decryption requires the private key at `~/.config/sops/age/keys.txt`
|
||||
(same key used for `sops -d`). No additional key management needed.
|
||||
|
||||
## Extension Point EP-RAIL-005
|
||||
|
||||
Once all five OAS layers implement this interface, the custodian can
|
||||
orchestrate a full-stack backup with:
|
||||
|
||||
```bash
|
||||
for repo in railiance-infra railiance-cluster railiance-platform \
|
||||
railiance-enablement railiance-apps; do
|
||||
make -C ~/$repo backup
|
||||
done
|
||||
```
|
||||
|
||||
No special protocol needed — just the standard interface.
|
||||
|
||||
---
|
||||
|
||||
## Tasks
|
||||
|
||||
### T01 — Define backup directory and encryption wrapper
|
||||
|
||||
```task
|
||||
id: T01
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
|
||||
```
|
||||
|
||||
Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
|
||||
|
||||
- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
|
||||
- Encrypt each artifact with age using public key from `.sops.yaml`
|
||||
- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
|
||||
- Keep last 7 of each type
|
||||
- Write `.last-backup` stamp
|
||||
- Exit 0 on success, non-zero on any failure
|
||||
- No network required
|
||||
|
||||
Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
|
||||
custodian DB — wrong scope, not applicable to this server).
|
||||
|
||||
**Done when:** `make backup` runs on COULOMBCORE without error and files
|
||||
appear in `/opt/backup/railiance/cluster/`.
|
||||
|
||||
---
|
||||
|
||||
### T02 — Back up k3s state (SQLite hot backup)
|
||||
|
||||
```task
|
||||
id: T02
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
|
||||
```
|
||||
|
||||
k3s has built-in etcd snapshot support:
|
||||
|
||||
```bash
|
||||
sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
|
||||
# Default location: /var/lib/rancher/k3s/server/db/snapshots/
|
||||
```
|
||||
|
||||
Add to the backup script: take a fresh snapshot, encrypt with age,
|
||||
copy to `/opt/backup/railiance/cluster/`.
|
||||
|
||||
> **Note — verify etcd is in use before implementing:**
|
||||
> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
|
||||
> Without it, k3s uses SQLite and this command will fail.
|
||||
> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
|
||||
|
||||
> **Note — sudo required:** etcd snapshot requires root. See T06 for how
|
||||
> this is resolved (backup runs under root's crontab).
|
||||
|
||||
**Done when:** backup includes a current etcd snapshot.
|
||||
|
||||
---
|
||||
|
||||
### T03 — Back up Helm release values
|
||||
|
||||
```task
|
||||
id: T03
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
|
||||
```
|
||||
|
||||
Capture current runtime Helm values for all releases:
|
||||
|
||||
```bash
|
||||
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
|
||||
jq -r '.[].name + " " + .namespace' | \
|
||||
while read name ns; do
|
||||
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
|
||||
done
|
||||
```
|
||||
|
||||
Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
|
||||
|
||||
> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
|
||||
> only by default. The backup script must either run as root (see T06) or k3s
|
||||
> must be configured with `--write-kubeconfig-mode=644`. Running as root
|
||||
> (via root crontab) is the chosen approach — no config change needed.
|
||||
|
||||
**Done when:** backup includes a snapshot of all Helm release values.
|
||||
|
||||
---
|
||||
|
||||
### T04 — Back up kubeconfig
|
||||
|
||||
```task
|
||||
id: T04
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
|
||||
```
|
||||
|
||||
Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
|
||||
into `kubeconfig-<ts>.yaml.age` in the backup directory.
|
||||
|
||||
**Done when:** backup includes the encrypted kubeconfig.
|
||||
|
||||
---
|
||||
|
||||
### T05 — make restore target
|
||||
|
||||
```task
|
||||
id: T05
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
|
||||
```
|
||||
|
||||
Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
|
||||
backups, with guided restore for the etcd snapshot case.
|
||||
|
||||
Restore of etcd from snapshot:
|
||||
```bash
|
||||
sudo k3s server --cluster-reset \
|
||||
--cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
|
||||
```
|
||||
|
||||
**Done when:** `make restore` prints available backups and a restore guide.
|
||||
|
||||
---
|
||||
|
||||
### T06 — Install cron job and run restore drill
|
||||
|
||||
```task
|
||||
id: T06
|
||||
status: done
|
||||
priority: medium
|
||||
state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
|
||||
```
|
||||
|
||||
#### Solving the sudo problem
|
||||
|
||||
The backup script needs root for two reasons:
|
||||
- `k3s etcd-snapshot save` requires root
|
||||
- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
|
||||
|
||||
**Solution: run the cron under root's crontab.**
|
||||
|
||||
This is the correct pattern for system-level backup jobs. It avoids a
|
||||
proliferating sudoers whitelist (one entry per command, brittle to maintain)
|
||||
and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
|
||||
production. The backup writes to `/opt/backup/` which is root-owned anyway.
|
||||
|
||||
Install the cron as root:
|
||||
|
||||
```bash
|
||||
sudo crontab -e
|
||||
# Add:
|
||||
0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
|
||||
```
|
||||
|
||||
Note: use the absolute path to the repo — `~` does not expand reliably in
|
||||
root's crontab unless HOME is set.
|
||||
|
||||
Verify it is installed:
|
||||
```bash
|
||||
sudo crontab -l | grep railiance
|
||||
```
|
||||
|
||||
#### Restore drill
|
||||
|
||||
Once T01–T04 are done, run a decrypt-and-verify drill:
|
||||
|
||||
```bash
|
||||
# Decrypt the etcd snapshot and verify it is a valid snapshot file
|
||||
sudo age -d -i ~/.config/sops/age/keys.txt \
|
||||
/opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
|
||||
| file -
|
||||
|
||||
# Record the drill
|
||||
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
|
||||
>> /opt/backup/railiance/cluster/restore-drill.log
|
||||
```
|
||||
|
||||
**Done when:** cron installed under root, drill completes without error,
|
||||
log entry written.
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- Decision D4: Integrated backup per capability (`DECISIONS.md`)
|
||||
- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
|
||||
- OAS Q3: Operability & Resilience
|
||||
- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
|
||||
- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore
|
||||
52
tests/test_cluster_frozen_history.py
Normal file
52
tests/test_cluster_frozen_history.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import shutil
|
||||
from pathlib import Path
|
||||
import pytest
|
||||
from repo_manager.identifiers import ensure_missing_work_record_identifiers as ensure
|
||||
from repo_manager.record_identity import scan_record_identities
|
||||
|
||||
FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
|
||||
@pytest.fixture
|
||||
def repo(tmp_path):
|
||||
root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
|
||||
live=root/'workplans/RCLUSTER-WP-0018.md';live.write_text('---\nid: RCLUSTER-WP-0018\nstatus: ready\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: todo\n```\n')
|
||||
return root,live
|
||||
|
||||
def test_exact_three_histories_preserve_15_bindings_and_strict_findings(repo):
|
||||
root,live=repo;archive={p:p.read_bytes() for p in (root/'workplans/archived').glob('*.md')};before=scan_record_identities(root)
|
||||
assert len(before['invalid_identifiers'])==15 and len(before['identity_collisions'])==5
|
||||
preview=ensure(root,execute=False);assert len(preview['assignments'])==2
|
||||
assert preview['historical_invalid_identifiers']==before['invalid_identifiers']
|
||||
assert preview['historical_identity_collisions']==before['identity_collisions']
|
||||
result=ensure(root,execute=True);assert len(result['assignments'])==2
|
||||
assert all(p.read_bytes()==raw for p,raw in archive.items())
|
||||
assert not scan_record_identities(root)['ok']
|
||||
assert ensure(root,execute=False)['assignments']==[]
|
||||
|
||||
@pytest.mark.parametrize('change',['bytes','missing','symlink','live','nonterminal','uuid'])
|
||||
def test_unknown_history_change_refuses_before_any_allocation(repo,change):
|
||||
root,live=repo;p=next((root/'workplans/archived').glob('*.md'));before=live.read_bytes()
|
||||
raw=p.read_bytes()
|
||||
if change=='missing':p.unlink()
|
||||
elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
|
||||
elif change=='live':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
|
||||
elif change=='nonterminal':p.write_bytes(raw.replace(b'status: done',b'status: todo',1))
|
||||
elif change=='uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
|
||||
else:p.write_bytes(raw+b'\nunreviewed change\n')
|
||||
with pytest.raises(ValueError):ensure(root)
|
||||
assert live.read_bytes()==before
|
||||
|
||||
def test_any_unknown_or_live_source_in_collision_still_refuses(repo):
|
||||
root,live=repo;before=live.read_bytes();other=root/'workplans/unknown.md'
|
||||
other.write_text('---\nid: RCLUSTER-WP-0099\nstatus: active\n---\n\n```task\nid: T01\nstatus: todo\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
|
||||
with pytest.raises(ValueError,match='identities are not safe'):ensure(root)
|
||||
assert live.read_bytes()==before
|
||||
|
||||
def test_other_repo_does_not_inherit_exact_frozen_exception(repo):
|
||||
root,live=repo;other=root.with_name('other');root.rename(other)
|
||||
with pytest.raises(ValueError,match='identities are not safe'):ensure(other)
|
||||
|
||||
def test_qualified_collision_stays_global(repo):
|
||||
root,live=repo;before=live.read_bytes()
|
||||
(root/'workplans/other.md').write_text('---\nid: RCLUSTER-WP-0019\nstatus: active\n---\n\n```task\nid: RCLUSTER-WP-0018-T01\nstatus: done\nstate_hub_task_id: 11111111-1111-4111-8111-111111111111\n```\n')
|
||||
with pytest.raises(ValueError,match='collisions='):ensure(root)
|
||||
assert live.read_bytes()==before
|
||||
|
|
@ -4,10 +4,10 @@ type: workplan
|
|||
title: "Admit exact historical source views after receiving preservation support"
|
||||
domain: infotech
|
||||
repo: repo-manager
|
||||
status: finished
|
||||
status: active
|
||||
owner: codex
|
||||
created: "2026-10-04"
|
||||
updated: "2026-10-06"
|
||||
updated: "2026-10-17"
|
||||
flavor: implementation
|
||||
depends_on: []
|
||||
state_hub_workstream_id: "efe0ecb2-2f3f-5f49-baee-b90ad1d353ab"
|
||||
|
|
@ -146,3 +146,96 @@ e22516cc finished) holding 64 tasks (24+12+7+21). Receipt: Activity sync at e809
|
|||
primary/railiance01, applied, 0 created / 45 updated (ACTIVITY-WP only) / 0 retired
|
||||
/ 0 refused, 0 task changes, historical UUIDs preserved. Evidence:
|
||||
state-hub docs/evidence/2026-10-06-activity-stale-retirement-rollout.json.
|
||||
|
||||
|
||||
## Cluster exact historical source successor, 2026-10-17
|
||||
|
||||
This scoped extension preserves the completed Activity/Custodian work above.
|
||||
The cluster publisher now refuses three closed RAIL-BS source files containing
|
||||
15 existing UUID-bound bare task labels. Preserve all three source bytes, IDs,
|
||||
and UUIDs. No new alias or general syntax exemption is introduced. Evidence and
|
||||
exact receiver-first proposal: cluster-historical-publication-r1 in the Custodian
|
||||
NetKingdom change packet. The existing SHR-WP-0001 project driver owns the
|
||||
Repo Manager / State Hub cross-owner wait; NK-WP-0049-T35 benefits from the
|
||||
blocked cluster publication. No worker or human gate is woken here.
|
||||
|
||||
## Prepare exact cluster registrar successor
|
||||
|
||||
```task
|
||||
id: RMGR-WP-0019-T04
|
||||
status: done
|
||||
priority: high
|
||||
state_hub_task_id: "1ccb06c6-ad30-5847-8704-e5628ba81f35"
|
||||
```
|
||||
|
||||
Prepared full-file SHA256 bindings for three archives and 15 exact task
|
||||
occurrences; strict scans still return 15 invalid labels and five collisions.
|
||||
Unknown/live sources, changed/missing/symlinked histories, UUID drift and
|
||||
qualified/global collisions refuse before allocation. Candidate tests pass 47;
|
||||
canonical application remains a separate reviewed step. The read-only current
|
||||
cluster preview proposes seven ordinary UUID bindings in two files and changes
|
||||
no bytes. One pre-existing ADHOC field is included explicitly: task
|
||||
RCLUSTER-WP-ADHOC-2026-07-27-T01 in workplans/ADHOC-2026-07-27.md receives
|
||||
3e3f39d3-5070-5ba3-97db-19f5f361f264 through managed allocation only.
|
||||
|
||||
## Admit cluster registrar successor only after receiving verification
|
||||
|
||||
```task
|
||||
id: RMGR-WP-0019-T05
|
||||
status: done
|
||||
priority: high
|
||||
depends_on: [STATE-WP-0095-T06]
|
||||
state_hub_task_id: "36f47e21-42d0-5aef-b065-4a5dd3af0726"
|
||||
```
|
||||
|
||||
Retain the current guard until exact reviewed receiving source has been delivered
|
||||
and verified through State Hub's normal owner deployment. Match both manifests,
|
||||
then integrate reviewed registrar source and verify ordinary allocator preview.
|
||||
No direct API patch, history rename or bypass substitutes for this gate.
|
||||
|
||||
## Run ordinary cluster owner publication and verify preservation
|
||||
|
||||
```task
|
||||
id: RMGR-WP-0019-T06
|
||||
status: wait
|
||||
priority: high
|
||||
depends_on: [RMGR-WP-0019-T05]
|
||||
state_hub_task_id: "4d1675b6-f49f-5b01-8d76-bbf2030d1b34"
|
||||
```
|
||||
|
||||
Only then run normal owner consistency/publication with the reviewed expected
|
||||
file set: RCLUSTER-WP-0018 and the one-field ADHOC managed backfill, plus ordinary
|
||||
generated indexes. Preserve all three archived files and all 18 existing
|
||||
parent/task rows. Return actual receiving/publication receipts to STATE-WP-0095-T07.
|
||||
Do not infer TLS recovery, credential custody or native acceptance from sync.
|
||||
|
||||
|
||||
## Receiver-first registrar source follow-through, 2026-10-17
|
||||
|
||||
The exact receiving return is retained in Custodian
|
||||
`docs/changes/netkingdom-target-2026-10-17/receiver-rollout-r4/verified-receiving-return.json`,
|
||||
bound to the actual native rollout/preservation/refusal receipts. It verifies the
|
||||
reviewed receiver source/image/module, API/MCP readiness, expected additive schema,
|
||||
three existing historical parents/15 tasks and rolled-back safe refusal probes.
|
||||
Only after that return was the exact registrar patch
|
||||
`232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc`
|
||||
admitted in the Repo Manager owner source. T04 retains its source-preparation
|
||||
outcome; T05 records this receiving-gated registrar admission, not a substitute
|
||||
for receiving. Canonical focused registrar/identity checks passed 47 cases in 5.84 seconds.
|
||||
The actual bounded read-only current Cluster allocator preview agreed with the
|
||||
reviewed source: seven ordinary missing UUID bindings in exactly the new WP0018 and
|
||||
one-field existing ADHOC backfill, with 15 invalid labels/five collisions still
|
||||
reported by the strict conformance scan. The three historical file bytes, IDs,
|
||||
UUIDs and ownership are unchanged. No broad syntax relaxation or direct Hub UUID
|
||||
patch is introduced. Exact source application/test/preview receipts travel with
|
||||
the same packet; source publication/projection is verified separately.
|
||||
|
||||
T06 remains wait until actual ordinary Cluster owner publication and receiving
|
||||
preservation are verified. Registrar source readiness alone does not mark that
|
||||
publication complete. Preserve the 18 existing rows and all three archives; the
|
||||
root coordinator owns the normal seven-field managed allocation and dirty-source
|
||||
preservation. Own Repo Manager projection must show exactly T01–T05 done, T06
|
||||
wait and active workplan status, including the managed new T04/T05/T06 UUIDs and
|
||||
unchanged earlier task UUIDs. The separate scanner RMGR-WP-0021-T05 completion
|
||||
and unrelated owner work remain untouched. No native TLS recovery, custody,
|
||||
current registration/private admission or NetKingdom target acceptance follows.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue