coulomb/the-custodian
5
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

docs: preserve reviewed NetKingdom session artifacts
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
Details
CI Smoke / container-smoke (push) Successful in 1s
Details
Reject generated-only commits / generated-commit-guard (push) Successful in 15s
Details

Browse source
This commit is contained in:
codex 2026-11-01 22:55:48 +01:00
parent 70fcd0a6f0
commit 3402090e5a
50 changed files with 7717 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

930
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub.patch Normal file
View file

@ -0,0 +1,930 @@
--- a/api/services/historical_projection.py
+++ b/api/services/historical_projection.py
@@ -72,6 +72,111 @@
"record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED",
"uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e",
"parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED"
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
+ "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9",
+ "record_id": "RAIL-BS-WP-0002",
+ "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d",
+ "parent_record_id": "RAIL-BS-WP-0002",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "e8510646-46ed-4697-a345-f3d3009eea78",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
+ "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a",
+ "record_id": "RAIL-BS-WP-0003",
+ "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b",
+ "parent_record_id": "RAIL-BS-WP-0003",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "ab166073-30a7-4702-a037-4091e8706e20",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
+ "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3",
+ "record_id": "RAIL-BS-WP-0004",
+ "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625",
+ "parent_record_id": "RAIL-BS-WP-0004",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "08233868-d522-4117-bc4e-6c0f52545665",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8",
+ "status": "done"
+ },
+ {
+ "label": "T06",
+ "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07",
+ "status": "done"
+ }
+ ]
}
]''')
--- /dev/null
+++ b/tests/test_cluster_historical_projection.py
@@ -0,0 +1,67 @@
+import shutil
+import uuid
+from pathlib import Path
+import pytest
+from api.services import forge_projection as fp
+from api.services import historical_projection as hp
+from tests.test_forge_projection import _FakeSession,_Repo,_Row,_TaskRow
+
+FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
+@pytest.fixture
+def bound(tmp_path):
+ root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
+ derived=fp.derive_from_checkout(root,'railiance-cluster','candidate')
+ assert len(derived.workplans)==3
+ rows=[];tasks=[]
+ for parent in derived.workplans:
+ row=_Row(parent.record_id.lower(),'finished',parent.relative_path);row.id=uuid.UUID(parent.uuid);rows.append(row)
+ for source in parent.tasks:
+ task=_TaskRow(source.record_id,status='done',workplan_id=row.id);task.id=uuid.UUID(source.uuid);tasks.append(task)
+ assert len(tasks)==15
+ return root,derived,rows,tasks
+
+def test_exact_three_sources_derive_only_frozen_15_task_uuids(bound):
+ root,derived,rows,tasks=bound
+ assert all(parent.historical_preservation for parent in derived.workplans)
+ assert all(parent.uuid!=fp.derived_record_uuid(parent.record_id) for parent in derived.workplans)
+ expected={item['uuid'] for entry in hp.BINDINGS if entry['repo']=='railiance-cluster' for item in entry['tasks']}
+ assert {str(t.id) for t in tasks}==expected
+
+@pytest.mark.asyncio
+async def test_existing_rows_are_entirely_unchanged_on_normal_receiving(bound):
+ root,derived,rows,tasks=bound;before=[dict(x.__dict__) for x in rows+tasks]
+ session=_FakeSession(_Repo(),rows,task_rows=tasks)
+ outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
+ assert outcome.status=='noop' and not outcome.refused
+ assert [dict(x.__dict__) for x in rows+tasks]==before
+ assert not session.added and not session.deleted
+
+@pytest.mark.parametrize('defect',['missing-parent','missing-task','foreign-holder','nonterminal-task','nonterminal-parent','duplicate-current-label'])
+@pytest.mark.asyncio
+async def test_unsupported_actual_binding_refuses_before_any_mutation(bound,defect):
+ root,derived,rows,tasks=bound
+ if defect=='missing-parent':rows.pop()
+ elif defect=='missing-task':tasks.pop()
+ elif defect=='foreign-holder':tasks[0].workplan_id=uuid.uuid4()
+ elif defect=='nonterminal-task':tasks[0].status='progress'
+ elif defect=='duplicate-current-label':tasks.append(_TaskRow(tasks[0].record_id,status='done',workplan_id=tasks[0].workplan_id))
+ else:rows[0].status='active'
+ before=[dict(x.__dict__) for x in rows+tasks];session=_FakeSession(_Repo(),rows,task_rows=tasks)
+ outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
+ assert outcome.status=='refused';assert [dict(x.__dict__) for x in rows+tasks]==before
+ assert not session.added and not session.deleted
+
+@pytest.mark.parametrize('change',['hash-drift','missing','symlink','live-source','missing-source-uuid'])
+def test_frozen_source_drift_refuses_derivation(bound,change):
+ root,*_=bound;p=next((root/'workplans/archived').glob('*.md'));raw=p.read_bytes()
+ if change=='missing':p.unlink()
+ elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
+ elif change=='live-source':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
+ elif change=='missing-source-uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
+ else:p.write_bytes(raw+b'\nunreviewed\n')
+ with pytest.raises(fp.ForgeDeriveError,match='Historical source binding'):fp.derive_from_checkout(root,'railiance-cluster','candidate')
+
+def test_unknown_repository_never_trusts_frozen_source_uuids(bound):
+ root,derived,*_=bound;other=fp.derive_from_checkout(root,'different-repo','candidate')
+ assert not any(parent.historical_preservation for parent in other.workplans)
+ assert not {parent.uuid for parent in derived.workplans}&{parent.uuid for parent in other.workplans}
--- a/workplans/STATE-WP-0095-historical-source-receiving.md
+++ b/workplans/STATE-WP-0095-historical-source-receiving.md
@@ -4,10 +4,10 @@
title: "Preserve exact closed historical source bindings in Forge receiving"
domain: infotech
repo: state-hub
-status: finished
+status: active
owner: codex
created: "2026-10-04"
-updated: "2026-10-06"
+updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058"
@@ -167,3 +167,73 @@
[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json).
Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and
ACTIVITY-WP-0045-T02 remain with their owners.
+
+
+## Cluster exact historical receiving successor, 2026-10-17
+
+This scoped extension retains the completed Activity/Custodian outcomes above.
+Three frozen cluster archives contain three existing parent UUIDs and 15 existing
+closed task UUIDs. Extend the existing exact historical receiving path before
+registrar admission; never derive replacement identities or modify those
+sources. Cross-owner observation attaches to existing SHR-WP-0001, with
+RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly.
+The local derived case is waiting; no live worker injection is authorized.
+
+## Prepare exact cluster source preservation and refusal cases
+
+```task
+id: STATE-WP-0095-T05
+status: done
+priority: high
+state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b"
+```
+
+Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager.
+Candidate receiving tests pass 26: existing rows remain entirely unchanged;
+missing/foreign/nonterminal rows and unknown, changed, missing or symlinked
+source refuse. Read-only public metadata confirms all 18 existing rows have the
+expected ownership and terminal status. These are source and metadata checks,
+not delivered receiving or a committed native trial.
+
+## Deliver reviewed cluster receiving successor before registrar admission
+
+```task
+id: STATE-WP-0095-T06
+status: wait
+priority: high
+depends_on: [STATE-WP-0095-T05]
+needs_human: true
+intervention_note: "Review exact committed source, CI image and owner deployment proposal before rollout."
+state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4"
+```
+
+Use the normal CI image and Helm owner path after exact revision/image pin,
+fresh headroom and reviewed dry-run/diff. API/MCP currently run main-7d99282;
+Helm revision75 is the observed rollback baseline, requiring freshness before
+execution. No deployment assent exists for this successor. Verify delivered
+module hash, health, both readiness checks and expected migrated schema
+c4d5e6f7a8b9; verify exact
+three parents/15 tasks and rejection behavior without committed probe changes.
+Only the verified return unblocks RMGR-WP-0019-T05. The full current image also carries inbox-feed migration c4d5e6f7a8b9,
+although this receiver patch adds none. Helm rollback leaves that migration in
+place; unrelated migration acceptance and old-image compatibility need normal
+owner evidence before rollout approval. Do not run destructive default local
+DB tests; use only exact disposable DB containment. Costs and monetary risk are
+unknown pending exact rollout proposal; no zero-cost or autonomy grant is inferred.
+
+## Verify ordinary cluster receiving and immutable historical rows
+
+```task
+id: STATE-WP-0095-T07
+status: wait
+priority: high
+depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06]
+state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d"
+```
+
+Capture normal cluster owner publication/receiving receipts and compare entire
+existing row snapshots and exact source bytes. No creation, movement, relabel,
+unretirement or retirement is authorized for these histories. A rollback first
+fences cluster sync and restores registrar refusal before removing receiving
+support; it never rewrites archived IDs or UUIDs. This work does not establish
+original TLS recovery or owner-native admission.
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md
@@ -0,0 +1,177 @@
+---
+id: RAIL-BS-WP-0002
+type: workplan
+title: "k3s and Kubernetes Platform Baseline"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: railiance
+topic_slug: railiance
+repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
+state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
+created: "2026-03-09"
+updated: "2026-03-10"
+completed: "2026-03-10"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# k3s and Kubernetes Platform Baseline
+
+## Goal
+
+Install k3s, Helm, and the baseline Kubernetes services on the converged
+HostEurope node. This workplan picks up exactly where `railiance-hosts`
+leaves off: a hardened, verified OS node that is ready for Kubernetes.
+
+## Pre-condition
+
+`railiance-infra` converge + Goss verify must pass before any task here
+is executed:
+
+```bash
+# In railiance-infra/
+make converge
+make verify # must exit 0
+```
+
+## Boundary
+
+This repo owns everything from k3s upward. It must not re-configure items
+defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
+`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
+
+**Out of scope here:** platform services (PostgreSQL, storage, identity)
+→ `railiance-platform`. Application deployments (Gitea, coulomb services)
+→ `railiance-apps`.
+
+---
+
+## Tasks
+
+### T01 — Ansible playbook: install k3s (server mode)
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
+completed: "2026-03-10"
+```
+
+Harden `ansible/bootstrap.yml` to a production-ready k3s install:
+
+- Use the official k3s install script pinned to a specific version
+ (`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
+- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
+ (cluster-init enables embedded etcd for future HA expansion)
+- Wait for node `Ready` before proceeding:
+ ```bash
+ k3s kubectl wait node --all --for=condition=Ready --timeout=120s
+ ```
+- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
+
+**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
+and the control node (via kubeconfig).
+
+---
+
+### T02 — Helm installation
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
+completed: "2026-03-10"
+```
+
+Add a task (or a role `roles/helm/`) that:
+
+1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
+2. Verifies the checksum
+3. Confirms `helm version` succeeds
+
+**Done when:** `helm version` succeeds on the HostEurope node.
+
+---
+
+### T03 — Smoke test: k3s + Helm
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
+completed: "2026-03-10"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+- `k3s kubectl get nodes` → node in Ready state
+- `helm version` exits 0
+- CoreDNS pod running in `kube-system`
+- Traefik ingress controller pod running (default in k3s)
+
+Run via:
+```bash
+ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
+```
+or directly over SSH if the kubeconfig is available locally.
+
+**Done when:** all assertions pass and the script exits 0.
+
+---
+
+### T04 — Commit kubeconfig management notes
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
+completed: "2026-03-10"
+```
+
+Document in `docs/kubeconfig.md`:
+
+- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
+- How to merge it into `~/.kube/config`
+- How to switch context: `kubectl config use-context default`
+- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
+
+**Done when:** doc written and committed.
+
+---
+
+### T05 — Add `make k3s-install` and `make smoke` targets
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
+completed: "2026-03-10"
+```
+
+Add to Makefile (create one if none exists):
+
+```makefile
+k3s-install: ## Install k3s and Helm on all inventory hosts
+ ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
+
+smoke: ## Run Kubernetes smoke tests
+ bash tests/smoke_kube.sh
+```
+
+**Done when:** both targets work and are listed in `make help`.
+
+---
+
+## References
+
+- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
+- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
+- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
+- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
+- k3s releases: https://github.com/k3s-io/k3s/releases
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md
@@ -0,0 +1,196 @@
+---
+id: RAIL-BS-WP-0003
+type: workplan
+title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+created: "2026-03-10"
+updated: "2026-03-10"
+state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
+
+## Summary
+
+On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
+restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
+entered CrashLoopBackOff and produced no logs. As a result Gitea's login
+and all write operations hung indefinitely. The root page was still served
+(from Valkey cache) which masked the failure.
+
+The fix was to patch a missing key in a Kubernetes secret. The root cause is
+that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
+populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
+secret, even though the pgpool pod requires it at startup.
+
+---
+
+## Timeline
+
+| Time (UTC) | Event |
+|---|---|
+| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
+| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
+| ~11:00 | User noticed Gitea login hanging; home page still loading |
+| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
+| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
+| ~13:15 | Gitea fully operational |
+
+---
+
+## Root Cause
+
+The Bitnami `pgpool` container startup script reads the file
+`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
+`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
+mount. That secret key was never created by the Helm chart, so the file did
+not exist. The container exited immediately with no logs.
+
+The pod had been running for 20 days without a restart, so this gap was
+never discovered during initial deployment.
+
+---
+
+## Evidence
+
+```bash
+# Secret was missing the pgpool-password key
+sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
+# data: keys were password, postgres-password, repmgr-password only
+# pgpool-password was absent
+
+# pgpool pod describe showed 824 back-off restarts over 173 minutes
+# No logs in either current or --previous output
+sudo k3s kubectl logs -n default <pgpool-pod> --previous
+# (empty)
+
+# Gitea process had zero TCP connections to PostgreSQL port 5432
+# but many connections to Valkey port 6379
+cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
+```
+
+---
+
+## Immediate Fix Applied
+
+```bash
+# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
+sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
+ --type='json' \
+ -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
+
+# Restart pgpool
+sudo k3s kubectl delete pod -n default <pgpool-pod-name>
+```
+
+---
+
+## Risk: Fix Will Be Lost on helm upgrade
+
+The patched secret is managed by Helm (annotation:
+`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
+secret from the chart template, which does not include `pgpool-password`,
+and the bug will recur.
+
+---
+
+## Tasks
+
+### T01 — Add pgpool-password to Helm values
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
+```
+
+Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
+include the pgpool-password so it survives `helm upgrade`:
+
+```yaml
+postgresql-ha:
+ postgresql:
+ pgpoolPassword: <value matching sr-check-password>
+```
+
+**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
+without manual secret patching.
+
+---
+
+### T02 — Add pgpool health check to smoke test
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+```bash
+# All postgresql-ha pods Running
+kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
+
+# pgpool specifically not in CrashLoopBackOff
+kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
+ -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
+```
+
+**Done when:** the smoke test catches a pgpool failure within 5 minutes.
+
+---
+
+### T03 — Add HA failover test
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
+```
+
+Create `tests/test_ha_failover.sh` that:
+
+1. Records Gitea login response time (baseline)
+2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
+3. Waits for repmgr to promote a replica (max 60s)
+4. Asserts Gitea login POST still succeeds within 10s
+5. Asserts pgpool pod is Running (not CrashLoopBackOff)
+6. Asserts all postgresql pods return to Running
+
+This test must pass before any PostgreSQL HA deployment is considered done.
+
+**Done when:** script exits 0 against a live cluster.
+
+---
+
+### T04 — Document the incident in docs/
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
+```
+
+Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
+timeline, root cause, and fix, so future operators understand what happened
+and how to recover.
+
+**Done when:** doc committed and linked from `docs/README.md`.
+
+---
+
+## References
+
+- Bitnami postgresql-ha chart v16.2.2
+- Gitea Helm chart v12.2.0
+- Related decision: D3 (HA testing policy) in `DECISIONS.md`
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md
@@ -0,0 +1,275 @@
+---
+id: RAIL-BS-WP-0004
+type: workplan
+title: "Integrated Backup — S2 Kubernetes Runtime Layer"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+topic_slug: railiance
+state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
+created: "2026-02-25"
+updated: "2026-03-26"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Integrated Backup — S2 Kubernetes Runtime Layer
+
+## Goal
+
+Implement the Q3 (Operability & Resilience) integrated backup for
+railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
+encrypted with age, written to a local directory on the server. No external
+dependencies required.
+
+## Architecture (Decision D4)
+
+Each railiance repo implements its own backup for what it owns. No central
+backup service. See `DECISIONS.md` D4 for full rationale.
+
+**Standard interface every railiance repo must provide:**
+
+```bash
+make backup # encrypt + write to /opt/backup/railiance/<layer>/
+make restore # restore from most recent local backup
+```
+
+Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
+Output: `/opt/backup/railiance/cluster/` on the server.
+
+## What S2 (railiance-cluster) owns and must back up
+
+| Asset | Why it matters |
+|---|---|
+| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
+| Helm release values | Runtime values not in git (any manually applied overrides) |
+| kubeconfig | Admin access to the cluster |
+
+**Not S2's responsibility:**
+- Custodian State Hub DB → the-custodian owns this
+- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
+- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
+- PostgreSQL data volumes → S3 (railiance-platform) owns this
+
+## Encryption
+
+Reuse the age public key from `.sops.yaml`:
+
+```bash
+AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
+tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
+```
+
+Decryption requires the private key at `~/.config/sops/age/keys.txt`
+(same key used for `sops -d`). No additional key management needed.
+
+## Extension Point EP-RAIL-005
+
+Once all five OAS layers implement this interface, the custodian can
+orchestrate a full-stack backup with:
+
+```bash
+for repo in railiance-infra railiance-cluster railiance-platform \
+ railiance-enablement railiance-apps; do
+ make -C ~/$repo backup
+done
+```
+
+No special protocol needed — just the standard interface.
+
+---
+
+## Tasks
+
+### T01 — Define backup directory and encryption wrapper
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
+```
+
+Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
+
+- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
+- Encrypt each artifact with age using public key from `.sops.yaml`
+- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
+- Keep last 7 of each type
+- Write `.last-backup` stamp
+- Exit 0 on success, non-zero on any failure
+- No network required
+
+Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
+custodian DB — wrong scope, not applicable to this server).
+
+**Done when:** `make backup` runs on COULOMBCORE without error and files
+appear in `/opt/backup/railiance/cluster/`.
+
+---
+
+### T02 — Back up k3s state (SQLite hot backup)
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
+```
+
+k3s has built-in etcd snapshot support:
+
+```bash
+sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
+# Default location: /var/lib/rancher/k3s/server/db/snapshots/
+```
+
+Add to the backup script: take a fresh snapshot, encrypt with age,
+copy to `/opt/backup/railiance/cluster/`.
+
+> **Note — verify etcd is in use before implementing:**
+> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
+> Without it, k3s uses SQLite and this command will fail.
+> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
+
+> **Note — sudo required:** etcd snapshot requires root. See T06 for how
+> this is resolved (backup runs under root's crontab).
+
+**Done when:** backup includes a current etcd snapshot.
+
+---
+
+### T03 — Back up Helm release values
+
+```task
+id: T03
+status: done
+priority: medium
+state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
+```
+
+Capture current runtime Helm values for all releases:
+
+```bash
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
+ jq -r '.[].name + " " + .namespace' | \
+ while read name ns; do
+ KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
+ done
+```
+
+Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
+
+> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
+> only by default. The backup script must either run as root (see T06) or k3s
+> must be configured with `--write-kubeconfig-mode=644`. Running as root
+> (via root crontab) is the chosen approach — no config change needed.
+
+**Done when:** backup includes a snapshot of all Helm release values.
+
+---
+
+### T04 — Back up kubeconfig
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
+```
+
+Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
+into `kubeconfig-<ts>.yaml.age` in the backup directory.
+
+**Done when:** backup includes the encrypted kubeconfig.
+
+---
+
+### T05 — make restore target
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
+```
+
+Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
+backups, with guided restore for the etcd snapshot case.
+
+Restore of etcd from snapshot:
+```bash
+sudo k3s server --cluster-reset \
+ --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
+```
+
+**Done when:** `make restore` prints available backups and a restore guide.
+
+---
+
+### T06 — Install cron job and run restore drill
+
+```task
+id: T06
+status: done
+priority: medium
+state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
+```
+
+#### Solving the sudo problem
+
+The backup script needs root for two reasons:
+- `k3s etcd-snapshot save` requires root
+- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
+
+**Solution: run the cron under root's crontab.**
+
+This is the correct pattern for system-level backup jobs. It avoids a
+proliferating sudoers whitelist (one entry per command, brittle to maintain)
+and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
+production. The backup writes to `/opt/backup/` which is root-owned anyway.
+
+Install the cron as root:
+
+```bash
+sudo crontab -e
+# Add:
+0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
+```
+
+Note: use the absolute path to the repo — `~` does not expand reliably in
+root's crontab unless HOME is set.
+
+Verify it is installed:
+```bash
+sudo crontab -l | grep railiance
+```
+
+#### Restore drill
+
+Once T01–T04 are done, run a decrypt-and-verify drill:
+
+```bash
+# Decrypt the etcd snapshot and verify it is a valid snapshot file
+sudo age -d -i ~/.config/sops/age/keys.txt \
+ /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
+ | file -
+
+# Record the drill
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
+ >> /opt/backup/railiance/cluster/restore-drill.log
+```
+
+**Done when:** cron installed under root, drill completes without error,
+log entry written.
+
+---
+
+## References
+
+- Decision D4: Integrated backup per capability (`DECISIONS.md`)
+- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
+- OAS Q3: Operability & Resilience
+- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
+- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore

195
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/api/services/historical_projection.py Normal file
View file

@ -0,0 +1,195 @@
"""Exact closed source bindings predating canonical Forge identity.
These are frozen historical source views, not aliases or UUID allocation rules.
An altered/missing source requires explicit owner review; generic source UUIDs
remain untrusted. Shared Activity views represent the one archived parent.
"""
from pathlib import Path
from hashlib import sha256
import json
BINDINGS = json.loads(r'''[
{
"repo": "activity-core",
"path": "workplans/archived/260707-custodian-WP-0003-event-bridge.md",
"sha256": "83ea8bcb151e003419b73c59dbd028f4a1ce3d1120a82e4a128a772503ff3b76",
"record_id": "custodian-WP-0003",
"uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5",
"parent_record_id": "custodian-WP-0003"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0001-temporal-backbone.md",
"sha256": "020e2eac973f2daa41c1687f8e867da9d521f945547e6aa82c87993b3ff6fc64",
"record_id": "custodian-WP-0001",
"uuid": "e22516cc-4fdf-4dad-a928-a9e04a9fa7c7",
"parent_record_id": "custodian-WP-0001"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0002-triggers-ops.md",
"sha256": "d8cbc61631c01b6f11a03a63f2fb6275fc853ff7f7459ce65973ed1a233daa01",
"record_id": "custodian-WP-0002",
"uuid": "3a4f47d9-8bc1-434e-acb4-bed5d4dacda0",
"parent_record_id": "custodian-WP-0002"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0003a-event-bridge-model-rules-registry.md",
"sha256": "ffa6682e9ba3aaf559bd44203d561c90dea69bcf1b1e778209a7b209e6e5cdd3",
"record_id": "custodian-WP-0003a",
"uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5",
"parent_record_id": "custodian-WP-0003"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0003b-event-bridge-parser-workflow-triggers.md",
"sha256": "4fc34e0f821e2df51e5c08c7ab34da35b65690381d01f0cfdf55d00c970b414c",
"record_id": "custodian-WP-0003b",
"uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5",
"parent_record_id": "custodian-WP-0003"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0003c-event-bridge-context-integration.md",
"sha256": "b6cd62b26515e509670ca39736b6bcec9633beedd46c9bd8f7598c06bac7276b",
"record_id": "custodian-WP-0003c",
"uuid": "b4eb45a9-69e3-4ab0-b00c-67a53c3117c5",
"parent_record_id": "custodian-WP-0003"
},
{
"repo": "activity-core",
"path": "workplans/custodian-WP-0004-railiance-ops.md",
"sha256": "20df02eea740e109c937130e7f8801a636fc0990ed8c6d2ced55659f8957adb1",
"record_id": "custodian-WP-0004",
"uuid": "759b1255-aa78-42b5-8fab-c5fcf168f5f4",
"parent_record_id": "custodian-WP-0004"
},
{
"repo": "the-custodian",
"path": "workplans/ADHOC-2026-08-25-retired.md",
"sha256": "344a1293dbb3a67a944eb34ca4a0337487d8f047226027a2107922346fbab2dd",
"record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED",
"uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e",
"parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED"
},
{
"repo": "railiance-cluster",
"path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
"sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9",
"record_id": "RAIL-BS-WP-0002",
"uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d",
"parent_record_id": "RAIL-BS-WP-0002",
"tasks": [
{
"label": "T01",
"uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40",
"status": "done"
},
{
"label": "T02",
"uuid": "e8510646-46ed-4697-a345-f3d3009eea78",
"status": "done"
},
{
"label": "T03",
"uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5",
"status": "done"
},
{
"label": "T04",
"uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f",
"status": "done"
},
{
"label": "T05",
"uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f",
"status": "done"
}
]
},
{
"repo": "railiance-cluster",
"path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
"sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a",
"record_id": "RAIL-BS-WP-0003",
"uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b",
"parent_record_id": "RAIL-BS-WP-0003",
"tasks": [
{
"label": "T01",
"uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc",
"status": "done"
},
{
"label": "T02",
"uuid": "ab166073-30a7-4702-a037-4091e8706e20",
"status": "done"
},
{
"label": "T03",
"uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01",
"status": "done"
},
{
"label": "T04",
"uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d",
"status": "done"
}
]
},
{
"repo": "railiance-cluster",
"path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
"sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3",
"record_id": "RAIL-BS-WP-0004",
"uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625",
"parent_record_id": "RAIL-BS-WP-0004",
"tasks": [
{
"label": "T01",
"uuid": "4526a842-ea31-4874-9231-92ab556cfe7b",
"status": "done"
},
{
"label": "T02",
"uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880",
"status": "done"
},
{
"label": "T03",
"uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3",
"status": "done"
},
{
"label": "T04",
"uuid": "08233868-d522-4117-bc4e-6c0f52545665",
"status": "done"
},
{
"label": "T05",
"uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8",
"status": "done"
},
{
"label": "T06",
"uuid": "f8e4a094-c367-40eb-b895-da17bc144b07",
"status": "done"
}
]
}
]''')
def bound_sources(repo_root: Path, repo_slug: str):
entries = [entry for entry in BINDINGS if entry["repo"] == repo_slug]
result = []
for entry in entries:
path = repo_root / entry["path"]
if path.is_symlink() or not path.is_file():
raise ValueError("Historical source binding missing or symlinked: " + entry["path"])
data = path.read_bytes()
if sha256(data).hexdigest() != entry["sha256"]:
raise ValueError("Historical source binding changed: " + entry["path"])
result.append((entry, data.decode("utf-8")))
return result

177
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md vendored Normal file
View file

@ -0,0 +1,177 @@
---
id: RAIL-BS-WP-0002
type: workplan
title: "k3s and Kubernetes Platform Baseline"
domain: financials
repo: railiance-cluster
status: finished
owner: railiance
topic_slug: railiance
repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
created: "2026-03-09"
updated: "2026-03-10"
completed: "2026-03-10"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# k3s and Kubernetes Platform Baseline
## Goal
Install k3s, Helm, and the baseline Kubernetes services on the converged
HostEurope node. This workplan picks up exactly where `railiance-hosts`
leaves off: a hardened, verified OS node that is ready for Kubernetes.
## Pre-condition
`railiance-infra` converge + Goss verify must pass before any task here
is executed:
```bash
# In railiance-infra/
make converge
make verify # must exit 0
```
## Boundary
This repo owns everything from k3s upward. It must not re-configure items
defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
**Out of scope here:** platform services (PostgreSQL, storage, identity)
→ `railiance-platform`. Application deployments (Gitea, coulomb services)
→ `railiance-apps`.
---
## Tasks
### T01 — Ansible playbook: install k3s (server mode)
```task
id: T01
status: done
priority: high
state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
completed: "2026-03-10"
```
Harden `ansible/bootstrap.yml` to a production-ready k3s install:
- Use the official k3s install script pinned to a specific version
(`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
(cluster-init enables embedded etcd for future HA expansion)
- Wait for node `Ready` before proceeding:
```bash
k3s kubectl wait node --all --for=condition=Ready --timeout=120s
```
- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
and the control node (via kubeconfig).
---
### T02 — Helm installation
```task
id: T02
status: done
priority: high
state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
completed: "2026-03-10"
```
Add a task (or a role `roles/helm/`) that:
1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
2. Verifies the checksum
3. Confirms `helm version` succeeds
**Done when:** `helm version` succeeds on the HostEurope node.
---
### T03 — Smoke test: k3s + Helm
```task
id: T03
status: done
priority: high
state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
completed: "2026-03-10"
```
Extend `tests/smoke_kube.sh` to assert:
- `k3s kubectl get nodes` → node in Ready state
- `helm version` exits 0
- CoreDNS pod running in `kube-system`
- Traefik ingress controller pod running (default in k3s)
Run via:
```bash
ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
```
or directly over SSH if the kubeconfig is available locally.
**Done when:** all assertions pass and the script exits 0.
---
### T04 — Commit kubeconfig management notes
```task
id: T04
status: done
priority: medium
state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
completed: "2026-03-10"
```
Document in `docs/kubeconfig.md`:
- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
- How to merge it into `~/.kube/config`
- How to switch context: `kubectl config use-context default`
- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
**Done when:** doc written and committed.
---
### T05 — Add `make k3s-install` and `make smoke` targets
```task
id: T05
status: done
priority: medium
state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
completed: "2026-03-10"
```
Add to Makefile (create one if none exists):
```makefile
k3s-install: ## Install k3s and Helm on all inventory hosts
ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
smoke: ## Run Kubernetes smoke tests
bash tests/smoke_kube.sh
```
**Done when:** both targets work and are listed in `make help`.
---
## References
- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
- k3s releases: https://github.com/k3s-io/k3s/releases

196
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md vendored Normal file
View file

@ -0,0 +1,196 @@
---
id: RAIL-BS-WP-0003
type: workplan
title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
domain: financials
repo: railiance-cluster
status: finished
owner: tegwick
created: "2026-03-10"
updated: "2026-03-10"
state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
## Summary
On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
entered CrashLoopBackOff and produced no logs. As a result Gitea's login
and all write operations hung indefinitely. The root page was still served
(from Valkey cache) which masked the failure.
The fix was to patch a missing key in a Kubernetes secret. The root cause is
that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
secret, even though the pgpool pod requires it at startup.
---
## Timeline
| Time (UTC) | Event |
|---|---|
| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
| ~11:00 | User noticed Gitea login hanging; home page still loading |
| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
| ~13:15 | Gitea fully operational |
---
## Root Cause
The Bitnami `pgpool` container startup script reads the file
`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
mount. That secret key was never created by the Helm chart, so the file did
not exist. The container exited immediately with no logs.
The pod had been running for 20 days without a restart, so this gap was
never discovered during initial deployment.
---
## Evidence
```bash
# Secret was missing the pgpool-password key
sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
# data: keys were password, postgres-password, repmgr-password only
# pgpool-password was absent
# pgpool pod describe showed 824 back-off restarts over 173 minutes
# No logs in either current or --previous output
sudo k3s kubectl logs -n default <pgpool-pod> --previous
# (empty)
# Gitea process had zero TCP connections to PostgreSQL port 5432
# but many connections to Valkey port 6379
cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
```
---
## Immediate Fix Applied
```bash
# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
--type='json' \
-p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
# Restart pgpool
sudo k3s kubectl delete pod -n default <pgpool-pod-name>
```
---
## Risk: Fix Will Be Lost on helm upgrade
The patched secret is managed by Helm (annotation:
`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
secret from the chart template, which does not include `pgpool-password`,
and the bug will recur.
---
## Tasks
### T01 — Add pgpool-password to Helm values
```task
id: T01
status: done
priority: high
state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
```
Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
include the pgpool-password so it survives `helm upgrade`:
```yaml
postgresql-ha:
postgresql:
pgpoolPassword: <value matching sr-check-password>
```
**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
without manual secret patching.
---
### T02 — Add pgpool health check to smoke test
```task
id: T02
status: done
priority: high
state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
```
Extend `tests/smoke_kube.sh` to assert:
```bash
# All postgresql-ha pods Running
kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
# pgpool specifically not in CrashLoopBackOff
kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
-o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
```
**Done when:** the smoke test catches a pgpool failure within 5 minutes.
---
### T03 — Add HA failover test
```task
id: T03
status: done
priority: high
state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
```
Create `tests/test_ha_failover.sh` that:
1. Records Gitea login response time (baseline)
2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
3. Waits for repmgr to promote a replica (max 60s)
4. Asserts Gitea login POST still succeeds within 10s
5. Asserts pgpool pod is Running (not CrashLoopBackOff)
6. Asserts all postgresql pods return to Running
This test must pass before any PostgreSQL HA deployment is considered done.
**Done when:** script exits 0 against a live cluster.
---
### T04 — Document the incident in docs/
```task
id: T04
status: done
priority: medium
state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
```
Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
timeline, root cause, and fix, so future operators understand what happened
and how to recover.
**Done when:** doc committed and linked from `docs/README.md`.
---
## References
- Bitnami postgresql-ha chart v16.2.2
- Gitea Helm chart v12.2.0
- Related decision: D3 (HA testing policy) in `DECISIONS.md`

275
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md vendored Normal file
View file

@ -0,0 +1,275 @@
---
id: RAIL-BS-WP-0004
type: workplan
title: "Integrated Backup — S2 Kubernetes Runtime Layer"
domain: financials
repo: railiance-cluster
status: finished
owner: tegwick
topic_slug: railiance
state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
created: "2026-02-25"
updated: "2026-03-26"
---
> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
# Integrated Backup — S2 Kubernetes Runtime Layer
## Goal
Implement the Q3 (Operability & Resilience) integrated backup for
railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
encrypted with age, written to a local directory on the server. No external
dependencies required.
## Architecture (Decision D4)
Each railiance repo implements its own backup for what it owns. No central
backup service. See `DECISIONS.md` D4 for full rationale.
**Standard interface every railiance repo must provide:**
```bash
make backup # encrypt + write to /opt/backup/railiance/<layer>/
make restore # restore from most recent local backup
```
Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
Output: `/opt/backup/railiance/cluster/` on the server.
## What S2 (railiance-cluster) owns and must back up
| Asset | Why it matters |
|---|---|
| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
| Helm release values | Runtime values not in git (any manually applied overrides) |
| kubeconfig | Admin access to the cluster |
**Not S2's responsibility:**
- Custodian State Hub DB → the-custodian owns this
- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
- PostgreSQL data volumes → S3 (railiance-platform) owns this
## Encryption
Reuse the age public key from `.sops.yaml`:
```bash
AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
```
Decryption requires the private key at `~/.config/sops/age/keys.txt`
(same key used for `sops -d`). No additional key management needed.
## Extension Point EP-RAIL-005
Once all five OAS layers implement this interface, the custodian can
orchestrate a full-stack backup with:
```bash
for repo in railiance-infra railiance-cluster railiance-platform \
railiance-enablement railiance-apps; do
make -C ~/$repo backup
done
```
No special protocol needed — just the standard interface.
---
## Tasks
### T01 — Define backup directory and encryption wrapper
```task
id: T01
status: done
priority: high
state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
```
Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
- Encrypt each artifact with age using public key from `.sops.yaml`
- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
- Keep last 7 of each type
- Write `.last-backup` stamp
- Exit 0 on success, non-zero on any failure
- No network required
Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
custodian DB — wrong scope, not applicable to this server).
**Done when:** `make backup` runs on COULOMBCORE without error and files
appear in `/opt/backup/railiance/cluster/`.
---
### T02 — Back up k3s state (SQLite hot backup)
```task
id: T02
status: done
priority: high
state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
```
k3s has built-in etcd snapshot support:
```bash
sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
# Default location: /var/lib/rancher/k3s/server/db/snapshots/
```
Add to the backup script: take a fresh snapshot, encrypt with age,
copy to `/opt/backup/railiance/cluster/`.
> **Note — verify etcd is in use before implementing:**
> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
> Without it, k3s uses SQLite and this command will fail.
> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
> **Note — sudo required:** etcd snapshot requires root. See T06 for how
> this is resolved (backup runs under root's crontab).
**Done when:** backup includes a current etcd snapshot.
---
### T03 — Back up Helm release values
```task
id: T03
status: done
priority: medium
state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
```
Capture current runtime Helm values for all releases:
```bash
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
jq -r '.[].name + " " + .namespace' | \
while read name ns; do
KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
done
```
Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
> only by default. The backup script must either run as root (see T06) or k3s
> must be configured with `--write-kubeconfig-mode=644`. Running as root
> (via root crontab) is the chosen approach — no config change needed.
**Done when:** backup includes a snapshot of all Helm release values.
---
### T04 — Back up kubeconfig
```task
id: T04
status: done
priority: medium
state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
```
Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
into `kubeconfig-<ts>.yaml.age` in the backup directory.
**Done when:** backup includes the encrypted kubeconfig.
---
### T05 — make restore target
```task
id: T05
status: done
priority: medium
state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
```
Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
backups, with guided restore for the etcd snapshot case.
Restore of etcd from snapshot:
```bash
sudo k3s server --cluster-reset \
--cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
```
**Done when:** `make restore` prints available backups and a restore guide.
---
### T06 — Install cron job and run restore drill
```task
id: T06
status: done
priority: medium
state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
```
#### Solving the sudo problem
The backup script needs root for two reasons:
- `k3s etcd-snapshot save` requires root
- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
**Solution: run the cron under root's crontab.**
This is the correct pattern for system-level backup jobs. It avoids a
proliferating sudoers whitelist (one entry per command, brittle to maintain)
and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
production. The backup writes to `/opt/backup/` which is root-owned anyway.
Install the cron as root:
```bash
sudo crontab -e
# Add:
0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
```
Note: use the absolute path to the repo — `~` does not expand reliably in
root's crontab unless HOME is set.
Verify it is installed:
```bash
sudo crontab -l | grep railiance
```
#### Restore drill
Once T01–T04 are done, run a decrypt-and-verify drill:
```bash
# Decrypt the etcd snapshot and verify it is a valid snapshot file
sudo age -d -i ~/.config/sops/age/keys.txt \
/opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
| file -
# Record the drill
echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
>> /opt/backup/railiance/cluster/restore-drill.log
```
**Done when:** cron installed under root, drill completes without error,
log entry written.
---
## References
- Decision D4: Integrated backup per capability (`DECISIONS.md`)
- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
- OAS Q3: Operability & Resilience
- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore

67
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/tests/test_cluster_historical_projection.py Normal file
View file

@ -0,0 +1,67 @@
import shutil
import uuid
from pathlib import Path
import pytest
from api.services import forge_projection as fp
from api.services import historical_projection as hp
from tests.test_forge_projection import _FakeSession,_Repo,_Row,_TaskRow
FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
@pytest.fixture
def bound(tmp_path):
root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
derived=fp.derive_from_checkout(root,'railiance-cluster','candidate')
assert len(derived.workplans)==3
rows=[];tasks=[]
for parent in derived.workplans:
row=_Row(parent.record_id.lower(),'finished',parent.relative_path);row.id=uuid.UUID(parent.uuid);rows.append(row)
for source in parent.tasks:
task=_TaskRow(source.record_id,status='done',workplan_id=row.id);task.id=uuid.UUID(source.uuid);tasks.append(task)
assert len(tasks)==15
return root,derived,rows,tasks
def test_exact_three_sources_derive_only_frozen_15_task_uuids(bound):
root,derived,rows,tasks=bound
assert all(parent.historical_preservation for parent in derived.workplans)
assert all(parent.uuid!=fp.derived_record_uuid(parent.record_id) for parent in derived.workplans)
expected={item['uuid'] for entry in hp.BINDINGS if entry['repo']=='railiance-cluster' for item in entry['tasks']}
assert {str(t.id) for t in tasks}==expected
@pytest.mark.asyncio
async def test_existing_rows_are_entirely_unchanged_on_normal_receiving(bound):
root,derived,rows,tasks=bound;before=[dict(x.__dict__) for x in rows+tasks]
session=_FakeSession(_Repo(),rows,task_rows=tasks)
outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
assert outcome.status=='noop' and not outcome.refused
assert [dict(x.__dict__) for x in rows+tasks]==before
assert not session.added and not session.deleted
@pytest.mark.parametrize('defect',['missing-parent','missing-task','foreign-holder','nonterminal-task','nonterminal-parent','duplicate-current-label'])
@pytest.mark.asyncio
async def test_unsupported_actual_binding_refuses_before_any_mutation(bound,defect):
root,derived,rows,tasks=bound
if defect=='missing-parent':rows.pop()
elif defect=='missing-task':tasks.pop()
elif defect=='foreign-holder':tasks[0].workplan_id=uuid.uuid4()
elif defect=='nonterminal-task':tasks[0].status='progress'
elif defect=='duplicate-current-label':tasks.append(_TaskRow(tasks[0].record_id,status='done',workplan_id=tasks[0].workplan_id))
else:rows[0].status='active'
before=[dict(x.__dict__) for x in rows+tasks];session=_FakeSession(_Repo(),rows,task_rows=tasks)
outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
assert outcome.status=='refused';assert [dict(x.__dict__) for x in rows+tasks]==before
assert not session.added and not session.deleted
@pytest.mark.parametrize('change',['hash-drift','missing','symlink','live-source','missing-source-uuid'])
def test_frozen_source_drift_refuses_derivation(bound,change):
root,*_=bound;p=next((root/'workplans/archived').glob('*.md'));raw=p.read_bytes()
if change=='missing':p.unlink()
elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
elif change=='live-source':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
elif change=='missing-source-uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
else:p.write_bytes(raw+b'\nunreviewed\n')
with pytest.raises(fp.ForgeDeriveError,match='Historical source binding'):fp.derive_from_checkout(root,'railiance-cluster','candidate')
def test_unknown_repository_never_trusts_frozen_source_uuids(bound):
root,derived,*_=bound;other=fp.derive_from_checkout(root,'different-repo','candidate')
assert not any(parent.historical_preservation for parent in other.workplans)
assert not {parent.uuid for parent in derived.workplans}&{parent.uuid for parent in other.workplans}

239
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md Normal file
View file

@ -0,0 +1,239 @@
---
id: STATE-WP-0095
type: workplan
title: "Preserve exact closed historical source bindings in Forge receiving"
domain: infotech
repo: state-hub
status: active
owner: codex
created: "2026-10-04"
updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058"
---
## Goal
Normal Forge receiving must preserve the seven bound Activity legacy source
views and the Custodian retired ADHOC source without new identities, source
rewrites, task movement, deletion, relabeling or unretirement. This implements
receiving support before RMGR-WP-0019 registrar admission. CUST-WP-0077 owns
cross-owner coordination; deployment and replay remain explicit owner returns.
## Verify source bindings and shared views
```task
id: STATE-WP-0095-T01
status: done
priority: high
state_hub_task_id: "6a5f87bb-e63d-5e20-b9fc-c1b5cd403ff7"
```
Reviewed seven exact Activity files: four existing workplans and 64 UUID-bound
closed tasks. The archived custodian-WP-0003 parent and three split_from views
share one existing parent UUID; every repeated task binding/content agrees.
Custodian ADHOC-2026-08-25-retired retains parent
1c53d3db-6633-5e2d-987f-10706d73be9e and task
d8af3e6b-26ae-580c-bbeb-caa8ad419bec. No identity migration is authorized.
## Implement preservation receiving and global collision refusal
```task
id: STATE-WP-0095-T02
status: done
priority: high
state_hub_task_id: "ebb24194-e4cc-5130-aaa6-663f2ea2c822"
```
Prepared exact source digest manifest, existing-parent/task verification,
immutable receiving and UUID-based historical diff. Ordinary canonical sources
continue deriving their identities. Missing/changed/symlinked source, absent
bound rows or nonterminal hub histories refuse. Check task UUID holders globally,
including retired and acknowledged-stale parent workplans, before INSERT.
Offline preview tests and snapshot proof are evidence of source behavior only;
finish after applying reviewed source and its focused tests to the owning repo.
## Deliver receiving through normal deployment and verify owner return
```task
id: STATE-WP-0095-T03
status: done
priority: high
depends_on: [STATE-WP-0095-T02]
state_hub_task_id: "e8566341-cf6e-5b9c-ad80-8564d14cd52a"
decision_id: "36772026-f43d-4331-b466-5f4b1afe4eca"
```
No direct live code patch. This task is a deployment return, not policy authority
or permission to resolve identity decisions. Exact receiving fixture acceptance
and global collision rejection must hold in the delivered version.
## Verify normal historical replay preserves existing identities
```task
id: STATE-WP-0095-T04
status: done
priority: high
depends_on: [STATE-WP-0095-T03, RMGR-WP-0019-T03]
state_hub_task_id: "3bbb311f-194e-5862-8aed-358a580f22b4"
```
Verify Activity four historical parent UUIDs/64 task UUIDs and Custodian retired
parent/task unchanged; no historical row creation, movement, deletion,
relabeling or unretirement. Unrelated stale projections remain separate decisions.
2026-10-05: Source exact-set retirement support now exists. Repository projection
reconcile accepts `expected_retirement_uuids`; that set must equal the stale
projections, and any task or dependency edge on a stale row refuses the call.
Live apply still needs a reviewed rollout. On 2026-10-05 the founder accepted
review `review-1ece4f3e-d0e3-4fd6-abc8-272f1b1876f4`, which names only the three
empty projections and preserves parent `b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`.
That acceptance is not a live retirement and does not resolve
`91b7c633-a66f-5d43-9f4b-6565aefb9356`.
## Offline evidence, 2026-10-04
Prepared receiver suite: Forge projection regression plus historical preservation,
missing/changed source, terminality, immutable UUID diff, and global holder query.
Real-source offline snapshot proof preserves four Activity parents/64 tasks and
one Custodian parent/task without mutations; no API writes or deployment occurred.
Exact original bindings are maintained in api/services/historical_projection.py.
2026-10-04 actual owning-source validation: 155 focused receiver/qualifier/planner
cases passed; the independent offline current-source/current-hub-snapshot proof
retained five existing parents and 65 task UUIDs, no mutations, clean historical
diff. T02 is source-complete; T03/T04 remain wait for reviewed deployment and
normal registrar/replay receiving, respectively. Evidence: docs/evidence/2026-10-04-historical-receiving-source.json.
## Verified receiving rollout and remaining replay gate, 2026-10-04
T03 is done on actual founder approval of proposal
STATE-WP-0095-T03-historical-receiving revision 1, SHA256
`d6789359b3fe7988a6201fff6f367cb46948aa197e1cc0ca66c31f45e1ea9981`,
approval receipt reference `user_reply:call_o87uEfFs3AtXfxO5YqUnSpcg:0`.
The normal owner Helm path delivered `main-5dd3df9`, commit
`5dd3df9c69eb564af10ea5ea01e349786212501a`, revision 72; API and MCP are Ready,
primary/railiance01 health and schema a0b1c2d3e4f5 verified, running source
hashes match. Rollback remains revision 71; no receipt-table downgrade is needed.
Actual historical preservation and global task-holder refusal were verified in
the delivered version. Durable receipt:
[verified historical receiving rollout](../docs/evidence/2026-10-04-historical-receiving-rollout.json).
Custodian immutable replay applied while preserving the retired historical
parent/task UUIDs. Activity immutable replay retained all four historical parent
and 64 task UUIDs and historical row contents, but refused transactionally on
three unrelated stale projections: custodian-wp-0003 (archived),
custodian-wp-0003a (done), custodian-wp-0003b (done). No retirement acknowledgement
was issued and that refusal is not represented as normal replay success.
Registrar source `c41d36c2` has 38 actual passing tests and Repo Manager's own
normal sync succeeded; registrar admission does not authorize retiring Activity
projections. T04 therefore remains wait, retaining RMGR-WP-0019-T03 and awaiting
Activity Core's precise reviewed disposition of those three stale rows. No
rename, identity migration, new paid native trial or further live action is
included in this source status update.
## Activity Core return, 2026-10-06
Activity Core returned the exact disposition (message
`cf2d5847-8b3f-4653-9d9f-d5fcfbe3f1e4`, ACTIVITY-WP-0045, activity-core commit
`941fac0`). `expected_retirement_uuids`:
`683724d9-4249-5e0e-88e0-7baff4330d58` (custodian-wp-0003),
`644017c7-c5f2-5a64-81c3-d8ee806f8251` (custodian-wp-0003a),
`a5806df9-5b9a-50da-81c1-7e606076bed9` (custodian-wp-0003b). Each has zero
tasks and zero dependency edges. None is preserved parent
`b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. Repo Manager independently verified the
set (message `916841f6`, repo-manager commit `498682c`). Nothing was retired.
T04 now waits on decision `91b7c633` and a reviewed live rollout. After that,
Repo Manager re-runs the Activity owner sync and closes RMGR-WP-0019-T03.
## Retirement rollout and T04 verification, 2026-10-06
The founder resolved decision `91b7c633` (retire exactly the three projections)
and approved the rollout. The founder ran the normal Helm upgrade to
`main-968a811` (revision 74; rollback revision 73, `main-fad9b3f`). API and MCP
are Ready, health is ok on primary/railiance01, and schema `b3c4d5e6f7a8` is
unchanged; no migrations. The activity-core reconcile at commit
`67a92d2ff8e033baf2c36998d24187fe83edec84` with `expected_retirement_uuids` set
to the three UUIDs applied: 0 created, 3 retired (tombstoned with an
`@retired-20261006` slug; UUIDs and statuses kept), 0 refused, 0 tasks
cancelled. All 7 bound Activity source files still match their digests. The four
historical parents and 64 task UUIDs are present under their bound parents with
unchanged fields. The Custodian retired parent and task are unchanged. Activity
had 54 workplans and 288 tasks before and after. Evidence:
[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json).
Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and
ACTIVITY-WP-0045-T02 remain with their owners.
## Cluster exact historical receiving successor, 2026-10-17
This scoped extension retains the completed Activity/Custodian outcomes above.
Three frozen cluster archives contain three existing parent UUIDs and 15 existing
closed task UUIDs. Extend the existing exact historical receiving path before
registrar admission; never derive replacement identities or modify those
sources. Cross-owner observation attaches to existing SHR-WP-0001, with
RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly.
The local derived case is waiting; no live worker injection is authorized.
## Prepare exact cluster source preservation and refusal cases
```task
id: STATE-WP-0095-T05
status: done
priority: high
state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b"
```
Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager.
Candidate receiving tests pass 26: existing rows remain entirely unchanged;
missing/foreign/nonterminal rows and unknown, changed, missing or symlinked
source refuse. Read-only public metadata confirms all 18 existing rows have the
expected ownership and terminal status. These are source and metadata checks,
not delivered receiving or a committed native trial.
## Deliver reviewed cluster receiving successor before registrar admission
```task
id: STATE-WP-0095-T06
status: wait
priority: high
depends_on: [STATE-WP-0095-T05]
needs_human: true
intervention_note: "Review exact committed source, CI image and owner deployment proposal before rollout."
state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4"
```
Use the normal CI image and Helm owner path after exact revision/image pin,
fresh headroom and reviewed dry-run/diff. API/MCP currently run main-7d99282;
Helm revision75 is the observed rollback baseline, requiring freshness before
execution. No deployment assent exists for this successor. Verify delivered
module hash, health, both readiness checks and expected migrated schema
c4d5e6f7a8b9; verify exact
three parents/15 tasks and rejection behavior without committed probe changes.
Only the verified return unblocks RMGR-WP-0019-T05. The full current image also carries inbox-feed migration c4d5e6f7a8b9,
although this receiver patch adds none. Helm rollback leaves that migration in
place; unrelated migration acceptance and old-image compatibility need normal
owner evidence before rollout approval. Do not run destructive default local
DB tests; use only exact disposable DB containment. Costs and monetary risk are
unknown pending exact rollout proposal; no zero-cost or autonomy grant is inferred.
## Verify ordinary cluster receiving and immutable historical rows
```task
id: STATE-WP-0095-T07
status: wait
priority: high
depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06]
state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d"
```
Capture normal cluster owner publication/receiving receipts and compare entire
existing row snapshots and exact source bytes. No creation, movement, relabel,
unretirement or retirement is authorized for these histories. A rollback first
fences cluster sync and restores registrar refusal before removing receiving
support; it never rewrites archived IDs or UUIDs. This work does not establish
original TLS recovery or owner-native admission.

104
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/test_publication_helpers.py Normal file
View file

@ -0,0 +1,104 @@
import hashlib
import importlib.util
import json
from pathlib import Path
import subprocess
import sys
import pytest
def load(name):
path=Path(__file__).parent/(name+'.py')
spec=importlib.util.spec_from_file_location(name,path)
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
return module
pub=load('publish_receiver_source')
ci=load('lookup_ci_metadata')
def git(root,*args):
return subprocess.check_output(['git','-C',str(root),*args]).decode().strip()
@pytest.fixture
def repo(tmp_path):
root=tmp_path/'owner';root.mkdir()
git(root,'init','-b','main');git(root,'config','user.name','Synthetic Test');git(root,'config','user.email','synthetic@example.invalid')
for name in ['source.py','.custodian-brief.md','WORK-RECORDS.md','evidence.json']:
(root/name).write_text('old\n')
git(root,'add','.');git(root,'commit','-m','baseline')
base=git(root,'rev-parse','HEAD')
(root/'.custodian-brief.md').write_text('staged brief\n');git(root,'add','.custodian-brief.md')
(root/'.custodian-brief.md').write_text('working brief\n')
(root/'WORK-RECORDS.md').write_text('working index\n');(root/'evidence.json').write_text('working evidence\n')
(root/'new-evidence.json').write_text('new evidence\n')
(root/'source.py').write_text('reviewed source\n');(root/'new-test.py').write_text('reviewed new test\n')
expected={'base_revision':base,'files':[{'path':p,'candidate_sha256':hashlib.sha256((root/p).read_bytes()).hexdigest()} for p in ['source.py','new-test.py']]}
return root,expected
def test_exact_commit_preserves_staged_brief_and_worktree_evidence(repo,tmp_path):
root,expected=repo
remote=tmp_path/'remote.git';subprocess.run(['git','init','--bare',str(remote)],check=True,capture_output=True)
git(root,'remote','add','origin',str(remote))
owned={f['path'] for f in expected['files']};before=pub.preserved_state(root,owned)
result=pub.commit_source(root,expected,[sys.executable,'-c','raise SystemExit(0)'],execute=True,push=True)
assert result['status']=='source_published_verified'
assert pub.preserved_state(root,owned)==before
assert git(root,'show',':.custodian-brief.md')=='staged brief'
assert git(root,'show','HEAD:.custodian-brief.md')=='old'
assert set(git(root,'diff-tree','--no-commit-id','--name-only','-r','HEAD').splitlines())==owned
def test_default_is_read_only_and_drift_refuses_before_commit(repo):
root,expected=repo;head=git(root,'rev-parse','HEAD');stage=git(root,'ls-files','--stage')
assert pub.commit_source(root,expected,[])['status']=='validated_not_executed'
assert git(root,'rev-parse','HEAD')==head and git(root,'ls-files','--stage')==stage
(root/'source.py').write_text('unreviewed change')
with pytest.raises(pub.Refusal,match='reviewed_source_drift'):
pub.commit_source(root,expected,[],execute=True)
assert git(root,'rev-parse','HEAD')==head and git(root,'ls-files','--stage')==stage
def test_normal_scanner_refusal_does_not_commit_or_change_unrelated_state(repo):
root,expected=repo;head=git(root,'rev-parse','HEAD')
owned={f['path'] for f in expected['files']};before=pub.preserved_state(root,owned)
with pytest.raises(pub.Refusal,match='normal_source_scanner_refused'):
pub.commit_source(root,expected,[sys.executable,'-c','raise SystemExit(1)'],execute=True)
assert git(root,'rev-parse','HEAD')==head and pub.preserved_state(root,owned)==before
@pytest.mark.parametrize('task_status,digest,expected',[('success','sha256:'+'b'*64,'artifact_metadata_verified'),('failure','sha256:'+'b'*64,'artifact_unverified'),('success',None,'artifact_unverified')])
def test_ci_and_digest_both_required(monkeypatch,task_status,digest,expected):
revision='a'*40
tasks={'workflow_runs':[{'head_sha':revision,'name':'build-and-push','status':task_status,'id':12}]}
def response(url,*args):
return {'status':'ok','body':json.dumps(tasks).encode()} if url==ci.TASKS else {'status':'ok','digest':digest}
monkeypatch.setattr(ci,'request',response)
assert ci.lookup(revision)['status']==expected
def test_public_auth_refusal_does_not_attempt_credential_fallback(monkeypatch):
calls=[]
def refused(url,*args):calls.append(url);return {'status':'public_access_refused','http_status':401}
monkeypatch.setattr(ci,'request',refused)
result=ci.lookup('a'*40)
assert result['status']=='artifact_unverified' and len(calls)==2
assert 'image_digest' not in result
def test_exact_revision_required_before_network(monkeypatch):
monkeypatch.setattr(ci,'request',lambda *a:pytest.fail('network must not start'))
with pytest.raises(ValueError,match='exact40char_revision_required'):ci.lookup('a'*7)
preflight=load('preflight_receiver')
@pytest.mark.parametrize('target,headroom,status,expected_calls',[(False,True,1,1),(True,False,1,2),(True,True,0,3)])
def test_preflight_refuses_wrong_target_or_headroom_and_never_deploys(monkeypatch,capsys,target,headroom,status,expected_calls):
calls=[]
def command(argv,env,timeout):
calls.append(argv)
assert env['KUBECONFIG']==preflight.KUBECONFIG
if argv[0]=='kubectl':return 0,json.dumps({'items':[{'status':{'addresses':[{'address':'92.205.62.239' if target else '192.0.2.1'}]}}]}).encode()
if 'railiance-state-hub-headroom' in argv:return 0,json.dumps({'ok':headroom,'reasons':[]}).encode()
assert 'railiance-state-hub-server-dry-run' in argv
assert 'RAILIANCE_STATE_HUB_IMAGE_TAG=main-aaaaaaa' in argv
return 0,b'sensitive internal diagnostics must not be emitted'
monkeypatch.setattr(preflight,'command',command)
monkeypatch.setattr(sys,'argv',['preflight','--source-revision','a'*40])
assert preflight.main()==status and len(calls)==expected_calls
emitted=capsys.readouterr().out
assert 'sensitive internal diagnostics' not in emitted
assert json.loads(emitted)['production_mutations'] is False

17
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/candidate-make-serverdryrun-result.json Normal file
View file

@ -0,0 +1,17 @@
{
"schema": "receiver-policy-compatible-make-serverdryrun.v1",
"candidate_makefile_sha256": "sha256:5584ef94071044cca8215141ecd8aee8d6ae5cfe73e8f5719358c680eb8af0d4",
"image_tag": "main-51bc70d",
"target_kubeconfig_fixed": true,
"status": "passed",
"exit_code": 0,
"classification": null,
"production_mutations": false,
"actual_secret_writes": false,
"raw_diagnostics_retained": false,
"force_conflicts_used": false,
"prior_make_failure_preserved": true,
"limitations": [
"Server admission only; not exact Helm reuse-values upgrade equivalence or rollout assent."
]
}

39
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/image-selected-source-comparison.json Normal file
View file

@ -0,0 +1,39 @@
{
"schema": "receiver-selected-image-source-comparison.v1",
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"image_digest": "sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015",
"selected_files": [
{
"path": "api/services/historical_projection.py",
"expected_sha256": "sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2",
"image_sha256": "sha256:e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2",
"matches": true,
"comparison_status": "match"
},
{
"path": "api/models/agent_message.py",
"expected_sha256": "sha256:97c3bf5f66304d0ddefbb5f0fa7889e087b17b9c9064676f3a8221c8865914a0",
"image_sha256": null,
"matches": false,
"comparison_status": "not_selected_in_receipt"
},
{
"path": "api/routers/message_feed.py",
"expected_sha256": "sha256:8e71d80397cd8cfa446682a14e6f4b1aabc3db17c748a279f95da61acf9b9aa9",
"image_sha256": null,
"matches": false,
"comparison_status": "not_selected_in_receipt"
},
{
"path": "migrations/versions/c4d5e6f7a8b9_inbox_feed_revision.py",
"expected_sha256": "sha256:48f1730037be99b5623e878d7f20a06bec243ce27317fde81c82e16006d7b3ce",
"image_sha256": "sha256:48f1730037be99b5623e878d7f20a06bec243ce27317fde81c82e16006d7b3ce",
"matches": true,
"comparison_status": "match"
}
],
"image_executed": false,
"scope": "Selected source bytes only; package origin/hermetic or full runtime acceptance not inferred.",
"all_observed_match": true,
"full_requested_selection_observed": false
}

19
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/managed-task-uuid-assignment.json Normal file
View file

@ -0,0 +1,19 @@
{
"schema": "repo-manager.identifier-assignment.v1",
"ok": true,
"executed": true,
"repo": "state-hub",
"namespace": "helixforge",
"assignments": [
{
"kind": "task",
"record_id": "STATE-WP-0095-T08",
"uuid": "2dd9b04f-2250-5a8a-b5bc-c8a009872858",
"path": "workplans/STATE-WP-0095-historical-source-receiving.md"
}
],
"historical_invalid_identifiers": [],
"files_changed": [
"workplans/STATE-WP-0095-historical-source-receiving.md"
]
}

31
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/review-freeze.json Normal file
View file

@ -0,0 +1,31 @@
{
"schema": "statehub-readiness-review-freeze.v1",
"stable_proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1",
"revision": 3,
"files": [
{
"path": "ROLLOUT-SUPPLEMENT.md",
"sha256": "sha256:df2962fd1588784a987e37932bd67f03f6248ea6384cf4bbb74ad1fe2e26be2b"
},
{
"path": "source-patch-manifest.json",
"sha256": "sha256:98211e65bebeee2c25ca75c6b9f0bc45128093baaba02069b6c41a15be32fe74"
},
{
"path": "state-hub-readiness.patch",
"sha256": "sha256:4c52fd5a102df43ceb8e007d93d34efe861a3b321527cf5f3e195e35113c36a0"
},
{
"path": "managed-task-uuid-assignment.json",
"sha256": "sha256:16f1b720a38744734dbab524ed2496269b7a08796cb6f2d72272f922f8f067f8"
},
{
"path": "candidate-make-serverdryrun-result.json",
"sha256": "sha256:d251ca8f03146d95046ef723e364bde2413b223a5a2027cfc74101a180344a63"
},
{
"path": "image-selected-source-comparison.json",
"sha256": "sha256:8fc724b3f61119e78b0ce153b0f48e0d8658ec46411c1c555b7009a58e0ce477"
}
]
}

78
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/state-hub-readiness.patch Normal file
View file

@ -0,0 +1,78 @@
--- a/Makefile
+++ b/Makefile
@@ -174,12 +174,12 @@
--namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \
-f $(RAILIANCE_STATE_HUB_VALUES) \
--set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \
- $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \
+ $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \
if $(KUBECTL) get namespace $(RAILIANCE_STATE_HUB_NAMESPACE) >/dev/null 2>&1; then \
- $(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \
+ $(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \
$(KUBECTL) apply --dry-run=server -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \
else \
echo "Namespace $(RAILIANCE_STATE_HUB_NAMESPACE) does not exist; validating namespaced app manifests with client dry-run."; \
--- /dev/null
+++ b/tests/test_server_dryrun_commands.py
@@ -0,0 +1,30 @@
+"""Pure Make command fixture: run with --noconftest, no database or cluster."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import pytest
+
+@pytest.mark.parametrize('namespace_exists',[True,False])
+def test_secret_server_dryrun_uses_ssa_without_force_or_other_phase_changes(tmp_path,namespace_exists):
+ makefile=Path(__file__).resolve().parents[1]/'Makefile'
+ log=tmp_path/'commands.jsonl';helm=tmp_path/'helm';kubectl=tmp_path/'kubectl'
+ helm.write_text('#!'+sys.executable+'\nprint("apiVersion: v1\\nkind: ConfigMap\\nmetadata: {name: fixture}")\n')
+ kubectl.write_text('#!'+sys.executable+'\nimport json,os,sys\nwith open(os.environ["NK_DRYRUN_LOG"],"a") as log:log.write(json.dumps(sys.argv[1:])+"\\n")\nif sys.argv[1:3]==["get","namespace"] and os.environ["NK_NAMESPACE_EXISTS"]=="0":raise SystemExit(1)\n')
+ helm.chmod(0o700);kubectl.chmod(0o700)
+ env=dict(os.environ);env['NK_DRYRUN_LOG']=str(log);env['NK_NAMESPACE_EXISTS']='1' if namespace_exists else '0'
+ p=subprocess.run(['make','--no-print-directory','-f',str(makefile),'railiance-state-hub-server-dry-run','HELM='+str(helm),'KUBECTL='+str(kubectl),'RAILIANCE_STATE_HUB_IMAGE_TAG=main-51bc70d'],cwd=tmp_path,env=env,capture_output=True,timeout=10)
+ assert p.returncode==0
+ calls=[json.loads(line) for line in log.read_text().splitlines()]
+ applies=[args for args in calls if args[0]=='apply']
+ secrets=[args for args in applies if any(p.endswith('.sops.yaml.template') for p in args)]
+ assert len(secrets)==2
+ assert all('--force-conflicts' not in args and '--force' not in args for args in applies)
+ for args in secrets:
+ if '--dry-run=server' in args:assert '--server-side' in args
+ else:assert not namespace_exists and '--dry-run=client' in args and '--server-side' not in args
+ others=[args for args in applies if args not in secrets]
+ assert all('--server-side' not in args for args in others)
+ assert all('--dry-run=server' in args or '--dry-run=client' in args for args in applies)
+ assert not any(args[0] in {'create','replace','delete','patch'} for args in calls)
--- a/workplans/STATE-WP-0095-historical-source-receiving.md
+++ b/workplans/STATE-WP-0095-historical-source-receiving.md
@@ -237,3 +237,25 @@
fences cluster sync and restores registrar refusal before removing receiving
support; it never rewrites archived IDs or UUIDs. This work does not establish
original TLS recovery or owner-native admission.
+
+## Fix policy-compatible Secret-template server dry-run preparation
+
+```task
+id: STATE-WP-0095-T08
+status: done
+priority: high
+state_hub_task_id: "2dd9b04f-2250-5a8a-b5bc-c8a009872858"
+```
+
+Prepared the narrow Makefile successor after the existing readiness command
+refused its first Secret-template phase. Ordinary client-side apply adds the
+last-applied annotation, correctly denied by the existing Platform
+reject-secret-last-applied policy. Only the two Secret-template server
+dry-run requests now use server-side apply with dry-run; no force-conflicts,
+Secret value change, policy relaxation or actual Secret write is introduced.
+Command-fixture tests verify both requests and preserve other dry-run phases.
+Evidence and original refusal remain in the Custodian cluster historical
+publication packet, revisions/r2 and r3-readiness. This is completed source
+preparation; T06 remains the explicit native rollout gate. The runtime image
+continues to be main-51bc70d; Makefile/workplan changes do not create a new
+application image or native acceptance.

517
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/state-hub/Makefile Normal file
View file

@ -0,0 +1,517 @@
.PHONY: start install install-cli dashboard-install dashboard-check db db-tools migrate seed api dashboard dashboard-local sync check-primary primary-port-clear check check-local test test-python clean register-project register-codex-project register-mcp configure-codex bootstrap-env dev-hub edge-relay mcp-profile validate-adr add-domain rename-domain add-repo list-repos register-path register-from-classification register-from-classification-all cleanup-stale tunnels-up tunnels-status tunnels-check bridges install-hooks install-hooks-all gitea-inventory token-reconcile railiance-state-hub-render railiance-state-hub-client-dry-run railiance-state-hub-server-dry-run railiance-state-hub-headroom
COMPOSE = docker compose -f infra/docker-compose.yml --env-file .env
PYTHON ?= python3
HELM ?= $(shell command -v helm 2>/dev/null || if [ -x "$$HOME/.local/bin/helm" ]; then printf "%s" "$$HOME/.local/bin/helm"; else printf "%s" "helm"; fi)
KUBECTL ?= $(shell command -v kubectl 2>/dev/null || if [ -x "$$HOME/.local/bin/kubectl" ]; then printf "%s" "$$HOME/.local/bin/kubectl"; else printf "%s" "kubectl"; fi)
RAILIANCE_STATE_HUB_RELEASE ?= state-hub
RAILIANCE_STATE_HUB_NAMESPACE ?= state-hub
RAILIANCE_STATE_HUB_CHART ?= deploy/railiance/apps/charts/state-hub
RAILIANCE_STATE_HUB_VALUES ?= deploy/railiance/apps/helm/state-hub-values.yaml
RAILIANCE_STATE_HUB_IMAGE_TAG ?= main-0747b4d
RAILIANCE_STATE_HUB_PLATFORM_DIR ?= deploy/railiance/platform
RAILIANCE_STATE_HUB_APP_MANIFESTS ?= deploy/railiance/apps/manifests
# Codex/WSL non-login shells may not source ~/.profile; keep uv discoverable.
UV ?= $(shell command -v uv 2>/dev/null || if [ -x "$$HOME/.local/bin/uv" ]; then printf "%s" "$$HOME/.local/bin/uv"; else printf "%s" "uv"; fi)
RMGR ?= $(shell command -v rmgr 2>/dev/null || if [ -x "$$HOME/repo-manager/.venv/bin/rmgr" ]; then printf "%s" "$$HOME/repo-manager/.venv/bin/rmgr"; else printf "%s" "rmgr"; fi)
STATE_HUB_API_BASE ?= http://127.0.0.1:8000
SYNC_PATH ?= .
SYNC_PUSH ?= 1
start:
@echo "# Normal production UI access (run in order)"
@echo "make bridges # connect local :8000 to the railiance01 primary"
@echo "make dashboard # verify the primary, then serve the UI on :3000"
@echo "make sync # push file-backed records and reconcile the exact forge commit"
@echo ""
@echo "# Optional local MCP adapter"
@echo "make mcp-http # local SSE adapter on :8001; not needed by the UI"
@echo ""
@echo "# Local fallback/development only"
@echo "make api # local Postgres + API; conflicts with the production :8000 tunnel"
@echo "make dashboard-local # dashboard against the deliberate local API"
install:
$(UV) sync
dashboard/node_modules/.bin/observable: dashboard/package.json dashboard/package-lock.json
cd dashboard && npm ci
dashboard-install: dashboard/node_modules/.bin/observable
dashboard-check: dashboard-install
cd dashboard && npm run build
## Symlink the custodian CLI into ~/.local/bin so it's on PATH system-wide
install-cli: install
mkdir -p ~/.local/bin
ln -sf "$(shell pwd)/.venv/bin/custodian" ~/.local/bin/custodian
ln -sf "$(shell pwd)/.venv/bin/statehub" ~/.local/bin/statehub
@echo "Installed: custodian → $$(readlink -f ~/.local/bin/custodian)"
@echo "Installed: statehub → $$(readlink -f ~/.local/bin/statehub)"
@echo "Make sure ~/.local/bin is on your PATH:"
@echo " echo 'export PATH=\"\$$HOME/.local/bin:\$$PATH\"' >> ~/.bashrc && source ~/.bashrc"
db:
$(COMPOSE) up -d postgres
db-tools:
$(COMPOSE) --profile tools up -d
migrate:
$(UV) run alembic upgrade head
seed:
$(UV) run python scripts/seed.py
## Start (or restart) the MCP SSE server on :8001 — primary transport for Claude Code.
## Remote clients (e.g. COULOMBCORE) connect via the ops-bridge tunnel (port 18001).
## Registration: claude mcp add-json -s user state-hub '{"type":"sse","url":"http://127.0.0.1:8001/sse"}'
mcp-http:
@fuser -k 8001/tcp 2>/dev/null && echo "Stopped running MCP server" || true
MCP_TRANSPORT=sse MCP_PORT=8001 $(UV) run python mcp_server/server.py
## Require the production identity before serving the normal dashboard. This
## prevents an accidental local `make api` from presenting an empty fallback DB
## as the live State Hub.
check-primary:
@health="$$(curl -fsS --max-time 5 http://127.0.0.1:8000/state/health 2>/dev/null)" || { \
echo "ERROR: State Hub primary is not reachable on 127.0.0.1:8000." >&2; \
echo "Run 'make bridges' first." >&2; \
exit 1; \
}; \
identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \
if [ "$$identity" != "primary/railiance01" ]; then \
echo "ERROR: 127.0.0.1:8000 is '$$identity', not the primary/railiance01 State Hub." >&2; \
echo "Stop the local API, then run 'make bridges'. Use 'make dashboard-local' only for intentional local development." >&2; \
exit 1; \
fi
## Refuse to start the production tunnel when a non-primary API already owns
## port 8000. With no listener, bridge is free to establish the tunnel.
primary-port-clear:
@if health="$$(curl -fsS --max-time 2 http://127.0.0.1:8000/state/health 2>/dev/null)"; then \
identity="$$(printf '%s' "$$health" | $(PYTHON) -c 'import json, sys; d=json.load(sys.stdin); print("{}/{}".format(d.get("instance_role", ""), d.get("instance_label", "")))')"; \
if [ "$$identity" != "primary/railiance01" ]; then \
echo "ERROR: port 8000 is occupied by '$$identity'." >&2; \
echo "Stop the local 'make api' process before starting the production tunnel." >&2; \
exit 1; \
fi; \
fi
dashboard: check-primary
@fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true
$(MAKE) dashboard-install
cd dashboard && npm run dev
## Deliberate local-development dashboard; bypasses the production identity gate.
dashboard-local:
@echo "WARNING: serving the dashboard against the local/fallback API on :8000."
@fuser -k 3000/tcp 2>/dev/null && echo "Stopped running dashboard" || true
$(MAKE) dashboard-install
cd dashboard && npm run dev
## Fast work-record path. Repo Manager assigns only missing deterministic IDs,
## verifies the primary/railliance01 bridge, and asks central to derive the
## exact pushed Forgejo commit in one transactional request.
sync:
$(RMGR) sync --path "$(SYNC_PATH)" --api-base "$(STATE_HUB_API_BASE)" $(if $(filter 1 true yes,$(SYNC_PUSH)),--push,)
check: check-primary
@echo "State Hub primary/railiance01 is healthy."
check-local:
curl -sf http://127.0.0.1:8000/state/health | python3 -m json.tool
# CUST-WP-0067-T09. The chart ships a copy of the-custodian canon allowed-values
# because a container has no checkout to read it from. Regenerate after any
# canon vocabulary change; the -check target fails on drift so a release cannot
# validate classification against a stale vocabulary.
sync-classification-allowed:
python3 scripts/sync_classification_allowed.py
check-classification-allowed:
python3 scripts/sync_classification_allowed.py --check
# STATE-WP-0091: read-only surge-headroom check. Does not start Helm.
# Live: make railiance-state-hub-headroom
# Fixtures: make railiance-state-hub-headroom HEADROOM_FIXTURE=tests/fixtures/release_headroom/rev63-105m.json
HEADROOM_FIXTURE ?=
railiance-state-hub-headroom:
@if [ -n "$(HEADROOM_FIXTURE)" ]; then \
$(PYTHON) scripts/release_headroom_preflight.py --fixture "$(HEADROOM_FIXTURE)"; \
else \
$(PYTHON) scripts/release_headroom_preflight.py --kubectl "$(KUBECTL)"; \
fi
railiance-state-hub-render: check-classification-allowed
$(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \
--namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \
-f $(RAILIANCE_STATE_HUB_VALUES) \
--set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG)
railiance-state-hub-client-dry-run:
@set -e; \
tmpdir="$$(mktemp -d)"; \
trap 'rm -rf "$$tmpdir"' EXIT; \
$(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \
--namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \
-f $(RAILIANCE_STATE_HUB_VALUES) \
--set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \
$(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"
railiance-state-hub-server-dry-run:
@set -e; \
tmpdir="$$(mktemp -d)"; \
trap 'rm -rf "$$tmpdir"' EXIT; \
$(HELM) template $(RAILIANCE_STATE_HUB_RELEASE) $(RAILIANCE_STATE_HUB_CHART) \
--namespace $(RAILIANCE_STATE_HUB_NAMESPACE) \
-f $(RAILIANCE_STATE_HUB_VALUES) \
--set image.tag=$(RAILIANCE_STATE_HUB_IMAGE_TAG) > "$$tmpdir/state-hub.yaml"; \
$(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-credentials.sops.yaml.template; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-cluster.yaml; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_PLATFORM_DIR)/state-hub-db-networkpolicies.yaml; \
$(KUBECTL) apply --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \
if $(KUBECTL) get namespace $(RAILIANCE_STATE_HUB_NAMESPACE) >/dev/null 2>&1; then \
$(KUBECTL) apply --server-side --dry-run=server -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \
$(KUBECTL) apply --dry-run=server -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \
else \
echo "Namespace $(RAILIANCE_STATE_HUB_NAMESPACE) does not exist; validating namespaced app manifests with client dry-run."; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-namespace.yaml; \
$(KUBECTL) apply --dry-run=client -f $(RAILIANCE_STATE_HUB_APP_MANIFESTS)/state-hub-env.secret.sops.yaml.template; \
$(KUBECTL) apply --dry-run=client -n $(RAILIANCE_STATE_HUB_NAMESPACE) -f "$$tmpdir/state-hub.yaml"; \
fi
test: test-python dashboard-check
test-python:
TEST_DATABASE_URL=postgresql+asyncpg://custodian:changeme@127.0.0.1:5432/custodian_test \
$(UV) run pytest -x -q
## Benchmark /state/summary revision cache (API must be running on :8000)
benchmark-summary-cache:
$(UV) run python scripts/benchmark_summary_cache.py
## ops-bridge managed tunnels
## Requires ops-bridge: bridge is at /home/worsch/.local/bin/bridge
tunnels-up: primary-port-clear
bridge up
tunnels-status:
bridge status
## End-to-end check: verifies SSH process alive + remote port listening on COULOMBCORE.
## Exits non-zero if any tunnel is not fully operational.
tunnels-check:
bridge check
## Ensure all ops-bridge tunnels are up and healthy.
## Brings up any stopped/stale tunnels, shows final status, exits non-zero if anything is still down.
bridges: primary-port-clear
@echo "==> Bringing up all tunnels..."
bridge up
@echo ""
@echo "==> Tunnel status:"
bridge status
@echo ""
@echo "==> Checking tunnel health..."
bridge check
## Start (or restart) the LOCAL FALLBACK backend — db + migrate + uvicorn.
## This replaces anything on :8000, including the production State Hub tunnel.
api: db
@echo "WARNING: starting the local fallback API; this is not the railiance01 primary."
@echo "Waiting for postgres..."; \
for i in 1 2 3 4 5 6 7 8 9 10; do \
nc -z 127.0.0.1 5432 2>/dev/null && break; \
sleep 1; \
done
$(MAKE) migrate
@fuser -k 8000/tcp 2>/dev/null && echo "Stopped running API" || true
$(UV) run uvicorn api.main:app --reload --reload-dir api --reload-dir mcp_server --reload-dir task_flow_engine --host 127.0.0.1 --port 8000
## Register a project (Claude Code): make register-project DOMAIN=railiance PROJECT_PATH=/home/worsch/railiance
register-project:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-project DOMAIN=<domain> PROJECT_PATH=<path>"; exit 1)
@test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1)
scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)"
## Register a Codex project (AGENTS.md + HTTP API): make register-codex-project DOMAIN=capabilities PROJECT_PATH=/home/worsch/my-repo
register-codex-project:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required. Usage: make register-codex-project DOMAIN=<domain> PROJECT_PATH=<path>"; exit 1)
@test -n "$(PROJECT_PATH)" || (echo "ERROR: PROJECT_PATH is required."; exit 1)
scripts/register_project.sh "$(DOMAIN)" "$(PROJECT_PATH)" --codex
## Register State Hub MCP for Claude Code. Optional: make register-mcp MCP_URL=http://127.0.0.1:18001/sse
register-mcp:
scripts/register-mcp.sh \
$(if $(MCP_URL),--url "$(MCP_URL)",) \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(DRY_RUN),--dry-run,)
## Configure Codex workspace-write networking for local State Hub access.
configure-codex:
scripts/configure-codex.sh \
$(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(WITH_MCP),--with-mcp,) \
$(if $(DRY_RUN),--dry-run,)
## Bootstrap a new operator/collaborator environment. Optional: make bootstrap-env ARGS="--install-missing"
bootstrap-env:
scripts/bootstrap-env.sh $(ARGS)
## Files-first local dev hub (CUST-WP-0054-T07): postgres + migrate + seed + API + repo registration.
## Optional: WITH_EDGE=1 WITH_MCP=1 make dev-hub
## Fleet tunnel when reachable: make dev-hub PROFILE=fleet
dev-hub:
bash scripts/dev_hub_up.sh $(if $(PROFILE),$(PROFILE),dev)
## Local edge relay for offline write buffering (STATE-WP-0068).
edge-relay:
@fuser -k 18080/tcp 2>/dev/null && echo "Stopped running edge relay" || true
STATEHUB_UPSTREAM_URL=$${STATEHUB_UPSTREAM_URL:-http://127.0.0.1:8000} \
$(UV) run uvicorn api.edge.relay:app --host 127.0.0.1 --port 18080
## Switch MCP/API profile: make mcp-profile PROFILE=dev|fleet|status
mcp-profile:
@test -n "$(PROFILE)" || (echo "ERROR: PROFILE is required (dev|fleet|status)."; exit 1)
bash scripts/mcp_hub_profile.sh "$(PROFILE)"
## Add a second repo to an existing domain: make add-repo DOMAIN=railiance REPO_PATH=/home/worsch/railiance-infra
add-repo:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1)
@test -n "$(REPO_PATH)" || (echo "ERROR: REPO_PATH is required."; exit 1)
scripts/register_project.sh "$(DOMAIN)" "$(REPO_PATH)" --additional
## Create a new domain: make add-domain DOMAIN=my_domain NAME="My Domain"
add-domain:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required (slug)."; exit 1)
@test -n "$(NAME)" || (echo "ERROR: NAME is required (display name)."; exit 1)
curl -sf -X POST http://127.0.0.1:8000/domains/ \
-H "Content-Type: application/json" \
-d "{\"slug\": \"$(DOMAIN)\", \"name\": \"$(NAME)\"}" | python3 -m json.tool
## Rename a domain: make rename-domain DOMAIN=old_slug NEW_SLUG=new_slug NEW_NAME="New Name"
rename-domain:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN (old slug) is required."; exit 1)
@test -n "$(NEW_SLUG)" || (echo "ERROR: NEW_SLUG is required."; exit 1)
@test -n "$(NEW_NAME)" || (echo "ERROR: NEW_NAME is required."; exit 1)
curl -sf -X PATCH http://127.0.0.1:8000/domains/$(DOMAIN)/rename \
-H "Content-Type: application/json" \
-d "{\"new_slug\": \"$(NEW_SLUG)\", \"new_name\": \"$(NEW_NAME)\"}" | python3 -m json.tool
## Register this machine's local path for a repo: make register-path REPO=marki-docx PATH=/home/tegwick/marki-docx
register-path:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make register-path REPO=<slug> PATH=<path>"; exit 1)
@test -n "$(PATH)" || (echo "ERROR: PATH is required. Usage: make register-path REPO=<slug> PATH=<path>"; exit 1)
curl -sf -X POST "http://127.0.0.1:8000/repos/$(REPO)/paths" \
-H "Content-Type: application/json" \
-d "{\"host\": \"$$(hostname)\", \"path\": \"$(PATH)\"}" | python3 -m json.tool
## List repos for a domain: make list-repos DOMAIN=railiance
list-repos:
@test -n "$(DOMAIN)" || (echo "ERROR: DOMAIN is required."; exit 1)
curl -sf "http://127.0.0.1:8000/repos/?domain=$(DOMAIN)" | python3 -m json.tool
## Ingest SBOM data for a repo (all mechanisms: lockfiles + ansible + sbom-tools.yaml).
## Auto-detect all sources: make ingest-sbom REPO=the-custodian REPO_PATH=/home/worsch/the-custodian
## Single lockfile (explicit): make ingest-sbom REPO=the-custodian LOCKFILE=/path/to/uv.lock
## Dry-run (no submit): make ingest-sbom REPO=the-custodian REPO_PATH=... DRY_RUN=1
## Tip: run capture-tools first for repos with system-level tool dependencies.
ingest-sbom:
@test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1)
$(UV) run python scripts/ingest_sbom.py --repo "$(REPO)" \
$(if $(LOCKFILE),--lockfile "$(LOCKFILE)") \
$(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \
$(if $(DRY_RUN),--dry-run)
## Ingest capability declarations from SCOPE.md into the catalog.
## Usage: make ingest-capabilities REPO=the-custodian [REPO_PATH=/home/worsch/the-custodian]
## Or: make ingest-capabilities-all
## Add DRY_RUN=1 to preview without writing.
ingest-capabilities:
@test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1)
$(UV) run python scripts/ingest_capabilities.py --repo "$(REPO)" \
$(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \
$(if $(DRY_RUN),--dry-run)
ingest-capabilities-all:
$(UV) run python scripts/ingest_capabilities.py --all \
$(if $(DRY_RUN),--dry-run)
## Check Repository Definition of Integrated (DoI) criteria for a repo.
## Usage: make check-doi REPO=llm-connect
## Or: make check-doi-all
## Add JSON=1 for machine-readable output.
check-doi:
@test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1)
$(UV) run python scripts/check_doi.py --repo "$(REPO)" $(if $(JSON),--json)
check-doi-all:
$(UV) run python scripts/check_doi.py --all $(if $(JSON),--json)
## Ingest tpsc.yaml service declarations from a repo into the TPSC catalog.
## Usage: make ingest-tpsc REPO=llm-connect
## Or: make ingest-tpsc-all
## Add DRY_RUN=1 to preview without writing.
ingest-tpsc:
@test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1)
$(UV) run python scripts/ingest_tpsc.py --repo "$(REPO)" \
$(if $(DRY_RUN),--dry-run)
ingest-tpsc-all:
$(UV) run python scripts/ingest_tpsc.py --all \
$(if $(DRY_RUN),--dry-run)
## Run SBOM capture agent for a repo — generates/updates sbom-tools.yaml.
## Usage: make capture-tools REPO=railiance-infra [REPO_PATH=/home/worsch/railiance-infra]
## Add DRY_RUN=1 to preview without writing.
capture-tools:
@test -n "$(REPO)" || (echo "ERROR: REPO is required."; exit 1)
$(UV) run python scripts/capture_sbom_tools.py --repo "$(REPO)" \
$(if $(REPO_PATH),--repo-path "$(REPO_PATH)") \
$(if $(DRY_RUN),--dry-run)
## Check a repo for ADR-001 compliance: make validate-adr REPO=/path/to/repo [DOMAIN=custodian]
validate-adr:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make validate-adr REPO=<path> [DOMAIN=<slug>]"; exit 1)
$(UV) run python scripts/validate_repo_adr.py "$(REPO)" $(if $(DOMAIN),--domain "$(DOMAIN)",)
## Consistency exit contract:
## - Direct scripts/consistency_check.py: 0 clean, 2 warnings-only, 1 failures.
## - Agent/operator Make wrappers below normalize warning-only 2 to shell success
## while preserving visible WARN output and keeping real failures non-zero.
## Check a single repo for ADR-001 consistency: make check-consistency REPO=the-custodian [REPO_PATH=/override]
## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures
check-consistency:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make check-consistency REPO=<slug>"; exit 1)
$(UV) run python scripts/consistency_check.py --repo "$(REPO)" \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Check and auto-fix a single repo: make fix-consistency REPO=the-custodian [REPO_PATH=/override]
## Exit 0 = clean or warnings-only (warnings stay visible), exit 1 = failures
fix-consistency:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make fix-consistency REPO=<slug>"; exit 1)
$(UV) run python scripts/consistency_check.py --repo "$(REPO)" --fix \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(REPO_PATH),--repo-path "$(REPO_PATH)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Normalize workplan frontmatter and task status literals in attached repos.
## Usage: make normalize-attached-workplans REPO=artifact-store
## make normalize-attached-workplans DIRTY=1
normalize-attached-workplans:
$(UV) run python scripts/normalize_attached_repo_workplans.py \
$(if $(REPO),--repo "$(REPO)",) \
$(if $(DIRTY),--dirty,) \
$(if $(DRY_RUN),--dry-run,)
@test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO=<slug> or DIRTY=1"; exit 1)
## Regenerate AGENTS.md / CLAUDE.md / .claude/rules from templates.
## Usage: make update-agent-instructions REPO=artifact-store
## make update-agent-instructions DIRTY=1
update-agent-instructions:
$(UV) run python scripts/update_agent_instruction_files.py \
$(if $(REPO),--repo "$(REPO)",) \
$(if $(DIRTY),--dirty,)
@test -n "$(REPO)$(DIRTY)" || (echo "ERROR: set REPO=<slug> or DIRTY=1"; exit 1)
## Reconcile measured token sources against State Hub.
## Usage: make token-reconcile [SINCE=2026-05-19] [APPLY=1] [ZERO_FALLBACKS=1]
token-reconcile:
$(PYTHON) scripts/token_reconcile.py \
$(if $(SINCE),--since "$(SINCE)",) \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(CODEX_HOME),--codex-home "$(CODEX_HOME)",) \
$(if $(CLAUDE_HOME),--claude-home "$(CLAUDE_HOME)",) \
$(if $(APPLY),--apply,) \
$(if $(ZERO_FALLBACKS),--zero-superseded-fallbacks,)
## Pull then fix: single repo or all repos if REPO omitted
## make fix-consistency-remote — smart pull+fix all repos that need it
## make fix-consistency-remote REPO=slug — pull+fix one repo
fix-consistency-remote:
$(UV) run python scripts/consistency_check.py \
$(if $(REPO),--repo "$(REPO)",--all) \
--remote \
$(if $(API_BASE),--api-base "$(API_BASE)",) \
$(if $(NO_WRITEBACK),--no-writeback,); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Infer repo slug from git remote URL and check: make check-consistency-here [REPO_PATH=/path/to/repo]
## Omit REPO_PATH to use the Python script's CWD (i.e. pass an empty --here flag).
check-consistency-here:
$(UV) run python scripts/consistency_check.py \
--here $(if $(REPO_PATH),"$(REPO_PATH)",) \
$(if $(API_BASE),--api-base "$(API_BASE)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Infer repo slug from git remote URL and fix: make fix-consistency-here [REPO_PATH=/path/to/repo]
fix-consistency-here:
$(UV) run python scripts/consistency_check.py \
--here $(if $(REPO_PATH),"$(REPO_PATH)",) \
--fix \
$(if $(API_BASE),--api-base "$(API_BASE)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Check all registered repos for ADR-001 consistency
check-consistency-all:
$(UV) run python scripts/consistency_check.py --all $(if $(API_BASE),--api-base "$(API_BASE)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Check and auto-fix all registered repos
fix-consistency-all:
$(UV) run python scripts/consistency_check.py --all --fix $(if $(API_BASE),--api-base "$(API_BASE)",); \
e=$$?; [ $$e -eq 2 ] && exit 0 || exit $$e
## Cancel open tasks belonging to completed/archived workstreams.
## Safe to run at any time; also suitable for a daily cron job.
## Cron example: 0 3 * * * cd ~/state-hub && make cleanup-stale
cleanup-stale:
$(UV) run python scripts/cleanup_stale_tasks.py
## Install custodian post-commit sync hook into one repo: make install-hooks REPO=marki-docx
install-hooks:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make install-hooks REPO=<slug>"; exit 1)
bash scripts/install_hooks.sh --repo "$(REPO)"
## Install custodian post-commit sync hook into all active registered repos
install-hooks-all:
bash scripts/install_hooks.sh --all
## Remove custodian post-commit sync hook from one repo: make remove-hooks REPO=marki-docx
remove-hooks:
@test -n "$(REPO)" || (echo "ERROR: REPO is required. Usage: make remove-hooks REPO=<slug>"; exit 1)
bash scripts/install_hooks.sh --repo "$(REPO)" --remove
## Compare Gitea coulomb org repos against state-hub registered repos
## Requires GITEA_TOKEN in env or .env: make gitea-inventory GITEA_TOKEN=<token>
gitea-inventory:
$(UV) run python scripts/gitea_inventory.py $(if $(JSON),--json)
## Register/update one repo from .repo-classification.yaml:
## make register-from-classification REPO=state-hub
## make register-from-classification PATH=/path/to/repo
## Optional: DRY_RUN=1
register-from-classification:
@test -n "$(REPO)" -o -n "$(PATH)" || (echo "ERROR: REPO or PATH is required."; exit 1)
$(UV) run python scripts/register_from_classification.py \
$(if $(PATH),--repo-path "$(PATH)",--slug "$(REPO)") \
$(if $(DRY_RUN),--dry-run,)
## Bulk register/update all active repos with accessible local paths
register-from-classification-all:
$(UV) run python scripts/register_from_classification.py --bulk \
$(if $(DRY_RUN),--dry-run,)
clean:
$(COMPOSE) down -v

30
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/state-hub/tests/test_server_dryrun_commands.py Normal file
View file

@ -0,0 +1,30 @@
"""Pure Make command fixture: run with --noconftest, no database or cluster."""
import json
import os
from pathlib import Path
import subprocess
import sys
import pytest
@pytest.mark.parametrize('namespace_exists',[True,False])
def test_secret_server_dryrun_uses_ssa_without_force_or_other_phase_changes(tmp_path,namespace_exists):
makefile=Path(__file__).resolve().parents[1]/'Makefile'
log=tmp_path/'commands.jsonl';helm=tmp_path/'helm';kubectl=tmp_path/'kubectl'
helm.write_text('#!'+sys.executable+'\nprint("apiVersion: v1\\nkind: ConfigMap\\nmetadata: {name: fixture}")\n')
kubectl.write_text('#!'+sys.executable+'\nimport json,os,sys\nwith open(os.environ["NK_DRYRUN_LOG"],"a") as log:log.write(json.dumps(sys.argv[1:])+"\\n")\nif sys.argv[1:3]==["get","namespace"] and os.environ["NK_NAMESPACE_EXISTS"]=="0":raise SystemExit(1)\n')
helm.chmod(0o700);kubectl.chmod(0o700)
env=dict(os.environ);env['NK_DRYRUN_LOG']=str(log);env['NK_NAMESPACE_EXISTS']='1' if namespace_exists else '0'
p=subprocess.run(['make','--no-print-directory','-f',str(makefile),'railiance-state-hub-server-dry-run','HELM='+str(helm),'KUBECTL='+str(kubectl),'RAILIANCE_STATE_HUB_IMAGE_TAG=main-51bc70d'],cwd=tmp_path,env=env,capture_output=True,timeout=10)
assert p.returncode==0
calls=[json.loads(line) for line in log.read_text().splitlines()]
applies=[args for args in calls if args[0]=='apply']
secrets=[args for args in applies if any(p.endswith('.sops.yaml.template') for p in args)]
assert len(secrets)==2
assert all('--force-conflicts' not in args and '--force' not in args for args in applies)
for args in secrets:
if '--dry-run=server' in args:assert '--server-side' in args
else:assert not namespace_exists and '--dry-run=client' in args and '--server-side' not in args
others=[args for args in applies if args not in secrets]
assert all('--server-side' not in args for args in others)
assert all('--dry-run=server' in args or '--dry-run=client' in args for args in applies)
assert not any(args[0] in {'create','replace','delete','patch'} for args in calls)

261
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/state-hub/workplans/STATE-WP-0095-historical-source-receiving.md Normal file
View file

@ -0,0 +1,261 @@
---
id: STATE-WP-0095
type: workplan
title: "Preserve exact closed historical source bindings in Forge receiving"
domain: infotech
repo: state-hub
status: active
owner: codex
created: "2026-10-04"
updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058"
---
## Goal
Normal Forge receiving must preserve the seven bound Activity legacy source
views and the Custodian retired ADHOC source without new identities, source
rewrites, task movement, deletion, relabeling or unretirement. This implements
receiving support before RMGR-WP-0019 registrar admission. CUST-WP-0077 owns
cross-owner coordination; deployment and replay remain explicit owner returns.
## Verify source bindings and shared views
```task
id: STATE-WP-0095-T01
status: done
priority: high
state_hub_task_id: "6a5f87bb-e63d-5e20-b9fc-c1b5cd403ff7"
```
Reviewed seven exact Activity files: four existing workplans and 64 UUID-bound
closed tasks. The archived custodian-WP-0003 parent and three split_from views
share one existing parent UUID; every repeated task binding/content agrees.
Custodian ADHOC-2026-08-25-retired retains parent
1c53d3db-6633-5e2d-987f-10706d73be9e and task
d8af3e6b-26ae-580c-bbeb-caa8ad419bec. No identity migration is authorized.
## Implement preservation receiving and global collision refusal
```task
id: STATE-WP-0095-T02
status: done
priority: high
state_hub_task_id: "ebb24194-e4cc-5130-aaa6-663f2ea2c822"
```
Prepared exact source digest manifest, existing-parent/task verification,
immutable receiving and UUID-based historical diff. Ordinary canonical sources
continue deriving their identities. Missing/changed/symlinked source, absent
bound rows or nonterminal hub histories refuse. Check task UUID holders globally,
including retired and acknowledged-stale parent workplans, before INSERT.
Offline preview tests and snapshot proof are evidence of source behavior only;
finish after applying reviewed source and its focused tests to the owning repo.
## Deliver receiving through normal deployment and verify owner return
```task
id: STATE-WP-0095-T03
status: done
priority: high
depends_on: [STATE-WP-0095-T02]
state_hub_task_id: "e8566341-cf6e-5b9c-ad80-8564d14cd52a"
decision_id: "36772026-f43d-4331-b466-5f4b1afe4eca"
```
No direct live code patch. This task is a deployment return, not policy authority
or permission to resolve identity decisions. Exact receiving fixture acceptance
and global collision rejection must hold in the delivered version.
## Verify normal historical replay preserves existing identities
```task
id: STATE-WP-0095-T04
status: done
priority: high
depends_on: [STATE-WP-0095-T03, RMGR-WP-0019-T03]
state_hub_task_id: "3bbb311f-194e-5862-8aed-358a580f22b4"
```
Verify Activity four historical parent UUIDs/64 task UUIDs and Custodian retired
parent/task unchanged; no historical row creation, movement, deletion,
relabeling or unretirement. Unrelated stale projections remain separate decisions.
2026-10-05: Source exact-set retirement support now exists. Repository projection
reconcile accepts `expected_retirement_uuids`; that set must equal the stale
projections, and any task or dependency edge on a stale row refuses the call.
Live apply still needs a reviewed rollout. On 2026-10-05 the founder accepted
review `review-1ece4f3e-d0e3-4fd6-abc8-272f1b1876f4`, which names only the three
empty projections and preserves parent `b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`.
That acceptance is not a live retirement and does not resolve
`91b7c633-a66f-5d43-9f4b-6565aefb9356`.
## Offline evidence, 2026-10-04
Prepared receiver suite: Forge projection regression plus historical preservation,
missing/changed source, terminality, immutable UUID diff, and global holder query.
Real-source offline snapshot proof preserves four Activity parents/64 tasks and
one Custodian parent/task without mutations; no API writes or deployment occurred.
Exact original bindings are maintained in api/services/historical_projection.py.
2026-10-04 actual owning-source validation: 155 focused receiver/qualifier/planner
cases passed; the independent offline current-source/current-hub-snapshot proof
retained five existing parents and 65 task UUIDs, no mutations, clean historical
diff. T02 is source-complete; T03/T04 remain wait for reviewed deployment and
normal registrar/replay receiving, respectively. Evidence: docs/evidence/2026-10-04-historical-receiving-source.json.
## Verified receiving rollout and remaining replay gate, 2026-10-04
T03 is done on actual founder approval of proposal
STATE-WP-0095-T03-historical-receiving revision 1, SHA256
`d6789359b3fe7988a6201fff6f367cb46948aa197e1cc0ca66c31f45e1ea9981`,
approval receipt reference `user_reply:call_o87uEfFs3AtXfxO5YqUnSpcg:0`.
The normal owner Helm path delivered `main-5dd3df9`, commit
`5dd3df9c69eb564af10ea5ea01e349786212501a`, revision 72; API and MCP are Ready,
primary/railiance01 health and schema a0b1c2d3e4f5 verified, running source
hashes match. Rollback remains revision 71; no receipt-table downgrade is needed.
Actual historical preservation and global task-holder refusal were verified in
the delivered version. Durable receipt:
[verified historical receiving rollout](../docs/evidence/2026-10-04-historical-receiving-rollout.json).
Custodian immutable replay applied while preserving the retired historical
parent/task UUIDs. Activity immutable replay retained all four historical parent
and 64 task UUIDs and historical row contents, but refused transactionally on
three unrelated stale projections: custodian-wp-0003 (archived),
custodian-wp-0003a (done), custodian-wp-0003b (done). No retirement acknowledgement
was issued and that refusal is not represented as normal replay success.
Registrar source `c41d36c2` has 38 actual passing tests and Repo Manager's own
normal sync succeeded; registrar admission does not authorize retiring Activity
projections. T04 therefore remains wait, retaining RMGR-WP-0019-T03 and awaiting
Activity Core's precise reviewed disposition of those three stale rows. No
rename, identity migration, new paid native trial or further live action is
included in this source status update.
## Activity Core return, 2026-10-06
Activity Core returned the exact disposition (message
`cf2d5847-8b3f-4653-9d9f-d5fcfbe3f1e4`, ACTIVITY-WP-0045, activity-core commit
`941fac0`). `expected_retirement_uuids`:
`683724d9-4249-5e0e-88e0-7baff4330d58` (custodian-wp-0003),
`644017c7-c5f2-5a64-81c3-d8ee806f8251` (custodian-wp-0003a),
`a5806df9-5b9a-50da-81c1-7e606076bed9` (custodian-wp-0003b). Each has zero
tasks and zero dependency edges. None is preserved parent
`b4eb45a9-69e3-4ab0-b00c-67a53c3117c5`. Repo Manager independently verified the
set (message `916841f6`, repo-manager commit `498682c`). Nothing was retired.
T04 now waits on decision `91b7c633` and a reviewed live rollout. After that,
Repo Manager re-runs the Activity owner sync and closes RMGR-WP-0019-T03.
## Retirement rollout and T04 verification, 2026-10-06
The founder resolved decision `91b7c633` (retire exactly the three projections)
and approved the rollout. The founder ran the normal Helm upgrade to
`main-968a811` (revision 74; rollback revision 73, `main-fad9b3f`). API and MCP
are Ready, health is ok on primary/railiance01, and schema `b3c4d5e6f7a8` is
unchanged; no migrations. The activity-core reconcile at commit
`67a92d2ff8e033baf2c36998d24187fe83edec84` with `expected_retirement_uuids` set
to the three UUIDs applied: 0 created, 3 retired (tombstoned with an
`@retired-20261006` slug; UUIDs and statuses kept), 0 refused, 0 tasks
cancelled. All 7 bound Activity source files still match their digests. The four
historical parents and 64 task UUIDs are present under their bound parents with
unchanged fields. The Custodian retired parent and task are unchanged. Activity
had 54 workplans and 288 tasks before and after. Evidence:
[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json).
Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and
ACTIVITY-WP-0045-T02 remain with their owners.
## Cluster exact historical receiving successor, 2026-10-17
This scoped extension retains the completed Activity/Custodian outcomes above.
Three frozen cluster archives contain three existing parent UUIDs and 15 existing
closed task UUIDs. Extend the existing exact historical receiving path before
registrar admission; never derive replacement identities or modify those
sources. Cross-owner observation attaches to existing SHR-WP-0001, with
RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly.
The local derived case is waiting; no live worker injection is authorized.
## Prepare exact cluster source preservation and refusal cases
```task
id: STATE-WP-0095-T05
status: done
priority: high
state_hub_task_id: "339b9a37-181c-5599-9a89-0393e199bd4b"
```
Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager.
Candidate receiving tests pass 26: existing rows remain entirely unchanged;
missing/foreign/nonterminal rows and unknown, changed, missing or symlinked
source refuse. Read-only public metadata confirms all 18 existing rows have the
expected ownership and terminal status. These are source and metadata checks,
not delivered receiving or a committed native trial.
## Deliver reviewed cluster receiving successor before registrar admission
```task
id: STATE-WP-0095-T06
status: wait
priority: high
depends_on: [STATE-WP-0095-T05]
needs_human: true
intervention_note: "Review exact committed source, CI image and owner deployment proposal before rollout."
state_hub_task_id: "6af973b4-1964-5d4d-a2f8-8dbd4c01fda4"
```
Use the normal CI image and Helm owner path after exact revision/image pin,
fresh headroom and reviewed dry-run/diff. API/MCP currently run main-7d99282;
Helm revision75 is the observed rollback baseline, requiring freshness before
execution. No deployment assent exists for this successor. Verify delivered
module hash, health, both readiness checks and expected migrated schema
c4d5e6f7a8b9; verify exact
three parents/15 tasks and rejection behavior without committed probe changes.
Only the verified return unblocks RMGR-WP-0019-T05. The full current image also carries inbox-feed migration c4d5e6f7a8b9,
although this receiver patch adds none. Helm rollback leaves that migration in
place; unrelated migration acceptance and old-image compatibility need normal
owner evidence before rollout approval. Do not run destructive default local
DB tests; use only exact disposable DB containment. Costs and monetary risk are
unknown pending exact rollout proposal; no zero-cost or autonomy grant is inferred.
## Verify ordinary cluster receiving and immutable historical rows
```task
id: STATE-WP-0095-T07
status: wait
priority: high
depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06]
state_hub_task_id: "cfd1ea99-e75b-542e-a434-acefb62f3c7d"
```
Capture normal cluster owner publication/receiving receipts and compare entire
existing row snapshots and exact source bytes. No creation, movement, relabel,
unretirement or retirement is authorized for these histories. A rollback first
fences cluster sync and restores registrar refusal before removing receiving
support; it never rewrites archived IDs or UUIDs. This work does not establish
original TLS recovery or owner-native admission.
## Fix policy-compatible Secret-template server dry-run preparation
```task
id: STATE-WP-0095-T08
status: done
priority: high
state_hub_task_id: "2dd9b04f-2250-5a8a-b5bc-c8a009872858"
```
Prepared the narrow Makefile successor after the existing readiness command
refused its first Secret-template phase. Ordinary client-side apply adds the
last-applied annotation, correctly denied by the existing Platform
reject-secret-last-applied policy. Only the two Secret-template server
dry-run requests now use server-side apply with dry-run; no force-conflicts,
Secret value change, policy relaxation or actual Secret write is introduced.
Command-fixture tests verify both requests and preserve other dry-run phases.
Evidence and original refusal remain in the Custodian cluster historical
publication packet, revisions/r2 and r3-readiness. This is completed source
preparation; T06 remains the explicit native rollout gate. The runtime image
continues to be main-51bc70d; Makefile/workplan changes do not create a new
application image or native acceptance.

51
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/draft-before-final-evidence/pending-supervision-entry.json Normal file
View file

@ -0,0 +1,51 @@
{
"proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1",
"disposition": "pending",
"outcome": "pending",
"proposal_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/proposal-metadata.json",
"original_revision": 1,
"original_digest_sha256": "5bef9e7818ce1ebc64b81e300b6b40dcfc710a8625d6e0595534f8284dc2052d",
"original_digest": "5bef9e7818ce1ebc64b81e300b6b40dcfc710a8625d6e0595534f8284dc2052d",
"current_revision": 4,
"current_digest_sha256": "a2bee1709f1fe213318c7adc1b28ad04d6755616d59867781b7d44ad5b8e6bcb",
"target": "railiance01 state-hub release/API/MCP and exact-new-trial migration Job only",
"expected_result": "Normal owner image promotion makes exact frozen cluster receiving available; API/MCP ready, expected c4 schema and exact three historical parents/15 tasks preserved; only then is later registrar admission eligible. No original TLS/identity native acceptance is inferred.",
"verification_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/ROLLOUT-SUPPLEMENT.md",
"rollback_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/MIGRATION-JOB-RECOVERY.md",
"cost_estimate_eur": null,
"risk_estimate_eur": null,
"cost_estimate": "Unknown attributable execution/network/operator recovery cost on existing infrastructure; no zero-cost claim or budget reservation inferred.",
"risk_estimate": "Unpriced production coordination outage, message migration locks/data compatibility, retained active migration Job, uncertain connection/lock mapping and recovery time; three-minute Helm wait is not a bound on outage or DB effects. Explicit supervised risk disposition required.",
"supervisor": "Bernd Worsch",
"approval_ref": null,
"approved_revision": null,
"approved_digest_sha256": null,
"executed_revision": null,
"executed_digest_sha256": null,
"refinement_or_rescue": true,
"promotion_evidence": false,
"reason": "Exact native rollout proposal remains pending. Source preparation, synthetic compatibility and public artifact verification are evidence only; earlier preparation failures and revisions remain one trial.",
"revisions": [
{
"revision": 1,
"proposal_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/REVIEW.md",
"digest_sha256": "5bef9e7818ce1ebc64b81e300b6b40dcfc710a8625d6e0595534f8284dc2052d",
"disposition": "not_submitted_or_unadjudicated",
"executed": false
},
{
"revision": 2,
"proposal_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/REVIEW.md",
"digest_sha256": "c8f13ed5d67d4743ea8e81bf720367022c820a060d7b225e84338eab2907493f",
"disposition": "not_submitted_or_unadjudicated",
"executed": false
},
{
"revision": 3,
"proposal_ref": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/ROLLOUT-SUPPLEMENT.md",
"digest_sha256": "df2962fd1588784a987e37932bd67f03f6248ea6384cf4bbb74ad1fe2e26be2b",
"disposition": "not_submitted_or_unadjudicated",
"executed": false
}
]
}

21
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/draft-before-final-evidence/pending-supervision-record-addition.json Normal file
View file

@ -0,0 +1,21 @@
{
"schema": "custodian.pending-supervision-addition.v1",
"status": "candidate_only_not_applied",
"target_record": ".kaizen/agents/custodian-codex/supervision.json",
"expected_record_sha256": "407340493ffe584e9a245911d01f60e8f9ec1478f84478388f0210fd0c8b1a6d",
"operation": "append_one_proposal_if_ID_absent_and_record_preimage_matches",
"entry_ref": "pending-supervision-entry.json",
"entry_sha256": "f6335be4151c9d2cf913e6260c20a7444778cd662e368c035d33e5dc2c66234c",
"preserve_authority_fields": [
"agent_id",
"scope",
"mode",
"supervisor",
"runtime_enforcement",
"autopilot_grant",
"cost_budget_eur",
"risk_limit_eur"
],
"authority_changes": false,
"approval_or_execution_inferred": false
}

119
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/draft-before-final-evidence/proposal-metadata.json Normal file
View file

@ -0,0 +1,119 @@
{
"schema": "custodian.supervised-rollout-proposal.v1",
"proposal_id": "STATE-WP-0095-T06-cluster-frozen-receiving-r1",
"revision": 4,
"status": "prepared_pending_supervisor_review_not_authorized",
"prior_revisions_retained": [
1,
2,
3
],
"original_revision": 1,
"original_digest_sha256": "5bef9e7818ce1ebc64b81e300b6b40dcfc710a8625d6e0595534f8284dc2052d",
"target": {
"host": "railiance01",
"node_ip": "92.205.62.239",
"namespace": "state-hub",
"release": "state-hub",
"deployments": [
"state-hub",
"state-hub-mcp"
],
"migration_job_name": "state-hub-migrate",
"migration_job_uid": "must_capture_exact_new_trial_UID_after_approved_creation_before_any_conditional_cancellation"
},
"source_revision": "51bc70d1ff8f25450a8650d8e50caf053093a02e",
"readiness_source_patch_sha256": "4c52fd5a102df43ceb8e007d93d34efe861a3b321527cf5f3e195e35113c36a0",
"image": {
"tag": "main-51bc70d",
"digest": "sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015",
"ci_task": 12624,
"ci_status": "success",
"selected_source_files": 233,
"package_metadata_records": 86,
"floating_build_origins": "unknown_observed_bytes_bound_to_manifest"
},
"expected_result": "Normal owner image promotion makes exact frozen cluster receiving available; API/MCP ready, expected c4 schema and exact three historical parents/15 tasks preserved; only then is later registrar admission eligible. No original TLS/identity native acceptance is inferred.",
"verification_ref": "../r3-readiness/ROLLOUT-SUPPLEMENT.md; MIGRATION-JOB-RECOVERY.md",
"rollback_ref": "MIGRATION-JOB-RECOVERY.md",
"observed_rollback_baseline": {
"helm_revision": 75,
"image_tag": "main-7d99282",
"freshness": "must_refresh_before_trial",
"schema_downgrade": "forbidden_automatic",
"migration_job_quiescence_required": true
},
"requested_conditional_recovery_scope": [
"Exact-new-trial Job UID/image/ownership-bound foreground cancellation if still active after failed Helm, with UID precondition",
"Observation of only that bound Job and its owned Pod UIDs and inability to recreate",
"Private owner metadata-only verification of bound migration connections/transactions/locks quiescent",
"Image-only owner rollback only after quiescence and actual-schema assessment; no automatic schema downgrade"
],
"explicit_exclusions": [
"No deployment before exact supervisor disposition",
"No Repo Manager admission before delivered receiver proof",
"No Secret value read/write, native credential request or guard change",
"No broad backend cancellation, namespace cleanup, finalizer removal or old-data restore",
"No owner-native TLS/recovery/private-admission claim"
],
"cost_estimate_eur": null,
"risk_estimate_eur": null,
"cost_estimate": "Unknown attributable execution/network/operator recovery cost on existing infrastructure; no zero-cost claim or budget reservation inferred.",
"risk_estimate": "Unpriced production coordination outage, message migration locks/data compatibility, retained active migration Job, uncertain connection/lock mapping and recovery time; three-minute Helm wait is not a bound on outage or DB effects. Explicit supervised risk disposition required.",
"remaining_readiness": [
"Canonical review/integration/publication of exact r3 Make/test/T08 source",
"Exact reuse-values upgrade dry-run/diff with verified image identity",
"Fresh release/image/schema/headroom and prior/new Job identity baseline",
"Explicit supervisor disposition including conditional Job cancellation scope, recovery operator and unpriced risk"
],
"approval_ref": null,
"approved_revision": null,
"approved_digest_sha256": null,
"executed_revision": null,
"executed_digest_sha256": null,
"production_execution_started": false,
"refinement_or_rescue": true,
"promotion_evidence": false,
"artifacts": [
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/REVIEW.md",
"sha256": "sha256:5bef9e7818ce1ebc64b81e300b6b40dcfc710a8625d6e0595534f8284dc2052d"
},
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r2/REVIEW.md",
"sha256": "sha256:c8f13ed5d67d4743ea8e81bf720367022c820a060d7b225e84338eab2907493f"
},
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/ROLLOUT-SUPPLEMENT.md",
"sha256": "sha256:df2962fd1588784a987e37932bd67f03f6248ea6384cf4bbb74ad1fe2e26be2b"
},
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/IMAGE-RECEIPT-SUPPLEMENT.md",
"sha256": "sha256:dbd5d08cfe9e19499634a7f3de756704697712caa041b05814fc211b90820fe6"
},
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r3-readiness/source-patch-manifest.json",
"sha256": "sha256:98211e65bebeee2c25ca75c6b9f0bc45128093baaba02069b6c41a15be32fe74"
},
{
"path": "docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/MIGRATION-JOB-RECOVERY.md",
"sha256": "sha256:bfbd229bcb534c4f104b40090dba6c2dd0e2e040dce627e923143f81035880bc"
},
{
"path": "docs/changes/netkingdom-target-2026-10-17/statehub-public-image-inspection.json",
"sha256": "sha256:5d182e5b0a3e9462024c0b9a123d631215f1b2d0568f18060b10a310a8251741"
},
{
"path": "docs/changes/netkingdom-target-2026-10-17/statehub-anonymous-registry-metadata.json",
"sha256": "sha256:30f80aeb153bfe33076e2e5b94ae127e97ed38c0e1d033dabfd94934cef8be9a"
},
{
"path": "docs/changes/netkingdom-target-2026-10-17/migration-compat-r1-result.json",
"sha256": "sha256:c290afa92777c4635a5ee06c1bf6c3f28b1a265b6bff6a91f5aee45f799c2923"
},
{
"path": "docs/changes/netkingdom-target-2026-10-17/migration-compat-r1-cleanup.json",
"sha256": "sha256:1d7e0830460c0e2c537027f94242b01661702aec1b2f93aa9041af6eb14cde21"
}
]
}

6
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/revisions/r1-initial-literal-guard/preservation-note.json Normal file
View file

@ -0,0 +1,6 @@
{
"status": "initial_source_recovered_from_exact_recorded_two_edit_inverse",
"source_reconstruction": "Reversed only the explicit public-coordinate constant and exact-coordinate condition added after initial literal_sensitive_env_refused receipt. No independently retained pre-change digest existed.",
"safe_failure_receipt": "../../exact-reuse-values-dryrun-result.json",
"sha256": "da487e2a302143486588279984fc19bd617f8f73fe718e7bdb116b2931ac1e09"
}

113
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/revisions/r1-initial-literal-guard/reuse_values_dryrun.py Normal file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Read-only exact owner Helm reuse-values render/diff; no raw output retained."""
import copy
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import yaml
OWNER=Path('/home/worsch/state-hub')
KUBE='/home/worsch/.kube/config-railiance01'
CHART='deploy/railiance/apps/charts/state-hub'
TAG='main-51bc70d'
HELM=['helm','--kubeconfig',KUBE,'-n','state-hub']
DRYRUN=HELM+['upgrade','state-hub',CHART,'--reuse-values','--set-string','image.tag='+TAG,
'--dry-run=server','--hide-secret','--no-hooks','--timeout','3m']
ALLOWED={'ConfigMap','Deployment','Job','Service','ServiceAccount','Namespace','Ingress'}
LIMIT=2*1024*1024
class Refusal(RuntimeError):pass
def digest(obj):return 'sha256:'+hashlib.sha256(json.dumps(obj,sort_keys=True,separators=(',',':')).encode()).hexdigest()
def parse(raw):
if len(raw)>LIMIT:raise Refusal('render_size_refused')
result=[]
for obj in yaml.safe_load_all(raw):
if not obj:continue
if not isinstance(obj,dict) or obj.get('kind') not in ALLOWED:raise Refusal('unexpected_or_secret_resource_refused')
result.append(obj)
return result
def sections(raw):
text=raw.decode()
if '\nMANIFEST:\n' not in text:raise Refusal('helm_render_sections_unrecognized')
before,manifest=text.split('\nMANIFEST:\n',1)
manifest=manifest.split('\nNOTES:',1)[0]
hooks=before.split('\nHOOKS:\n',1)[1] if '\nHOOKS:\n' in before else ''
return parse(hooks)+parse(manifest)
def identity(obj):
meta=obj.get('metadata') or {}
return (obj['apiVersion'],obj['kind'],meta.get('namespace') or ('' if obj['kind']=='Namespace' else 'state-hub'),meta['name'])
def diff_paths(a,b,prefix=''):
if type(a)!=type(b):return [prefix]
if isinstance(a,dict):
paths=[]
for key in sorted(set(a)|set(b)):
path=prefix+'/'+key.replace('~','~0').replace('/','~1')
paths.extend([path] if key not in a or key not in b else diff_paths(a[key],b[key],path))
return paths
if isinstance(a,list):
if len(a)!=len(b):return [prefix]
return [p for i,(x,y) in enumerate(zip(a,b)) for p in diff_paths(x,y,prefix+'/'+str(i))]
return [] if a==b else [prefix]
def compare(old,new):
baseline={identity(o):o for o in old};intended={identity(o):o for o in new}
if len(baseline)!=len(old) or len(intended)!=len(new):raise Refusal('duplicate_object_identity')
rows=[]
for key in sorted(set(baseline)|set(intended)):
a=baseline.get(key);b=intended.get(key);paths=diff_paths(a,b)
images=[]
if b and b['kind'] in {'Deployment','Job'}:
spec=b.get('spec',{}).get('template',{}).get('spec',{})
for container in (spec.get('containers',[])+spec.get('initContainers',[])):
image=container.get('image','')
if not re.fullmatch(r'forgejo\.coulomb\.social/coulomb/state-hub:main-[0-9a-f]{7}(?:@sha256:[0-9a-f]{64})?',image):raise Refusal('unexpected_image_refused')
images.append(image)
for env in container.get('env',[]):
if 'value' in env and re.search(r'PASSWORD|TOKEN|SECRET|DATABASE_URL',env.get('name',''),re.I):raise Refusal('literal_sensitive_env_refused')
rows.append({'api_version':key[0],'kind':key[1],'namespace':key[2],'name':key[3],
'intended_images':images,'intended_spec_sha256':digest(b.get('spec',{})) if b else None,
'baseline_object_sha256':digest(a) if a else None,'intended_object_sha256':digest(b) if b else None,
'changed_paths':paths})
image_path=re.compile(r'^/spec/template/spec/(?:initContainers|containers)/[0-9]+/image$')
only_images=all(image_path.fullmatch(p) for row in rows for p in row['changed_paths'])
return {'objects':rows,'changed_object_count':sum(bool(r['changed_paths']) for r in rows),
'intended_diff_only_images':only_images,'object_identity_set_unchanged':set(baseline)==set(intended)}
def command(args):
env=dict(os.environ);env['KUBECONFIG']=KUBE
p=subprocess.run(args,cwd=OWNER,env=env,capture_output=True,timeout=45)
if len(p.stdout)>LIMIT:raise Refusal('command_output_size_refused')
if p.returncode:
lower=p.stderr.decode(errors='replace').lower()
label=next((label for marker,label in [('reject-secret-last-applied','secret_annotation_policy_refused'),('forbidden','admission_or_auth_refused'),('cannot patch','patch_validation_refused'),('is invalid','resource_validation_refused'),('timeout','bounded_timeout'),('connection refused','api_unavailable')] if marker in lower),'unknown_refusal')
raise Refusal(label)
return p.stdout
def main():
result={'schema':'statehub.exact-reuse-values-readonly-diff.v1','status':'failed',
'dryrun_argv':DRYRUN,'production_mutations':False,'hooks_executed':False,
'raw_helm_values_release_content_logs_retained':False,
'limitations':['Dry-run admission/render only, not an actual image pull/migration/deployment.',
'Helm timeout does not stop a later live migration Job; exact scoped cancellation/quiescence remains required.']}
try:
nodes=json.loads(command(['kubectl','--kubeconfig',KUBE,'get','nodes','-o','json']))
if not any(a.get('address')=='92.205.62.239' for n in nodes.get('items',[]) for a in n.get('status',{}).get('addresses',[])):raise Refusal('target_identity_refused')
result['stage']='baseline_release_manifest'
old=parse(command(HELM+['get','manifest','state-hub']))
result['stage']='baseline_release_hooks'
old+=parse(command(HELM+['get','hooks','state-hub']))
result['stage']='exact_upgrade_server_dryrun'
new=sections(command(DRYRUN))
result.update(compare(old,new));result['status']='passed_images_only' if result['intended_diff_only_images'] and result['object_identity_set_unchanged'] else 'review_nonimage_delta_required'
except (Refusal,subprocess.TimeoutExpired,ValueError,yaml.YAMLError) as error:
result['error_class']=type(error).__name__
result['error_label']=str(error) if isinstance(error,Refusal) else 'bounded_parse_or_command_failure'
print(json.dumps(result,indent=2))
return int(result['status']!='passed_images_only')
if __name__=='__main__':raise SystemExit(main())

117
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/revisions/r2-source-coordinate-guard/reuse_values_dryrun.py Normal file
View file

@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Read-only exact owner Helm reuse-values render/diff; no raw output retained."""
import copy
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import yaml
OWNER=Path('/home/worsch/state-hub')
KUBE='/home/worsch/.kube/config-railiance01'
CHART='deploy/railiance/apps/charts/state-hub'
TAG='main-51bc70d'
HELM=['helm','--kubeconfig',KUBE,'-n','state-hub']
DRYRUN=HELM+['upgrade','state-hub',CHART,'--reuse-values','--set-string','image.tag='+TAG,
'--dry-run=server','--hide-secret','--no-hooks','--timeout','3m']
PUBLIC_COORDINATES={'OPENBAO_K8S_TOKEN_PATH':'/var/run/secrets/openbao/token',
'FORGE_READ_SECRET_PATH':'platform/data/workloads/state-hub/forge-derivation',
'FORGE_READ_SECRET_KEY':'FORGE_READ_TOKEN'}
ALLOWED={'ConfigMap','Deployment','Job','Service','ServiceAccount','Namespace','Ingress'}
LIMIT=2*1024*1024
class Refusal(RuntimeError):pass
def digest(obj):return 'sha256:'+hashlib.sha256(json.dumps(obj,sort_keys=True,separators=(',',':')).encode()).hexdigest()
def parse(raw):
if len(raw)>LIMIT:raise Refusal('render_size_refused')
result=[]
for obj in yaml.safe_load_all(raw):
if not obj:continue
if not isinstance(obj,dict) or obj.get('kind') not in ALLOWED:raise Refusal('unexpected_or_secret_resource_refused')
result.append(obj)
return result
def sections(raw):
text=raw.decode()
if '\nMANIFEST:\n' not in text:raise Refusal('helm_render_sections_unrecognized')
before,manifest=text.split('\nMANIFEST:\n',1)
manifest=manifest.split('\nNOTES:',1)[0]
hooks=before.split('\nHOOKS:\n',1)[1] if '\nHOOKS:\n' in before else ''
return parse(hooks)+parse(manifest)
def identity(obj):
meta=obj.get('metadata') or {}
return (obj['apiVersion'],obj['kind'],meta.get('namespace') or ('' if obj['kind']=='Namespace' else 'state-hub'),meta['name'])
def diff_paths(a,b,prefix=''):
if type(a)!=type(b):return [prefix]
if isinstance(a,dict):
paths=[]
for key in sorted(set(a)|set(b)):
path=prefix+'/'+key.replace('~','~0').replace('/','~1')
paths.extend([path] if key not in a or key not in b else diff_paths(a[key],b[key],path))
return paths
if isinstance(a,list):
if len(a)!=len(b):return [prefix]
return [p for i,(x,y) in enumerate(zip(a,b)) for p in diff_paths(x,y,prefix+'/'+str(i))]
return [] if a==b else [prefix]
def compare(old,new):
baseline={identity(o):o for o in old};intended={identity(o):o for o in new}
if len(baseline)!=len(old) or len(intended)!=len(new):raise Refusal('duplicate_object_identity')
rows=[]
for key in sorted(set(baseline)|set(intended)):
a=baseline.get(key);b=intended.get(key);paths=diff_paths(a,b)
images=[]
if b and b['kind'] in {'Deployment','Job'}:
spec=b.get('spec',{}).get('template',{}).get('spec',{})
for container in (spec.get('containers',[])+spec.get('initContainers',[])):
image=container.get('image','')
if not re.fullmatch(r'forgejo\.coulomb\.social/coulomb/state-hub:main-[0-9a-f]{7}(?:@sha256:[0-9a-f]{64})?',image):raise Refusal('unexpected_image_refused')
images.append(image)
for env in container.get('env',[]):
if 'value' in env and re.search(r'PASSWORD|TOKEN|SECRET|DATABASE_URL',env.get('name',''),re.I):
if env.get('name') not in PUBLIC_COORDINATES or env['value']!=PUBLIC_COORDINATES[env['name']]:raise Refusal('literal_sensitive_env_refused')
rows.append({'api_version':key[0],'kind':key[1],'namespace':key[2],'name':key[3],
'intended_images':images,'intended_spec_sha256':digest(b.get('spec',{})) if b else None,
'baseline_object_sha256':digest(a) if a else None,'intended_object_sha256':digest(b) if b else None,
'changed_paths':paths})
image_path=re.compile(r'^/spec/template/spec/(?:initContainers|containers)/[0-9]+/image$')
only_images=all(image_path.fullmatch(p) for row in rows for p in row['changed_paths'])
return {'objects':rows,'changed_object_count':sum(bool(r['changed_paths']) for r in rows),
'intended_diff_only_images':only_images,'object_identity_set_unchanged':set(baseline)==set(intended)}
def command(args):
env=dict(os.environ);env['KUBECONFIG']=KUBE
p=subprocess.run(args,cwd=OWNER,env=env,capture_output=True,timeout=45)
if len(p.stdout)>LIMIT:raise Refusal('command_output_size_refused')
if p.returncode:
lower=p.stderr.decode(errors='replace').lower()
label=next((label for marker,label in [('reject-secret-last-applied','secret_annotation_policy_refused'),('forbidden','admission_or_auth_refused'),('cannot patch','patch_validation_refused'),('is invalid','resource_validation_refused'),('timeout','bounded_timeout'),('connection refused','api_unavailable')] if marker in lower),'unknown_refusal')
raise Refusal(label)
return p.stdout
def main():
result={'schema':'statehub.exact-reuse-values-readonly-diff.v1','status':'failed',
'dryrun_argv':DRYRUN,'production_mutations':False,'hooks_executed':False,
'raw_helm_values_release_content_logs_retained':False,
'limitations':['Dry-run admission/render only, not an actual image pull/migration/deployment.',
'Helm timeout does not stop a later live migration Job; exact scoped cancellation/quiescence remains required.']}
try:
nodes=json.loads(command(['kubectl','--kubeconfig',KUBE,'get','nodes','-o','json']))
if not any(a.get('address')=='92.205.62.239' for n in nodes.get('items',[]) for a in n.get('status',{}).get('addresses',[])):raise Refusal('target_identity_refused')
result['stage']='baseline_release_manifest'
old=parse(command(HELM+['get','manifest','state-hub']))
result['stage']='baseline_release_hooks'
old+=parse(command(HELM+['get','hooks','state-hub']))
result['stage']='exact_upgrade_server_dryrun'
new=sections(command(DRYRUN))
result.update(compare(old,new));result['status']='passed_images_only' if result['intended_diff_only_images'] and result['object_identity_set_unchanged'] else 'review_nonimage_delta_required'
except (Refusal,subprocess.TimeoutExpired,ValueError,yaml.YAMLError) as error:
result['error_class']=type(error).__name__
result['error_label']=str(error) if isinstance(error,Refusal) else 'bounded_parse_or_command_failure'
print(json.dumps(result,indent=2))
return int(result['status']!='passed_images_only')
if __name__=='__main__':raise SystemExit(main())

117
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/revisions/r4-rollout-review/revisions/r3-digest-pinned-reference/reuse_values_dryrun.py Normal file
View file

@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Read-only exact owner Helm reuse-values render/diff; no raw output retained."""
import copy
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import yaml
OWNER=Path('/home/worsch/state-hub')
KUBE='/home/worsch/.kube/config-railiance01'
CHART='deploy/railiance/apps/charts/state-hub'
TAG='main-51bc70d@sha256:c802690ee1811b3d77df4077936f9cdced1385514e8673fcbc9232b84d645015'
HELM=['helm','--kubeconfig',KUBE,'-n','state-hub']
DRYRUN=HELM+['upgrade','state-hub',CHART,'--reuse-values','--set-string','image.tag='+TAG,
'--dry-run=server','--hide-secret','--no-hooks','--timeout','3m']
PUBLIC_COORDINATES={'OPENBAO_K8S_TOKEN_PATH':'/var/run/secrets/openbao/token',
'FORGE_READ_SECRET_PATH':'platform/data/workloads/state-hub/forge-derivation',
'FORGE_READ_SECRET_KEY':'FORGE_READ_TOKEN'}
ALLOWED={'ConfigMap','Deployment','Job','Service','ServiceAccount','Namespace','Ingress'}
LIMIT=2*1024*1024
class Refusal(RuntimeError):pass
def digest(obj):return 'sha256:'+hashlib.sha256(json.dumps(obj,sort_keys=True,separators=(',',':')).encode()).hexdigest()
def parse(raw):
if len(raw)>LIMIT:raise Refusal('render_size_refused')
result=[]
for obj in yaml.safe_load_all(raw):
if not obj:continue
if not isinstance(obj,dict) or obj.get('kind') not in ALLOWED:raise Refusal('unexpected_or_secret_resource_refused')
result.append(obj)
return result
def sections(raw):
text=raw.decode()
if '\nMANIFEST:\n' not in text:raise Refusal('helm_render_sections_unrecognized')
before,manifest=text.split('\nMANIFEST:\n',1)
manifest=manifest.split('\nNOTES:',1)[0]
hooks=before.split('\nHOOKS:\n',1)[1] if '\nHOOKS:\n' in before else ''
return parse(hooks)+parse(manifest)
def identity(obj):
meta=obj.get('metadata') or {}
return (obj['apiVersion'],obj['kind'],meta.get('namespace') or ('' if obj['kind']=='Namespace' else 'state-hub'),meta['name'])
def diff_paths(a,b,prefix=''):
if type(a)!=type(b):return [prefix]
if isinstance(a,dict):
paths=[]
for key in sorted(set(a)|set(b)):
path=prefix+'/'+key.replace('~','~0').replace('/','~1')
paths.extend([path] if key not in a or key not in b else diff_paths(a[key],b[key],path))
return paths
if isinstance(a,list):
if len(a)!=len(b):return [prefix]
return [p for i,(x,y) in enumerate(zip(a,b)) for p in diff_paths(x,y,prefix+'/'+str(i))]
return [] if a==b else [prefix]
def compare(old,new):
baseline={identity(o):o for o in old};intended={identity(o):o for o in new}
if len(baseline)!=len(old) or len(intended)!=len(new):raise Refusal('duplicate_object_identity')
rows=[]
for key in sorted(set(baseline)|set(intended)):
a=baseline.get(key);b=intended.get(key);paths=diff_paths(a,b)
images=[]
if b and b['kind'] in {'Deployment','Job'}:
spec=b.get('spec',{}).get('template',{}).get('spec',{})
for container in (spec.get('containers',[])+spec.get('initContainers',[])):
image=container.get('image','')
if not re.fullmatch(r'forgejo\.coulomb\.social/coulomb/state-hub:main-[0-9a-f]{7}(?:@sha256:[0-9a-f]{64})?',image):raise Refusal('unexpected_image_refused')
images.append(image)
for env in container.get('env',[]):
if 'value' in env and re.search(r'PASSWORD|TOKEN|SECRET|DATABASE_URL',env.get('name',''),re.I):
if env.get('name') not in PUBLIC_COORDINATES or env['value']!=PUBLIC_COORDINATES[env['name']]:raise Refusal('literal_sensitive_env_refused')
rows.append({'api_version':key[0],'kind':key[1],'namespace':key[2],'name':key[3],
'intended_images':images,'intended_spec_sha256':digest(b.get('spec',{})) if b else None,
'baseline_object_sha256':digest(a) if a else None,'intended_object_sha256':digest(b) if b else None,
'changed_paths':paths})
image_path=re.compile(r'^/spec/template/spec/(?:initContainers|containers)/[0-9]+/image$')
only_images=all(image_path.fullmatch(p) for row in rows for p in row['changed_paths'])
return {'objects':rows,'changed_object_count':sum(bool(r['changed_paths']) for r in rows),
'intended_diff_only_images':only_images,'object_identity_set_unchanged':set(baseline)==set(intended)}
def command(args):
env=dict(os.environ);env['KUBECONFIG']=KUBE
p=subprocess.run(args,cwd=OWNER,env=env,capture_output=True,timeout=45)
if len(p.stdout)>LIMIT:raise Refusal('command_output_size_refused')
if p.returncode:
lower=p.stderr.decode(errors='replace').lower()
label=next((label for marker,label in [('reject-secret-last-applied','secret_annotation_policy_refused'),('forbidden','admission_or_auth_refused'),('cannot patch','patch_validation_refused'),('is invalid','resource_validation_refused'),('timeout','bounded_timeout'),('connection refused','api_unavailable')] if marker in lower),'unknown_refusal')
raise Refusal(label)
return p.stdout
def main():
result={'schema':'statehub.exact-reuse-values-readonly-diff.v1','status':'failed',
'dryrun_argv':DRYRUN,'production_mutations':False,'hooks_executed':False,
'raw_helm_values_release_content_logs_retained':False,
'limitations':['Dry-run admission/render only, not an actual image pull/migration/deployment.',
'Helm timeout does not stop a later live migration Job; exact scoped cancellation/quiescence remains required.']}
try:
nodes=json.loads(command(['kubectl','--kubeconfig',KUBE,'get','nodes','-o','json']))
if not any(a.get('address')=='92.205.62.239' for n in nodes.get('items',[]) for a in n.get('status',{}).get('addresses',[])):raise Refusal('target_identity_refused')
result['stage']='baseline_release_manifest'
old=parse(command(HELM+['get','manifest','state-hub']))
result['stage']='baseline_release_hooks'
old+=parse(command(HELM+['get','hooks','state-hub']))
result['stage']='exact_upgrade_server_dryrun'
new=sections(command(DRYRUN))
result.update(compare(old,new));result['status']='passed_images_only' if result['intended_diff_only_images'] and result['object_identity_set_unchanged'] else 'review_nonimage_delta_required'
except (Refusal,subprocess.TimeoutExpired,ValueError,yaml.YAMLError) as error:
result['error_class']=type(error).__name__
result['error_label']=str(error) if isinstance(error,Refusal) else 'bounded_parse_or_command_failure'
print(json.dumps(result,indent=2))
return int(result['status']!='passed_images_only')
if __name__=='__main__':raise SystemExit(main())

11
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/rollback-image-metadata.json Normal file
View file

@ -0,0 +1,11 @@
{
"status": "metadata_observed",
"release": "state-hub",
"namespace": "state-hub",
"revision": 75,
"image": {
"repository": "forgejo.coulomb.social/coulomb/state-hub",
"tag": "main-7d99282",
"pullPolicy": "IfNotPresent"
}
}

97
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/source-patch-manifest.json Normal file
View file

@ -0,0 +1,97 @@
{
"schema": "cluster-historical-source-patches.v1",
"status": "isolated_review_candidate_not_applied",
"binding_manifests_identical": true,
"repos": {
"repo-manager": {
"base_revision": "9ebde0a9bd60d5c82910494fbe6bd1b891455cee",
"files": [
{
"path": "src/repo_manager/historical_sources.py",
"action": "modify",
"base_sha256": "a465aa595867c5e80918168158f8a27bf429d469b2e5249eca98b4e5ef84ce03",
"candidate_sha256": "838a0422b81e1215e3ec148c3104dd603d9dfc476610bb5617d98ccf5d3be218"
},
{
"path": "src/repo_manager/identifiers.py",
"action": "modify",
"base_sha256": "68dbbd3f137f326a49d20ba3f5855d6b00b4d62eb01dc904b51a08d77552d2bf",
"candidate_sha256": "4e35fcd009c20822574fd33171176bd14a1ff2dd5af8b0bb2820954974593a2a"
},
{
"path": "tests/test_cluster_frozen_history.py",
"action": "add",
"base_sha256": null,
"candidate_sha256": "0490ef3eb474f653e4a6a52b711999876bef061c07de3e7a6d327a8ba54acaf3"
},
{
"path": "workplans/RMGR-WP-0019-historical-source-admission.md",
"action": "modify",
"base_sha256": "638af4000aa442a57df8eddef2d8b5aa5dffb9dbc9ccf547d9e04cf25012ffe4",
"candidate_sha256": "6378d96b70259ae81435b6ce8f97864698bdc5b6679cabbe9dc61a1fc1d09699"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3"
}
],
"patch_sha256": "232552c3d601d185984be5a8e5bc9937bc9ddda7327923b5f0440f6deb2c87fc"
},
"state-hub": {
"base_revision": "4ab66e363ed00a67ad1ea0434854bc4284c7a26e",
"files": [
{
"path": "api/services/historical_projection.py",
"action": "modify",
"base_sha256": "4d7cd127462bedaaf1c0915c6a11f58b25d28ae6f000eb89c95d903bf4a0e94e",
"candidate_sha256": "e3c4bf965764b32a6df5d61c71272ee22569df9a9ddae2a3fc3df97ec554e0b2"
},
{
"path": "tests/test_cluster_historical_projection.py",
"action": "add",
"base_sha256": null,
"candidate_sha256": "183f0574aa01f69269a06d162b29b5b95c3415f6fce0560bb92188c09be26507"
},
{
"path": "workplans/STATE-WP-0095-historical-source-receiving.md",
"action": "modify",
"base_sha256": "9be4b0661f0cc817d55dcf02f959339cb6c13f56956b111a6ecd87d1bec79405",
"candidate_sha256": "706c7113bf13c124e0281ce9838141d50ceb8fc6007879b4bb52b6f4a346f0a9"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a"
},
{
"path": "tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
"action": "add",
"base_sha256": null,
"candidate_sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3"
}
],
"patch_sha256": "258e936a06b11446cbf131c48b9965be4d0800f753ba3a3bf180c53a631b8e94"
}
}
}

922
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub.patch Normal file
View file

@ -0,0 +1,922 @@
--- a/api/services/historical_projection.py
+++ b/api/services/historical_projection.py
@@ -72,6 +72,111 @@
"record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED",
"uuid": "1c53d3db-6633-5e2d-987f-10706d73be9e",
"parent_record_id": "CUST-WP-ADHOC-2026-08-25-RETIRED"
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md",
+ "sha256": "8d6f8533c119117686bc0d5bb9ed5349726aa6c3189ff98b53409c68ac685ca9",
+ "record_id": "RAIL-BS-WP-0002",
+ "uuid": "4c63dfc6-9eac-4e79-9f77-8f644ad7147d",
+ "parent_record_id": "RAIL-BS-WP-0002",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "3f042630-eab0-4c6a-9167-e2b28ff20e40",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "e8510646-46ed-4697-a345-f3d3009eea78",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "dab2c07f-8aa0-4635-8df6-857e87e93fc5",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "5c3d40e4-239b-488e-9519-6f7a38d2325f",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "7f9e0e58-a130-467a-a2d0-b3f2564e496f",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md",
+ "sha256": "4dc2f61e15adc3c5c81f9ecf8b933512275b95e8f829fac52ae7cde1385cea4a",
+ "record_id": "RAIL-BS-WP-0003",
+ "uuid": "7ee9ee22-1fae-4567-9194-8d70a9e0f45b",
+ "parent_record_id": "RAIL-BS-WP-0003",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "ab166073-30a7-4702-a037-4091e8706e20",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "140da396-8e30-4f4d-b88c-c42c0cd46c01",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d",
+ "status": "done"
+ }
+ ]
+ },
+ {
+ "repo": "railiance-cluster",
+ "path": "workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md",
+ "sha256": "04f787078232395bd2d43861de16c521848559fa286f06e1caa9cb779aa71ac3",
+ "record_id": "RAIL-BS-WP-0004",
+ "uuid": "7e8b0c20-51eb-40c9-9e3b-85dd380d7625",
+ "parent_record_id": "RAIL-BS-WP-0004",
+ "tasks": [
+ {
+ "label": "T01",
+ "uuid": "4526a842-ea31-4874-9231-92ab556cfe7b",
+ "status": "done"
+ },
+ {
+ "label": "T02",
+ "uuid": "a6313e06-1976-46a7-8e31-df4eb2eca880",
+ "status": "done"
+ },
+ {
+ "label": "T03",
+ "uuid": "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3",
+ "status": "done"
+ },
+ {
+ "label": "T04",
+ "uuid": "08233868-d522-4117-bc4e-6c0f52545665",
+ "status": "done"
+ },
+ {
+ "label": "T05",
+ "uuid": "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8",
+ "status": "done"
+ },
+ {
+ "label": "T06",
+ "uuid": "f8e4a094-c367-40eb-b895-da17bc144b07",
+ "status": "done"
+ }
+ ]
}
]''')
--- /dev/null
+++ b/tests/test_cluster_historical_projection.py
@@ -0,0 +1,67 @@
+import shutil
+import uuid
+from pathlib import Path
+import pytest
+from api.services import forge_projection as fp
+from api.services import historical_projection as hp
+from tests.test_forge_projection import _FakeSession,_Repo,_Row,_TaskRow
+
+FIXTURE=Path(__file__).parent/'fixtures/railiance-cluster'
+@pytest.fixture
+def bound(tmp_path):
+ root=tmp_path/'railiance-cluster';shutil.copytree(FIXTURE,root)
+ derived=fp.derive_from_checkout(root,'railiance-cluster','candidate')
+ assert len(derived.workplans)==3
+ rows=[];tasks=[]
+ for parent in derived.workplans:
+ row=_Row(parent.record_id.lower(),'finished',parent.relative_path);row.id=uuid.UUID(parent.uuid);rows.append(row)
+ for source in parent.tasks:
+ task=_TaskRow(source.record_id,status='done',workplan_id=row.id);task.id=uuid.UUID(source.uuid);tasks.append(task)
+ assert len(tasks)==15
+ return root,derived,rows,tasks
+
+def test_exact_three_sources_derive_only_frozen_15_task_uuids(bound):
+ root,derived,rows,tasks=bound
+ assert all(parent.historical_preservation for parent in derived.workplans)
+ assert all(parent.uuid!=fp.derived_record_uuid(parent.record_id) for parent in derived.workplans)
+ expected={item['uuid'] for entry in hp.BINDINGS if entry['repo']=='railiance-cluster' for item in entry['tasks']}
+ assert {str(t.id) for t in tasks}==expected
+
+@pytest.mark.asyncio
+async def test_existing_rows_are_entirely_unchanged_on_normal_receiving(bound):
+ root,derived,rows,tasks=bound;before=[dict(x.__dict__) for x in rows+tasks]
+ session=_FakeSession(_Repo(),rows,task_rows=tasks)
+ outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
+ assert outcome.status=='noop' and not outcome.refused
+ assert [dict(x.__dict__) for x in rows+tasks]==before
+ assert not session.added and not session.deleted
+
+@pytest.mark.parametrize('defect',['missing-parent','missing-task','foreign-holder','nonterminal-task','nonterminal-parent','duplicate-current-label'])
+@pytest.mark.asyncio
+async def test_unsupported_actual_binding_refuses_before_any_mutation(bound,defect):
+ root,derived,rows,tasks=bound
+ if defect=='missing-parent':rows.pop()
+ elif defect=='missing-task':tasks.pop()
+ elif defect=='foreign-holder':tasks[0].workplan_id=uuid.uuid4()
+ elif defect=='nonterminal-task':tasks[0].status='progress'
+ elif defect=='duplicate-current-label':tasks.append(_TaskRow(tasks[0].record_id,status='done',workplan_id=tasks[0].workplan_id))
+ else:rows[0].status='active'
+ before=[dict(x.__dict__) for x in rows+tasks];session=_FakeSession(_Repo(),rows,task_rows=tasks)
+ outcome=await fp.reset_repository_projection(session,'railiance-cluster',derived=derived)
+ assert outcome.status=='refused';assert [dict(x.__dict__) for x in rows+tasks]==before
+ assert not session.added and not session.deleted
+
+@pytest.mark.parametrize('change',['hash-drift','missing','symlink','live-source','missing-source-uuid'])
+def test_frozen_source_drift_refuses_derivation(bound,change):
+ root,*_=bound;p=next((root/'workplans/archived').glob('*.md'));raw=p.read_bytes()
+ if change=='missing':p.unlink()
+ elif change=='symlink':p.unlink();target=root/'saved';target.write_bytes(raw);p.symlink_to(target)
+ elif change=='live-source':p.write_bytes(raw.replace(b'status: finished',b'status: active',1))
+ elif change=='missing-source-uuid':p.write_bytes(raw.replace(b'state_hub_task_id:',b'old_task_id:',1))
+ else:p.write_bytes(raw+b'\nunreviewed\n')
+ with pytest.raises(fp.ForgeDeriveError,match='Historical source binding'):fp.derive_from_checkout(root,'railiance-cluster','candidate')
+
+def test_unknown_repository_never_trusts_frozen_source_uuids(bound):
+ root,derived,*_=bound;other=fp.derive_from_checkout(root,'different-repo','candidate')
+ assert not any(parent.historical_preservation for parent in other.workplans)
+ assert not {parent.uuid for parent in derived.workplans}&{parent.uuid for parent in other.workplans}
--- a/workplans/STATE-WP-0095-historical-source-receiving.md
+++ b/workplans/STATE-WP-0095-historical-source-receiving.md
@@ -4,10 +4,10 @@
title: "Preserve exact closed historical source bindings in Forge receiving"
domain: infotech
repo: state-hub
-status: finished
+status: active
owner: codex
created: "2026-10-04"
-updated: "2026-10-06"
+updated: "2026-10-17"
flavor: implementation
depends_on: []
state_hub_workstream_id: "79579572-8120-5d3b-aca5-dd4aff60d058"
@@ -167,3 +167,65 @@
[activity stale retirement rollout](../docs/evidence/2026-10-06-activity-stale-retirement-rollout.json).
Repo Manager's re-run of the Activity owner sync (RMGR-WP-0019-T03) and
ACTIVITY-WP-0045-T02 remain with their owners.
+
+
+## Cluster exact historical receiving successor, 2026-10-17
+
+This scoped extension retains the completed Activity/Custodian outcomes above.
+Three frozen cluster archives contain three existing parent UUIDs and 15 existing
+closed task UUIDs. Extend the existing exact historical receiving path before
+registrar admission; never derive replacement identities or modify those
+sources. Cross-owner observation attaches to existing SHR-WP-0001, with
+RMGR-WP-0019 as obligor/beneficiary and NK-WP-0049-T35 benefiting indirectly.
+The local derived case is waiting; no live worker injection is authorized.
+
+## Prepare exact cluster source preservation and refusal cases
+
+```task
+id: STATE-WP-0095-T05
+status: done
+priority: high
+```
+
+Prepared the same full-file SHA256 and 15 task binding manifest as Repo Manager.
+Candidate receiving tests pass 26: existing rows remain entirely unchanged;
+missing/foreign/nonterminal rows and unknown, changed, missing or symlinked
+source refuse. Read-only public metadata confirms all 18 existing rows have the
+expected ownership and terminal status. These are source and metadata checks,
+not delivered receiving or a committed native trial.
+
+## Deliver reviewed cluster receiving successor before registrar admission
+
+```task
+id: STATE-WP-0095-T06
+status: wait
+priority: high
+depends_on: [STATE-WP-0095-T05]
+needs_human: true
+intervention_note: "Review exact committed source, CI image and owner deployment proposal before rollout."
+```
+
+Use the normal CI image and Helm owner path after exact revision/image pin,
+fresh headroom and reviewed dry-run/diff. API/MCP currently run main-7d99282;
+Helm revision75 is the observed rollback baseline, requiring freshness before
+execution. No deployment assent exists for this successor. Verify delivered
+module hash, health, both readiness checks and unchanged schema; verify exact
+three parents/15 tasks and rejection behavior without committed probe changes.
+Only the verified return unblocks RMGR-WP-0019-T05. Costs and monetary risk are
+unknown pending exact rollout proposal; no zero-cost or autonomy grant is inferred.
+
+## Verify ordinary cluster receiving and immutable historical rows
+
+```task
+id: STATE-WP-0095-T07
+status: wait
+priority: high
+depends_on: [STATE-WP-0095-T06, RMGR-WP-0019-T06]
+```
+
+Capture normal cluster owner publication/receiving receipts and compare entire
+existing row snapshots and exact source bytes. No creation, movement, relabel,
+unretirement or retirement is authorized for these histories. A rollback first
+fences cluster sync and restores registrar refusal before removing receiving
+support; it never rewrites archived IDs or UUIDs. This work does not establish
+original TLS recovery or owner-native admission.
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0002-k3s-baseline.md
@@ -0,0 +1,177 @@
+---
+id: RAIL-BS-WP-0002
+type: workplan
+title: "k3s and Kubernetes Platform Baseline"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: railiance
+topic_slug: railiance
+repo_goal_id: "70ab2379-fb9d-4fec-a09d-b2a717e4ace8"
+state_hub_workstream_id: "4c63dfc6-9eac-4e79-9f77-8f644ad7147d"
+created: "2026-03-09"
+updated: "2026-03-10"
+completed: "2026-03-10"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# k3s and Kubernetes Platform Baseline
+
+## Goal
+
+Install k3s, Helm, and the baseline Kubernetes services on the converged
+HostEurope node. This workplan picks up exactly where `railiance-hosts`
+leaves off: a hardened, verified OS node that is ready for Kubernetes.
+
+## Pre-condition
+
+`railiance-infra` converge + Goss verify must pass before any task here
+is executed:
+
+```bash
+# In railiance-infra/
+make converge
+make verify # must exit 0
+```
+
+## Boundary
+
+This repo owns everything from k3s upward. It must not re-configure items
+defined in `railiance-infra/spec/server-baseline.yaml`. See ADR-003:
+`railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`.
+
+**Out of scope here:** platform services (PostgreSQL, storage, identity)
+→ `railiance-platform`. Application deployments (Gitea, coulomb services)
+→ `railiance-apps`.
+
+---
+
+## Tasks
+
+### T01 — Ansible playbook: install k3s (server mode)
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "3f042630-eab0-4c6a-9167-e2b28ff20e40"
+completed: "2026-03-10"
+```
+
+Harden `ansible/bootstrap.yml` to a production-ready k3s install:
+
+- Use the official k3s install script pinned to a specific version
+ (`INSTALL_K3S_VERSION=v1.35.1+k3s1`)
+- `INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644"`
+ (cluster-init enables embedded etcd for future HA expansion)
+- Wait for node `Ready` before proceeding:
+ ```bash
+ k3s kubectl wait node --all --for=condition=Ready --timeout=120s
+ ```
+- Fetch kubeconfig to the control node as `~/.kube/config-hosteurope`
+
+**Done when:** `k3s kubectl get nodes` returns `Ready` from both the server
+and the control node (via kubeconfig).
+
+---
+
+### T02 — Helm installation
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "e8510646-46ed-4697-a345-f3d3009eea78"
+completed: "2026-03-10"
+```
+
+Add a task (or a role `roles/helm/`) that:
+
+1. Downloads the Helm binary (pinned version) to `/usr/local/bin/helm`
+2. Verifies the checksum
+3. Confirms `helm version` succeeds
+
+**Done when:** `helm version` succeeds on the HostEurope node.
+
+---
+
+### T03 — Smoke test: k3s + Helm
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "dab2c07f-8aa0-4635-8df6-857e87e93fc5"
+completed: "2026-03-10"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+- `k3s kubectl get nodes` → node in Ready state
+- `helm version` exits 0
+- CoreDNS pod running in `kube-system`
+- Traefik ingress controller pod running (default in k3s)
+
+Run via:
+```bash
+ansible-playbook -i ansible/hosts.ini ansible/smoke.yml
+```
+or directly over SSH if the kubeconfig is available locally.
+
+**Done when:** all assertions pass and the script exits 0.
+
+---
+
+### T04 — Commit kubeconfig management notes
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "5c3d40e4-239b-488e-9519-6f7a38d2325f"
+completed: "2026-03-10"
+```
+
+Document in `docs/kubeconfig.md`:
+
+- Where the kubeconfig is fetched to (`~/.kube/config-hosteurope`)
+- How to merge it into `~/.kube/config`
+- How to switch context: `kubectl config use-context default`
+- Security note: kubeconfig is gitignored (contains cluster CA + client cert)
+
+**Done when:** doc written and committed.
+
+---
+
+### T05 — Add `make k3s-install` and `make smoke` targets
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "7f9e0e58-a130-467a-a2d0-b3f2564e496f"
+completed: "2026-03-10"
+```
+
+Add to Makefile (create one if none exists):
+
+```makefile
+k3s-install: ## Install k3s and Helm on all inventory hosts
+ ansible-playbook -i ansible/hosts.ini ansible/bootstrap.yml
+
+smoke: ## Run Kubernetes smoke tests
+ bash tests/smoke_kube.sh
+```
+
+**Done when:** both targets work and are listed in `make help`.
+
+---
+
+## References
+
+- Repo goal: `70ab2379-fb9d-4fec-a09d-b2a717e4ace8` (Install k3s and Kubernetes Baseline)
+- Domain goal: `6f96c712-60e6-4ea9-ab06-168878eafbce` (Three-Phoenix Secure Kubernetes Infrastructure)
+- Pre-condition: railiance-infra WP-0001 (Secure Single-Server Bootstrap) — completed 2026-03-09
+- Boundary ADR: `railiance-infra/docs/adr/ADR-003-railiance-5repo-stack-architecture.md`
+- k3s releases: https://github.com/k3s-io/k3s/releases
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0003-pgpool-ha-failover-fix.md
@@ -0,0 +1,196 @@
+---
+id: RAIL-BS-WP-0003
+type: workplan
+title: "pgpool CrashLoopBackOff on PostgreSQL HA failover — missing secret key"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+created: "2026-03-10"
+updated: "2026-03-10"
+state_hub_workstream_id: "7ee9ee22-1fae-4567-9194-8d70a9e0f45b"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Bug Report: pgpool CrashLoopBackOff on PostgreSQL HA failover
+
+## Summary
+
+On 2026-03-10 a PostgreSQL HA failover caused all three postgresql pods to
+restart. pgpool — the connection pooler between Gitea and PostgreSQL — then
+entered CrashLoopBackOff and produced no logs. As a result Gitea's login
+and all write operations hung indefinitely. The root page was still served
+(from Valkey cache) which masked the failure.
+
+The fix was to patch a missing key in a Kubernetes secret. The root cause is
+that the `gitea-12.2.0` Helm chart (postgresql-ha subchart v16.2.2) does not
+populate the `pgpool-password` key in the `gitea-postgresql-ha-postgresql`
+secret, even though the pgpool pod requires it at startup.
+
+---
+
+## Timeline
+
+| Time (UTC) | Event |
+|---|---|
+| ~09:45 | postgresql-0, postgresql-2 pods restarted (repmgr failover) |
+| ~09:45 | pgpool pod restarted and entered CrashLoopBackOff |
+| ~11:00 | User noticed Gitea login hanging; home page still loading |
+| ~13:00 | Root cause identified: missing `pgpool-password` secret key |
+| ~13:10 | Secret patched; pgpool pod deleted and restarted cleanly |
+| ~13:15 | Gitea fully operational |
+
+---
+
+## Root Cause
+
+The Bitnami `pgpool` container startup script reads the file
+`/opt/bitnami/pgpool/secrets/pgpool-password`, which is mounted from the
+`gitea-postgresql-ha-postgresql` Kubernetes Secret via a `subPath` volume
+mount. That secret key was never created by the Helm chart, so the file did
+not exist. The container exited immediately with no logs.
+
+The pod had been running for 20 days without a restart, so this gap was
+never discovered during initial deployment.
+
+---
+
+## Evidence
+
+```bash
+# Secret was missing the pgpool-password key
+sudo k3s kubectl get secret -n default gitea-postgresql-ha-postgresql -o yaml
+# data: keys were password, postgres-password, repmgr-password only
+# pgpool-password was absent
+
+# pgpool pod describe showed 824 back-off restarts over 173 minutes
+# No logs in either current or --previous output
+sudo k3s kubectl logs -n default <pgpool-pod> --previous
+# (empty)
+
+# Gitea process had zero TCP connections to PostgreSQL port 5432
+# but many connections to Valkey port 6379
+cat /proc/<gitea-pid>/net/tcp | grep 1538 # 1538 = 5432 hex — no results
+```
+
+---
+
+## Immediate Fix Applied
+
+```bash
+# Add the missing key (value = sr-check-password = changeme4 = base64: Y2hhbmdlbWU0)
+sudo k3s kubectl patch secret -n default gitea-postgresql-ha-postgresql \
+ --type='json' \
+ -p='[{"op":"add","path":"/data/pgpool-password","value":"Y2hhbmdlbWU0"}]'
+
+# Restart pgpool
+sudo k3s kubectl delete pod -n default <pgpool-pod-name>
+```
+
+---
+
+## Risk: Fix Will Be Lost on helm upgrade
+
+The patched secret is managed by Helm (annotation:
+`meta.helm.sh/release-name: gitea`). A `helm upgrade` will regenerate the
+secret from the chart template, which does not include `pgpool-password`,
+and the bug will recur.
+
+---
+
+## Tasks
+
+### T01 — Add pgpool-password to Helm values
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "6841c93a-f146-47eb-9f7c-8fa0e02c1bbc"
+```
+
+Create or update `helm/gitea-values.yaml` (or equivalent) to permanently
+include the pgpool-password so it survives `helm upgrade`:
+
+```yaml
+postgresql-ha:
+ postgresql:
+ pgpoolPassword: <value matching sr-check-password>
+```
+
+**Done when:** `helm upgrade gitea` completes and pgpool starts cleanly
+without manual secret patching.
+
+---
+
+### T02 — Add pgpool health check to smoke test
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "ab166073-30a7-4702-a037-4091e8706e20"
+```
+
+Extend `tests/smoke_kube.sh` to assert:
+
+```bash
+# All postgresql-ha pods Running
+kubectl get pods -n default | grep gitea-postgresql-ha | grep -v Running && exit 1
+
+# pgpool specifically not in CrashLoopBackOff
+kubectl get pod -n default -l app.kubernetes.io/component=pgpool \
+ -o jsonpath='{.items[0].status.containerStatuses[0].state}' | grep -v crash
+```
+
+**Done when:** the smoke test catches a pgpool failure within 5 minutes.
+
+---
+
+### T03 — Add HA failover test
+
+```task
+id: T03
+status: done
+priority: high
+state_hub_task_id: "140da396-8e30-4f4d-b88c-c42c0cd46c01"
+```
+
+Create `tests/test_ha_failover.sh` that:
+
+1. Records Gitea login response time (baseline)
+2. Kills the primary PostgreSQL pod: `kubectl delete pod gitea-postgresql-ha-postgresql-0 -n default`
+3. Waits for repmgr to promote a replica (max 60s)
+4. Asserts Gitea login POST still succeeds within 10s
+5. Asserts pgpool pod is Running (not CrashLoopBackOff)
+6. Asserts all postgresql pods return to Running
+
+This test must pass before any PostgreSQL HA deployment is considered done.
+
+**Done when:** script exits 0 against a live cluster.
+
+---
+
+### T04 — Document the incident in docs/
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "d8a3ba40-fda0-4c1f-a9f1-ffcd621a5b3d"
+```
+
+Add `docs/incidents/2026-03-10-pgpool-missing-secret.md` with the full
+timeline, root cause, and fix, so future operators understand what happened
+and how to recover.
+
+**Done when:** doc committed and linked from `docs/README.md`.
+
+---
+
+## References
+
+- Bitnami postgresql-ha chart v16.2.2
+- Gitea Helm chart v12.2.0
+- Related decision: D3 (HA testing policy) in `DECISIONS.md`
--- /dev/null
+++ b/tests/fixtures/railiance-cluster/workplans/archived/260622-RAIL-BS-WP-0004-safety-net.md
@@ -0,0 +1,275 @@
+---
+id: RAIL-BS-WP-0004
+type: workplan
+title: "Integrated Backup — S2 Kubernetes Runtime Layer"
+domain: financials
+repo: railiance-cluster
+status: finished
+owner: tegwick
+topic_slug: railiance
+state_hub_workstream_id: "7e8b0c20-51eb-40c9-9e3b-85dd380d7625"
+created: "2026-02-25"
+updated: "2026-03-26"
+---
+
+> **Terminology note:** Historical text in this archived workplan may use the legacy term "workstream". The fleet term is **workplan** (`canon/standards/workplan-terminology-fleet_v0.1.md`).
+
+# Integrated Backup — S2 Kubernetes Runtime Layer
+
+## Goal
+
+Implement the Q3 (Operability & Resilience) integrated backup for
+railiance-cluster (S2). Backs up what S2 owns — the Kubernetes runtime state —
+encrypted with age, written to a local directory on the server. No external
+dependencies required.
+
+## Architecture (Decision D4)
+
+Each railiance repo implements its own backup for what it owns. No central
+backup service. See `DECISIONS.md` D4 for full rationale.
+
+**Standard interface every railiance repo must provide:**
+
+```bash
+make backup # encrypt + write to /opt/backup/railiance/<layer>/
+make restore # restore from most recent local backup
+```
+
+Encryption: age, same key pair as SOPS secrets (`.sops.yaml` public key).
+Output: `/opt/backup/railiance/cluster/` on the server.
+
+## What S2 (railiance-cluster) owns and must back up
+
+| Asset | Why it matters |
+|---|---|
+| k3s etcd snapshots | Full cluster state — all workloads, configs, secrets |
+| Helm release values | Runtime values not in git (any manually applied overrides) |
+| kubeconfig | Admin access to the cluster |
+
+**Not S2's responsibility:**
+- Custodian State Hub DB → the-custodian owns this
+- Operator workstation config (`.claude/`, `.gitconfig`) → operator's own concern
+- Application data (Gitea repos, uploads) → S5 (railiance-apps) owns this
+- PostgreSQL data volumes → S3 (railiance-platform) owns this
+
+## Encryption
+
+Reuse the age public key from `.sops.yaml`:
+
+```bash
+AGE_PUBLIC_KEY=$(grep 'age:' .sops.yaml | awk '{print $2}')
+tar -czf - <assets> | age -r "${AGE_PUBLIC_KEY}" -o backup.tar.gz.age
+```
+
+Decryption requires the private key at `~/.config/sops/age/keys.txt`
+(same key used for `sops -d`). No additional key management needed.
+
+## Extension Point EP-RAIL-005
+
+Once all five OAS layers implement this interface, the custodian can
+orchestrate a full-stack backup with:
+
+```bash
+for repo in railiance-infra railiance-cluster railiance-platform \
+ railiance-enablement railiance-apps; do
+ make -C ~/$repo backup
+done
+```
+
+No special protocol needed — just the standard interface.
+
+---
+
+## Tasks
+
+### T01 — Define backup directory and encryption wrapper
+
+```task
+id: T01
+status: done
+priority: high
+state_hub_task_id: "4526a842-ea31-4874-9231-92ab556cfe7b"
+```
+
+Create `tools/cmd/railiance-backup-s2` (replacing the old `railiance-backup`):
+
+- Backup dir: `/opt/backup/railiance/cluster/` (create with `mkdir -p`)
+- Encrypt each artifact with age using public key from `.sops.yaml`
+- Write timestamp-named files: `etcd-<ts>.snap.age`, `helm-values-<ts>.tar.gz.age`, `kubeconfig-<ts>.yaml.age`
+- Keep last 7 of each type
+- Write `.last-backup` stamp
+- Exit 0 on success, non-zero on any failure
+- No network required
+
+Also remove the old `tools/cmd/railiance-backup` (backed up Docker-based
+custodian DB — wrong scope, not applicable to this server).
+
+**Done when:** `make backup` runs on COULOMBCORE without error and files
+appear in `/opt/backup/railiance/cluster/`.
+
+---
+
+### T02 — Back up k3s state (SQLite hot backup)
+
+```task
+id: T02
+status: done
+priority: high
+state_hub_task_id: "a6313e06-1976-46a7-8e31-df4eb2eca880"
+```
+
+k3s has built-in etcd snapshot support:
+
+```bash
+sudo k3s etcd-snapshot save --name railiance-$(date -u +%Y%m%dT%H%M%SZ)
+# Default location: /var/lib/rancher/k3s/server/db/snapshots/
+```
+
+Add to the backup script: take a fresh snapshot, encrypt with age,
+copy to `/opt/backup/railiance/cluster/`.
+
+> **Note — verify etcd is in use before implementing:**
+> `k3s etcd-snapshot` only works if k3s was started with `--cluster-init`.
+> Without it, k3s uses SQLite and this command will fail.
+> Verify first: `sudo k3s etcd-snapshot ls 2>&1`
+
+> **Note — sudo required:** etcd snapshot requires root. See T06 for how
+> this is resolved (backup runs under root's crontab).
+
+**Done when:** backup includes a current etcd snapshot.
+
+---
+
+### T03 — Back up Helm release values
+
+```task
+id: T03
+status: done
+priority: medium
+state_hub_task_id: "05d42a55-921f-4aa7-bb76-e8af9c7e0ac3"
+```
+
+Capture current runtime Helm values for all releases:
+
+```bash
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm list -A -o json | \
+ jq -r '.[].name + " " + .namespace' | \
+ while read name ns; do
+ KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm get values "$name" -n "$ns" -o yaml
+ done
+```
+
+Tar and age-encrypt into `helm-values-<ts>.tar.gz.age`.
+
+> **Note — kubeconfig permissions:** `/etc/rancher/k3s/k3s.yaml` is root-readable
+> only by default. The backup script must either run as root (see T06) or k3s
+> must be configured with `--write-kubeconfig-mode=644`. Running as root
+> (via root crontab) is the chosen approach — no config change needed.
+
+**Done when:** backup includes a snapshot of all Helm release values.
+
+---
+
+### T04 — Back up kubeconfig
+
+```task
+id: T04
+status: done
+priority: medium
+state_hub_task_id: "08233868-d522-4117-bc4e-6c0f52545665"
+```
+
+Age-encrypt `~/.kube/config-hosteurope` (or `/etc/rancher/k3s/k3s.yaml`)
+into `kubeconfig-<ts>.yaml.age` in the backup directory.
+
+**Done when:** backup includes the encrypted kubeconfig.
+
+---
+
+### T05 — make restore target
+
+```task
+id: T05
+status: done
+priority: medium
+state_hub_task_id: "2d5acff7-4a4e-4ddd-ad06-08237ad3dac8"
+```
+
+Add `tools/cmd/railiance-restore-s2` that decrypts and lists available
+backups, with guided restore for the etcd snapshot case.
+
+Restore of etcd from snapshot:
+```bash
+sudo k3s server --cluster-reset \
+ --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/<name>
+```
+
+**Done when:** `make restore` prints available backups and a restore guide.
+
+---
+
+### T06 — Install cron job and run restore drill
+
+```task
+id: T06
+status: done
+priority: medium
+state_hub_task_id: "f8e4a094-c367-40eb-b895-da17bc144b07"
+```
+
+#### Solving the sudo problem
+
+The backup script needs root for two reasons:
+- `k3s etcd-snapshot save` requires root
+- `/etc/rancher/k3s/k3s.yaml` (kubeconfig) is root-readable only
+
+**Solution: run the cron under root's crontab.**
+
+This is the correct pattern for system-level backup jobs. It avoids a
+proliferating sudoers whitelist (one entry per command, brittle to maintain)
+and matches how tools like `rsnapshot`, `bacula`, and `borgbackup` work in
+production. The backup writes to `/opt/backup/` which is root-owned anyway.
+
+Install the cron as root:
+
+```bash
+sudo crontab -e
+# Add:
+0 2 * * * make -C /home/tegwick/railiance-cluster backup >> /opt/backup/railiance/cluster/backup.log 2>&1
+```
+
+Note: use the absolute path to the repo — `~` does not expand reliably in
+root's crontab unless HOME is set.
+
+Verify it is installed:
+```bash
+sudo crontab -l | grep railiance
+```
+
+#### Restore drill
+
+Once T01–T04 are done, run a decrypt-and-verify drill:
+
+```bash
+# Decrypt the etcd snapshot and verify it is a valid snapshot file
+sudo age -d -i ~/.config/sops/age/keys.txt \
+ /opt/backup/railiance/cluster/etcd-$(ls /opt/backup/railiance/cluster/etcd-*.snap.age | sort -r | head -1 | xargs basename | sed 's/etcd-//;s/.snap.age//').snap.age \
+ | file -
+
+# Record the drill
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) restore drill OK" \
+ >> /opt/backup/railiance/cluster/restore-drill.log
+```
+
+**Done when:** cron installed under root, drill completes without error,
+log entry written.
+
+---
+
+## References
+
+- Decision D4: Integrated backup per capability (`DECISIONS.md`)
+- Decision D2: Nextcloud as optional offsite extension (still valid, not a requirement)
+- OAS Q3: Operability & Resilience
+- Extension point EP-RAIL-005: Custodian full-stack backup orchestration
+- k3s etcd snapshots: https://docs.k3s.io/datastore/backup-restore

309
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/classification.py Normal file
View file

@ -0,0 +1,309 @@
"""Repo classification validation for State Hub registration (STATE-WP-0065 P1).
Loads allowed values from the custodian canon standard and validates classification
blocks against controlled vocabularies.
"""
from __future__ import annotations
import re
from dataclasses import dataclass, field
from pathlib import Path
import os
import yaml
# Explicit override first — every other candidate below is a developer
# workstation path, so in a container none of them exist and classification
# validation fails outright. Set REPO_CLASSIFICATION_ALLOWED_PATH in any
# deployment that does not carry a the-custodian checkout (CUST-WP-0067-T04).
_ENV_ALLOWED = os.environ.get("REPO_CLASSIFICATION_ALLOWED_PATH")
# Workstation checkout, railiance01 fleet checkout, then state-hub sibling fallback.
_PRIMARY_ALLOWED = Path(
"/home/worsch/the-custodian/canon/standards/repo-classification.allowed.yaml"
)
_RAILIANCE_ALLOWED = Path(
"/home/tegwick/the-custodian/canon/standards/repo-classification.allowed.yaml"
)
_FALLBACK_ALLOWED = (
Path(__file__).resolve().parent.parent.parent
/ "the-custodian"
/ "canon"
/ "standards"
/ "repo-classification.allowed.yaml"
)
@dataclass
class ClassificationData:
"""Normalized classification fields stored on ``managed_repos``."""
category: str
domain: str
secondary_domains: list[str] = field(default_factory=list)
capability_tags: list[str] = field(default_factory=list)
business_stake: list[str] = field(default_factory=list)
business_mechanics: list[str] = field(default_factory=list)
classified_at: str | None = None
classified_by: str | None = None
standard_version: str | None = None
def to_dict(self) -> dict:
return {
"category": self.category,
"domain": self.domain,
"secondary_domains": list(self.secondary_domains),
"capability_tags": list(self.capability_tags),
"business_stake": list(self.business_stake),
"business_mechanics": list(self.business_mechanics),
"classified_at": self.classified_at,
"classified_by": self.classified_by,
"standard_version": self.standard_version,
}
@classmethod
def from_block(cls, block: dict) -> ClassificationData:
return cls(
category=block["category"],
domain=block["domain"],
secondary_domains=list(block.get("secondary_domains") or []),
capability_tags=list(block.get("capability_tags") or []),
business_stake=list(block.get("business_stake") or []),
business_mechanics=list(block.get("business_mechanics") or []),
classified_at=block.get("classified_at"),
classified_by=block.get("classified_by"),
standard_version=block.get("version") or block.get("standard_version"),
)
def _allowed_path() -> Path:
if _ENV_ALLOWED:
candidate = Path(_ENV_ALLOWED)
if candidate.is_file():
return candidate
raise FileNotFoundError(
f"REPO_CLASSIFICATION_ALLOWED_PATH is set to {candidate}, which is not a file"
)
for candidate in (_PRIMARY_ALLOWED, _RAILIANCE_ALLOWED, _FALLBACK_ALLOWED):
if candidate.is_file():
return candidate
raise FileNotFoundError(
"repo-classification.allowed.yaml not found at "
f"{_PRIMARY_ALLOWED}, {_RAILIANCE_ALLOWED}, or {_FALLBACK_ALLOWED}. "
"Set REPO_CLASSIFICATION_ALLOWED_PATH when running without a "
"the-custodian checkout (for example in a container)."
)
def load_allowed_values(path: Path | None = None) -> dict:
"""Load the machine-readable allowed-values YAML."""
target = path or _allowed_path()
with target.open(encoding="utf-8") as fh:
return yaml.safe_load(fh)
def _known_capability_tags(allowed: dict) -> set[str]:
tags: set[str] = set()
for fam in (allowed.get("capability_families") or {}).values():
tags.update(fam or [])
return tags
def validate_classification(block: dict) -> tuple[list[str], list[str]]:
"""Validate a ``repo_classification`` block.
Returns ``(errors, warnings)``. *block* should be the inner mapping (not the
full YAML document with the ``repo_classification`` wrapper).
"""
allowed = load_allowed_values()
errors: list[str] = []
warnings: list[str] = []
if not isinstance(block, dict):
return (["classification block must be a mapping"], [])
categories = set(allowed["categories"])
domains = set(allowed["domains"])
stakes = set(allowed["business_stake"])
mechanics = set(allowed["business_mechanics"])
guidance = allowed.get("guidance", {})
pattern = re.compile(
guidance.get("capability_tag_pattern", r"^[a-z0-9]+(-[a-z0-9]+)*$")
)
category = block.get("category")
if category is None:
errors.append("`category` is required")
elif category not in categories:
errors.append(f"`category` '{category}' not in {sorted(categories)}")
domain = block.get("domain")
if domain is None:
errors.append("`domain` is required")
elif domain not in domains:
errors.append(f"`domain` '{domain}' not in allowed domains")
secondary = block.get("secondary_domains") or []
if not isinstance(secondary, list):
errors.append("`secondary_domains` must be a list")
secondary = []
for d in secondary:
if d not in domains:
errors.append(f"secondary domain '{d}' not in allowed domains")
if d == domain:
errors.append(f"secondary domain '{d}' repeats the primary domain")
if len(secondary) != len(set(secondary)):
errors.append("`secondary_domains` contains duplicates")
smax = guidance.get("secondary_domains_max", 3)
if len(secondary) > smax:
warnings.append(
f"{len(secondary)} secondary_domains exceeds recommended max {smax}"
)
tags = block.get("capability_tags") or []
if not isinstance(tags, list):
errors.append("`capability_tags` must be a list")
tags = []
known = _known_capability_tags(allowed)
for t in tags:
if not isinstance(t, str) or not pattern.match(t):
errors.append(f"capability_tag '{t}' is not lowercase kebab-case")
elif t not in known:
warnings.append(
f"capability_tag '{t}' is not a recommended family tag "
"(allowed, check for synonym)"
)
stake = block.get("business_stake") or []
if not isinstance(stake, list):
errors.append("`business_stake` must be a list")
stake = []
for s in stake:
if s not in stakes:
errors.append(f"business_stake '{s}' not in {sorted(stakes)}")
if stake:
lo = guidance.get("business_stake_recommended_min", 2)
hi = guidance.get("business_stake_recommended_max", 6)
if not (lo <= len(stake) <= hi):
warnings.append(
f"{len(stake)} business_stake values; {lo}-{hi} recommended"
)
mech = block.get("business_mechanics") or []
if not isinstance(mech, list):
errors.append("`business_mechanics` must be a list")
mech = []
for m in mech:
if m not in mechanics:
errors.append(f"business_mechanics '{m}' not in {sorted(mechanics)}")
return errors, warnings
CLASSIFICATION_FILENAME = ".repo-classification.yaml"
# Market-domain slugs (Repo Classification Standard v1.0 §6).
MARKET_DOMAIN_SLUGS: frozenset[str] = frozenset({
"infotech",
"financials",
"communication",
"consumer",
"health",
"industrials",
"energy",
"utilities",
"materials",
"realestate",
"crypto",
"agents",
"space",
"government",
})
# Legacy coordination-domain slugs still found in workplan frontmatter ``domain:``.
# Maps to market-domain slugs used by the Hub ``domains`` table post-migration.
LEGACY_COORDINATION_TO_MARKET: dict[str, str] = {
"custodian": "infotech",
"railiance": "financials",
"markitect": "communication",
"coulomb_social": "communication",
"personhood": "government",
"foerster_capabilities": "agents",
"capabilities": "agents",
"canon": "infotech",
"citation_evidence": "infotech",
"helix_forge": "infotech",
"inter_hub": "infotech",
"netkingdom": "communication",
"stack": "infotech",
"vergabe_teilnahme": "government",
"whynot": "consumer",
"test_domain_v2": "infotech",
}
def resolve_topic_domain_slug(
workplan_domain: str,
*,
repo_market_domain: str | None = None,
) -> str:
"""Map a workplan frontmatter ``domain`` value to a market-domain slug.
Workplans may still carry legacy coordination slugs (e.g. ``custodian``)
after the spine migration; topic lookup must use the market domain stored
on ``domains.slug``.
"""
domain = (workplan_domain or "").strip()
if not domain:
return repo_market_domain or ""
if domain in MARKET_DOMAIN_SLUGS:
return domain
mapped = LEGACY_COORDINATION_TO_MARKET.get(domain)
if mapped:
return mapped
return repo_market_domain or domain
def load_classification_document(path: Path) -> dict | None:
"""Load and return the YAML document, or ``None`` if missing/unreadable."""
if not path.is_file():
return None
try:
with path.open(encoding="utf-8") as fh:
doc = yaml.safe_load(fh)
except (OSError, yaml.YAMLError):
return None
return doc if isinstance(doc, dict) else None
def extract_classification_block(doc: dict | None) -> dict | None:
"""Return the inner ``repo_classification`` mapping from a loaded document."""
if not doc:
return None
block = doc.get("repo_classification")
return block if isinstance(block, dict) else None
def load_classification_file(
repo_path: Path | str,
*,
filename: str = CLASSIFICATION_FILENAME,
) -> tuple[ClassificationData | None, list[str], list[str]]:
"""Load ``.repo-classification.yaml`` from a repo root and validate it.
Returns ``(data, errors, warnings)``. *data* is ``None`` when the file is
missing, unreadable, or has blocking validation errors.
"""
root = Path(repo_path)
doc = load_classification_document(root / filename)
block = extract_classification_block(doc)
if block is None:
if doc is None:
return (None, [f"{filename} missing or unreadable"], [])
return (None, [f"{filename} has no repo_classification block"], [])
errors, warnings = validate_classification(block)
if errors:
return (None, errors, warnings)
return (ClassificationData.from_block(block), [], warnings)

47
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/config.py Normal file
View file

@ -0,0 +1,47 @@
from typing import Literal
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
)
database_url: str = "postgresql+asyncpg://custodian:changeme@127.0.0.1:5432/custodian"
api_base: str = "http://127.0.0.1:8000"
# What this instance claims to be. Deliberately defaults to "unknown":
# an instance that has not declared itself is not the primary, and callers
# that need the authoritative hub must be able to tell the difference.
# Answering on a port is not evidence of authority — that assumption cost
# seven weeks of onboarding (CUST-WP-0067-T03, ADR-010).
state_hub_instance_role: Literal["primary", "cache", "unknown"] = "unknown"
# Env vars are STATE_HUB_INSTANCE_ROLE / STATE_HUB_INSTANCE_LABEL — the
# field name *is* the binding, so renaming either breaks the deployment
# silently. Covered by tests/test_instance_identity.py.
# Free-form label to name *which* instance answered, e.g. "railiance01".
state_hub_instance_label: str | None = None
debug: bool = False
state_hub_report_dir: str = "reports/recently-on-scope"
state_hub_markitect_cli_path: str | None = None
activity_core_url: str | None = None
activity_core_worker_token: str | None = None
ops_run_projection_ttl_seconds: float = 15.0
ops_run_sla_hours: float = 1.0
sbom_nexus_url: str | None = None
sbom_nexus_read_mode: Literal["legacy", "nexus"] = "legacy"
sbom_nexus_write_mode: Literal["legacy", "nexus"] = "legacy"
sbom_nexus_timeout_seconds: float = 5.0
# Repository renames are central-only, compare-and-set operations. The
# secret signs short-lived, non-persistent preflight evidence; it must be
# supplied by the deployment (normally through OpenBao), never recorded in
# State Hub. With no secret the read-only report remains available but no
# mutation token can be issued.
repository_rename_preflight_secret: str | None = None
repository_rename_preflight_ttl_seconds: int = 900
repository_rename_forge_timeout_seconds: float = 10.0
settings = Settings()

24
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/database.py Normal file
View file

@ -0,0 +1,24 @@
from collections.abc import AsyncGenerator
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from api.config import settings
engine = create_async_engine(
settings.database_url,
echo=settings.debug,
pool_pre_ping=True,
pool_size=10,
max_overflow=20,
)
async_session_factory = async_sessionmaker(
engine,
class_=AsyncSession,
expire_on_commit=False,
)
async def get_session() -> AsyncGenerator[AsyncSession, None]:
async with async_session_factory() as session:
yield session

542
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/doi_engine.py Normal file
View file

@ -0,0 +1,542 @@
"""DoI engine — evaluates all 14 Repository Definition of Integrated criteria.
Shared by the API endpoint (async) and the CLI check script (asyncio.run).
All checks use only the repo dict from /repos/{slug} + HTTP calls to the API
+ local filesystem reads. No direct DB access.
"""
from __future__ import annotations
import asyncio
import json
import os
import re
import socket
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Literal
import yaml
CriterionStatus = Literal["pass", "fail", "warn", "skip"]
Tier = Literal["none", "core", "standard", "full"]
# Criteria that belong to each tier (in check order)
CORE_IDS = {"C1", "C2", "C3", "C4"}
STANDARD_IDS = {"C5a", "C5b", "C5c", "C6", "C7", "C8", "C9"}
FULL_IDS = {"C10", "C11", "C12", "C13", "C14"}
STANDARD_SCOPE_SECTIONS = [
"One-liner",
"Core Idea",
"In Scope",
"Out of Scope",
"Relevant When",
"Not Relevant When",
"Current State",
"How It Fits",
"Terminology",
"Related / Overlapping",
"Provided Capabilities",
]
_CAPABILITY_BLOCK_RE = re.compile(r"```capability\s*\n(.*?)```", re.DOTALL | re.IGNORECASE)
_H2_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE)
@dataclass
class CriterionResult:
id: str
label: str
tier: str
status: CriterionStatus
detail: str = ""
@dataclass
class DoIReport:
repo_slug: str
tier: Tier
core_pass: bool
standard_pass: bool
full_pass: bool
criteria: list[CriterionResult] = field(default_factory=list)
checked_at: str = field(default_factory=lambda: datetime.now(tz=timezone.utc).isoformat())
def evaluate_scope_health(repo: dict) -> list[dict[str, Any]]:
"""Return machine-readable SCOPE.md health issues for C5a/C5b/C5c.
The returned records intentionally mirror DoI criterion IDs while carrying
section-level hints that downstream repo-scoping can use to refresh only
the affected parts of SCOPE.md.
"""
repo_path = _resolve_path(repo)
if not repo_path:
return [
{
"id": "C5a",
"label": "SCOPE.md present",
"status": "skip",
"detail": "Local path unavailable",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": [],
},
{
"id": "C5b",
"label": "SCOPE.md standard sections",
"status": "skip",
"detail": "Local path unavailable",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": [],
},
{
"id": "C5c",
"label": "SCOPE.md capability blocks",
"status": "skip",
"detail": "Local path unavailable",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": [],
},
]
scope_path = Path(repo_path) / "SCOPE.md"
if not scope_path.exists():
return [
{
"id": "C5a",
"label": "SCOPE.md present",
"status": "fail",
"detail": "SCOPE.md not found at repo root",
"missing_sections": STANDARD_SCOPE_SECTIONS.copy(),
"invalid_capability_blocks": [],
"needs_refresh_sections": STANDARD_SCOPE_SECTIONS.copy(),
},
{
"id": "C5b",
"label": "SCOPE.md standard sections",
"status": "skip",
"detail": "SCOPE.md absent",
"missing_sections": STANDARD_SCOPE_SECTIONS.copy(),
"invalid_capability_blocks": [],
"needs_refresh_sections": STANDARD_SCOPE_SECTIONS.copy(),
},
{
"id": "C5c",
"label": "SCOPE.md capability blocks",
"status": "skip",
"detail": "SCOPE.md absent",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": ["Provided Capabilities"],
},
]
text = scope_path.read_text()
issues: list[dict[str, Any]] = [{
"id": "C5a",
"label": "SCOPE.md present",
"status": "pass",
"detail": "",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": [],
}]
headings = {h.strip() for h in _H2_RE.findall(text)}
missing_sections = [section for section in STANDARD_SCOPE_SECTIONS if section not in headings]
if missing_sections:
issues.append({
"id": "C5b",
"label": "SCOPE.md standard sections",
"status": "warn",
"detail": f"Missing H2 section(s): {', '.join(missing_sections)}",
"missing_sections": missing_sections,
"invalid_capability_blocks": [],
"needs_refresh_sections": missing_sections,
})
else:
issues.append({
"id": "C5b",
"label": "SCOPE.md standard sections",
"status": "pass",
"detail": f"All {len(STANDARD_SCOPE_SECTIONS)} standard sections present",
"missing_sections": [],
"invalid_capability_blocks": [],
"needs_refresh_sections": [],
})
capability_blocks = _CAPABILITY_BLOCK_RE.findall(text)
valid_blocks = 0
invalid_blocks: list[dict[str, Any]] = []
for index, block in enumerate(capability_blocks, start=1):
try:
parsed = yaml.safe_load(block) or {}
if isinstance(parsed, dict) and parsed.get("type") and parsed.get("title"):
valid_blocks += 1
else:
invalid_blocks.append({
"index": index,
"reason": "Capability block must be YAML with type and title",
})
except yaml.YAMLError as exc:
invalid_blocks.append({"index": index, "reason": str(exc)})
if valid_blocks > 0:
issues.append({
"id": "C5c",
"label": "SCOPE.md capability blocks",
"status": "pass",
"detail": f"{valid_blocks} valid capability block(s)",
"missing_sections": [],
"invalid_capability_blocks": invalid_blocks,
"needs_refresh_sections": [],
})
else:
detail = "No fenced capability block found"
if invalid_blocks:
detail = "No valid capability block found"
issues.append({
"id": "C5c",
"label": "SCOPE.md capability blocks",
"status": "warn",
"detail": detail,
"missing_sections": [],
"invalid_capability_blocks": invalid_blocks,
"needs_refresh_sections": ["Provided Capabilities"],
})
return issues
def compute_fingerprint(
repo: dict,
latest_tpsc_snap_at: str | None,
latest_goal_updated_at: str | None,
) -> str:
"""Compute a pipe-joined fingerprint of all inputs that affect DoI criteria.
If any component changes, the fingerprint changes and the cache is invalidated:
- repo.updated_at → covers last_sbom_at, remote_url, host_paths, domain changes
- latest_tpsc_snap_at → C9 (TPSC snapshot exists)
- latest_goal_updated_at → C10 (active repo goal)
- mtime of SCOPE.md, CLAUDE.md, tpsc.yaml → C5, C6, C9, C11, C12
"""
parts = [
str(repo.get("updated_at") or ""),
str(latest_tpsc_snap_at or ""),
str(latest_goal_updated_at or ""),
]
repo_path = _resolve_path(repo)
if repo_path:
for fname in ("SCOPE.md", "CLAUDE.md", "tpsc.yaml"):
f = Path(repo_path) / fname
try:
parts.append(f"{fname}:{f.stat().st_mtime:.3f}")
except FileNotFoundError:
parts.append(f"{fname}:absent")
return "|".join(parts)
def _sweep_hostname() -> str:
"""Hostname for host_paths resolution; override when pod hostname != fleet FQDN."""
return os.environ.get("STATE_HUB_SWEEP_HOSTNAME", "").strip() or socket.gethostname()
def _resolve_path(repo: dict) -> str:
hostname = _sweep_hostname()
host_paths = repo.get("host_paths") or {}
candidates = []
if host_paths.get(hostname):
candidates.append(host_paths[hostname])
if repo.get("local_path"):
candidates.append(repo["local_path"])
for raw in candidates:
p = Path(raw).expanduser()
if p.is_dir():
return str(p)
return ""
def resolve_repo_path(repo: dict) -> str:
"""Resolve the repo path using the same host-aware rules as DoI checks."""
return _resolve_path(repo)
def _get_sync(api_base: str, path: str, params: dict | None = None) -> object:
url = f"{api_base}{path}"
if params:
q = "&".join(f"{k}={v}" for k, v in params.items() if v is not None)
if q:
url = f"{url}?{q}"
req = urllib.request.Request(url, headers={"Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=5) as r:
return json.loads(r.read())
except Exception:
return None
async def _get(api_base: str, path: str, params: dict | None = None) -> object:
"""Async wrapper — runs blocking urllib in a thread so the event loop stays free."""
return await asyncio.to_thread(_get_sync, api_base, path, params)
async def _run_consistency(repo_slug: str, api_base: str) -> tuple[int, int, int]:
"""Run consistency_check.py and return (fail, warn, info) counts."""
script = Path(__file__).parent.parent / "scripts" / "consistency_check.py"
proc = await asyncio.create_subprocess_exec(
"uv", "run", "python", str(script),
"--repo", repo_slug,
"--api-base", api_base,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=str(Path(__file__).parent.parent),
)
stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=30)
text = stdout.decode()
fail = warn = info = 0
for line in text.splitlines():
if "Summary:" in line:
parts = line.split("|")
for p in parts:
p = p.strip()
if "fail" in p:
try: fail = int(p.split()[0])
except ValueError: pass
elif "warn" in p:
try: warn = int(p.split()[0])
except ValueError: pass
elif "info" in p:
try: info = int(p.split()[0])
except ValueError: pass
return fail, warn, info
async def evaluate(
repo: dict,
api_base: str = "http://127.0.0.1:8000",
skip_consistency: bool = False,
prefetch: dict | None = None,
) -> DoIReport:
"""Evaluate all 14 DoI criteria for a repo.
Args:
repo: Repo dict (slug, domain_slug, local_path, remote_url, host_paths, last_sbom_at).
api_base: API base URL — only used when prefetch is absent.
skip_consistency: Skip C7/C13 subprocess calls (used in summary mode).
prefetch: Optional pre-fetched bulk data to avoid HTTP self-calls:
{
"domain_status": {"custodian": "active", ...}, # slug → status
"tpsc_snap_counts": {"llm-connect": 1, ...}, # repo_slug → count
"active_goal_counts": {"llm-connect": 0, ...}, # repo_slug → count
}
"""
slug = repo.get("slug", "unknown")
results: list[CriterionResult] = []
def _r(id: str, label: str, tier: str, status: CriterionStatus, detail: str = "") -> CriterionResult:
r = CriterionResult(id=id, label=label, tier=tier, status=status, detail=detail)
results.append(r)
return r
# ── Tier 1: Core ─────────────────────────────────────────────────────────
# C1: registered
_r("C1", "Registered in state-hub", "core", "pass", "Repo record exists")
# C2: domain assigned and active
domain_slug = repo.get("domain_slug") or ""
if not domain_slug:
_r("C2", "Domain assigned", "core", "fail", "No domain_slug on repo record")
else:
if prefetch and "domain_status" in prefetch:
dom_status = prefetch["domain_status"].get(domain_slug)
else:
d = await _get(api_base, f"/domains/{domain_slug}/")
dom_status = d.get("status") if d else None
if dom_status == "active":
_r("C2", "Domain assigned", "core", "pass", f"domain: {domain_slug}")
elif dom_status:
_r("C2", "Domain assigned", "core", "warn", f"Domain '{domain_slug}' status: {dom_status}")
else:
_r("C2", "Domain assigned", "core", "fail", f"Domain '{domain_slug}' not found")
# C3: local path resolves
repo_path = _resolve_path(repo)
if repo_path:
_r("C3", "Local path resolves", "core", "pass", repo_path)
else:
raw = repo.get("local_path") or "(none)"
_r("C3", "Local path resolves", "core", "fail", f"Path not accessible: {raw}")
# C4: remote URL set
remote = repo.get("remote_url") or ""
if remote.strip():
_r("C4", "Remote URL set", "core", "pass", remote)
else:
_r("C4", "Remote URL set", "core", "fail", "remote_url is empty")
# ── Tier 2: Standard ─────────────────────────────────────────────────────
# C5a/C5b/C5c: SCOPE.md structure and capability declarations
for issue in evaluate_scope_health(repo):
_r(issue["id"], issue["label"], "standard", issue["status"], issue["detail"])
# C6: CLAUDE.md
if not repo_path:
_r("C6", "CLAUDE.md present", "standard", "skip", "Local path unavailable")
elif (Path(repo_path) / "CLAUDE.md").exists():
_r("C6", "CLAUDE.md present", "standard", "pass")
else:
_r("C6", "CLAUDE.md present", "standard", "fail", "CLAUDE.md not found at repo root")
# C7: workplan convention — consistency check 0 FAIL
if skip_consistency:
_r("C7", "Workplan convention (0 FAIL)", "standard", "skip", "Not checked in summary mode — use /repos/{slug}/doi for full check")
else:
try:
fail, warn, _ = await _run_consistency(slug, api_base)
if fail == 0:
_r("C7", "Workplan convention (0 FAIL)", "standard", "pass", f"consistency: {fail} fail / {warn} warn")
else:
_r("C7", "Workplan convention (0 FAIL)", "standard", "fail", f"consistency: {fail} fail / {warn} warn")
except Exception as e:
_r("C7", "Workplan convention (0 FAIL)", "standard", "skip", f"Could not run consistency check: {e}")
# C8: SBOM ingested
last_sbom = repo.get("last_sbom_at")
if last_sbom:
_r("C8", "SBOM ingested", "standard", "pass", f"last ingested: {last_sbom[:10]}")
else:
_r("C8", "SBOM ingested", "standard", "fail", "last_sbom_at not set — run make ingest-sbom")
# C9: TPSC declared (tpsc.yaml present + snapshot exists)
tpsc_file_ok = repo_path and (Path(repo_path) / "tpsc.yaml").exists()
if prefetch and "tpsc_snap_counts" in prefetch:
has_snap = (prefetch["tpsc_snap_counts"].get(slug, 0) > 0)
snap_count = prefetch["tpsc_snap_counts"].get(slug, 0)
else:
tpsc_snaps = await _get(api_base, "/tpsc/snapshots/", {"repo_slug": slug}) or []
has_snap = len(tpsc_snaps) > 0
snap_count = len(tpsc_snaps)
if not repo_path:
_r("C9", "TPSC declared", "standard", "skip", "Local path unavailable")
elif tpsc_file_ok and has_snap:
_r("C9", "TPSC declared", "standard", "pass", f"{snap_count} snapshot(s)")
elif tpsc_file_ok and not has_snap:
_r("C9", "TPSC declared", "standard", "warn", "tpsc.yaml exists but not yet ingested — run make ingest-tpsc")
elif not tpsc_file_ok:
_r("C9", "TPSC declared", "standard", "fail", "tpsc.yaml missing at repo root")
# ── Tier 3: Full ─────────────────────────────────────────────────────────
# C10: active repo goal
if prefetch and "active_goal_counts" in prefetch:
active_goal_count = prefetch["active_goal_counts"].get(slug, 0)
else:
goals = await _get(api_base, "/repo-goals/", {"repo_slug": slug}) or []
active_goal_count = sum(1 for g in goals if g.get("status") == "active")
if active_goal_count > 0:
_r("C10", "Active repo goal", "full", "pass", f"{active_goal_count} active goal(s)")
else:
_r("C10", "Active repo goal", "full", "fail", "No active repo goal — create one with create_repo_goal()")
# C11: Provided Capabilities declared in SCOPE.md
if not repo_path:
_r("C11", "Provided Capabilities declared", "full", "skip", "Local path unavailable")
else:
scope = Path(repo_path) / "SCOPE.md"
if not scope.exists():
_r("C11", "Provided Capabilities declared", "full", "skip", "SCOPE.md absent")
else:
text = scope.read_text()
has_cap_block = "```capability" in text
has_none_explicit = "## Provided Capabilities" in text and (
"none" in text.lower().split("## provided capabilities")[-1][:200]
or "no capabilities" in text.lower().split("## provided capabilities")[-1][:200]
)
if has_cap_block:
_r("C11", "Provided Capabilities declared", "full", "pass", "capability block(s) found in SCOPE.md")
elif has_none_explicit:
_r("C11", "Provided Capabilities declared", "full", "pass", "Explicitly declared none in SCOPE.md")
elif "## Provided Capabilities" in text:
_r("C11", "Provided Capabilities declared", "full", "warn",
"Section present but no capability block or explicit none — add blocks or state 'none'")
else:
_r("C11", "Provided Capabilities declared", "full", "fail",
"No '## Provided Capabilities' section in SCOPE.md")
# C12: agents template applied (CLAUDE.md mentions kaizen)
if not repo_path:
_r("C12", "Agents template applied", "full", "skip", "Local path unavailable")
else:
claude_md = Path(repo_path) / "CLAUDE.md"
if not claude_md.exists():
_r("C12", "Agents template applied", "full", "skip", "CLAUDE.md absent")
else:
text = claude_md.read_text()
if "get_kaizen_agent" in text or "kaizen" in text.lower():
_r("C12", "Agents template applied", "full", "pass")
else:
_r("C12", "Agents template applied", "full", "fail",
"CLAUDE.md has no kaizen agent reference")
# C13: consistency check clean (0 FAIL, 0 WARN — C-12 exempt)
if skip_consistency:
_r("C13", "Consistency check clean (0 FAIL/WARN)", "full", "skip", "Not checked in summary mode — use /repos/{slug}/doi for full check")
else:
try:
fail, warn, _ = await _run_consistency(slug, api_base)
if fail == 0 and warn == 0:
_r("C13", "Consistency check clean (0 FAIL/WARN)", "full", "pass")
elif fail == 0 and warn > 0:
_r("C13", "Consistency check clean (0 FAIL/WARN)", "full", "warn",
f"{warn} warn(s) — C-12 legacy tasks may be exempt")
else:
_r("C13", "Consistency check clean (0 FAIL/WARN)", "full", "fail",
f"{fail} fail(s), {warn} warn(s)")
except Exception as e:
_r("C13", "Consistency check clean (0 FAIL/WARN)", "full", "skip", f"Could not run: {e}")
# C14: host paths registered
host_paths = repo.get("host_paths") or {}
if host_paths:
_r("C14", "Host paths registered", "full", "pass",
f"{len(host_paths)} host(s): {', '.join(host_paths.keys())}")
else:
_r("C14", "Host paths registered", "full", "fail",
"host_paths empty — run update_repo_path() for each active machine")
# ── Compute tier ─────────────────────────────────────────────────────────
by_id = {r.id: r for r in results}
def _tier_pass(ids: set[str]) -> bool:
return all(by_id[i].status in ("pass", "warn") for i in ids if i in by_id)
core_pass = _tier_pass(CORE_IDS)
standard_pass = core_pass and _tier_pass(STANDARD_IDS)
full_pass = standard_pass and _tier_pass(FULL_IDS)
if full_pass:
tier: Tier = "full"
elif standard_pass:
tier = "standard"
elif core_pass:
tier = "core"
else:
tier = "none"
return DoIReport(
repo_slug=slug,
tier=tier,
core_pass=core_pass,
standard_pass=standard_pass,
full_pass=full_pass,
criteria=results,
)

1
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/edge/__init__.py Normal file
View file

@ -0,0 +1 @@
"""State Hub edge relay and durable outbox helpers."""

358
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/edge/outbox.py Normal file
View file

@ -0,0 +1,358 @@
from __future__ import annotations
import json
import os
import sqlite3
import stat
import uuid
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
from api.services.write_idempotency import route_class_for
DEFAULT_OUTBOX_PATH = Path(os.environ.get("STATEHUB_OUTBOX_PATH", "~/.statehub/edge-outbox.sqlite3")).expanduser()
MAX_PAYLOAD_BYTES = 64 * 1024
SECRET_FIELD_NAMES = {
"authorization",
"cookie",
"set-cookie",
"password",
"passwd",
"secret",
"api_key",
"apikey",
"access_token",
"refresh_token",
"bearer_token",
"client_secret",
"private_key",
"credential",
"credentials",
}
@dataclass(frozen=True)
class OutboxEnvelope:
id: str
idempotency_key: str
method: str
path: str
body: dict[str, Any] | list[Any] | None
route_class: str
source_agent: str | None
source_host: str | None
repo_slug: str | None
session_id: str | None
observed_revision: dict[str, Any] | None
status: str
attempt_count: int
next_retry_at: str | None
last_error: str | None
response_status: int | None
response_body: dict[str, Any] | list[Any] | str | None
created_at: str
updated_at: str
acked_at: str | None
class PayloadRejected(ValueError):
pass
def utcnow() -> str:
return datetime.now(tz=timezone.utc).isoformat()
def default_outbox_path() -> Path:
return DEFAULT_OUTBOX_PATH
def scrub_payload(value: Any) -> Any:
if isinstance(value, dict):
scrubbed: dict[str, Any] = {}
for key, item in value.items():
normalized = str(key).lower().replace("-", "_")
if normalized in SECRET_FIELD_NAMES:
scrubbed[key] = "[redacted]"
else:
scrubbed[key] = scrub_payload(item)
return scrubbed
if isinstance(value, list):
return [scrub_payload(item) for item in value]
return value
def _json_loads(raw: str | None) -> Any:
if raw is None:
return None
return json.loads(raw)
def _json_dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, sort_keys=True, separators=(",", ":"))
def _parse_dt(value: str | None) -> datetime | None:
if not value:
return None
return datetime.fromisoformat(value)
class OutboxStore:
def __init__(self, path: str | Path | None = None) -> None:
self.path = Path(path).expanduser() if path is not None else default_outbox_path()
self.path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
self._chmod_private()
def _connect(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
return conn
def _init_db(self) -> None:
with self._connect() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS outbox_envelopes (
id TEXT PRIMARY KEY,
idempotency_key TEXT NOT NULL UNIQUE,
method TEXT NOT NULL,
path TEXT NOT NULL,
body_json TEXT,
route_class TEXT NOT NULL,
source_agent TEXT,
source_host TEXT,
repo_slug TEXT,
session_id TEXT,
observed_revision_json TEXT,
status TEXT NOT NULL,
attempt_count INTEGER NOT NULL DEFAULT 0,
next_retry_at TEXT,
last_error TEXT,
response_status INTEGER,
response_body_json TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
acked_at TEXT
)
"""
)
conn.execute("CREATE INDEX IF NOT EXISTS ix_outbox_status ON outbox_envelopes(status)")
conn.execute("CREATE INDEX IF NOT EXISTS ix_outbox_next_retry ON outbox_envelopes(next_retry_at)")
conn.commit()
def _chmod_private(self) -> None:
try:
os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR)
except OSError:
pass
def enqueue(
self,
*,
method: str,
path: str,
body: Any,
idempotency_key: str | None = None,
source_agent: str | None = None,
source_host: str | None = None,
repo_slug: str | None = None,
session_id: str | None = None,
observed_revision: dict[str, Any] | None = None,
) -> OutboxEnvelope:
route_class = route_class_for(method, path)
if route_class is None:
raise PayloadRejected(f"{method.upper()} {path} is not queueable")
scrubbed = scrub_payload(body)
encoded = _json_dumps(scrubbed)
if encoded is not None and len(encoded.encode("utf-8")) > MAX_PAYLOAD_BYTES:
raise PayloadRejected("payload exceeds offline outbox size limit")
now = utcnow()
envelope_id = str(uuid.uuid4())
key = idempotency_key or f"statehub-edge:{envelope_id}"
method_upper = method.upper()
with self._connect() as conn:
if route_class == "replace":
conn.execute(
"""
UPDATE outbox_envelopes
SET status = 'cancelled', updated_at = ?, last_error = ?
WHERE status = 'queued'
AND route_class = 'replace'
AND method = ?
AND path = ?
""",
(now, f"superseded by {envelope_id}", method_upper, path),
)
conn.execute(
"""
INSERT INTO outbox_envelopes (
id, idempotency_key, method, path, body_json, route_class,
source_agent, source_host, repo_slug, session_id,
observed_revision_json, status, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'queued', ?, ?)
""",
(
envelope_id,
key,
method_upper,
path,
encoded,
route_class,
source_agent,
source_host,
repo_slug,
session_id,
_json_dumps(observed_revision),
now,
now,
),
)
conn.commit()
return self.get(envelope_id)
def get(self, envelope_id: str) -> OutboxEnvelope:
with self._connect() as conn:
row = conn.execute("SELECT * FROM outbox_envelopes WHERE id = ?", (envelope_id,)).fetchone()
if row is None:
raise KeyError(envelope_id)
return self._row_to_envelope(row)
def list(self, *, status: str | None = None, limit: int = 100) -> list[OutboxEnvelope]:
with self._connect() as conn:
if status:
rows = conn.execute(
"SELECT * FROM outbox_envelopes WHERE status = ? ORDER BY created_at LIMIT ?",
(status, limit),
).fetchall()
else:
rows = conn.execute(
"SELECT * FROM outbox_envelopes ORDER BY created_at LIMIT ?",
(limit,),
).fetchall()
return [self._row_to_envelope(row) for row in rows]
def due(self, *, limit: int = 50) -> list[OutboxEnvelope]:
now = utcnow()
with self._connect() as conn:
rows = conn.execute(
"""
SELECT * FROM outbox_envelopes
WHERE status = 'queued' AND (next_retry_at IS NULL OR next_retry_at <= ?)
ORDER BY created_at
LIMIT ?
""",
(now, limit),
).fetchall()
return [self._row_to_envelope(row) for row in rows]
def summary(self) -> dict[str, Any]:
with self._connect() as conn:
rows = conn.execute(
"SELECT status, COUNT(*) AS count, MIN(created_at) AS oldest FROM outbox_envelopes GROUP BY status"
).fetchall()
by_status = {row["status"]: row["count"] for row in rows}
oldest_pending = None
for row in rows:
if row["status"] in {"queued", "sending", "conflict"} and row["oldest"]:
oldest_pending = min(filter(None, [oldest_pending, row["oldest"]])) if oldest_pending else row["oldest"]
return {
"path": str(self.path),
"by_status": by_status,
"pending_count": sum(by_status.get(status, 0) for status in ("queued", "sending")),
"conflict_count": by_status.get("conflict", 0),
"oldest_pending_at": oldest_pending,
}
def mark_sending(self, envelope_id: str) -> None:
self._update(envelope_id, status="sending", updated_at=utcnow())
def mark_acked(self, envelope_id: str, *, response_status: int, response_body: Any) -> None:
now = utcnow()
self._update(
envelope_id,
status="acked",
response_status=response_status,
response_body_json=_json_dumps(response_body),
updated_at=now,
acked_at=now,
last_error=None,
next_retry_at=None,
)
def mark_conflict(self, envelope_id: str, *, response_status: int, response_body: Any) -> None:
self._update(
envelope_id,
status="conflict",
response_status=response_status,
response_body_json=_json_dumps(response_body),
updated_at=utcnow(),
last_error="conflict",
)
def mark_dead(self, envelope_id: str, *, error: str, response_status: int | None = None, response_body: Any = None) -> None:
self._update(
envelope_id,
status="dead",
response_status=response_status,
response_body_json=_json_dumps(response_body),
updated_at=utcnow(),
last_error=error,
)
def mark_retry(self, envelope_id: str, *, error: str, attempt_count: int) -> None:
delay_seconds = min(3600, 2 ** min(attempt_count, 10))
next_retry = datetime.now(tz=timezone.utc) + timedelta(seconds=delay_seconds)
self._update(
envelope_id,
status="queued",
attempt_count=attempt_count,
next_retry_at=next_retry.isoformat(),
updated_at=utcnow(),
last_error=error[:500],
)
def retry(self, envelope_id: str) -> None:
self._update(envelope_id, status="queued", next_retry_at=None, updated_at=utcnow())
def cancel(self, envelope_id: str) -> None:
self._update(envelope_id, status="cancelled", updated_at=utcnow())
def export(self, *, status: str | None = None, limit: int = 1000) -> list[dict[str, Any]]:
return [envelope.__dict__ for envelope in self.list(status=status, limit=limit)]
def _update(self, envelope_id: str, **values: Any) -> None:
assignments = ", ".join(f"{key} = ?" for key in values)
params = list(values.values()) + [envelope_id]
with self._connect() as conn:
conn.execute(f"UPDATE outbox_envelopes SET {assignments} WHERE id = ?", params)
conn.commit()
def _row_to_envelope(self, row: sqlite3.Row) -> OutboxEnvelope:
return OutboxEnvelope(
id=row["id"],
idempotency_key=row["idempotency_key"],
method=row["method"],
path=row["path"],
body=_json_loads(row["body_json"]),
route_class=row["route_class"],
source_agent=row["source_agent"],
source_host=row["source_host"],
repo_slug=row["repo_slug"],
session_id=row["session_id"],
observed_revision=_json_loads(row["observed_revision_json"]),
status=row["status"],
attempt_count=row["attempt_count"],
next_retry_at=row["next_retry_at"],
last_error=row["last_error"],
response_status=row["response_status"],
response_body=_json_loads(row["response_body_json"]),
created_at=row["created_at"],
updated_at=row["updated_at"],
acked_at=row["acked_at"],
)

152
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/edge/read_cache.py Normal file
View file

@ -0,0 +1,152 @@
from __future__ import annotations
import os
import sqlite3
import stat
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlencode
DEFAULT_READ_CACHE_PATH = Path(
os.environ.get("STATEHUB_READ_CACHE_PATH", "~/.statehub/edge-read-cache.sqlite3")
).expanduser()
MAX_CACHE_ENTRY_BYTES = 1024 * 1024
CACHEABLE_GET_PREFIXES = (
"/state/",
"/workplans/",
"/messages/",
"/decisions/",
"/tasks/",
"/progress/",
"/sbom/",
"/repos",
"/legacy-meter/",
)
@dataclass(frozen=True)
class ReadCacheEntry:
cache_key: str
status_code: int
content: bytes
content_type: str | None
cached_at: str
def default_read_cache_path() -> Path:
return DEFAULT_READ_CACHE_PATH
def utcnow() -> str:
return datetime.now(tz=timezone.utc).isoformat()
def build_cache_key(method: str, path: str, query_items: list[tuple[str, str]]) -> str:
normalized_path = path if path.startswith("/") else f"/{path}"
query = urlencode(sorted(query_items))
if query:
return f"{method.upper()}:{normalized_path}?{query}"
return f"{method.upper()}:{normalized_path}"
def is_cacheable_get(path: str) -> bool:
normalized = path if path.startswith("/") else f"/{path}"
if normalized.startswith("/edge/"):
return False
return any(normalized.startswith(prefix) for prefix in CACHEABLE_GET_PREFIXES)
class ReadCacheStore:
def __init__(self, path: str | Path | None = None) -> None:
self.path = Path(path).expanduser() if path is not None else default_read_cache_path()
self.path.parent.mkdir(parents=True, exist_ok=True)
self._init_db()
self._chmod_private()
def _connect(self) -> sqlite3.Connection:
conn = sqlite3.connect(self.path)
conn.row_factory = sqlite3.Row
return conn
def _init_db(self) -> None:
with self._connect() as conn:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS read_cache_entries (
cache_key TEXT PRIMARY KEY,
status_code INTEGER NOT NULL,
content BLOB NOT NULL,
content_type TEXT,
cached_at TEXT NOT NULL
)
"""
)
conn.execute(
"CREATE INDEX IF NOT EXISTS ix_read_cache_cached_at ON read_cache_entries(cached_at)"
)
conn.commit()
def _chmod_private(self) -> None:
try:
os.chmod(self.path, stat.S_IRUSR | stat.S_IWUSR)
except OSError:
pass
def put(
self,
*,
cache_key: str,
status_code: int,
content: bytes,
content_type: str | None,
) -> None:
if len(content) > MAX_CACHE_ENTRY_BYTES:
return
now = utcnow()
with self._connect() as conn:
conn.execute(
"""
INSERT INTO read_cache_entries (
cache_key, status_code, content, content_type, cached_at
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET
status_code = excluded.status_code,
content = excluded.content,
content_type = excluded.content_type,
cached_at = excluded.cached_at
""",
(cache_key, status_code, content, content_type, now),
)
conn.commit()
def get(self, cache_key: str) -> ReadCacheEntry | None:
with self._connect() as conn:
row = conn.execute(
"SELECT * FROM read_cache_entries WHERE cache_key = ?",
(cache_key,),
).fetchone()
if row is None:
return None
return ReadCacheEntry(
cache_key=row["cache_key"],
status_code=row["status_code"],
content=row["content"],
content_type=row["content_type"],
cached_at=row["cached_at"],
)
def summary(self) -> dict[str, int | str | None]:
with self._connect() as conn:
row = conn.execute(
"""
SELECT COUNT(*) AS entry_count,
MIN(cached_at) AS oldest_cached_at
FROM read_cache_entries
"""
).fetchone()
return {
"entry_count": int(row["entry_count"] or 0),
"oldest_cached_at": row["oldest_cached_at"],
}

271
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/edge/relay.py Normal file
View file

@ -0,0 +1,271 @@
from __future__ import annotations
import os
import socket
from datetime import datetime, timezone
from typing import Any
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from api.edge.outbox import OutboxEnvelope, OutboxStore, PayloadRejected, default_outbox_path
from api.edge.read_cache import (
ReadCacheStore,
build_cache_key,
default_read_cache_path,
is_cacheable_get,
)
from api.services.write_idempotency import route_class_for
HOP_BY_HOP_HEADERS = {
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailer",
"transfer-encoding",
"upgrade",
"content-encoding",
"content-length",
}
def _safe_response_headers(headers: httpx.Headers) -> dict[str, str]:
return {key: value for key, value in headers.items() if key.lower() not in HOP_BY_HOP_HEADERS}
def _body_summary(response: httpx.Response) -> Any:
try:
return response.json()
except ValueError:
return {"text": response.text[:500]}
def queued_receipt(envelope: OutboxEnvelope, upstream_error: str) -> dict[str, Any]:
return {
"queued": True,
"outbox_id": envelope.id,
"idempotency_key": envelope.idempotency_key,
"upstream": "unreachable",
"upstream_error": upstream_error,
"route_class": envelope.route_class,
}
async def replay_pending(
store: OutboxStore,
*,
upstream_url: str,
limit: int = 50,
timeout: float = 10.0,
) -> dict[str, int]:
counts = {"sent": 0, "acked": 0, "conflict": 0, "retry": 0, "dead": 0}
async with httpx.AsyncClient(base_url=upstream_url.rstrip("/"), timeout=timeout) as client:
for envelope in store.due(limit=limit):
counts["sent"] += 1
store.mark_sending(envelope.id)
try:
response = await client.request(
envelope.method,
envelope.path,
json=envelope.body,
headers={
"Idempotency-Key": envelope.idempotency_key,
"X-StateHub-Source-Agent": envelope.source_agent or "statehub-edge",
"X-StateHub-Source-Host": envelope.source_host or socket.gethostname(),
},
)
except httpx.HTTPError as exc:
counts["retry"] += 1
store.mark_retry(envelope.id, error=str(exc), attempt_count=envelope.attempt_count + 1)
continue
response_body = _body_summary(response)
if response.status_code == 409:
counts["conflict"] += 1
store.mark_conflict(envelope.id, response_status=response.status_code, response_body=response_body)
elif 200 <= response.status_code < 300:
counts["acked"] += 1
store.mark_acked(envelope.id, response_status=response.status_code, response_body=response_body)
elif response.status_code >= 500:
counts["retry"] += 1
store.mark_retry(
envelope.id,
error=f"HTTP {response.status_code}: {response.text[:300]}",
attempt_count=envelope.attempt_count + 1,
)
else:
counts["dead"] += 1
store.mark_dead(
envelope.id,
error=f"HTTP {response.status_code}: not retryable",
response_status=response.status_code,
response_body=response_body,
)
return counts
def _cache_age_seconds(cached_at: str) -> int:
try:
cached = datetime.fromisoformat(cached_at)
except ValueError:
return 0
if cached.tzinfo is None:
cached = cached.replace(tzinfo=timezone.utc)
return max(0, int((datetime.now(timezone.utc) - cached).total_seconds()))
def _stale_cache_headers(cached_at: str) -> dict[str, str]:
return {
"X-StateHub-Edge-Cache": "stale",
"X-StateHub-Edge-Cache-Age": str(_cache_age_seconds(cached_at)),
}
def create_app(
*,
upstream_url: str | None = None,
outbox_path: str | None = None,
read_cache_path: str | None = None,
timeout: float = 10.0,
) -> FastAPI:
upstream = (upstream_url or os.environ.get("STATEHUB_UPSTREAM_URL") or os.environ.get("API_BASE") or "http://127.0.0.1:8000").rstrip("/")
store_path = outbox_path or default_outbox_path()
cache_path = read_cache_path or str(default_read_cache_path())
store_instance: OutboxStore | None = None
cache_instance: ReadCacheStore | None = None
def get_store() -> OutboxStore:
nonlocal store_instance
if store_instance is None:
store_instance = OutboxStore(store_path)
return store_instance
def get_cache() -> ReadCacheStore:
nonlocal cache_instance
if cache_instance is None:
cache_instance = ReadCacheStore(cache_path)
return cache_instance
app = FastAPI(title="State Hub Edge Relay", version="0.1.0")
@app.get("/edge/health")
async def edge_health() -> dict[str, Any]:
reachable = False
error = None
try:
async with httpx.AsyncClient(base_url=upstream, timeout=2.0) as client:
response = await client.get("/state/health")
reachable = response.status_code < 500
except httpx.HTTPError as exc:
error = str(exc)
return {
"status": "ok",
"upstream": upstream,
"upstream_reachable": reachable,
"upstream_error": error,
"outbox": get_store().summary(),
"read_cache": get_cache().summary(),
}
@app.post("/edge/replay")
async def edge_replay(limit: int = 50) -> dict[str, int]:
return await replay_pending(get_store(), upstream_url=upstream, limit=limit, timeout=timeout)
@app.api_route("/{path:path}", methods=["GET", "POST", "PATCH", "PUT", "DELETE"])
async def proxy(path: str, request: Request) -> Response:
api_path = "/" + path
body: Any = None
if request.method in {"POST", "PATCH", "PUT"}:
try:
body = await request.json()
except ValueError:
body = None
headers = {}
if idempotency_key := request.headers.get("idempotency-key"):
headers["Idempotency-Key"] = idempotency_key
if request.headers.get("content-type"):
headers["Content-Type"] = request.headers["content-type"]
cache_key = None
if request.method == "GET" and is_cacheable_get(api_path):
cache_key = build_cache_key(
request.method,
api_path,
list(request.query_params.multi_items()),
)
try:
async with httpx.AsyncClient(base_url=upstream, timeout=timeout) as client:
response = await client.request(
request.method,
api_path,
params=request.query_params,
json=body if body is not None else None,
headers=headers,
)
if cache_key is not None and 200 <= response.status_code < 300:
get_cache().put(
cache_key=cache_key,
status_code=response.status_code,
content=response.content,
content_type=response.headers.get("content-type"),
)
response_headers = _safe_response_headers(response.headers)
if cache_key is not None:
response_headers["X-StateHub-Edge-Cache"] = "hit"
return Response(
content=response.content,
status_code=response.status_code,
headers=response_headers,
media_type=response.headers.get("content-type"),
)
except httpx.HTTPError as exc:
if cache_key is not None:
cached = get_cache().get(cache_key)
if cached is not None:
stale_headers = _stale_cache_headers(cached.cached_at)
if cached.content_type:
stale_headers["Content-Type"] = cached.content_type
return Response(
content=cached.content,
status_code=cached.status_code,
headers=stale_headers,
media_type=cached.content_type,
)
route_class = route_class_for(request.method, api_path)
if route_class is None or request.method not in {"POST", "PATCH"}:
return JSONResponse(
status_code=503,
content={
"error": "upstream unreachable and route is not queueable",
"method": request.method,
"path": api_path,
"upstream": upstream,
"detail": str(exc),
},
)
try:
envelope = get_store().enqueue(
method=request.method,
path=api_path,
body=body,
idempotency_key=request.headers.get("idempotency-key"),
source_agent=request.headers.get("x-statehub-source-agent"),
source_host=request.headers.get("x-statehub-source-host") or socket.gethostname(),
repo_slug=request.headers.get("x-statehub-repo-slug"),
session_id=request.headers.get("x-statehub-session-id"),
observed_revision=None,
)
except PayloadRejected as reject:
return JSONResponse(status_code=422, content={"error": str(reject)})
return JSONResponse(status_code=202, content=queued_receipt(envelope, str(exc)))
return app
app = create_app()

13
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/events/__init__.py Normal file
View file

@ -0,0 +1,13 @@
from api.events.envelope import EventEnvelope
from api.events.nats_publisher import (
publish_event,
publish_event_sync,
shutdown_publisher,
)
__all__ = [
"EventEnvelope",
"publish_event",
"publish_event_sync",
"shutdown_publisher",
]

55
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/events/envelope.py Normal file
View file

@ -0,0 +1,55 @@
"""EventEnvelope — schema for state-hub lifecycle events published to NATS.
Mirrors the EventEnvelope contract defined in activity-core
(`src/activity_core/models.py`). The state-hub publishes; activity-core
consumes and routes to ActivityDefinitions.
Subject naming convention (see docs/nats-event-subjects.md):
org.statehub.{noun}.{verb}
Examples:
org.statehub.repo.registered
org.statehub.workstream.completed
org.statehub.decision.resolved
org.statehub.domain.goal.activated
org.statehub.task.stale
"""
from __future__ import annotations
import uuid
from datetime import datetime, timezone
from typing import Any
from pydantic import BaseModel, Field
PUBLISHER = "state-hub"
class EventEnvelope(BaseModel):
"""Standard envelope shared with activity-core. Do not break compatibility.
All inbound events on activity-core's side are normalised into this shape.
"""
id: str = Field(description="UUID v4 — stable unique ID for deduplication.")
type: str = Field(description="Dot-namespaced event type, e.g. 'org.statehub.repo.registered'.")
version: str = Field(default="1.0", description="Schema version string.")
timestamp: datetime = Field(description="When the event occurred (UTC).")
publisher: str = Field(default=PUBLISHER, description="Originating service.")
attributes: dict[str, Any] = Field(
default_factory=dict,
description="Event-specific attributes; structure varies by event type.",
)
@classmethod
def new(cls, event_type: str, attributes: dict[str, Any] | None = None) -> "EventEnvelope":
"""Construct an envelope with a fresh UUID and current UTC timestamp."""
return cls(
id=str(uuid.uuid4()),
type=event_type,
timestamp=datetime.now(tz=timezone.utc),
publisher=PUBLISHER,
attributes=attributes or {},
)

139
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/events/nats_publisher.py Normal file
View file

@ -0,0 +1,139 @@
"""NATS JetStream publisher for state-hub lifecycle events.
Design:
- One process-wide publisher (`_Publisher` singleton).
- Connects lazily on first publish; reuses the connection thereafter.
- When ``NATS_URL`` is unset or empty, every publish is a logged no-op
so the state hub remains usable in environments without NATS.
- All publishes are fire-and-forget from the caller's perspective.
Failures are logged but never raise — losing a lifecycle event must
never break the API request that triggered it.
Stream + subject conventions live in ``docs/nats-event-subjects.md``.
Envelope schema lives in :mod:`api.events.envelope`.
"""
from __future__ import annotations
import asyncio
import logging
import os
from typing import TYPE_CHECKING
from api.events.envelope import EventEnvelope
if TYPE_CHECKING: # pragma: no cover — import-only for typing
from nats.aio.client import Client as NATSClient
from nats.js.client import JetStreamContext
logger = logging.getLogger("state_hub.events.nats")
_STREAM_NAME = "ACTIVITY_EVENTS"
_STREAM_SUBJECT_PATTERN = "org.>"
def _nats_url() -> str | None:
"""Resolve NATS_URL at call time so tests / configs can override it."""
url = os.environ.get("NATS_URL", "").strip()
return url or None
class _Publisher:
"""Singleton holding the live NATS connection + JetStream context."""
def __init__(self) -> None:
self._nc: "NATSClient | None" = None
self._js: "JetStreamContext | None" = None
self._connect_lock = asyncio.Lock()
self._ensured_stream = False
async def _connect(self, url: str) -> None:
# Imported inside the method so module import works without the dep.
import nats
import nats.js.api
async with self._connect_lock:
if self._nc is not None and self._nc.is_connected:
return
self._nc = await nats.connect(url, connect_timeout=2)
self._js = self._nc.jetstream()
logger.info("nats: connected to %s", url)
if not self._ensured_stream:
try:
await self._js.find_stream_name_by_subject("org.statehub.repo.registered")
self._ensured_stream = True
except Exception:
try:
await self._js.add_stream(
nats.js.api.StreamConfig(
name=_STREAM_NAME,
subjects=[_STREAM_SUBJECT_PATTERN],
)
)
logger.info("nats: created JetStream stream %r", _STREAM_NAME)
except Exception as exc: # pragma: no cover — defensive
logger.warning("nats: could not ensure stream %r: %s", _STREAM_NAME, exc)
self._ensured_stream = True
async def publish(self, subject: str, envelope: EventEnvelope) -> None:
url = _nats_url()
if url is None:
logger.debug("nats: NATS_URL unset — skipping publish %s (id=%s)", subject, envelope.id)
return
try:
if self._nc is None or not self._nc.is_connected:
await self._connect(url)
assert self._js is not None
payload = envelope.model_dump_json().encode()
ack = await self._js.publish(subject, payload)
logger.info(
"nats: published %s id=%s stream=%s seq=%s",
subject,
envelope.id,
getattr(ack, "stream", "?"),
getattr(ack, "seq", "?"),
)
except Exception as exc:
logger.warning("nats: publish failed %s id=%s err=%s", subject, envelope.id, exc)
async def shutdown(self) -> None:
if self._nc is not None:
try:
await self._nc.drain()
except Exception: # pragma: no cover — defensive
pass
self._nc = None
self._js = None
self._ensured_stream = False
_PUBLISHER = _Publisher()
async def publish_event(subject: str, envelope: EventEnvelope) -> None:
"""Publish ``envelope`` on ``subject``. Logs but never raises on failure.
No-op when ``NATS_URL`` is not configured.
"""
await _PUBLISHER.publish(subject, envelope)
def publish_event_sync(subject: str, envelope: EventEnvelope) -> None:
"""Fire-and-forget variant for sync callers (scripts, cron jobs).
Runs the publish in a short-lived event loop. Intended for one-shot CLI
callers that aren't already inside an async context. Server code should
prefer :func:`publish_event` with ``asyncio.create_task``.
"""
try:
asyncio.run(publish_event(subject, envelope))
except RuntimeError:
# Already inside a running loop — schedule and forget.
asyncio.get_event_loop().create_task(publish_event(subject, envelope))
async def shutdown_publisher() -> None:
"""Drain the NATS connection on app shutdown."""
await _PUBLISHER.shutdown()

88
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/flow_defs.py Normal file
View file

@ -0,0 +1,88 @@
from __future__ import annotations
from functools import lru_cache
from pathlib import Path
from typing import Any
import yaml
from task_flow_engine import AssertionDef, AssertionResult, FlowDef, FlowEngine, FlowResult
FLOW_DIR = Path(__file__).resolve().parents[1] / "flows"
@lru_cache
def load_flow(entity_type: str) -> FlowDef:
path = FLOW_DIR / f"{entity_type}.yaml"
data = yaml.safe_load(path.read_text(encoding="utf-8"))
return FlowDef.from_dict(data)
def evaluate_transition(
entity_type: str,
current_workstation: str,
target_workstation: str,
extra: dict[str, Any] | None = None,
) -> tuple[bool, list[AssertionResult], FlowResult]:
flow = load_flow(entity_type)
obj = {
"status": current_workstation,
"workstation": current_workstation,
"previous_workstation": current_workstation,
**(extra or {}),
}
engine = create_flow_engine()
result = engine.evaluate(obj, flow)
can_reach, failures = engine.can_reach(obj, flow, target_workstation)
return can_reach, failures, result
def create_flow_engine() -> FlowEngine:
return FlowEngine(
custom_ops={
"dependencies.any_incomplete": _dependencies_any_incomplete,
}
)
def _dependencies_any_incomplete(
assertion: AssertionDef,
obj: dict[str, Any],
values: list[Any],
) -> bool:
expected = assertion.value
if isinstance(expected, list):
return bool(values) and any(value not in expected for value in values)
return bool(values) and any(value != expected for value in values)
def assertion_result_to_dict(result: AssertionResult) -> dict[str, Any]:
return {
"id": result.id,
"passed": result.passed,
"target": result.target,
"op": result.op,
"expected": result.expected,
"actual": result.actual,
"description": result.description,
"reason": result.reason,
}
def flow_result_to_dict(result: FlowResult) -> dict[str, Any]:
return {
"current_workstation": result.current_workstation,
"exit_blocked": result.exit_blocked,
"blocking_assertions": [
assertion_result_to_dict(item) for item in result.blocking_assertions
],
"reachable": result.reachable,
"unreachable": [
{
"workstation": item.workstation,
"blocking": assertion_result_to_dict(item.blocking),
}
for item in result.unreachable
],
}

157
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/main.py Normal file
View file

@ -0,0 +1,157 @@
import hashlib
import os
import time
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response as StarletteResponse
from api.database import engine
from api.events import shutdown_publisher
from api.services.write_idempotency import WriteIdempotencyMiddleware
from api.routers import decisions, extension_points, intake, ops_runs, progress, state, suggestions, tasks, technical_debt, topics, workstreams, workstream_dependencies
from api.routers import domains, repos, contributions, sbom, policy, domain_goals, repo_goals, messages, memo_notifications, capability_requests, tpsc, services
from api.routers import token_events
from api.routers import message_feed
from api.routers import interface_changes
from api.routers import flows
from api.routers import recently_on_scope
from api.routers import consistency_sweep
from api.routers import reconciliation
from api.routers import execution
from api.routers import fabric
from api.routers import legacy_meter
from api.routers import review_contracts
from api.routers import identifier_migrations
from api.routers import repository_renames
from api.routers import work_record_projection
class ETagMiddleware(BaseHTTPMiddleware):
"""Add ETag + conditional-GET (304) support to all JSON GET responses."""
async def dispatch(self, request: Request, call_next):
started = time.perf_counter()
response = await call_next(request)
if request.method != "GET":
response.headers["X-StateHub-Elapsed-Ms"] = f"{(time.perf_counter() - started) * 1000:.1f}"
return response
if "application/json" not in response.headers.get("content-type", ""):
response.headers["X-StateHub-Elapsed-Ms"] = f"{(time.perf_counter() - started) * 1000:.1f}"
return response
body_parts = []
async for chunk in response.body_iterator:
body_parts.append(chunk)
body = b"".join(body_parts)
elapsed_ms = f"{(time.perf_counter() - started) * 1000:.1f}"
etag = '"' + hashlib.md5(body, usedforsecurity=False).hexdigest() + '"'
if request.headers.get("if-none-match") == etag:
return StarletteResponse(
status_code=304,
headers={
"ETag": etag,
"Cache-Control": "no-cache",
"X-StateHub-Elapsed-Ms": elapsed_ms,
"X-StateHub-Response-Bytes": "0",
},
)
headers = {k: v for k, v in response.headers.items() if k.lower() != "content-length"}
headers["ETag"] = etag
headers["X-StateHub-Elapsed-Ms"] = elapsed_ms
headers["X-StateHub-Response-Bytes"] = str(len(body))
if not any(k.lower() == "cache-control" for k in headers):
headers["Cache-Control"] = "no-cache"
return StarletteResponse(
content=body,
status_code=response.status_code,
headers=headers,
media_type=response.media_type,
)
@asynccontextmanager
async def lifespan(app: FastAPI):
yield
await shutdown_publisher()
await engine.dispose()
app = FastAPI(
title="Custodian State Hub",
description="Local-first state API for the Custodian agent system.",
version="0.6.0",
lifespan=lifespan,
)
_default_dashboard_origins = [
*(f"http://localhost:{port}" for port in range(3000, 3006)),
*(f"http://127.0.0.1:{port}" for port in range(3000, 3006)),
*(f"http://[::1]:{port}" for port in range(3000, 3006)),
]
_cors_env = os.getenv("CORS_ORIGINS", ",".join(_default_dashboard_origins))
_cors_origins = [o.strip() for o in _cors_env.split(",") if o.strip()]
app.add_middleware(WriteIdempotencyMiddleware)
app.add_middleware(ETagMiddleware)
app.add_middleware(
CORSMiddleware,
allow_origins=_cors_origins,
allow_methods=["GET", "POST", "PATCH", "DELETE", "PUT"],
allow_headers=["Content-Type", "If-None-Match", "Idempotency-Key", "X-StateHub-Source-Agent", "X-StateHub-Source-Host"],
expose_headers=["ETag", "X-StateHub-Elapsed-Ms", "X-StateHub-Response-Bytes", "X-StateHub-Cache", "X-StateHub-Idempotency-Replay"],
)
app.include_router(domains.router)
app.include_router(recently_on_scope.hourly_router)
app.include_router(recently_on_scope.router)
app.include_router(consistency_sweep.router)
app.include_router(repos.router)
app.include_router(repository_renames.router)
app.include_router(repository_renames.operation_router)
app.include_router(work_record_projection.router)
app.include_router(topics.router)
app.include_router(workstreams.router)
app.include_router(workstreams.workplan_router)
app.include_router(workstream_dependencies.router)
app.include_router(workstream_dependencies.workplan_router)
app.include_router(tasks.router)
app.include_router(decisions.router)
app.include_router(intake.router)
app.include_router(extension_points.router)
app.include_router(technical_debt.router)
app.include_router(progress.router)
app.include_router(domain_goals.router)
app.include_router(repo_goals.router)
app.include_router(contributions.router)
app.include_router(sbom.router)
# Before messages.router so /messages/feed is not captured by /messages/{id}.
app.include_router(message_feed.router)
app.include_router(messages.router)
app.include_router(memo_notifications.router)
app.include_router(capability_requests.router)
app.include_router(suggestions.router)
app.include_router(tpsc.router)
app.include_router(services.router)
app.include_router(token_events.router)
app.include_router(interface_changes.router)
app.include_router(flows.router)
app.include_router(reconciliation.router)
app.include_router(execution.router)
app.include_router(fabric.router)
app.include_router(legacy_meter.router)
app.include_router(review_contracts.router)
app.include_router(identifier_migrations.router)
app.include_router(state.router)
app.include_router(ops_runs.router)
app.include_router(policy.router)
@app.get("/", include_in_schema=False)
async def root():
return {"service": "dev-hub", "docs": "/docs"}

98
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/__init__.py Normal file
View file

@ -0,0 +1,98 @@
from api.models.base import Base
from api.models.domain import Domain
from api.models.domain_goal import DomainGoal, DomainGoalStatus
from api.models.topic import Topic, TopicStatus
from api.models.managed_repo import ManagedRepo
from api.models.repo_goal import RepoGoal, RepoGoalStatus
from api.models.workplan import Workplan
from api.models.workplan_dependency import WorkplanDependency
from api.models.workstream import Workstream
from api.models.workstream_dependency import WorkstreamDependency
from api.models.task import Task, TaskStatus, TaskPriority
from api.models.decision import Decision, DecisionType, DecisionStatus
from api.models.intake import (
Intake,
IntakeNote,
IntakeLane,
IntakeStatus,
IntakeOutcome,
)
from api.models.progress_event import ProgressEvent
from api.models.extension_point import ExtensionPoint, EPStatus
from api.models.technical_debt import TechnicalDebt, TDStatus
from api.models.contribution import Contribution, ContributionType, ContributionStatus
from api.models.sbom_snapshot import SBOMSnapshot
from api.models.sbom_entry import SBOMEntry, Ecosystem
from api.models.agent_message import AgentMessage, MessageReceipt, MessageTombstone
from api.models.memo_notification import MemoNotificationReceipt
from api.models.capability_catalog import CapabilityCatalog
from api.models.capability_request import CapabilityRequest
from api.models.tpsc import TPSCCatalog, TPSCSnapshot, TPSCEntry
from api.models.service_catalog import (
ServiceCatalog,
ServiceThirdParty,
ServiceFirstParty,
ServiceCloud,
ServiceSelfHosted,
)
from api.models.doi_cache import DOICache
from api.models.token_event import TokenEvent
from api.models.interface_change import InterfaceChange
from api.models.workplan_launch_request import WorkplanLaunchRequest
from api.models.fabric_graph import FabricGraphImport, FabricGraphNode, FabricGraphEdge
from api.models.legacy_meter import LegacyInterface, LegacyInterfaceUsageBucket
from api.models.write_idempotency_key import WriteIdempotencyKey
from api.models.work_record_identifier_alias import WorkRecordIdentifierAlias
from api.models.review_contract import ReviewContract, ReviewReceipt
from api.models.repository_rename import (
RepositoryForgeIdentity,
RepositoryRenameOperation,
RepositorySlug,
)
from api.models.suggestion import (
Suggestion,
SuggestionNote,
SuggestionRelevanceBump,
SuggestionStage,
)
__all__ = [
"Base",
"Domain",
"DomainGoal", "DomainGoalStatus",
"Topic", "TopicStatus",
"ManagedRepo",
"RepoGoal", "RepoGoalStatus",
"Workplan",
"WorkplanDependency",
"Workstream",
"WorkstreamDependency",
"Task", "TaskStatus", "TaskPriority",
"Decision", "DecisionType", "DecisionStatus",
"Intake", "IntakeNote", "IntakeLane", "IntakeStatus", "IntakeOutcome",
"ProgressEvent",
"ExtensionPoint", "EPStatus",
"TechnicalDebt", "TDStatus",
"Contribution", "ContributionType", "ContributionStatus",
"SBOMSnapshot",
"SBOMEntry", "Ecosystem",
"AgentMessage",
"MessageReceipt",
"MemoNotificationReceipt",
"CapabilityCatalog",
"CapabilityRequest",
"TPSCCatalog", "TPSCSnapshot", "TPSCEntry",
"ServiceCatalog", "ServiceThirdParty", "ServiceFirstParty",
"ServiceCloud", "ServiceSelfHosted",
"DOICache",
"TokenEvent",
"InterfaceChange",
"WorkplanLaunchRequest",
"FabricGraphImport", "FabricGraphNode", "FabricGraphEdge",
"LegacyInterface", "LegacyInterfaceUsageBucket",
"WriteIdempotencyKey",
"WorkRecordIdentifierAlias",
"ReviewContract", "ReviewReceipt",
"RepositoryForgeIdentity", "RepositoryRenameOperation", "RepositorySlug",
"Suggestion", "SuggestionNote", "SuggestionRelevanceBump", "SuggestionStage",
]

220
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/agent_message.py Normal file
View file

@ -0,0 +1,220 @@
import uuid
from datetime import datetime
from sqlalchemy import (
DDL,
BigInteger,
DateTime,
FetchedValue,
ForeignKey,
Index,
Sequence,
String,
Text,
event,
text,
)
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, new_uuid
BROADCAST = "broadcast"
MESSAGE_KINDS = ("message", "news", "standing")
# STATE-WP-0098-T05: inbox change feed (docs/inbox-change-feed-contract.md).
# ``revision`` is owned by the database trigger below; application code never
# sets it. Every inbox write transaction gets exactly one revision.
INBOX_REVISION_SEQ = Sequence("inbox_revision_seq", metadata=Base.metadata)
def _revision_column() -> Mapped[int]:
return mapped_column(
BigInteger,
nullable=False,
index=True,
server_default=FetchedValue(),
server_onupdate=FetchedValue(),
)
class AgentMessage(Base):
__tablename__ = "agent_messages"
__table_args__ = (
Index("ix_agent_messages_to_kind_archived", "to_agent", "kind", "archived_at"),
)
__mapper_args__ = {"eager_defaults": True}
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
from_agent: Mapped[str] = mapped_column(String(100), nullable=False)
to_agent: Mapped[str] = mapped_column(String(100), nullable=False, index=True)
subject: Mapped[str] = mapped_column(String(500), nullable=False)
body: Mapped[str] = mapped_column(Text, nullable=False)
thread_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("agent_messages.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
read_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
archived_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
# STATE-WP-0093: kind is only meaningful for broadcasts
# (message | news | standing); direct messages stay "message".
kind: Mapped[str] = mapped_column(
String(20), nullable=False, default="message", server_default="message"
)
expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
supersedes_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("agent_messages.id", ondelete="SET NULL"),
nullable=True,
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=text("now()"),
nullable=False,
)
revision: Mapped[int] = _revision_column()
thread_root: Mapped["AgentMessage | None"] = relationship(
"AgentMessage",
remote_side="AgentMessage.id",
foreign_keys=[thread_id],
lazy="select",
)
class MessageReceipt(Base):
"""Per-reader state for broadcast messages (STATE-WP-0093).
Written only for broadcasts; direct messages keep the global
``AgentMessage.read_at``.
"""
__tablename__ = "message_receipts"
__mapper_args__ = {"eager_defaults": True}
message_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("agent_messages.id", ondelete="CASCADE"),
primary_key=True,
)
agent: Mapped[str] = mapped_column(String(100), primary_key=True, index=True)
delivered_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
read_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
acknowledged_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
revision: Mapped[int] = _revision_column()
class MessageTombstone(Base):
"""A deleted ``agent_messages`` row, written by the delete trigger.
Bodies are never kept. Served by the inbox feed within the 90-day horizon
(contract D1).
"""
__tablename__ = "message_tombstones"
message_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True)
revision: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True)
deleted_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
reason: Mapped[str | None] = mapped_column(Text, nullable=True)
# Trigger DDL for ``Base.metadata.create_all`` (the test suite). Production
# gets the same objects from migration c4d5e6f7a8b9_inbox_feed_revision, which
# keeps its own frozen copy; tests/test_message_feed.py checks they agree.
INBOX_ADVISORY_LOCK_KEY = 5_348_490_098
INBOX_FEED_FUNCTIONS_SQL = f"""
CREATE OR REPLACE FUNCTION inbox_feed_revision() RETURNS bigint
LANGUAGE plpgsql AS $fn$
DECLARE
tx text := txid_current()::text;
cached text := coalesce(current_setting('statehub.inbox_rev', true), '');
rev bigint;
BEGIN
PERFORM pg_advisory_xact_lock({INBOX_ADVISORY_LOCK_KEY});
IF split_part(cached, ':', 1) = tx THEN
RETURN split_part(cached, ':', 2)::bigint;
END IF;
rev := nextval('inbox_revision_seq');
PERFORM set_config('statehub.inbox_rev', tx || ':' || rev::text, true);
RETURN rev;
END
$fn$;
CREATE OR REPLACE FUNCTION inbox_feed_stamp_revision() RETURNS trigger
LANGUAGE plpgsql AS $fn$
BEGIN
IF TG_OP = 'UPDATE' THEN
IF NEW IS NOT DISTINCT FROM OLD THEN
RETURN NEW;
END IF;
END IF;
NEW.revision := inbox_feed_revision();
RETURN NEW;
END
$fn$;
CREATE OR REPLACE FUNCTION inbox_feed_tombstone() RETURNS trigger
LANGUAGE plpgsql AS $fn$
BEGIN
INSERT INTO message_tombstones (message_id, revision, deleted_at, reason)
VALUES (
OLD.id,
inbox_feed_revision(),
now(),
nullif(current_setting('statehub.delete_reason', true), '')
)
ON CONFLICT (message_id) DO UPDATE
SET revision = EXCLUDED.revision,
deleted_at = EXCLUDED.deleted_at,
reason = EXCLUDED.reason;
RETURN OLD;
END
$fn$;
"""
event.listen(
AgentMessage.__table__,
"after_create",
DDL(
INBOX_FEED_FUNCTIONS_SQL
+ """
CREATE TRIGGER agent_messages_inbox_revision
BEFORE INSERT OR UPDATE ON agent_messages
FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision();
CREATE TRIGGER agent_messages_inbox_tombstone
BEFORE DELETE ON agent_messages
FOR EACH ROW EXECUTE FUNCTION inbox_feed_tombstone();
"""
),
)
event.listen(
MessageReceipt.__table__,
"after_create",
DDL(
INBOX_FEED_FUNCTIONS_SQL
+ """
CREATE TRIGGER message_receipts_inbox_revision
BEFORE INSERT OR UPDATE ON message_receipts
FOR EACH ROW EXECUTE FUNCTION inbox_feed_stamp_revision();
"""
),
)

53
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/base.py Normal file
View file

@ -0,0 +1,53 @@
import os
import time
import uuid
from datetime import datetime
from sqlalchemy import DateTime, func
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
class Base(DeclarativeBase):
pass
class TimestampMixin:
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), nullable=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
server_default=func.now(),
onupdate=func.now(),
nullable=False,
)
def new_uuid() -> uuid.UUID:
return uuid.uuid4()
def new_uuid7() -> uuid.UUID:
"""Generate a UUIDv7 (RFC 9562): 48-bit big-endian ms timestamp, version
and variant bits, remaining bits random. Time-sortable, so primary keys
generated with this helper order chronologically without a separate
created_at index lookup — the identity layering canon
(work-record-types_v0.1.md) calls this out as the primary internal key
for new work-record entities.
Dependency-free (no uuid7 in stdlib before Python 3.14, no third-party
lib added for a ~15-line, non-cryptographic layout).
"""
unix_ts_ms = int(time.time() * 1000)
rand = int.from_bytes(os.urandom(10), "big")
rand_a = (rand >> 62) & 0x0FFF # top 12 bits of the 80 random bits
rand_b = rand & 0x3FFFFFFFFFFFFFFF # bottom 62 bits
value = (
(unix_ts_ms << 80)
| (0x7 << 76) # version 7
| (rand_a << 64)
| (0x2 << 62) # variant 10
| rand_b
)
return uuid.UUID(int=value)

50
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/capability_catalog.py Normal file
View file

@ -0,0 +1,50 @@
import uuid
from sqlalchemy import ARRAY, ForeignKey, String, Text, UniqueConstraint
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class CapabilityCatalog(Base, TimestampMixin):
__tablename__ = "capability_catalog"
__table_args__ = (
UniqueConstraint("domain_id", "capability_type", "title", name="uq_catalog_domain_type_title"),
)
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
domain_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("domains.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
repo_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("managed_repos.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
capability_type: Mapped[str] = mapped_column(String(50), nullable=False)
title: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
keywords: Mapped[list[str]] = mapped_column(
ARRAY(String), nullable=False, server_default="{}"
)
status: Mapped[str] = mapped_column(
String(20), nullable=False, default="active", server_default="active"
)
domain: Mapped["Domain"] = relationship("Domain", lazy="selectin") # noqa: F821
repo: Mapped["ManagedRepo | None"] = relationship("ManagedRepo", lazy="selectin") # noqa: F821
@property
def domain_slug(self) -> str:
return self.domain.slug if self.domain is not None else ""
@property
def repo_slug(self) -> str | None:
return self.repo.slug if self.repo is not None else None

101
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/capability_request.py Normal file
View file

@ -0,0 +1,101 @@
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class CapabilityRequest(Base, TimestampMixin):
__tablename__ = "capability_requests"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
title: Mapped[str] = mapped_column(String(500), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
capability_type: Mapped[str] = mapped_column(String(50), nullable=False)
priority: Mapped[str] = mapped_column(
String(20), nullable=False, default="medium", server_default="medium"
)
status: Mapped[str] = mapped_column(
String(20), nullable=False, default="requested", server_default="requested"
)
# Requester side
requesting_domain_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("domains.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
requesting_workplan_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("workplans.id", ondelete="SET NULL", onupdate="CASCADE"),
nullable=True,
)
requesting_agent: Mapped[str] = mapped_column(String(100), nullable=False)
# Fulfiller side (populated on accept / auto-route)
fulfilling_domain_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("domains.id", ondelete="SET NULL"),
nullable=True,
index=True,
)
fulfilling_workplan_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("workplans.id", ondelete="SET NULL", onupdate="CASCADE"),
nullable=True,
)
fulfilling_agent: Mapped[str | None] = mapped_column(String(100), nullable=True)
# Links
blocking_task_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("tasks.id", ondelete="SET NULL", onupdate="CASCADE"),
nullable=True,
)
catalog_entry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("capability_catalog.id", ondelete="SET NULL"),
nullable=True,
)
resolution_note: Mapped[str | None] = mapped_column(Text, nullable=True)
routing_note: Mapped[str | None] = mapped_column(Text, nullable=True)
# Dispute fields (populated when status = routing_disputed)
dispute_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
disputed_by: Mapped[str | None] = mapped_column(String(100), nullable=True)
dispute_suggested_domain: Mapped[str | None] = mapped_column(String(100), nullable=True)
disputed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
accepted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
completed_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
# Relationships
requesting_domain: Mapped["Domain"] = relationship( # noqa: F821
"Domain", foreign_keys=[requesting_domain_id], lazy="selectin"
)
fulfilling_domain: Mapped["Domain | None"] = relationship( # noqa: F821
"Domain", foreign_keys=[fulfilling_domain_id], lazy="selectin"
)
blocking_task: Mapped["Task | None"] = relationship("Task", lazy="selectin") # noqa: F821
catalog_entry: Mapped["CapabilityCatalog | None"] = relationship( # noqa: F821
"CapabilityCatalog", lazy="selectin"
)
@property
def requesting_domain_slug(self) -> str:
return self.requesting_domain.slug if self.requesting_domain else ""
@property
def fulfilling_domain_slug(self) -> str | None:
return self.fulfilling_domain.slug if self.fulfilling_domain else None

68
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/contribution.py Normal file
View file

@ -0,0 +1,68 @@
import enum
import uuid
from datetime import datetime
from sqlalchemy import DateTime, Enum, ForeignKey, String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class ContributionType(str, enum.Enum):
br = "br"
fr = "fr"
ep = "ep"
upr = "upr"
class ContributionStatus(str, enum.Enum):
draft = "draft"
submitted = "submitted"
acknowledged = "acknowledged"
accepted = "accepted"
rejected = "rejected"
merged = "merged"
withdrawn = "withdrawn"
class Contribution(Base, TimestampMixin):
__tablename__ = "contributions"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
type: Mapped[ContributionType] = mapped_column(
Enum(ContributionType, name="contributiontype"), nullable=False
)
target_org: Mapped[str | None] = mapped_column(String(200), nullable=True)
target_repo: Mapped[str | None] = mapped_column(String(200), nullable=True)
slug: Mapped[str | None] = mapped_column(String(200), nullable=True)
title: Mapped[str] = mapped_column(String(500), nullable=False)
status: Mapped[ContributionStatus] = mapped_column(
Enum(ContributionStatus, name="contributionstatus"),
nullable=False, default=ContributionStatus.draft,
)
body_path: Mapped[str | None] = mapped_column(Text, nullable=True)
related_topic_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("topics.id", ondelete="SET NULL"), nullable=True
)
related_workplan_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("workplans.id", ondelete="SET NULL", onupdate="CASCADE"),
nullable=True,
)
repo_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("managed_repos.id", ondelete="SET NULL"), nullable=True
)
submitted_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
resolved_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
topic: Mapped["Topic"] = relationship("Topic", lazy="selectin") # noqa: F821
workplan: Mapped["Workplan"] = relationship("Workplan", lazy="selectin") # noqa: F821
repo: Mapped["ManagedRepo"] = relationship("ManagedRepo", lazy="selectin") # noqa: F821

66
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/decision.py Normal file
View file

@ -0,0 +1,66 @@
import enum
import uuid
from datetime import datetime
from sqlalchemy import CheckConstraint, DateTime, Enum, ForeignKey, String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class DecisionType(str, enum.Enum):
made = "made"
pending = "pending"
class DecisionStatus(str, enum.Enum):
open = "open"
resolved = "resolved"
escalated = "escalated"
superseded = "superseded"
class Decision(Base, TimestampMixin):
__tablename__ = "decisions"
__table_args__ = (
CheckConstraint(
"topic_id IS NOT NULL OR workplan_id IS NOT NULL",
name="ck_decisions_topic_or_workplan",
),
)
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
topic_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("topics.id", ondelete="RESTRICT"), nullable=True, index=True
)
workplan_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("workplans.id", ondelete="RESTRICT", onupdate="CASCADE"),
nullable=True,
index=True,
)
title: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
decision_type: Mapped[DecisionType] = mapped_column(
Enum(DecisionType), nullable=False, default=DecisionType.pending
)
status: Mapped[DecisionStatus] = mapped_column(
Enum(DecisionStatus), nullable=False, default=DecisionStatus.open
)
rationale: Mapped[str | None] = mapped_column(Text, nullable=True)
decided_by: Mapped[str | None] = mapped_column(String(100), nullable=True)
decided_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
deadline: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
escalation_note: Mapped[str | None] = mapped_column(Text, nullable=True)
superseded_by: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("decisions.id", ondelete="SET NULL"), nullable=True
)
topic: Mapped["Topic | None"] = relationship("Topic", back_populates="decisions") # noqa: F821
workplan: Mapped["Workplan | None"] = relationship("Workplan", back_populates="decisions") # noqa: F821
progress_events: Mapped[list["ProgressEvent"]] = relationship( # noqa: F821
"ProgressEvent", back_populates="decision", lazy="selectin"
)

27
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/doi_cache.py Normal file
View file

@ -0,0 +1,27 @@
import uuid
from datetime import datetime
from sqlalchemy import Boolean, DateTime, ForeignKey, String, Text, func
from sqlalchemy.dialects.postgresql import JSON, UUID
from sqlalchemy.orm import Mapped, mapped_column
from api.models.base import Base
class DOICache(Base):
__tablename__ = "doi_cache"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
repo_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("managed_repos.id", ondelete="CASCADE"),
nullable=False, unique=True, index=True,
)
tier: Mapped[str] = mapped_column(String(20), nullable=False)
core_pass: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false")
standard_pass: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false")
full_pass: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false")
criteria: Mapped[list | None] = mapped_column(JSON, nullable=True)
# Pipe-joined string of timestamps/mtimes used to detect staleness
fingerprint: Mapped[str] = mapped_column(Text, nullable=False)
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())

29
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/domain.py Normal file
View file

@ -0,0 +1,29 @@
import uuid
from sqlalchemy import String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class Domain(Base, TimestampMixin):
__tablename__ = "domains"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
slug: Mapped[str] = mapped_column(String(50), unique=True, nullable=False, index=True)
name: Mapped[str] = mapped_column(String(200), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="active")
topics: Mapped[list["Topic"]] = relationship( # noqa: F821
"Topic", back_populates="domain", lazy="selectin"
)
repos: Mapped[list["ManagedRepo"]] = relationship( # noqa: F821
"ManagedRepo", back_populates="domain", lazy="selectin"
)
goals: Mapped[list["DomainGoal"]] = relationship( # noqa: F821
"DomainGoal", back_populates="domain", lazy="selectin"
)

41
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/domain_goal.py Normal file
View file

@ -0,0 +1,41 @@
import enum
import uuid
from sqlalchemy import ForeignKey, String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class DomainGoalStatus(str, enum.Enum):
active = "active"
archived = "archived"
superseded = "superseded"
class DomainGoal(Base, TimestampMixin):
__tablename__ = "domain_goals"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
domain_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("domains.id", ondelete="RESTRICT"), nullable=False, index=True
)
title: Mapped[str] = mapped_column(String(500), nullable=False)
description: Mapped[str] = mapped_column(Text, nullable=False)
status: Mapped[str] = mapped_column(
String(20), nullable=False, default=DomainGoalStatus.active.value, server_default="active"
)
domain: Mapped["Domain"] = relationship( # noqa: F821
"Domain", back_populates="goals", lazy="selectin"
)
repo_goals: Mapped[list["RepoGoal"]] = relationship( # noqa: F821
"RepoGoal", back_populates="domain_goal", lazy="selectin"
)
@property
def domain_slug(self) -> str:
return self.domain.slug if self.domain is not None else ""

59
docs/changes/netkingdom-target-2026-10-16/cluster-historical-publication-r1/state-hub/api/models/extension_point.py Normal file
View file

@ -0,0 +1,59 @@
import enum
import uuid
from sqlalchemy import Enum, ForeignKey, String, Text
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.orm import Mapped, mapped_column, relationship
from api.models.base import Base, TimestampMixin, new_uuid
class EPStatus(str, enum.Enum):
open = "open"
in_progress = "in_progress"
addressed = "addressed"
deferred = "deferred"
wont_fix = "wont_fix"
class ExtensionPoint(Base, TimestampMixin):
__tablename__ = "extension_points"
id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), primary_key=True, default=new_uuid
)
ep_id: Mapped[str | None] = mapped_column(
String(30), nullable=True, unique=True, index=True
) # human-readable ref, e.g. EP-CUST-001
domain_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True),
ForeignKey("domains.id", ondelete="RESTRICT"),
nullable=False,
index=True,
)
title: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
location: Mapped[str | None] = mapped_column(String(500), nullable=True)
ep_type: Mapped[str] = mapped_column(
String(50), nullable=False, default="other"
) # api | schema | mcp | dashboard | architecture | integration | other
status: Mapped[EPStatus] = mapped_column(
Enum(EPStatus, name="epstatus"), nullable=False, default=EPStatus.open
)
priority: Mapped[str] = mapped_column(String(20), nullable=False, default="medium")
topic_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("topics.id", ondelete="SET NULL"), nullable=True
)
workplan_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("workplans.id", ondelete="SET NULL", onupdate="CASCADE"),
nullable=True,
)
domain: Mapped["Domain"] = relationship("Domain", lazy="selectin") # noqa: F821
topic: Mapped["Topic"] = relationship("Topic", lazy="selectin") # noqa: F821
workplan: Mapped["Workplan"] = relationship("Workplan", lazy="selectin") # noqa: F821
@property
def domain_slug(self) -> str:
return self.domain.slug if self.domain is not None else ""