RAILIANCE-WP-0012/0013/0014: Core Hub S5 release, CNPG observability, and follow-up workplans

Deliver the inbox-suggestion implementation from WP-0012: Core Hub Helm chart and Makefile targets, CNPG backup status tooling, and updated backup handoff docs. Archive the finished WP-0012 workplan and register ready follow-ups WP-0013 (CNPG backup wiring + restore drill) and WP-0014 (Core Hub Helm cutover and vergabe-teilnahme image refresh).
2026-07-10 15:14:20 +02:00 · 2026-07-10 15:14:20 +02:00 · df7225dd3e
commit df7225dd3e
parent 85f81714f1
19 changed files with 1025 additions and 2 deletions
--- a/97
+++ b/97
@ -55,6 +55,19 @@ ESO_CHART_VERSION ?= 0.16.1
 INTER_HUB_IMAGE_REF = $(INTER_HUB_IMAGE_REPOSITORY):$(INTER_HUB_IMAGE_TAG)
 INTER_HUB_IMAGE_SET_ARG = $(if $(strip $(INTER_HUB_IMAGE_TAG)),--set image.tag=$(INTER_HUB_IMAGE_TAG),)

+CORE_HUB_RELEASE   ?= core-hub
+CORE_HUB_NAMESPACE ?= core-hub
+CORE_HUB_CHART     ?= charts/core-hub
+CORE_HUB_VALUES    ?= helm/core-hub-values.yaml
+CORE_HUB_STAGING_NAMESPACE ?= core-hub-staging
+CORE_HUB_STAGING_VALUES ?= helm/core-hub-staging-values.yaml
+CORE_HUB_IMAGE_REPOSITORY ?= forgejo.coulomb.social/coulomb/core-hub
+CORE_HUB_IMAGE_TAG ?=
+CORE_HUB_BASE_URL  ?= https://hub.coulomb.social
+CORE_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG)
+CORE_HUB_IMAGE_REF = $(CORE_HUB_IMAGE_REPOSITORY):$(CORE_HUB_IMAGE_TAG)
+CORE_HUB_IMAGE_SET_ARG = $(if $(strip $(CORE_HUB_IMAGE_TAG)),--set image.tag=$(CORE_HUB_IMAGE_TAG),)
+
 SOPS_SENTINEL ?= $(FORGEJO_SECRETS)
 DRY_RUN_CREATE_NAMESPACES ?= false

@ -96,6 +109,19 @@ check-inter-hub-image-tag: ## Require an explicit inter-hub image tag for produc
 check-inter-hub-image: check-inter-hub-image-tag ## Verify the inter-hub OCI image tag exists before deploy
 	tools/check-oci-image.sh "$(INTER_HUB_IMAGE_REF)"

+check-core-hub-image-tag: ## Require an explicit core-hub image tag for production deploys
+	@test -n "$(CORE_HUB_IMAGE_TAG)" || { \
+		echo "Set CORE_HUB_IMAGE_TAG=<sha> for core-hub production deploys." >&2; \
+		echo "Example: CORE_HUB_IMAGE_TAG=59bf1e4 make core-hub-deploy" >&2; \
+		exit 1; \
+	}
+
+check-core-hub-image: check-core-hub-image-tag ## Verify the core-hub OCI image tag exists before deploy
+	tools/check-oci-image.sh "$(CORE_HUB_IMAGE_REF)"
+
+cnpg-backup-status: check-railiance01-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage
+	KUBECONFIG="$(RAILIANCE01_KUBECONFIG)" tools/cnpg-backup-status.sh
+
 ##@ Vergabe Teilnahme

 vergabe-dry-run: ## helm template render (no apply) for inspection
@ -173,6 +199,77 @@ inter-hub-smoke: ## Verify public inter-hub v2 route and OpenAPI surface after r
 inter-hub-logs: check-railiance01-kubeconfig ## Tail inter-hub app logs from Railiance01
 	KUBECONFIG="$(INTER_HUB_KUBECONFIG)" kubectl logs -n $(INTER_HUB_NAMESPACE) -l app=$(INTER_HUB_RELEASE) -f --tail=50

+##@ Core Hub
+
+core-hub-render-baseline: ## Render checked-in core-hub values for chart validation only
+	helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_NAMESPACE) \
+		-f $(CORE_HUB_VALUES)
+
+core-hub-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_NAMESPACE) \
+		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG)
+
+core-hub-server-dry-run: check-railiance01-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on Railiance01
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_NAMESPACE) --create-namespace \
+		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --dry-run=server --timeout 5m
+
+core-hub-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on Railiance01
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_NAMESPACE) --create-namespace \
+		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m
+
+core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) delete job $(CORE_HUB_RELEASE)-prod-migrate --ignore-not-found
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_NAMESPACE) \
+		-f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) \
+		--set migration.enabled=true | KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl apply -f -
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) wait --for=condition=complete \
+		job/$(CORE_HUB_RELEASE)-prod-migrate --timeout=180s
+
+core-hub-status: check-railiance01-kubeconfig ## Show core-hub pod / svc / ingress / cert state on Railiance01
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(CORE_HUB_NAMESPACE) \
+		-l app.kubernetes.io/name=core-hub --ignore-not-found
+
+core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm history, values, deployment, and pods
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm history $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE)
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm get values $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE)
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl describe deploy/$(CORE_HUB_RELEASE)-api -n $(CORE_HUB_NAMESPACE)
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods -n $(CORE_HUB_NAMESPACE) -o wide
+
+core-hub-smoke: ## Verify public Core Hub health and v2 API surface after rollout
+	CORE_HUB_BASE_URL="$(CORE_HUB_BASE_URL)" tools/core-hub-smoke.sh
+
+core-hub-logs: check-railiance01-kubeconfig ## Tail core-hub app logs from Railiance01
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl logs -n $(CORE_HUB_NAMESPACE) \
+		-l app.kubernetes.io/name=core-hub,app.kubernetes.io/component=api -f --tail=50
+
+core-hub-staging-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_STAGING_NAMESPACE) \
+		-f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG)
+
+core-hub-staging-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_STAGING_NAMESPACE) --create-namespace \
+		-f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m
+
+core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) delete job $(CORE_HUB_RELEASE)-staging-migrate --ignore-not-found
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \
+		--namespace $(CORE_HUB_STAGING_NAMESPACE) \
+		-f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) \
+		--set migration.enabled=true | KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl apply -f -
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) wait --for=condition=complete \
+		job/$(CORE_HUB_RELEASE)-staging-migrate --timeout=180s
+
+core-hub-staging-status: check-railiance01-kubeconfig ## Show core-hub staging pod / svc state
+	KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc -n $(CORE_HUB_STAGING_NAMESPACE) \
+		-l app.kubernetes.io/name=core-hub --ignore-not-found
+
 ##@ Forgejo (forgejo.coulomb.social)

 eso-deploy: check-railiance01-kubeconfig ## Install External Secrets Operator on railiance01
--- a/charts/core-hub/Chart.yaml
+++ b/charts/core-hub/Chart.yaml
@ -0,0 +1,16 @@
+apiVersion: v2
+name: core-hub
+description: |
+  Core Hub API deployment for Railiance01 (hub.coulomb.social).
+type: application
+version: 0.1.0
+appVersion: "0.1.0"
+keywords:
+  - core-hub
+  - hub
+  - railiance
+home: https://forgejo.coulomb.social/coulomb/core-hub
+sources:
+  - https://forgejo.coulomb.social/coulomb/core-hub
+maintainers:
+  - name: railiance-apps
--- a/charts/core-hub/templates/_helpers.tpl
+++ b/charts/core-hub/templates/_helpers.tpl
@ -0,0 +1,33 @@
+{{- define "corehub.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "corehub.labels" -}}
+app.kubernetes.io/name: core-hub
+app.kubernetes.io/component: api
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/part-of: railiance-apps
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
+{{- with .Values.runtime.coreHubEnv }}
+environment: {{ . }}
+{{- end }}
+{{- end -}}
+
+{{- define "corehub.selectorLabels" -}}
+app.kubernetes.io/name: core-hub
+app.kubernetes.io/component: api
+{{- end -}}
+
+{{- define "corehub.image" -}}
+{{- if not .Values.image.tag -}}
+{{- fail "image.tag is required - pin it in helm/core-hub-values.yaml or pass --set image.tag=<sha>" -}}
+{{- end -}}
+{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
+{{- end -}}
+
+{{- define "corehub.runtimeConfigMapName" -}}
+{{- printf "%s-runtime" .Release.Name -}}
+{{- end -}}
--- a/charts/core-hub/templates/configmap.yaml
+++ b/charts/core-hub/templates/configmap.yaml
@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "corehub.runtimeConfigMapName" . }}
+  labels: {{- include "corehub.labels" . | nindent 4 }}
+data:
+  CORE_HUB_ENV: {{ .Values.runtime.coreHubEnv | quote }}
+  CORE_HUB_AUTO_CREATE_TABLES: {{ .Values.runtime.autoCreateTables | quote }}
--- a/charts/core-hub/templates/deployment.yaml
+++ b/charts/core-hub/templates/deployment.yaml
@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "corehub.fullname" . }}
+  labels: {{- include "corehub.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels: {{- include "corehub.selectorLabels" . | nindent 6 }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels: {{- include "corehub.labels" . | nindent 8 }}
+    spec:
+      securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: api
+          image: {{ include "corehub.image" . | quote }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext: {{- toYaml .Values.securityContext | nindent 12 }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.targetPort }}
+              protocol: TCP
+          envFrom:
+            - configMapRef:
+                name: {{ include "corehub.runtimeConfigMapName" . }}
+            - secretRef:
+                name: {{ .Values.envSecretName | quote }}
+          readinessProbe:
+            httpGet:
+              path: {{ .Values.probes.readiness.path }}
+              port: http
+            initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.probes.readiness.periodSeconds }}
+            timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }}
+            failureThreshold: {{ .Values.probes.readiness.failureThreshold }}
+          livenessProbe:
+            httpGet:
+              path: {{ .Values.probes.liveness.path }}
+              port: http
+            initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.probes.liveness.periodSeconds }}
+            timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }}
+            failureThreshold: {{ .Values.probes.liveness.failureThreshold }}
+          resources: {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{- toYaml . | nindent 8 }}
+      {{- end }}
--- a/charts/core-hub/templates/ingress.yaml
+++ b/charts/core-hub/templates/ingress.yaml
@ -0,0 +1,28 @@
+{{- if .Values.ingress.enabled }}
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: core-hub
+  labels: {{- include "corehub.labels" . | nindent 4 }}
+  annotations:
+    {{- toYaml .Values.ingress.annotations | nindent 4 }}
+spec:
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    - hosts:
+        - {{ .Values.ingress.host }}
+      secretName: core-hub-tls
+  {{- end }}
+  rules:
+    - host: {{ .Values.ingress.host }}
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ include "corehub.fullname" . }}
+                port:
+                  number: {{ .Values.service.port }}
+{{- end }}
--- a/charts/core-hub/templates/migrate-job.yaml
+++ b/charts/core-hub/templates/migrate-job.yaml
@ -0,0 +1,35 @@
+{{- if .Values.migration.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Values.migration.jobName }}
+  labels:
+    app.kubernetes.io/name: core-hub
+    app.kubernetes.io/component: migration
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/part-of: railiance-apps
+    {{- with .Values.runtime.coreHubEnv }}
+    environment: {{ . }}
+    {{- end }}
+spec:
+  backoffLimit: {{ .Values.migration.backoffLimit }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: core-hub
+        app.kubernetes.io/component: migration
+        app.kubernetes.io/part-of: railiance-apps
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: migrate
+          image: {{ include "corehub.image" . | quote }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["alembic", "upgrade", "head"]
+          envFrom:
+            - configMapRef:
+                name: {{ include "corehub.runtimeConfigMapName" . }}
+            - secretRef:
+                name: {{ .Values.envSecretName | quote }}
+{{- end }}
--- a/charts/core-hub/templates/service.yaml
+++ b/charts/core-hub/templates/service.yaml
@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "corehub.fullname" . }}
+  labels: {{- include "corehub.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - name: http
+      port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+  selector: {{- include "corehub.selectorLabels" . | nindent 4 }}
--- a/charts/core-hub/values.yaml
+++ b/charts/core-hub/values.yaml
@ -0,0 +1,63 @@
+nameOverride: core-hub-api
+
+image:
+  repository: forgejo.coulomb.social/coulomb/core-hub
+  tag: ""
+  pullPolicy: IfNotPresent
+
+replicaCount: 1
+
+runtime:
+  coreHubEnv: production
+  autoCreateTables: "0"
+
+envSecretName: core-hub-prod-env
+
+service:
+  type: ClusterIP
+  port: 8010
+  targetPort: 8010
+
+resources:
+  requests:
+    cpu: 250m
+    memory: 512Mi
+  limits:
+    cpu: 1000m
+    memory: 1Gi
+
+probes:
+  liveness:
+    path: /healthz
+    initialDelaySeconds: 15
+    periodSeconds: 20
+    timeoutSeconds: 5
+    failureThreshold: 3
+  readiness:
+    path: /readyz
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 3
+
+migration:
+  enabled: false
+  jobName: core-hub-migrate
+  backoffLimit: 1
+
+ingress:
+  enabled: true
+  className: traefik
+  host: hub.coulomb.social
+  tls: true
+  annotations:
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+    traefik.ingress.kubernetes.io/router.tls: "true"
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+
+podSecurityContext: {}
+securityContext: {}
+
+nodeSelector: {}
+tolerations: []
+affinity: {}
--- a/docs/app-data-backup-restore-handoff.md
+++ b/docs/app-data-backup-restore-handoff.md
@ -45,6 +45,32 @@ If this gate is missing, the app can still be used for smoke, development, or
 migration validation, but promotion beyond that should create or link a
 `railiance-platform` workplan.

+## Production cluster inventory (2026-07-10)
+
+Custodian delivery-lane snapshot
+(`the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`)
+confirmed all four production CNPG clusters on railiance01 have
+`spec.backup=null` and no `ScheduledBackup` resources:
+
+| Cluster | Consumer impact |
+| --- | --- |
+| `apps-pg` | `vergabe_db` and future S5 app databases |
+| `forgejo-db` | Forgejo metadata and package registry state |
+| `net-kingdom-pg` | Net Kingdom relational data |
+| `state-hub-db` | State Hub API persistence |
+
+Check current posture:
+
+```bash
+make cnpg-backup-status
+```
+
+Operator-ready barman `ObjectStore` and `ScheduledBackup` templates live in
+`manifests/cnpg-backup-readiness.yaml`. Apply only after platform supplies
+object-store credentials and retention decisions. Phase 1 platform backup lane
+may still use logical `pg_dump` for some clusters; see
+`disaster-control/BackupPickupQueue.md` item 6.
+
 ## `vergabe-teilnahme` Gate

 Current posture:
@ -52,8 +78,9 @@ Current posture:
 - database: `vergabe_db` on `databases/apps-pg`;
 - app role Secret: `vergabe-app-credentials`;
 - env Secret: `vergabe-teilnahme-env`;
- current backup status: platform docs state that `apps-pg` backup coverage is
-  follow-up work;
+- current backup status: no CNPG `ScheduledBackup` on `apps-pg` as of
+  2026-07-10; `make cnpg-backup-status` reports degraded until platform or
+  operator wiring completes;
 - restore status: no app-level restore drill evidence is recorded in this repo.

 Minimum evidence before production-trust use:
--- a/docs/core-hub-on-railiance01.md
+++ b/docs/core-hub-on-railiance01.md
@ -0,0 +1,93 @@
+# Core Hub on railiance01
+
+`hub.coulomb.social` serves Core Hub in production. This repo owns the S5 Helm
+release path; Core Hub owns the application image, migrations, and API behavior.
+
+## Hosts and release surface
+
+| Server | IP | Role |
+|---|---|---|
+| Railiance01 | `92.205.62.239` | Production k3s; deploy S5 apps here |
+
+| Item | Production | Staging |
+|---|---|---|
+| Namespace | `core-hub` | `core-hub-staging` |
+| Helm release | `core-hub` | `core-hub` |
+| Chart | `charts/core-hub` | `charts/core-hub` |
+| Values | `helm/core-hub-values.yaml` | `helm/core-hub-staging-values.yaml` |
+| Image | `forgejo.coulomb.social/coulomb/core-hub:<tag>` | same |
+| Runtime Secret | `core-hub-prod-env` | `core-hub-staging-env` |
+| Public host | `hub.coulomb.social` | cluster-local / port-forward |
+
+## Secret custody
+
+Do not commit decrypted secrets. Runtime secrets are operator-owned and should
+reference OpenBao or approved platform custody paths:
+
+| Secret key | Purpose |
+|---|---|
+| `CORE_HUB_DATABASE_URL` | Async SQLAlchemy URL for the environment database |
+| `CORE_HUB_API_TOKEN` | Operator token for protected API smokes |
+
+`CORE_HUB_AUTO_CREATE_TABLES` must remain `0` in cluster environments. Alembic
+is the only schema path.
+
+## Deploy
+
+Use the Railiance01 kubeconfig. The Makefile defaults to
+`~/.kube/config-hosteurope` and fails fast when it is missing.
+
+```bash
+make core-hub-render-baseline
+CORE_HUB_IMAGE_TAG=<sha> make check-core-hub-image
+CORE_HUB_IMAGE_TAG=<sha> make core-hub-dry-run
+CORE_HUB_IMAGE_TAG=<sha> make core-hub-server-dry-run
+CORE_HUB_IMAGE_TAG=<sha> make core-hub-migrate
+CORE_HUB_IMAGE_TAG=<sha> make core-hub-deploy
+make core-hub-status
+make core-hub-smoke
+```
+
+Production deploys require an explicit image tag. The checked-in values file
+carries a render baseline only.
+
+Staging follows the same pattern with `core-hub-staging-*` targets and
+`helm/core-hub-staging-values.yaml`.
+
+## Ingress cutover
+
+The chart can render the production ingress for `hub.coulomb.social`. During an
+initial cutover from the legacy Inter-Hub ingress, delete the conflicting
+`inter-hub` ingress before applying the Core Hub ingress. See
+`core-hub/k8s/railiance-production/README.md` for the historical flip sequence.
+
+## Migration boundary
+
+`make core-hub-migrate` renders and applies a one-off Job using the same image
+tag as deploy. Wait for Job completion before rolling out a schema-dependent
+image change.
+
+## Smoke gate
+
+`make core-hub-smoke` checks:
+
+- `/healthz` and `/readyz`
+- `/api/v2/openapi.json` resource coverage
+- public `/api/v2/widget-types`
+- protected `/api/v2/hubs`, `/api/v2/widgets`, `/api/v2/hub-registry`
+
+For operator-authenticated smokes and activity-core sink proof, use the Core Hub
+service-repo targets documented in `core-hub/docs/deployment/staging-profile.md`.
+
+## Rollback
+
+- Scale the Core Hub Deployment to zero or roll back to the last known good tag.
+- Re-apply the legacy Inter-Hub ingress if public routing must return to the
+  Haskell service during stabilization.
+- Do not delete legacy Inter-Hub runtime secrets during rollback.
+
+## Related docs
+
+- `the-custodian/docs/core-hub-helixforge-build-alignment.md`
+- `core-hub/docs/deployment/staging-profile.md`
+- `docs/inter-hub-on-railiance01.md` (retired Inter-Hub path)
--- a/helm/core-hub-staging-values.yaml
+++ b/helm/core-hub-staging-values.yaml
@ -0,0 +1,17 @@
+# Staging overrides for the core-hub Helm chart.
+# Non-secret values only; runtime secrets live in core-hub-staging-env.
+
+image:
+  tag: "staging-placeholder"  # Deploy requires CORE_HUB_IMAGE_TAG=<sha>.
+
+runtime:
+  coreHubEnv: staging
+  autoCreateTables: "0"
+
+envSecretName: core-hub-staging-env
+
+migration:
+  jobName: core-hub-staging-migrate
+
+ingress:
+  enabled: false
--- a/helm/core-hub-values.yaml
+++ b/helm/core-hub-values.yaml
@ -0,0 +1,15 @@
+# Production overrides for the core-hub Helm chart.
+# Non-secret values only; CORE_HUB_DATABASE_URL and CORE_HUB_API_TOKEN come from
+# the Secret 'core-hub-prod-env' in the core-hub namespace.
+
+image:
+  tag: "production-placeholder"  # Render baseline only; deploy requires CORE_HUB_IMAGE_TAG=<sha>.
+
+runtime:
+  coreHubEnv: production
+  autoCreateTables: "0"
+
+envSecretName: core-hub-prod-env
+
+migration:
+  jobName: core-hub-prod-migrate
--- a/manifests/cnpg-backup-readiness.yaml
+++ b/manifests/cnpg-backup-readiness.yaml
@ -0,0 +1,52 @@
+# Operator-ready CNPG backup wiring template for railiance01 production DB clusters.
+# Do not apply until platform supplies object-store credentials and destinationPath.
+# Secrets referenced below are placeholders — create them via OpenBao/operator paths.
+#
+# Clusters covered: apps-pg, forgejo-db, net-kingdom-pg, state-hub-db
+# Check posture: make cnpg-backup-status
+#
+# Apply sequence (after credentials exist):
+#   1. Create Secret cnpg-backup-s3 in namespace databases
+#   2. Uncomment and patch ObjectStore + cluster backup patches per cluster
+#   3. Apply ScheduledBackup resources
+#   4. Run restore drill in isolated namespace; record non-secret evidence
+---
+# apiVersion: barmancloud.cnpg.io/v1
+# kind: ObjectStore
+# metadata:
+#   name: railiance-cnpg-backups
+#   namespace: databases
+#   labels:
+#     app.kubernetes.io/part-of: railiance-apps
+# spec:
+#   configuration:
+#     destinationPath: s3://REPLACE_ME/railiance-cnpg/
+#     endpointURL: https://REPLACE_ME
+#     s3Credentials:
+#       accessKeyId:
+#         name: cnpg-backup-s3
+#         key: ACCESS_KEY_ID
+#       secretAccessKey:
+#         name: cnpg-backup-s3
+#         key: SECRET_ACCESS_KEY
+#     wal:
+#       compression: gzip
+#     data:
+#       compression: gzip
+#       immediateCheckpoint: true
+#   retentionPolicy: 14d
+---
+# Example ScheduledBackup for apps-pg (repeat per cluster with spec.cluster.name)
+# apiVersion: postgresql.cnpg.io/v1
+# kind: ScheduledBackup
+# metadata:
+#   name: apps-pg-daily
+#   namespace: databases
+#   labels:
+#     app.kubernetes.io/part-of: railiance-apps
+# spec:
+#   schedule: "0 0 2 * * *"
+#   backupOwnerReference: self
+#   cluster:
+#     name: apps-pg
+#   immediate: true
--- a/tools/cnpg-backup-status.sh
+++ b/tools/cnpg-backup-status.sh
@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace="${CNPG_NAMESPACE:-databases}"
+clusters=(
+  apps-pg
+  forgejo-db
+  net-kingdom-pg
+  state-hub-db
+)
+
+if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
+  echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2
+  exit 1
+fi
+
+echo "CNPG backup posture in namespace ${namespace}"
+echo
+
+missing=0
+
+for cluster in "${clusters[@]}"; do
+  if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
+    echo "${cluster}: MISSING (Cluster CR not found)"
+    missing=$((missing + 1))
+    echo
+    continue
+  fi
+
+  backup_spec="$(kubectl get cluster "$cluster" -n "$namespace" \
+    -o jsonpath='{.spec.backup}' 2>/dev/null || true)"
+  if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then
+    echo "${cluster}: backup spec = null"
+  else
+    echo "${cluster}: backup spec configured"
+  fi
+
+  scheduled="$(kubectl get scheduledbackup -n "$namespace" \
+    -o jsonpath="{range .items[?(@.spec.cluster.name=='${cluster}')]}{.metadata.name}{' '}{end}" 2>/dev/null || true)"
+  scheduled="${scheduled%" "}"
+  if [[ -z "$scheduled" ]]; then
+    echo "${cluster}: ScheduledBackup = none"
+    missing=$((missing + 1))
+  else
+    echo "${cluster}: ScheduledBackup = ${scheduled}"
+  fi
+
+  if command -v kubectl >/dev/null 2>&1 && kubectl cnpg status "$cluster" -n "$namespace" >/dev/null 2>&1; then
+    echo "${cluster}: kubectl cnpg status available"
+    kubectl cnpg status "$cluster" -n "$namespace" | sed -n '1,8p'
+  else
+    phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+    instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)"
+    echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}"
+  fi
+
+  echo
+done
+
+if (( missing > 0 )); then
+  echo "RESULT: degraded — ${missing} cluster(s) lack ScheduledBackup coverage"
+  echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-backup-readiness.yaml"
+  exit 1
+fi
+
+echo "RESULT: ok — all tracked clusters have ScheduledBackup resources"
--- a/tools/core-hub-smoke.sh
+++ b/tools/core-hub-smoke.sh
@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+base_url="${CORE_HUB_BASE_URL:-https://hub.coulomb.social}"
+base_url="${base_url%/}"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+request() {
+  local name="$1"
+  local url="$2"
+  local expected_status="$3"
+  local body="$tmpdir/${name}.json"
+  local status
+
+  status="$(curl -sS -o "$body" -w "%{http_code}" "$url")"
+  if [[ "$status" != "$expected_status" ]]; then
+    echo "ERROR: expected $url to return $expected_status, got $status" >&2
+    cat "$body" >&2
+    echo >&2
+    exit 1
+  fi
+
+  echo "$body"
+}
+
+health_body="$(request healthz "${base_url}/healthz" 200)"
+ready_body="$(request readyz "${base_url}/readyz" 200)"
+
+python3 - "$health_body" "$ready_body" <<'PY'
+import json
+import sys
+
+for path, filename in (("/healthz", sys.argv[1]), ("/readyz", sys.argv[2])):
+    with open(filename, encoding="utf-8") as fh:
+        payload = json.load(fh)
+    if not isinstance(payload, dict) or payload.get("status") not in {"ok", "ready"}:
+        raise SystemExit(f"{path} did not return an ok/ready health payload")
+    print(f"ok: {path} returned healthy JSON")
+PY
+
+openapi_body="$(request openapi "${base_url}/api/v2/openapi.json" 200)"
+python3 - "$openapi_body" <<'PY'
+import json
+import sys
+
+required_paths = {
+    "/hubs",
+    "/hub-capability-manifests",
+    "/api-consumers",
+    "/policy-scopes",
+    "/widgets",
+    "/hub-registry",
+    "/widget-types",
+}
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+paths = payload.get("paths")
+if not isinstance(paths, dict):
+    raise SystemExit("/api/v2/openapi.json did not include an OpenAPI paths object")
+
+missing = sorted(required_paths - set(paths))
+if missing:
+    raise SystemExit("OpenAPI missing paths: " + ", ".join(missing))
+
+print("ok: /api/v2/openapi.json lists expected v2 resources")
+PY
+
+widget_types_body="$(request widget-types "${base_url}/api/v2/widget-types" 200)"
+python3 - "$widget_types_body" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+if not isinstance(payload, list) or not payload:
+    raise SystemExit("/api/v2/widget-types did not return a non-empty JSON list")
+
+print("ok: /api/v2/widget-types returned public discovery JSON")
+PY
+
+for endpoint in hubs hub-registry widgets; do
+  body="$(request "${endpoint}-protected" "${base_url}/api/v2/${endpoint}" 401)"
+  python3 - "$body" "/api/v2/${endpoint}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+path = sys.argv[2]
+if not isinstance(payload, dict):
+    raise SystemExit(f"{path} returned 401 but not JSON")
+
+code = payload.get("code")
+detail = payload.get("detail")
+if code == "invalid_api_key":
+    print(f"ok: {path} requires an API key")
+elif isinstance(detail, dict) and detail.get("code") == "unauthorized":
+    print(f"ok: {path} requires bearer authentication")
+else:
+    raise SystemExit(f"{path} returned 401 but not a recognized auth error payload")
+PY
+done
+
+echo "ok: Core Hub public smoke checks passed for ${base_url}"
--- a/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md
+++ b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md
@ -0,0 +1,103 @@
+---
+id: RAILIANCE-WP-0013
+type: workplan
+title: "CNPG backup wiring and restore drill"
+domain: financials
+repo: railiance-apps
+status: ready
+owner: codex
+topic_slug: railiance
+created: "2026-07-10"
+updated: "2026-07-10"
+state_hub_workstream_id: "764c2447-0713-45d5-a14d-810a13027b8b"
+---
+
+# CNPG backup wiring and restore drill
+
+Follow-up to `RAILIANCE-WP-0012-T04` and custodian inbox `a1957cce`. Observability
+and manifest templates exist; production clusters on railiance01 still report
+degraded backup posture via `make cnpg-backup-status`.
+
+Evidence baseline:
+`the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`
+(`s4_2_backup_restore: FAIL`).
+
+Coordinate with `railiance-platform` for object-store credentials and retention.
+Phase 1 may use logical `pg_dump` for some clusters per
+`disaster-control/BackupPickupQueue.md` item 6; barman `ScheduledBackup` remains
+the longer-term target in `manifests/cnpg-backup-readiness.yaml`.
+
+## Confirm backup lane and credential path
+
+```task
+id: RAILIANCE-WP-0013-T01
+status: todo
+priority: high
+state_hub_task_id: "f2051213-0a9f-418e-82ad-9bd7c79fd0dc"
+```
+
+Record the approved backup mechanism per cluster: Phase 1 logical dump vs barman
+`ObjectStore` + `ScheduledBackup`. Identify OpenBao/operator paths for
+`cnpg-backup-s3` or equivalent without copying secret values into Git,
+workplans, or State Hub notes.
+
+## Wire apps-pg backup coverage
+
+```task
+id: RAILIANCE-WP-0013-T02
+status: wait
+priority: high
+state_hub_task_id: "2d78019e-f040-422d-89d0-0d0e8c6af718"
+```
+
+Enable backup coverage for `apps-pg` first because `vergabe_db` is
+production-facing. Apply platform-approved manifests or Makefile-driven operator
+steps. `make cnpg-backup-status` must show a non-null backup spec or active
+`ScheduledBackup` for `apps-pg`.
+
+Blocked on: `RAILIANCE-WP-0013-T01`.
+
+## Wire remaining production DB clusters
+
+```task
+id: RAILIANCE-WP-0013-T03
+status: wait
+priority: medium
+state_hub_task_id: "d3e41204-98a5-4ccb-ae91-526e7a606baa"
+```
+
+Extend the same backup lane to `forgejo-db`, `net-kingdom-pg`, and
+`state-hub-db`. Re-run `make cnpg-backup-status` until all four clusters pass.
+
+Blocked on: `RAILIANCE-WP-0013-T02`.
+
+## Run isolated restore drill with evidence
+
+```task
+id: RAILIANCE-WP-0013-T04
+status: wait
+priority: high
+state_hub_task_id: "15602bb9-d1f6-456e-9af3-bc7c27ebb36f"
+```
+
+Restore the latest `apps-pg` backup into an isolated namespace or test database.
+Verify row counts, run `vergabe-teilnahme` migrations, and capture non-secret
+evidence (timestamp, cluster, backup id, restore namespace, post-restore smoke
+result). Do not store decrypted dumps or credentials in Git.
+
+Blocked on: `RAILIANCE-WP-0013-T02`.
+
+## Close vergabe production-trust backup gate
+
+```task
+id: RAILIANCE-WP-0013-T05
+status: wait
+priority: medium
+state_hub_task_id: "666c0a9b-4f9b-4ea3-bf0b-14aa76b8ea93"
+```
+
+Update `docs/app-data-backup-restore-handoff.md` and `docs/vergabe-teilnahme.md`
+with backup/restore evidence references. Mark the S5 gate satisfied only after
+T02 and T04 evidence exists.
+
+Blocked on: `RAILIANCE-WP-0013-T04`.
--- a/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md
+++ b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md
@ -0,0 +1,98 @@
+---
+id: RAILIANCE-WP-0014
+type: workplan
+title: "Core Hub Helm cutover and vergabe-teilnahme image refresh"
+domain: financials
+repo: railiance-apps
+status: ready
+owner: codex
+topic_slug: railiance
+created: "2026-07-10"
+updated: "2026-07-10"
+state_hub_workstream_id: "780c8d1d-ea75-479e-9733-10b518dde242"
+---
+
+# Core Hub Helm cutover and vergabe-teilnahme image refresh
+
+Follow-up to `RAILIANCE-WP-0012` and custodian inbox `3ced85c7`. The S5 Helm
+chart, values, Makefile targets, and runbook exist; the live cluster may still
+run Core Hub from `core-hub/k8s/railiance-production/` manifests.
+
+Separately, the 2026-07-10 delivery-lane snapshot shows `vergabe-teilnahme`
+deployed at `f95de14` while the source repo head is `1061f3e` (14-commit drift).
+
+Production-touching deploy steps require explicit image tags and operator
+approval. Do not store secrets in workplans.
+
+## Core Hub server dry-run on Railiance01
+
+```task
+id: RAILIANCE-WP-0014-T01
+status: todo
+priority: high
+state_hub_task_id: "b9dea9f3-f0c0-41c1-b9c4-ec4c44d43d55"
+```
+
+Run `CORE_HUB_IMAGE_TAG=<sha> make check-core-hub-image`,
+`core-hub-server-dry-run`, and `core-hub-dry-run` against Railiance01. Record the
+chosen image tag and any schema or secret prerequisites before apply.
+
+## Adopt Helm-managed Core Hub release
+
+```task
+id: RAILIANCE-WP-0014-T02
+status: wait
+priority: high
+state_hub_task_id: "0e591104-0123-4e4a-84ff-5deb435e8441"
+```
+
+Run `core-hub-migrate` then `core-hub-deploy` with an explicit
+`CORE_HUB_IMAGE_TAG`. Confirm the Helm release owns `core-hub-api`, runtime
+ConfigMap, and Service in namespace `core-hub`. Retire duplicate resources from
+the service-repo `k8s/railiance-production/` apply path once stable.
+
+Blocked on: `RAILIANCE-WP-0014-T01`.
+
+## Verify Core Hub post-cutover smoke
+
+```task
+id: RAILIANCE-WP-0014-T03
+status: wait
+priority: high
+state_hub_task_id: "eca3ce8b-68a8-44d0-90c7-2cdefa482200"
+```
+
+Run `make core-hub-status`, `make core-hub-smoke`, and `make core-hub-release-info`.
+Ingress for `hub.coulomb.social` must remain healthy. Document rollback steps in
+`docs/core-hub-on-railiance01.md` if Helm values drift from prior manifests.
+
+Blocked on: `RAILIANCE-WP-0014-T02`.
+
+## Refresh vergabe-teilnahme production image
+
+```task
+id: RAILIANCE-WP-0014-T04
+status: todo
+priority: medium
+state_hub_task_id: "d547f68d-52ec-4ff1-952b-ec70c834e0d3"
+```
+
+Identify the current published image tag for `vergabe-teilnahme` in the Forgejo
+registry, verify the manifest with `tools/check-oci-image.sh`, run
+`vergabe-dry-run` and `vergabe-server-dry-run` if available, then deploy with an
+explicit tag. Close the 14-commit drift noted in the delivery-lane evidence.
+
+## Post-deploy vergabe smoke and runbook evidence
+
+```task
+id: RAILIANCE-WP-0014-T05
+status: wait
+priority: medium
+state_hub_task_id: "4eaaa345-4252-46c1-bd8a-49d654ec48f4"
+```
+
+After deploy, run `make vergabe-status`, migrations if required, and HTTPS smoke
+against `vergabe-teilnahme.whywhynot.de`. Update `docs/vergabe-teilnahme.md`
+with the deployed tag, date, and smoke result (non-secret).
+
+Blocked on: `RAILIANCE-WP-0014-T04`.
--- a/workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md
+++ b/workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md
@ -0,0 +1,90 @@
+---
+id: RAILIANCE-WP-0012
+type: workplan
+title: "Inbox suggestions — Core Hub S5 release and CNPG backup observability"
+domain: financials
+repo: railiance-apps
+status: finished
+owner: codex
+topic_slug: railiance
+created: "2026-07-10"
+updated: "2026-07-10"
+state_hub_workstream_id: "25cee4ac-d1e4-441a-bc33-cf8c90ee67a6"
+---
+
+# Inbox suggestions — Core Hub S5 release and CNPG backup observability
+
+Sources:
+
+- State Hub inbox `a1957cce` (HIGH): no CNPG backups on railiance01 production DB
+  clusters; evidence `the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`.
+- State Hub inbox `3ced85c7`: Core Hub release ownership follow-up from
+  `CUST-WP-0052`; cite `the-custodian/docs/core-hub-helixforge-build-alignment.md`
+  and `core-hub/docs/deployment/staging-profile.md`.
+
+## Core Hub Helm chart and production values
+
+```task
+id: RAILIANCE-WP-0012-T01
+status: done
+priority: high
+state_hub_task_id: "0795f034-eacc-41d5-af8f-e5909f72f79f"
+```
+
+Promote the temporary `core-hub/k8s/railiance-production/` profile into
+`charts/core-hub/` with non-secret production and staging values under `helm/`.
+Secrets remain operator-owned (`core-hub-prod-env`, `core-hub-staging-env`).
+
+## Makefile release targets and smoke gate
+
+```task
+id: RAILIANCE-WP-0012-T02
+status: done
+priority: high
+state_hub_task_id: "f975d9d3-e9b4-45ff-8088-03b2a5a855c6"
+```
+
+Add `core-hub-dry-run`, `core-hub-server-dry-run`, `core-hub-deploy`,
+`core-hub-status`, `core-hub-migrate`, `core-hub-smoke`, and staging
+counterparts. Require explicit `CORE_HUB_IMAGE_TAG=<sha>` for production deploys.
+
+## Operator runbook for Core Hub on railiance01
+
+```task
+id: RAILIANCE-WP-0012-T03
+status: done
+priority: medium
+state_hub_task_id: "41cba5e9-2c19-4b90-9dbe-517a3413eac7"
+```
+
+Document the S5 release path in `docs/core-hub-on-railiance01.md`, including
+migration boundary, ingress cutover notes, rollback, and links to Core Hub
+service-repo smokes.
+
+## CNPG backup posture observability
+
+```task
+id: RAILIANCE-WP-0012-T04
+status: done
+priority: high
+state_hub_task_id: "7c9794ca-478c-4da9-ae77-b3aa62924e98"
+```
+
+Add `make cnpg-backup-status` and `tools/cnpg-backup-status.sh` to report backup
+spec and `ScheduledBackup` coverage for `apps-pg`, `forgejo-db`,
+`net-kingdom-pg`, and `state-hub-db`. Include operator-ready manifest templates
+for barman `ObjectStore` and `ScheduledBackup` wiring once platform supplies
+object-store credentials.
+
+## Update app data backup handoff with delivery-lane evidence
+
+```task
+id: RAILIANCE-WP-0012-T05
+status: done
+priority: medium
+state_hub_task_id: "126f8f26-2536-40d6-8166-060f93ada85b"
+```
+
+Refresh `docs/app-data-backup-restore-handoff.md` with the 2026-07-10 custodian
+conformance snapshot, the cluster inventory, and the platform coordination path
+for backup implementation.