From df7225dd3e1aadbbc50a0a793082eb6066d49a07 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 10 Jul 2026 15:14:20 +0200 Subject: [PATCH] RAILIANCE-WP-0012/0013/0014: Core Hub S5 release, CNPG observability, and follow-up workplans Deliver the inbox-suggestion implementation from WP-0012: Core Hub Helm chart and Makefile targets, CNPG backup status tooling, and updated backup handoff docs. Archive the finished WP-0012 workplan and register ready follow-ups WP-0013 (CNPG backup wiring + restore drill) and WP-0014 (Core Hub Helm cutover and vergabe-teilnahme image refresh). --- Makefile | 97 +++++++++++++++ charts/core-hub/Chart.yaml | 16 +++ charts/core-hub/templates/_helpers.tpl | 33 ++++++ charts/core-hub/templates/configmap.yaml | 8 ++ charts/core-hub/templates/deployment.yaml | 59 ++++++++++ charts/core-hub/templates/ingress.yaml | 28 +++++ charts/core-hub/templates/migrate-job.yaml | 35 ++++++ charts/core-hub/templates/service.yaml | 13 +++ charts/core-hub/values.yaml | 63 ++++++++++ docs/app-data-backup-restore-handoff.md | 31 ++++- docs/core-hub-on-railiance01.md | 93 +++++++++++++++ helm/core-hub-staging-values.yaml | 17 +++ helm/core-hub-values.yaml | 15 +++ manifests/cnpg-backup-readiness.yaml | 52 +++++++++ tools/cnpg-backup-status.sh | 66 +++++++++++ tools/core-hub-smoke.sh | 110 ++++++++++++++++++ ...P-0013-cnpg-backup-wiring-restore-drill.md | 103 ++++++++++++++++ ...4-core-hub-helm-cutover-vergabe-refresh.md | 98 ++++++++++++++++ ...-inbox-suggestions-core-hub-cnpg-backup.md | 90 ++++++++++++++ 19 files changed, 1025 insertions(+), 2 deletions(-) create mode 100644 charts/core-hub/Chart.yaml create mode 100644 charts/core-hub/templates/_helpers.tpl create mode 100644 charts/core-hub/templates/configmap.yaml create mode 100644 charts/core-hub/templates/deployment.yaml create mode 100644 charts/core-hub/templates/ingress.yaml create mode 100644 charts/core-hub/templates/migrate-job.yaml create mode 100644 charts/core-hub/templates/service.yaml create mode 100644 charts/core-hub/values.yaml create mode 100644 docs/core-hub-on-railiance01.md create mode 100644 helm/core-hub-staging-values.yaml create mode 100644 helm/core-hub-values.yaml create mode 100644 manifests/cnpg-backup-readiness.yaml create mode 100755 tools/cnpg-backup-status.sh create mode 100755 tools/core-hub-smoke.sh create mode 100644 workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md create mode 100644 workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md create mode 100644 workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md diff --git a/Makefile b/Makefile index 4396ab4..797e922 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,19 @@ ESO_CHART_VERSION ?= 0.16.1 INTER_HUB_IMAGE_REF = $(INTER_HUB_IMAGE_REPOSITORY):$(INTER_HUB_IMAGE_TAG) INTER_HUB_IMAGE_SET_ARG = $(if $(strip $(INTER_HUB_IMAGE_TAG)),--set image.tag=$(INTER_HUB_IMAGE_TAG),) +CORE_HUB_RELEASE ?= core-hub +CORE_HUB_NAMESPACE ?= core-hub +CORE_HUB_CHART ?= charts/core-hub +CORE_HUB_VALUES ?= helm/core-hub-values.yaml +CORE_HUB_STAGING_NAMESPACE ?= core-hub-staging +CORE_HUB_STAGING_VALUES ?= helm/core-hub-staging-values.yaml +CORE_HUB_IMAGE_REPOSITORY ?= forgejo.coulomb.social/coulomb/core-hub +CORE_HUB_IMAGE_TAG ?= +CORE_HUB_BASE_URL ?= https://hub.coulomb.social +CORE_HUB_KUBECONFIG ?= $(RAILIANCE01_KUBECONFIG) +CORE_HUB_IMAGE_REF = $(CORE_HUB_IMAGE_REPOSITORY):$(CORE_HUB_IMAGE_TAG) +CORE_HUB_IMAGE_SET_ARG = $(if $(strip $(CORE_HUB_IMAGE_TAG)),--set image.tag=$(CORE_HUB_IMAGE_TAG),) + SOPS_SENTINEL ?= $(FORGEJO_SECRETS) DRY_RUN_CREATE_NAMESPACES ?= false @@ -96,6 +109,19 @@ check-inter-hub-image-tag: ## Require an explicit inter-hub image tag for produc check-inter-hub-image: check-inter-hub-image-tag ## Verify the inter-hub OCI image tag exists before deploy tools/check-oci-image.sh "$(INTER_HUB_IMAGE_REF)" +check-core-hub-image-tag: ## Require an explicit core-hub image tag for production deploys + @test -n "$(CORE_HUB_IMAGE_TAG)" || { \ + echo "Set CORE_HUB_IMAGE_TAG= for core-hub production deploys." >&2; \ + echo "Example: CORE_HUB_IMAGE_TAG=59bf1e4 make core-hub-deploy" >&2; \ + exit 1; \ + } + +check-core-hub-image: check-core-hub-image-tag ## Verify the core-hub OCI image tag exists before deploy + tools/check-oci-image.sh "$(CORE_HUB_IMAGE_REF)" + +cnpg-backup-status: check-railiance01-kubeconfig ## Report CNPG backup spec and ScheduledBackup coverage + KUBECONFIG="$(RAILIANCE01_KUBECONFIG)" tools/cnpg-backup-status.sh + ##@ Vergabe Teilnahme vergabe-dry-run: ## helm template render (no apply) for inspection @@ -173,6 +199,77 @@ inter-hub-smoke: ## Verify public inter-hub v2 route and OpenAPI surface after r inter-hub-logs: check-railiance01-kubeconfig ## Tail inter-hub app logs from Railiance01 KUBECONFIG="$(INTER_HUB_KUBECONFIG)" kubectl logs -n $(INTER_HUB_NAMESPACE) -l app=$(INTER_HUB_RELEASE) -f --tail=50 +##@ Core Hub + +core-hub-render-baseline: ## Render checked-in core-hub values for chart validation only + helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_NAMESPACE) \ + -f $(CORE_HUB_VALUES) + +core-hub-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render with an explicit production image tag + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_NAMESPACE) \ + -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) + +core-hub-server-dry-run: check-railiance01-kubeconfig check-core-hub-image ## Helm server dry-run core-hub upgrade on Railiance01 + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_NAMESPACE) --create-namespace \ + -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --dry-run=server --timeout 5m + +core-hub-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub Helm release on Railiance01 + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_NAMESPACE) --create-namespace \ + -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m + +core-hub-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations via a one-off Job + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) delete job $(CORE_HUB_RELEASE)-prod-migrate --ignore-not-found + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_NAMESPACE) \ + -f $(CORE_HUB_VALUES) $(CORE_HUB_IMAGE_SET_ARG) \ + --set migration.enabled=true | KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl apply -f - + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_NAMESPACE) wait --for=condition=complete \ + job/$(CORE_HUB_RELEASE)-prod-migrate --timeout=180s + +core-hub-status: check-railiance01-kubeconfig ## Show core-hub pod / svc / ingress / cert state on Railiance01 + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc,ingress,certificate -n $(CORE_HUB_NAMESPACE) \ + -l app.kubernetes.io/name=core-hub --ignore-not-found + +core-hub-release-info: check-railiance01-kubeconfig ## Show core-hub Helm history, values, deployment, and pods + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm history $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE) + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm get values $(CORE_HUB_RELEASE) -n $(CORE_HUB_NAMESPACE) + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl describe deploy/$(CORE_HUB_RELEASE)-api -n $(CORE_HUB_NAMESPACE) + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods -n $(CORE_HUB_NAMESPACE) -o wide + +core-hub-smoke: ## Verify public Core Hub health and v2 API surface after rollout + CORE_HUB_BASE_URL="$(CORE_HUB_BASE_URL)" tools/core-hub-smoke.sh + +core-hub-logs: check-railiance01-kubeconfig ## Tail core-hub app logs from Railiance01 + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl logs -n $(CORE_HUB_NAMESPACE) \ + -l app.kubernetes.io/name=core-hub,app.kubernetes.io/component=api -f --tail=50 + +core-hub-staging-dry-run: check-railiance01-kubeconfig check-core-hub-image-tag ## helm template render for core-hub staging + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_STAGING_NAMESPACE) \ + -f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) + +core-hub-staging-deploy: check-railiance01-kubeconfig check-core-hub-image ## Deploy / upgrade core-hub staging release + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm upgrade --install $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_STAGING_NAMESPACE) --create-namespace \ + -f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) --wait --timeout 5m + +core-hub-staging-migrate: check-railiance01-kubeconfig check-core-hub-image ## Run Alembic migrations for core-hub staging + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) delete job $(CORE_HUB_RELEASE)-staging-migrate --ignore-not-found + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" helm template $(CORE_HUB_RELEASE) $(CORE_HUB_CHART) \ + --namespace $(CORE_HUB_STAGING_NAMESPACE) \ + -f $(CORE_HUB_STAGING_VALUES) $(CORE_HUB_IMAGE_SET_ARG) \ + --set migration.enabled=true | KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl apply -f - + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl -n $(CORE_HUB_STAGING_NAMESPACE) wait --for=condition=complete \ + job/$(CORE_HUB_RELEASE)-staging-migrate --timeout=180s + +core-hub-staging-status: check-railiance01-kubeconfig ## Show core-hub staging pod / svc state + KUBECONFIG="$(CORE_HUB_KUBECONFIG)" kubectl get pods,svc -n $(CORE_HUB_STAGING_NAMESPACE) \ + -l app.kubernetes.io/name=core-hub --ignore-not-found + ##@ Forgejo (forgejo.coulomb.social) eso-deploy: check-railiance01-kubeconfig ## Install External Secrets Operator on railiance01 diff --git a/charts/core-hub/Chart.yaml b/charts/core-hub/Chart.yaml new file mode 100644 index 0000000..c6414bb --- /dev/null +++ b/charts/core-hub/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +name: core-hub +description: | + Core Hub API deployment for Railiance01 (hub.coulomb.social). +type: application +version: 0.1.0 +appVersion: "0.1.0" +keywords: + - core-hub + - hub + - railiance +home: https://forgejo.coulomb.social/coulomb/core-hub +sources: + - https://forgejo.coulomb.social/coulomb/core-hub +maintainers: + - name: railiance-apps \ No newline at end of file diff --git a/charts/core-hub/templates/_helpers.tpl b/charts/core-hub/templates/_helpers.tpl new file mode 100644 index 0000000..4b6958d --- /dev/null +++ b/charts/core-hub/templates/_helpers.tpl @@ -0,0 +1,33 @@ +{{- define "corehub.fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "corehub.labels" -}} +app.kubernetes.io/name: core-hub +app.kubernetes.io/component: api +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/part-of: railiance-apps +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }} +{{- with .Values.runtime.coreHubEnv }} +environment: {{ . }} +{{- end }} +{{- end -}} + +{{- define "corehub.selectorLabels" -}} +app.kubernetes.io/name: core-hub +app.kubernetes.io/component: api +{{- end -}} + +{{- define "corehub.image" -}} +{{- if not .Values.image.tag -}} +{{- fail "image.tag is required - pin it in helm/core-hub-values.yaml or pass --set image.tag=" -}} +{{- end -}} +{{- printf "%s:%s" .Values.image.repository .Values.image.tag -}} +{{- end -}} + +{{- define "corehub.runtimeConfigMapName" -}} +{{- printf "%s-runtime" .Release.Name -}} +{{- end -}} \ No newline at end of file diff --git a/charts/core-hub/templates/configmap.yaml b/charts/core-hub/templates/configmap.yaml new file mode 100644 index 0000000..af31cb3 --- /dev/null +++ b/charts/core-hub/templates/configmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "corehub.runtimeConfigMapName" . }} + labels: {{- include "corehub.labels" . | nindent 4 }} +data: + CORE_HUB_ENV: {{ .Values.runtime.coreHubEnv | quote }} + CORE_HUB_AUTO_CREATE_TABLES: {{ .Values.runtime.autoCreateTables | quote }} \ No newline at end of file diff --git a/charts/core-hub/templates/deployment.yaml b/charts/core-hub/templates/deployment.yaml new file mode 100644 index 0000000..afcad8b --- /dev/null +++ b/charts/core-hub/templates/deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "corehub.fullname" . }} + labels: {{- include "corehub.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: {{- include "corehub.selectorLabels" . | nindent 6 }} + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + template: + metadata: + labels: {{- include "corehub.labels" . | nindent 8 }} + spec: + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: api + image: {{ include "corehub.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: http + containerPort: {{ .Values.service.targetPort }} + protocol: TCP + envFrom: + - configMapRef: + name: {{ include "corehub.runtimeConfigMapName" . }} + - secretRef: + name: {{ .Values.envSecretName | quote }} + readinessProbe: + httpGet: + path: {{ .Values.probes.readiness.path }} + port: http + initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.readiness.failureThreshold }} + livenessProbe: + httpGet: + path: {{ .Values.probes.liveness.path }} + port: http + initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.probes.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} + failureThreshold: {{ .Values.probes.liveness.failureThreshold }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} \ No newline at end of file diff --git a/charts/core-hub/templates/ingress.yaml b/charts/core-hub/templates/ingress.yaml new file mode 100644 index 0000000..9cc86ca --- /dev/null +++ b/charts/core-hub/templates/ingress.yaml @@ -0,0 +1,28 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: core-hub + labels: {{- include "corehub.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.ingress.annotations | nindent 4 }} +spec: + ingressClassName: {{ .Values.ingress.className }} + {{- if .Values.ingress.tls }} + tls: + - hosts: + - {{ .Values.ingress.host }} + secretName: core-hub-tls + {{- end }} + rules: + - host: {{ .Values.ingress.host }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "corehub.fullname" . }} + port: + number: {{ .Values.service.port }} +{{- end }} \ No newline at end of file diff --git a/charts/core-hub/templates/migrate-job.yaml b/charts/core-hub/templates/migrate-job.yaml new file mode 100644 index 0000000..77ccd41 --- /dev/null +++ b/charts/core-hub/templates/migrate-job.yaml @@ -0,0 +1,35 @@ +{{- if .Values.migration.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Values.migration.jobName }} + labels: + app.kubernetes.io/name: core-hub + app.kubernetes.io/component: migration + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/part-of: railiance-apps + {{- with .Values.runtime.coreHubEnv }} + environment: {{ . }} + {{- end }} +spec: + backoffLimit: {{ .Values.migration.backoffLimit }} + template: + metadata: + labels: + app.kubernetes.io/name: core-hub + app.kubernetes.io/component: migration + app.kubernetes.io/part-of: railiance-apps + spec: + restartPolicy: Never + containers: + - name: migrate + image: {{ include "corehub.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["alembic", "upgrade", "head"] + envFrom: + - configMapRef: + name: {{ include "corehub.runtimeConfigMapName" . }} + - secretRef: + name: {{ .Values.envSecretName | quote }} +{{- end }} \ No newline at end of file diff --git a/charts/core-hub/templates/service.yaml b/charts/core-hub/templates/service.yaml new file mode 100644 index 0000000..37bc8ed --- /dev/null +++ b/charts/core-hub/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "corehub.fullname" . }} + labels: {{- include "corehub.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + selector: {{- include "corehub.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/charts/core-hub/values.yaml b/charts/core-hub/values.yaml new file mode 100644 index 0000000..2384ecf --- /dev/null +++ b/charts/core-hub/values.yaml @@ -0,0 +1,63 @@ +nameOverride: core-hub-api + +image: + repository: forgejo.coulomb.social/coulomb/core-hub + tag: "" + pullPolicy: IfNotPresent + +replicaCount: 1 + +runtime: + coreHubEnv: production + autoCreateTables: "0" + +envSecretName: core-hub-prod-env + +service: + type: ClusterIP + port: 8010 + targetPort: 8010 + +resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 1Gi + +probes: + liveness: + path: /healthz + initialDelaySeconds: 15 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 3 + readiness: + path: /readyz + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + +migration: + enabled: false + jobName: core-hub-migrate + backoffLimit: 1 + +ingress: + enabled: true + className: traefik + host: hub.coulomb.social + tls: true + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + cert-manager.io/cluster-issuer: letsencrypt-prod + +podSecurityContext: {} +securityContext: {} + +nodeSelector: {} +tolerations: [] +affinity: {} \ No newline at end of file diff --git a/docs/app-data-backup-restore-handoff.md b/docs/app-data-backup-restore-handoff.md index a152808..9f6f55f 100644 --- a/docs/app-data-backup-restore-handoff.md +++ b/docs/app-data-backup-restore-handoff.md @@ -45,6 +45,32 @@ If this gate is missing, the app can still be used for smoke, development, or migration validation, but promotion beyond that should create or link a `railiance-platform` workplan. +## Production cluster inventory (2026-07-10) + +Custodian delivery-lane snapshot +(`the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`) +confirmed all four production CNPG clusters on railiance01 have +`spec.backup=null` and no `ScheduledBackup` resources: + +| Cluster | Consumer impact | +| --- | --- | +| `apps-pg` | `vergabe_db` and future S5 app databases | +| `forgejo-db` | Forgejo metadata and package registry state | +| `net-kingdom-pg` | Net Kingdom relational data | +| `state-hub-db` | State Hub API persistence | + +Check current posture: + +```bash +make cnpg-backup-status +``` + +Operator-ready barman `ObjectStore` and `ScheduledBackup` templates live in +`manifests/cnpg-backup-readiness.yaml`. Apply only after platform supplies +object-store credentials and retention decisions. Phase 1 platform backup lane +may still use logical `pg_dump` for some clusters; see +`disaster-control/BackupPickupQueue.md` item 6. + ## `vergabe-teilnahme` Gate Current posture: @@ -52,8 +78,9 @@ Current posture: - database: `vergabe_db` on `databases/apps-pg`; - app role Secret: `vergabe-app-credentials`; - env Secret: `vergabe-teilnahme-env`; -- current backup status: platform docs state that `apps-pg` backup coverage is - follow-up work; +- current backup status: no CNPG `ScheduledBackup` on `apps-pg` as of + 2026-07-10; `make cnpg-backup-status` reports degraded until platform or + operator wiring completes; - restore status: no app-level restore drill evidence is recorded in this repo. Minimum evidence before production-trust use: diff --git a/docs/core-hub-on-railiance01.md b/docs/core-hub-on-railiance01.md new file mode 100644 index 0000000..c91e6bc --- /dev/null +++ b/docs/core-hub-on-railiance01.md @@ -0,0 +1,93 @@ +# Core Hub on railiance01 + +`hub.coulomb.social` serves Core Hub in production. This repo owns the S5 Helm +release path; Core Hub owns the application image, migrations, and API behavior. + +## Hosts and release surface + +| Server | IP | Role | +|---|---|---| +| Railiance01 | `92.205.62.239` | Production k3s; deploy S5 apps here | + +| Item | Production | Staging | +|---|---|---| +| Namespace | `core-hub` | `core-hub-staging` | +| Helm release | `core-hub` | `core-hub` | +| Chart | `charts/core-hub` | `charts/core-hub` | +| Values | `helm/core-hub-values.yaml` | `helm/core-hub-staging-values.yaml` | +| Image | `forgejo.coulomb.social/coulomb/core-hub:` | same | +| Runtime Secret | `core-hub-prod-env` | `core-hub-staging-env` | +| Public host | `hub.coulomb.social` | cluster-local / port-forward | + +## Secret custody + +Do not commit decrypted secrets. Runtime secrets are operator-owned and should +reference OpenBao or approved platform custody paths: + +| Secret key | Purpose | +|---|---| +| `CORE_HUB_DATABASE_URL` | Async SQLAlchemy URL for the environment database | +| `CORE_HUB_API_TOKEN` | Operator token for protected API smokes | + +`CORE_HUB_AUTO_CREATE_TABLES` must remain `0` in cluster environments. Alembic +is the only schema path. + +## Deploy + +Use the Railiance01 kubeconfig. The Makefile defaults to +`~/.kube/config-hosteurope` and fails fast when it is missing. + +```bash +make core-hub-render-baseline +CORE_HUB_IMAGE_TAG= make check-core-hub-image +CORE_HUB_IMAGE_TAG= make core-hub-dry-run +CORE_HUB_IMAGE_TAG= make core-hub-server-dry-run +CORE_HUB_IMAGE_TAG= make core-hub-migrate +CORE_HUB_IMAGE_TAG= make core-hub-deploy +make core-hub-status +make core-hub-smoke +``` + +Production deploys require an explicit image tag. The checked-in values file +carries a render baseline only. + +Staging follows the same pattern with `core-hub-staging-*` targets and +`helm/core-hub-staging-values.yaml`. + +## Ingress cutover + +The chart can render the production ingress for `hub.coulomb.social`. During an +initial cutover from the legacy Inter-Hub ingress, delete the conflicting +`inter-hub` ingress before applying the Core Hub ingress. See +`core-hub/k8s/railiance-production/README.md` for the historical flip sequence. + +## Migration boundary + +`make core-hub-migrate` renders and applies a one-off Job using the same image +tag as deploy. Wait for Job completion before rolling out a schema-dependent +image change. + +## Smoke gate + +`make core-hub-smoke` checks: + +- `/healthz` and `/readyz` +- `/api/v2/openapi.json` resource coverage +- public `/api/v2/widget-types` +- protected `/api/v2/hubs`, `/api/v2/widgets`, `/api/v2/hub-registry` + +For operator-authenticated smokes and activity-core sink proof, use the Core Hub +service-repo targets documented in `core-hub/docs/deployment/staging-profile.md`. + +## Rollback + +- Scale the Core Hub Deployment to zero or roll back to the last known good tag. +- Re-apply the legacy Inter-Hub ingress if public routing must return to the + Haskell service during stabilization. +- Do not delete legacy Inter-Hub runtime secrets during rollback. + +## Related docs + +- `the-custodian/docs/core-hub-helixforge-build-alignment.md` +- `core-hub/docs/deployment/staging-profile.md` +- `docs/inter-hub-on-railiance01.md` (retired Inter-Hub path) \ No newline at end of file diff --git a/helm/core-hub-staging-values.yaml b/helm/core-hub-staging-values.yaml new file mode 100644 index 0000000..12c6553 --- /dev/null +++ b/helm/core-hub-staging-values.yaml @@ -0,0 +1,17 @@ +# Staging overrides for the core-hub Helm chart. +# Non-secret values only; runtime secrets live in core-hub-staging-env. + +image: + tag: "staging-placeholder" # Deploy requires CORE_HUB_IMAGE_TAG=. + +runtime: + coreHubEnv: staging + autoCreateTables: "0" + +envSecretName: core-hub-staging-env + +migration: + jobName: core-hub-staging-migrate + +ingress: + enabled: false \ No newline at end of file diff --git a/helm/core-hub-values.yaml b/helm/core-hub-values.yaml new file mode 100644 index 0000000..550a3e7 --- /dev/null +++ b/helm/core-hub-values.yaml @@ -0,0 +1,15 @@ +# Production overrides for the core-hub Helm chart. +# Non-secret values only; CORE_HUB_DATABASE_URL and CORE_HUB_API_TOKEN come from +# the Secret 'core-hub-prod-env' in the core-hub namespace. + +image: + tag: "production-placeholder" # Render baseline only; deploy requires CORE_HUB_IMAGE_TAG=. + +runtime: + coreHubEnv: production + autoCreateTables: "0" + +envSecretName: core-hub-prod-env + +migration: + jobName: core-hub-prod-migrate \ No newline at end of file diff --git a/manifests/cnpg-backup-readiness.yaml b/manifests/cnpg-backup-readiness.yaml new file mode 100644 index 0000000..c27fd0e --- /dev/null +++ b/manifests/cnpg-backup-readiness.yaml @@ -0,0 +1,52 @@ +# Operator-ready CNPG backup wiring template for railiance01 production DB clusters. +# Do not apply until platform supplies object-store credentials and destinationPath. +# Secrets referenced below are placeholders — create them via OpenBao/operator paths. +# +# Clusters covered: apps-pg, forgejo-db, net-kingdom-pg, state-hub-db +# Check posture: make cnpg-backup-status +# +# Apply sequence (after credentials exist): +# 1. Create Secret cnpg-backup-s3 in namespace databases +# 2. Uncomment and patch ObjectStore + cluster backup patches per cluster +# 3. Apply ScheduledBackup resources +# 4. Run restore drill in isolated namespace; record non-secret evidence +--- +# apiVersion: barmancloud.cnpg.io/v1 +# kind: ObjectStore +# metadata: +# name: railiance-cnpg-backups +# namespace: databases +# labels: +# app.kubernetes.io/part-of: railiance-apps +# spec: +# configuration: +# destinationPath: s3://REPLACE_ME/railiance-cnpg/ +# endpointURL: https://REPLACE_ME +# s3Credentials: +# accessKeyId: +# name: cnpg-backup-s3 +# key: ACCESS_KEY_ID +# secretAccessKey: +# name: cnpg-backup-s3 +# key: SECRET_ACCESS_KEY +# wal: +# compression: gzip +# data: +# compression: gzip +# immediateCheckpoint: true +# retentionPolicy: 14d +--- +# Example ScheduledBackup for apps-pg (repeat per cluster with spec.cluster.name) +# apiVersion: postgresql.cnpg.io/v1 +# kind: ScheduledBackup +# metadata: +# name: apps-pg-daily +# namespace: databases +# labels: +# app.kubernetes.io/part-of: railiance-apps +# spec: +# schedule: "0 0 2 * * *" +# backupOwnerReference: self +# cluster: +# name: apps-pg +# immediate: true \ No newline at end of file diff --git a/tools/cnpg-backup-status.sh b/tools/cnpg-backup-status.sh new file mode 100755 index 0000000..d2848aa --- /dev/null +++ b/tools/cnpg-backup-status.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +namespace="${CNPG_NAMESPACE:-databases}" +clusters=( + apps-pg + forgejo-db + net-kingdom-pg + state-hub-db +) + +if ! kubectl get ns "$namespace" >/dev/null 2>&1; then + echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2 + exit 1 +fi + +echo "CNPG backup posture in namespace ${namespace}" +echo + +missing=0 + +for cluster in "${clusters[@]}"; do + if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then + echo "${cluster}: MISSING (Cluster CR not found)" + missing=$((missing + 1)) + echo + continue + fi + + backup_spec="$(kubectl get cluster "$cluster" -n "$namespace" \ + -o jsonpath='{.spec.backup}' 2>/dev/null || true)" + if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then + echo "${cluster}: backup spec = null" + else + echo "${cluster}: backup spec configured" + fi + + scheduled="$(kubectl get scheduledbackup -n "$namespace" \ + -o jsonpath="{range .items[?(@.spec.cluster.name=='${cluster}')]}{.metadata.name}{' '}{end}" 2>/dev/null || true)" + scheduled="${scheduled%" "}" + if [[ -z "$scheduled" ]]; then + echo "${cluster}: ScheduledBackup = none" + missing=$((missing + 1)) + else + echo "${cluster}: ScheduledBackup = ${scheduled}" + fi + + if command -v kubectl >/dev/null 2>&1 && kubectl cnpg status "$cluster" -n "$namespace" >/dev/null 2>&1; then + echo "${cluster}: kubectl cnpg status available" + kubectl cnpg status "$cluster" -n "$namespace" | sed -n '1,8p' + else + phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)" + echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}" + fi + + echo +done + +if (( missing > 0 )); then + echo "RESULT: degraded — ${missing} cluster(s) lack ScheduledBackup coverage" + echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-backup-readiness.yaml" + exit 1 +fi + +echo "RESULT: ok — all tracked clusters have ScheduledBackup resources" \ No newline at end of file diff --git a/tools/core-hub-smoke.sh b/tools/core-hub-smoke.sh new file mode 100755 index 0000000..b9c917e --- /dev/null +++ b/tools/core-hub-smoke.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +base_url="${CORE_HUB_BASE_URL:-https://hub.coulomb.social}" +base_url="${base_url%/}" + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +request() { + local name="$1" + local url="$2" + local expected_status="$3" + local body="$tmpdir/${name}.json" + local status + + status="$(curl -sS -o "$body" -w "%{http_code}" "$url")" + if [[ "$status" != "$expected_status" ]]; then + echo "ERROR: expected $url to return $expected_status, got $status" >&2 + cat "$body" >&2 + echo >&2 + exit 1 + fi + + echo "$body" +} + +health_body="$(request healthz "${base_url}/healthz" 200)" +ready_body="$(request readyz "${base_url}/readyz" 200)" + +python3 - "$health_body" "$ready_body" <<'PY' +import json +import sys + +for path, filename in (("/healthz", sys.argv[1]), ("/readyz", sys.argv[2])): + with open(filename, encoding="utf-8") as fh: + payload = json.load(fh) + if not isinstance(payload, dict) or payload.get("status") not in {"ok", "ready"}: + raise SystemExit(f"{path} did not return an ok/ready health payload") + print(f"ok: {path} returned healthy JSON") +PY + +openapi_body="$(request openapi "${base_url}/api/v2/openapi.json" 200)" +python3 - "$openapi_body" <<'PY' +import json +import sys + +required_paths = { + "/hubs", + "/hub-capability-manifests", + "/api-consumers", + "/policy-scopes", + "/widgets", + "/hub-registry", + "/widget-types", +} + +with open(sys.argv[1], encoding="utf-8") as fh: + payload = json.load(fh) + +paths = payload.get("paths") +if not isinstance(paths, dict): + raise SystemExit("/api/v2/openapi.json did not include an OpenAPI paths object") + +missing = sorted(required_paths - set(paths)) +if missing: + raise SystemExit("OpenAPI missing paths: " + ", ".join(missing)) + +print("ok: /api/v2/openapi.json lists expected v2 resources") +PY + +widget_types_body="$(request widget-types "${base_url}/api/v2/widget-types" 200)" +python3 - "$widget_types_body" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as fh: + payload = json.load(fh) + +if not isinstance(payload, list) or not payload: + raise SystemExit("/api/v2/widget-types did not return a non-empty JSON list") + +print("ok: /api/v2/widget-types returned public discovery JSON") +PY + +for endpoint in hubs hub-registry widgets; do + body="$(request "${endpoint}-protected" "${base_url}/api/v2/${endpoint}" 401)" + python3 - "$body" "/api/v2/${endpoint}" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as fh: + payload = json.load(fh) + +path = sys.argv[2] +if not isinstance(payload, dict): + raise SystemExit(f"{path} returned 401 but not JSON") + +code = payload.get("code") +detail = payload.get("detail") +if code == "invalid_api_key": + print(f"ok: {path} requires an API key") +elif isinstance(detail, dict) and detail.get("code") == "unauthorized": + print(f"ok: {path} requires bearer authentication") +else: + raise SystemExit(f"{path} returned 401 but not a recognized auth error payload") +PY +done + +echo "ok: Core Hub public smoke checks passed for ${base_url}" \ No newline at end of file diff --git a/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md new file mode 100644 index 0000000..79bc4c3 --- /dev/null +++ b/workplans/RAILIANCE-WP-0013-cnpg-backup-wiring-restore-drill.md @@ -0,0 +1,103 @@ +--- +id: RAILIANCE-WP-0013 +type: workplan +title: "CNPG backup wiring and restore drill" +domain: financials +repo: railiance-apps +status: ready +owner: codex +topic_slug: railiance +created: "2026-07-10" +updated: "2026-07-10" +state_hub_workstream_id: "764c2447-0713-45d5-a14d-810a13027b8b" +--- + +# CNPG backup wiring and restore drill + +Follow-up to `RAILIANCE-WP-0012-T04` and custodian inbox `a1957cce`. Observability +and manifest templates exist; production clusters on railiance01 still report +degraded backup posture via `make cnpg-backup-status`. + +Evidence baseline: +`the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json` +(`s4_2_backup_restore: FAIL`). + +Coordinate with `railiance-platform` for object-store credentials and retention. +Phase 1 may use logical `pg_dump` for some clusters per +`disaster-control/BackupPickupQueue.md` item 6; barman `ScheduledBackup` remains +the longer-term target in `manifests/cnpg-backup-readiness.yaml`. + +## Confirm backup lane and credential path + +```task +id: RAILIANCE-WP-0013-T01 +status: todo +priority: high +state_hub_task_id: "f2051213-0a9f-418e-82ad-9bd7c79fd0dc" +``` + +Record the approved backup mechanism per cluster: Phase 1 logical dump vs barman +`ObjectStore` + `ScheduledBackup`. Identify OpenBao/operator paths for +`cnpg-backup-s3` or equivalent without copying secret values into Git, +workplans, or State Hub notes. + +## Wire apps-pg backup coverage + +```task +id: RAILIANCE-WP-0013-T02 +status: wait +priority: high +state_hub_task_id: "2d78019e-f040-422d-89d0-0d0e8c6af718" +``` + +Enable backup coverage for `apps-pg` first because `vergabe_db` is +production-facing. Apply platform-approved manifests or Makefile-driven operator +steps. `make cnpg-backup-status` must show a non-null backup spec or active +`ScheduledBackup` for `apps-pg`. + +Blocked on: `RAILIANCE-WP-0013-T01`. + +## Wire remaining production DB clusters + +```task +id: RAILIANCE-WP-0013-T03 +status: wait +priority: medium +state_hub_task_id: "d3e41204-98a5-4ccb-ae91-526e7a606baa" +``` + +Extend the same backup lane to `forgejo-db`, `net-kingdom-pg`, and +`state-hub-db`. Re-run `make cnpg-backup-status` until all four clusters pass. + +Blocked on: `RAILIANCE-WP-0013-T02`. + +## Run isolated restore drill with evidence + +```task +id: RAILIANCE-WP-0013-T04 +status: wait +priority: high +state_hub_task_id: "15602bb9-d1f6-456e-9af3-bc7c27ebb36f" +``` + +Restore the latest `apps-pg` backup into an isolated namespace or test database. +Verify row counts, run `vergabe-teilnahme` migrations, and capture non-secret +evidence (timestamp, cluster, backup id, restore namespace, post-restore smoke +result). Do not store decrypted dumps or credentials in Git. + +Blocked on: `RAILIANCE-WP-0013-T02`. + +## Close vergabe production-trust backup gate + +```task +id: RAILIANCE-WP-0013-T05 +status: wait +priority: medium +state_hub_task_id: "666c0a9b-4f9b-4ea3-bf0b-14aa76b8ea93" +``` + +Update `docs/app-data-backup-restore-handoff.md` and `docs/vergabe-teilnahme.md` +with backup/restore evidence references. Mark the S5 gate satisfied only after +T02 and T04 evidence exists. + +Blocked on: `RAILIANCE-WP-0013-T04`. \ No newline at end of file diff --git a/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md new file mode 100644 index 0000000..b7815db --- /dev/null +++ b/workplans/RAILIANCE-WP-0014-core-hub-helm-cutover-vergabe-refresh.md @@ -0,0 +1,98 @@ +--- +id: RAILIANCE-WP-0014 +type: workplan +title: "Core Hub Helm cutover and vergabe-teilnahme image refresh" +domain: financials +repo: railiance-apps +status: ready +owner: codex +topic_slug: railiance +created: "2026-07-10" +updated: "2026-07-10" +state_hub_workstream_id: "780c8d1d-ea75-479e-9733-10b518dde242" +--- + +# Core Hub Helm cutover and vergabe-teilnahme image refresh + +Follow-up to `RAILIANCE-WP-0012` and custodian inbox `3ced85c7`. The S5 Helm +chart, values, Makefile targets, and runbook exist; the live cluster may still +run Core Hub from `core-hub/k8s/railiance-production/` manifests. + +Separately, the 2026-07-10 delivery-lane snapshot shows `vergabe-teilnahme` +deployed at `f95de14` while the source repo head is `1061f3e` (14-commit drift). + +Production-touching deploy steps require explicit image tags and operator +approval. Do not store secrets in workplans. + +## Core Hub server dry-run on Railiance01 + +```task +id: RAILIANCE-WP-0014-T01 +status: todo +priority: high +state_hub_task_id: "b9dea9f3-f0c0-41c1-b9c4-ec4c44d43d55" +``` + +Run `CORE_HUB_IMAGE_TAG= make check-core-hub-image`, +`core-hub-server-dry-run`, and `core-hub-dry-run` against Railiance01. Record the +chosen image tag and any schema or secret prerequisites before apply. + +## Adopt Helm-managed Core Hub release + +```task +id: RAILIANCE-WP-0014-T02 +status: wait +priority: high +state_hub_task_id: "0e591104-0123-4e4a-84ff-5deb435e8441" +``` + +Run `core-hub-migrate` then `core-hub-deploy` with an explicit +`CORE_HUB_IMAGE_TAG`. Confirm the Helm release owns `core-hub-api`, runtime +ConfigMap, and Service in namespace `core-hub`. Retire duplicate resources from +the service-repo `k8s/railiance-production/` apply path once stable. + +Blocked on: `RAILIANCE-WP-0014-T01`. + +## Verify Core Hub post-cutover smoke + +```task +id: RAILIANCE-WP-0014-T03 +status: wait +priority: high +state_hub_task_id: "eca3ce8b-68a8-44d0-90c7-2cdefa482200" +``` + +Run `make core-hub-status`, `make core-hub-smoke`, and `make core-hub-release-info`. +Ingress for `hub.coulomb.social` must remain healthy. Document rollback steps in +`docs/core-hub-on-railiance01.md` if Helm values drift from prior manifests. + +Blocked on: `RAILIANCE-WP-0014-T02`. + +## Refresh vergabe-teilnahme production image + +```task +id: RAILIANCE-WP-0014-T04 +status: todo +priority: medium +state_hub_task_id: "d547f68d-52ec-4ff1-952b-ec70c834e0d3" +``` + +Identify the current published image tag for `vergabe-teilnahme` in the Forgejo +registry, verify the manifest with `tools/check-oci-image.sh`, run +`vergabe-dry-run` and `vergabe-server-dry-run` if available, then deploy with an +explicit tag. Close the 14-commit drift noted in the delivery-lane evidence. + +## Post-deploy vergabe smoke and runbook evidence + +```task +id: RAILIANCE-WP-0014-T05 +status: wait +priority: medium +state_hub_task_id: "4eaaa345-4252-46c1-bd8a-49d654ec48f4" +``` + +After deploy, run `make vergabe-status`, migrations if required, and HTTPS smoke +against `vergabe-teilnahme.whywhynot.de`. Update `docs/vergabe-teilnahme.md` +with the deployed tag, date, and smoke result (non-secret). + +Blocked on: `RAILIANCE-WP-0014-T04`. \ No newline at end of file diff --git a/workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md b/workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md new file mode 100644 index 0000000..599a8c4 --- /dev/null +++ b/workplans/archived/260710-RAILIANCE-WP-0012-inbox-suggestions-core-hub-cnpg-backup.md @@ -0,0 +1,90 @@ +--- +id: RAILIANCE-WP-0012 +type: workplan +title: "Inbox suggestions — Core Hub S5 release and CNPG backup observability" +domain: financials +repo: railiance-apps +status: finished +owner: codex +topic_slug: railiance +created: "2026-07-10" +updated: "2026-07-10" +state_hub_workstream_id: "25cee4ac-d1e4-441a-bc33-cf8c90ee67a6" +--- + +# Inbox suggestions — Core Hub S5 release and CNPG backup observability + +Sources: + +- State Hub inbox `a1957cce` (HIGH): no CNPG backups on railiance01 production DB + clusters; evidence `the-custodian/docs/evidence/vergabe-teilnahme-delivery-lane-20260710.json`. +- State Hub inbox `3ced85c7`: Core Hub release ownership follow-up from + `CUST-WP-0052`; cite `the-custodian/docs/core-hub-helixforge-build-alignment.md` + and `core-hub/docs/deployment/staging-profile.md`. + +## Core Hub Helm chart and production values + +```task +id: RAILIANCE-WP-0012-T01 +status: done +priority: high +state_hub_task_id: "0795f034-eacc-41d5-af8f-e5909f72f79f" +``` + +Promote the temporary `core-hub/k8s/railiance-production/` profile into +`charts/core-hub/` with non-secret production and staging values under `helm/`. +Secrets remain operator-owned (`core-hub-prod-env`, `core-hub-staging-env`). + +## Makefile release targets and smoke gate + +```task +id: RAILIANCE-WP-0012-T02 +status: done +priority: high +state_hub_task_id: "f975d9d3-e9b4-45ff-8088-03b2a5a855c6" +``` + +Add `core-hub-dry-run`, `core-hub-server-dry-run`, `core-hub-deploy`, +`core-hub-status`, `core-hub-migrate`, `core-hub-smoke`, and staging +counterparts. Require explicit `CORE_HUB_IMAGE_TAG=` for production deploys. + +## Operator runbook for Core Hub on railiance01 + +```task +id: RAILIANCE-WP-0012-T03 +status: done +priority: medium +state_hub_task_id: "41cba5e9-2c19-4b90-9dbe-517a3413eac7" +``` + +Document the S5 release path in `docs/core-hub-on-railiance01.md`, including +migration boundary, ingress cutover notes, rollback, and links to Core Hub +service-repo smokes. + +## CNPG backup posture observability + +```task +id: RAILIANCE-WP-0012-T04 +status: done +priority: high +state_hub_task_id: "7c9794ca-478c-4da9-ae77-b3aa62924e98" +``` + +Add `make cnpg-backup-status` and `tools/cnpg-backup-status.sh` to report backup +spec and `ScheduledBackup` coverage for `apps-pg`, `forgejo-db`, +`net-kingdom-pg`, and `state-hub-db`. Include operator-ready manifest templates +for barman `ObjectStore` and `ScheduledBackup` wiring once platform supplies +object-store credentials. + +## Update app data backup handoff with delivery-lane evidence + +```task +id: RAILIANCE-WP-0012-T05 +status: done +priority: medium +state_hub_task_id: "126f8f26-2536-40d6-8166-060f93ada85b" +``` + +Refresh `docs/app-data-backup-restore-handoff.md` with the 2026-07-10 custodian +conformance snapshot, the cluster inventory, and the platform coordination path +for backup implementation. \ No newline at end of file