RAILIANCE-WP-0012/0013/0014: Core Hub S5 release, CNPG observability, and follow-up workplans

Deliver the inbox-suggestion implementation from WP-0012: Core Hub Helm chart
and Makefile targets, CNPG backup status tooling, and updated backup handoff
docs. Archive the finished WP-0012 workplan and register ready follow-ups
WP-0013 (CNPG backup wiring + restore drill) and WP-0014 (Core Hub Helm
cutover and vergabe-teilnahme image refresh).

tools/cnpg-backup-status.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace="${CNPG_NAMESPACE:-databases}"
+clusters=(
+  apps-pg
+  forgejo-db
+  net-kingdom-pg
+  state-hub-db
+)
+
+if ! kubectl get ns "$namespace" >/dev/null 2>&1; then
+  echo "ERROR: namespace $namespace is not reachable from the current kubeconfig" >&2
+  exit 1
+fi
+
+echo "CNPG backup posture in namespace ${namespace}"
+echo
+
+missing=0
+
+for cluster in "${clusters[@]}"; do
+  if ! kubectl get cluster "$cluster" -n "$namespace" >/dev/null 2>&1; then
+    echo "${cluster}: MISSING (Cluster CR not found)"
+    missing=$((missing + 1))
+    echo
+    continue
+  fi
+
+  backup_spec="$(kubectl get cluster "$cluster" -n "$namespace" \
+    -o jsonpath='{.spec.backup}' 2>/dev/null || true)"
+  if [[ -z "$backup_spec" || "$backup_spec" == "null" ]]; then
+    echo "${cluster}: backup spec = null"
+  else
+    echo "${cluster}: backup spec configured"
+  fi
+
+  scheduled="$(kubectl get scheduledbackup -n "$namespace" \
+    -o jsonpath="{range .items[?(@.spec.cluster.name=='${cluster}')]}{.metadata.name}{' '}{end}" 2>/dev/null || true)"
+  scheduled="${scheduled%" "}"
+  if [[ -z "$scheduled" ]]; then
+    echo "${cluster}: ScheduledBackup = none"
+    missing=$((missing + 1))
+  else
+    echo "${cluster}: ScheduledBackup = ${scheduled}"
+  fi
+
+  if command -v kubectl >/dev/null 2>&1 && kubectl cnpg status "$cluster" -n "$namespace" >/dev/null 2>&1; then
+    echo "${cluster}: kubectl cnpg status available"
+    kubectl cnpg status "$cluster" -n "$namespace" | sed -n '1,8p'
+  else
+    phase="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)"
+    instances="$(kubectl get cluster "$cluster" -n "$namespace" -o jsonpath='{.status.instances}' 2>/dev/null || true)"
+    echo "${cluster}: phase=${phase:-unknown} instances=${instances:-unknown}"
+  fi
+
+  echo
+done
+
+if (( missing > 0 )); then
+  echo "RESULT: degraded — ${missing} cluster(s) lack ScheduledBackup coverage"
+  echo "See docs/app-data-backup-restore-handoff.md and manifests/cnpg-backup-readiness.yaml"
+  exit 1
+fi
+
+echo "RESULT: ok — all tracked clusters have ScheduledBackup resources"

tools/core-hub-smoke.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+base_url="${CORE_HUB_BASE_URL:-https://hub.coulomb.social}"
+base_url="${base_url%/}"
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+request() {
+  local name="$1"
+  local url="$2"
+  local expected_status="$3"
+  local body="$tmpdir/${name}.json"
+  local status
+
+  status="$(curl -sS -o "$body" -w "%{http_code}" "$url")"
+  if [[ "$status" != "$expected_status" ]]; then
+    echo "ERROR: expected $url to return $expected_status, got $status" >&2
+    cat "$body" >&2
+    echo >&2
+    exit 1
+  fi
+
+  echo "$body"
+}
+
+health_body="$(request healthz "${base_url}/healthz" 200)"
+ready_body="$(request readyz "${base_url}/readyz" 200)"
+
+python3 - "$health_body" "$ready_body" <<'PY'
+import json
+import sys
+
+for path, filename in (("/healthz", sys.argv[1]), ("/readyz", sys.argv[2])):
+    with open(filename, encoding="utf-8") as fh:
+        payload = json.load(fh)
+    if not isinstance(payload, dict) or payload.get("status") not in {"ok", "ready"}:
+        raise SystemExit(f"{path} did not return an ok/ready health payload")
+    print(f"ok: {path} returned healthy JSON")
+PY
+
+openapi_body="$(request openapi "${base_url}/api/v2/openapi.json" 200)"
+python3 - "$openapi_body" <<'PY'
+import json
+import sys
+
+required_paths = {
+    "/hubs",
+    "/hub-capability-manifests",
+    "/api-consumers",
+    "/policy-scopes",
+    "/widgets",
+    "/hub-registry",
+    "/widget-types",
+}
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+paths = payload.get("paths")
+if not isinstance(paths, dict):
+    raise SystemExit("/api/v2/openapi.json did not include an OpenAPI paths object")
+
+missing = sorted(required_paths - set(paths))
+if missing:
+    raise SystemExit("OpenAPI missing paths: " + ", ".join(missing))
+
+print("ok: /api/v2/openapi.json lists expected v2 resources")
+PY
+
+widget_types_body="$(request widget-types "${base_url}/api/v2/widget-types" 200)"
+python3 - "$widget_types_body" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+if not isinstance(payload, list) or not payload:
+    raise SystemExit("/api/v2/widget-types did not return a non-empty JSON list")
+
+print("ok: /api/v2/widget-types returned public discovery JSON")
+PY
+
+for endpoint in hubs hub-registry widgets; do
+  body="$(request "${endpoint}-protected" "${base_url}/api/v2/${endpoint}" 401)"
+  python3 - "$body" "/api/v2/${endpoint}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fh:
+    payload = json.load(fh)
+
+path = sys.argv[2]
+if not isinstance(payload, dict):
+    raise SystemExit(f"{path} returned 401 but not JSON")
+
+code = payload.get("code")
+detail = payload.get("detail")
+if code == "invalid_api_key":
+    print(f"ok: {path} requires an API key")
+elif isinstance(detail, dict) and detail.get("code") == "unauthorized":
+    print(f"ok: {path} requires bearer authentication")
+else:
+    raise SystemExit(f"{path} returned 401 but not a recognized auth error payload")
+PY
+done
+
+echo "ok: Core Hub public smoke checks passed for ${base_url}"