diff --git a/Makefile b/Makefile index 2d19ee6..140f7af 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,10 @@ apps-pg-backup-dry-run: check-production-kubeconfig ## Logical pg_dump + age enc KUBECONFIG="$(PRODUCTION_KUBECONFIG)" PLATFORM_REPO="$(PLATFORM_REPO)" RAILIANCE_BACKUP_DRY_RUN=1 \ tools/apps-pg-backup-dry-run.sh +apps-pg-restore-drill: check-production-kubeconfig ## Isolated vergabe_db restore drill with row-count gate + chmod +x tools/apps-pg-restore-drill.sh + KUBECONFIG="$(PRODUCTION_KUBECONFIG)" tools/apps-pg-restore-drill.sh + apps-pg-backup: check-production-kubeconfig ## Logical pg_dump + age encrypt + Nextcloud upload for vergabe_db chmod +x tools/apps-pg-backup-dry-run.sh @KUBECONFIG="$(PRODUCTION_KUBECONFIG)" PLATFORM_REPO="$(PLATFORM_REPO)" bao kv metadata get \ @@ -489,4 +493,4 @@ help: ## Show this help /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } \ /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST) -.PHONY: check-tools check-sops k8s-server-dry-run apps-pg-status check-railiance01-kubeconfig check-inter-hub-image-tag check-inter-hub-image check-vergabe-image-tag check-vergabe-image vergabe-dry-run vergabe-deploy vergabe-ingress-deploy vergabe-status vergabe-migrate vergabe-seed vergabe-superuser vergabe-logs vergabe-db-url-secret eso-deploy forgejo-openbao-eso-token-apply forgejo-openbao-store-deploy forgejo-mailer-es-deploy forgejo-mailer-es-status forgejo-dry-run forgejo-server-dry-run forgejo-deploy forgejo-ingress-deploy forgejo-ssh-nodeport-deploy forgejo-status forgejo-smoke forgejo-npm-smoke forgejo-verify forgejo-operator-bootstrap forgejo-runner-registration-sops-bootstrap forgejo-secrets-check forgejo-logs forgejo-runner-registration-deploy forgejo-runner-deploy forgejo-runner-status forgejo-runner-logs inter-hub-render-baseline inter-hub-dry-run inter-hub-server-dry-run inter-hub-deploy inter-hub-status inter-hub-release-info inter-hub-smoke inter-hub-logs reuse-dry-run reuse-deploy reuse-status reuse-smoke reuse-logs reuse-forgejo-webhook reuse-openbao-eso-token-apply reuse-openbao-store-deploy reuse-runtime-es-deploy reuse-runtime-es-status reuse-webhook-smoke help +.PHONY: check-tools check-sops k8s-server-dry-run apps-pg-status apps-pg-restore-drill check-railiance01-kubeconfig check-inter-hub-image-tag check-inter-hub-image check-vergabe-image-tag check-vergabe-image vergabe-dry-run vergabe-deploy vergabe-ingress-deploy vergabe-status vergabe-migrate vergabe-seed vergabe-superuser vergabe-logs vergabe-db-url-secret eso-deploy forgejo-openbao-eso-token-apply forgejo-openbao-store-deploy forgejo-mailer-es-deploy forgejo-mailer-es-status forgejo-dry-run forgejo-server-dry-run forgejo-deploy forgejo-ingress-deploy forgejo-ssh-nodeport-deploy forgejo-status forgejo-smoke forgejo-npm-smoke forgejo-verify forgejo-operator-bootstrap forgejo-runner-registration-sops-bootstrap forgejo-secrets-check forgejo-logs forgejo-runner-registration-deploy forgejo-runner-deploy forgejo-runner-status forgejo-runner-logs inter-hub-render-baseline inter-hub-dry-run inter-hub-server-dry-run inter-hub-deploy inter-hub-status inter-hub-release-info inter-hub-smoke inter-hub-logs reuse-dry-run reuse-deploy reuse-status reuse-smoke reuse-logs reuse-forgejo-webhook reuse-openbao-eso-token-apply reuse-openbao-store-deploy reuse-runtime-es-deploy reuse-runtime-es-status reuse-webhook-smoke help diff --git a/docs/evidence/apps-pg-restore-drill-20260711T230725Z.json b/docs/evidence/apps-pg-restore-drill-20260711T230725Z.json new file mode 100644 index 0000000..adcc0ef --- /dev/null +++ b/docs/evidence/apps-pg-restore-drill-20260711T230725Z.json @@ -0,0 +1,12 @@ +{ + "captured_at": "20260711T230725Z", + "workplan": "RAILIANCE-WP-0013", + "cluster": "apps-pg", + "source_database": "vergabe_db", + "restore_database": "vergabe_restore_drill", + "backup_file": "apps-pg-vergabe_db-20260711T221353Z.dump", + "production_row_estimate": 195, + "restored_row_estimate": 195, + "row_count_gate": "pass", + "notes": "Isolated restore on live CNPG primary; drill DB dropped after evidence write." +} diff --git a/tools/apps-pg-restore-drill.sh b/tools/apps-pg-restore-drill.sh new file mode 100755 index 0000000..0dc09f1 --- /dev/null +++ b/tools/apps-pg-restore-drill.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Isolated apps-pg restore drill for vergabe_db (RAILIANCE-WP-0013-T04). +# Restores a local pg_dump into a temporary database on the apps-pg cluster. +set -euo pipefail + +APPS_PG_NAMESPACE="${APPS_PG_NAMESPACE:-databases}" +APPS_PG_CLUSTER="${APPS_PG_CLUSTER:-apps-pg}" +APPS_PG_POD="${APPS_PG_POD:-}" +SOURCE_DB="${APPS_PG_DATABASE:-vergabe_db}" +BACKUP_DIR="${BACKUP_DIR:-$HOME/.cache/railiance/backups/apps-pg}" +BACKUP_FILE="${APPS_PG_BACKUP_FILE:-}" +DRILL_DB="${APPS_PG_RESTORE_DB:-vergabe_restore_drill}" +EVIDENCE_DIR="${EVIDENCE_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/docs/evidence}" + +if [[ -z "$APPS_PG_POD" ]]; then + APPS_PG_POD="$( + kubectl get pods -n "$APPS_PG_NAMESPACE" \ + -l "cnpg.io/cluster=${APPS_PG_CLUSTER},role=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true + )" +fi +if [[ -z "$APPS_PG_POD" ]]; then + echo "ERROR: primary pod for ${APPS_PG_CLUSTER} not found" >&2 + exit 1 +fi + +if [[ -z "$BACKUP_FILE" ]]; then + BACKUP_FILE="$( + find "$BACKUP_DIR" -maxdepth 1 -name "apps-pg-${SOURCE_DB}-*.dump" -type f 2>/dev/null \ + | sort -r | head -1 + )" +fi +if [[ -z "$BACKUP_FILE" || ! -f "$BACKUP_FILE" ]]; then + echo "ERROR: no backup file found for ${SOURCE_DB} under ${BACKUP_DIR}" >&2 + echo "Run: make apps-pg-backup-dry-run" >&2 + exit 1 +fi + +psql_exec() { + kubectl exec -n "$APPS_PG_NAMESPACE" "$APPS_PG_POD" -c postgres -- \ + psql -U postgres -v ON_ERROR_STOP=1 "$@" +} + +count_tables() { + local db="$1" + psql_exec -d "$db" -Atc \ + "SELECT COALESCE(SUM(n_live_tup),0)::bigint FROM pg_stat_user_tables;" +} + +echo "apps-pg restore drill" +echo " cluster: ${APPS_PG_CLUSTER}" +echo " pod: ${APPS_PG_POD}" +echo " source: ${SOURCE_DB}" +echo " backup: ${BACKUP_FILE}" +echo " drill db: ${DRILL_DB}" +echo + +psql_exec -d postgres -c "DROP DATABASE IF EXISTS \"${DRILL_DB}\";" +psql_exec -d postgres -c "CREATE DATABASE \"${DRILL_DB}\";" +echo "ok: created drill database ${DRILL_DB}" + +kubectl exec -i -n "$APPS_PG_NAMESPACE" "$APPS_PG_POD" -c postgres -- \ + pg_restore -U postgres -d "$DRILL_DB" --no-owner --no-acl --clean --if-exists \ + < "$BACKUP_FILE" +echo "ok: pg_restore completed" + +PROD_ROWS="$(count_tables "$SOURCE_DB")" +DRILL_ROWS="$(count_tables "$DRILL_DB")" +echo "row estimate — production ${SOURCE_DB}: ${PROD_ROWS}" +echo "row estimate — drill ${DRILL_DB}: ${DRILL_ROWS}" + +if [[ "$PROD_ROWS" != "$DRILL_ROWS" ]]; then + echo "ERROR: row-count mismatch between production and restore drill" >&2 + exit 1 +fi +echo "ok: row-count gate passed" + +TS="$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "$EVIDENCE_DIR" +EVIDENCE_FILE="${EVIDENCE_DIR}/apps-pg-restore-drill-${TS}.json" +python3 - <