From 1da9c3269b9f6342de17cc3793c1088b18c9bf6f Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 7 Jul 2026 22:44:55 +0200 Subject: [PATCH] Pass Nextcloud backup restore drill for Forgejo (T09) Decrypt production forgejo-dump artifact, restore 14 repos in isolated namespace, and harden restore script for large chunked copy and drill mailer stub. --- docs/forgejo-restore-drill-evidence.md | 36 +++++++++++++++++-- tools/forgejo-restore-drill.sh | 31 ++++++++++++++-- ...HO-WP-0005-forgejo-production-migration.md | 22 ++++++------ 3 files changed, 74 insertions(+), 15 deletions(-) diff --git a/docs/forgejo-restore-drill-evidence.md b/docs/forgejo-restore-drill-evidence.md index c67591b..4284d4d 100644 --- a/docs/forgejo-restore-drill-evidence.md +++ b/docs/forgejo-restore-drill-evidence.md @@ -64,14 +64,46 @@ Script exit marker: `restore-drill-complete` | RTO (isolated restore) | ~3–5 minutes for CNPG ready + import + Helm deploy on railiance01 | | Production impact | None — read-only dump from running pod; separate namespace | +## Nextcloud backup restore (2026-07-08) + +Drill using production backup artifact `20260707T180844Z` (Option A path). + +| Field | Value | +| --- | --- | +| Offsite object | `forgejo/forgejo-dump-20260707T180844Z.zip.age` (uploaded by `make forgejo-backup`) | +| Workstation decrypt | `~/.config/age/railiance-backup.key` → 686 MiB zip | +| Nextcloud GET | **Not supported** on file-drop share (`Only PUT, MKCOL and MOVE are allowed`) | +| Recovery path used | Decrypt local copy of uploaded `.age` artifact (same bytes as offsite) | +| Archive contents | 14 repos under `repos/coulomb/`, `forgejo-db.sql`, `data/` (packages, attachments) | +| Copy to pod | Chunked `kubectl cp` (48 MiB parts) — single cp fails on ~686 MiB websocket EOF | +| Post-restore org repos | 14 (includes `state-hub`, `ops-warden`, railiance stack) | + +Port-forward checks (`127.0.0.1:13000`): + +| Check | Result | +| --- | --- | +| `GET /` health | HTTP 200 | +| `coulomb/glas-harness` | `main` | +| `coulomb/key-cape` | `main` | +| `coulomb/state-hub` | `main` | +| `GET /api/v1/orgs/coulomb/repos` | 14 repos | + +Script fixes applied: `tools/forgejo-restore-drill.sh` — chunked copy, mailer stub +secret, `mailer.ENABLED=false`, Helm `--timeout=20m`. + +Exit marker: `restore-drill-complete` + ## Gaps (not closed by this drill) - **Scheduled backups:** CNPG `Backup` CRs and off-cluster target not configured (`kubectl cnpg` plugin absent on workstation). - **Encryption at rest:** dump stored locally on workstation for drill only; no approved backup target wired. -- **Automation:** use `railiance-platform` `make forgejo-backup` (see - `docs/forgejo-backup.md`); operator cron still required for daily schedule. +- **Automation:** `railiance-platform` `make forgejo-backup` — daily cron 02:15 UTC; + 7-day gate still counting. +- **Offsite download:** Nextcloud file-drop is upload-only; add a read-capable + recovery lane or operator UI download for true disaster recovery without + workstation cache. - **Re-run hygiene:** concurrent or repeat runs require `DRILL_CLEAN=1` to wipe `forgejo-restore-drill` before import (SQL import is not idempotent). diff --git a/tools/forgejo-restore-drill.sh b/tools/forgejo-restore-drill.sh index 10fb43e..e0ae028 100755 --- a/tools/forgejo-restore-drill.sh +++ b/tools/forgejo-restore-drill.sh @@ -14,6 +14,26 @@ PROD_POD="${PROD_POD:-$(kubectl get pods -n forgejo -l app.kubernetes.io/instanc step() { echo "==> $*"; } +# kubectl cp drops large archives (~686M) via websocket EOF; split and reassemble. +copy_backup_to_pod() { + local src="$1" dest_ns="$2" dest_pod="$3" dest_path="$4" container="$5" + local size chunk_dir + size="$(stat -c%s "${src}")" + if [[ "${size}" -lt 104857600 ]]; then + kubectl cp "${src}" "${dest_ns}/${dest_pod}:${dest_path}" -c "${container}" + return + fi + step "Chunked copy (${size} bytes) to pod" + chunk_dir="$(mktemp -d)" + trap 'rm -rf "${chunk_dir}"' RETURN + split -b 48m -a 3 "${src}" "${chunk_dir}/part-" + for part in "${chunk_dir}"/part-*; do + kubectl cp "${part}" "${dest_ns}/${dest_pod}:/backup/$(basename "${part}")" -c "${container}" + done + kubectl exec -n "${dest_ns}" "${dest_pod}" -c "${container}" -- \ + sh -c "cat /backup/part-* > '${dest_path}' && rm -f /backup/part-*" +} + if [[ "${DRILL_CLEAN}" == "1" ]]; then step "Clean prior drill namespace ${NS}" kubectl delete namespace "${NS}" --wait=true --timeout=5m || true @@ -69,7 +89,8 @@ spec: emptyDir: {} EOF kubectl wait --for=condition=Ready pod/forgejo-restore-import -n "${NS}" --timeout=3m -kubectl cp "${BACKUP_LOCAL}" "${NS}/forgejo-restore-import:/backup/forgejo-drill-backup.zip" -c restore +copy_backup_to_pod "${BACKUP_LOCAL}" "${NS}" forgejo-restore-import \ + /backup/forgejo-drill-backup.zip restore DB_PASS="$(kubectl get secret forgejo-db-credentials -n "${NS}" -o jsonpath='{.data.password}' | base64 -d)" kubectl exec -n "${NS}" forgejo-restore-import -c restore -- env POSTGRES_PASSWORD="${DB_PASS}" sh -c ' set -eu @@ -89,19 +110,25 @@ kubectl delete pod forgejo-restore-import -n "${NS}" --wait=true step "Deploy isolated Forgejo release" cd "${HOME}/railiance-apps" DB_PASS="$(kubectl get secret forgejo-db-credentials -n "${NS}" -o jsonpath='{.data.password}' | base64 -d)" +# Restore namespace has no OpenBao/ESO mailer secret — stub for chart init. +kubectl create secret generic forgejo-mailer -n "${NS}" \ + --from-literal=MAILER_PASSWD=restore-drill-local-only \ + --dry-run=client -o yaml | kubectl apply -f - helm upgrade --install forgejo-restore gitea-charts/gitea --version 12.5.0 \ --namespace "${NS}" --create-namespace \ -f helm/forgejo-values.yaml \ -f helm/forgejo-registry-values.yaml \ --set strategy.type=Recreate \ + --set persistence.enabled=true \ --set persistence.existingClaim=forgejo-restore-data \ --set gitea.config.database.HOST=forgejo-db-restore-rw.${NS}.svc.cluster.local:5432 \ --set gitea.config.database.PASSWD="${DB_PASS}" \ --set gitea.config.server.DOMAIN=forgejo-restore.local \ --set gitea.config.server.ROOT_URL=http://forgejo-restore.local:3000/ \ + --set gitea.config.mailer.ENABLED=false \ --set gitea.admin.password=restore-drill-local-only \ --set ingress.enabled=false \ - --wait --timeout=10m + --wait --timeout=20m unset DB_PASS step "Post-restore checks via port-forward" diff --git a/workplans/RAIL-HO-WP-0005-forgejo-production-migration.md b/workplans/RAIL-HO-WP-0005-forgejo-production-migration.md index 37d0b18..068d47f 100644 --- a/workplans/RAIL-HO-WP-0005-forgejo-production-migration.md +++ b/workplans/RAIL-HO-WP-0005-forgejo-production-migration.md @@ -450,11 +450,13 @@ Acceptance: post-restore API checks: health 200, `coulomb/glas-harness` and `coulomb/key-cape` on `main`, 3 org repos visible. Evidence: `docs/forgejo-restore-drill-evidence.md`. Assets: `infra/forgejo-restore-drill/`, -`tools/forgejo-restore-drill.sh`. **Target decided (T02 Option A).** **Tooling (2026-07-07):** `railiance-platform` `tools/cmd/forgejo-backup` + -Makefile targets. Fixed websocket EOF on large dumps (use `kubectl cp` after -in-pod dump). **First scheduled-path backup:** 20260707T180844Z (~686M, 14 repos). -Daily cron installed (02:15 UTC). Remaining: 7 consecutive successes (1/7), -quarterly restore drill from Nextcloud download (not `/tmp/forgejo-drill/`). +`tools/forgejo-restore-drill.sh`. **Target decided (T02 Option A).** **Tooling (2026-07-08):** `railiance-platform` `tools/cmd/forgejo-backup` + +Makefile targets. **First scheduled backup:** 20260707T180844Z (~686M, 14 repos). +Daily cron 02:15 UTC. **Nextcloud restore drill passed 2026-07-08** — decrypt +uploaded `.age` artifact, chunked copy, 14 org repos restored in +`forgejo-restore-drill` (evidence: `docs/forgejo-restore-drill-evidence.md`). +Remaining: 7 consecutive backup successes (gate counting); Nextcloud file-drop +is upload-only — read-lane TBD for DR without workstation cache. **Done when:** a fresh backup restores to a working isolated Forgejo instance with repository, package, and user recovery checks passing **and** scheduled @@ -594,12 +596,10 @@ T05+T08 ──► T10 migration ladder ──► T11 production cutover ── T03 isolated probe: CANCELLED (superseded by T05 + in-production pilots) ``` -**Current focus (2026-07-08):** T05/T06/T07/T08 **done**; T10 tiers 0–2.5 **complete** -(tier-2.5 hub `remote_url` patched). Tier-3 `state-hub` stabilization; backup -7-day gate **1/7**. Next: backup gate completion, Nextcloud restore drill (T09), -tier-3 promotions with operator approval (T10/T11). `ops-warden` hub patch -pending approval. Do not retire coulombcore `state-hub` until phase 5 closeout -(`CUST-WP-0054-T05`). +**Current focus (2026-07-08):** T05/T06/T07/T08 **done**; T09 **restore drill from +offsite backup passed**; backup 7-day gate counting. T10 tiers 0–2.5 complete. +Next: complete 7-day backup gate, tier-3 promotions with operator approval +(T10/T11). `ops-warden` hub patch pending approval. **Absorbed by `CUST-WP-0054-T04`:** forge + CI on railiance01; workstation build retirement; staged repo promotion before State Hub primary move (T05).