tegwick
de8f11e721
Long forgejo dump sessions were aborting kubectl exec with websocket close 1006. Launch dump via nohup inside the pod and poll for the zip.
159 lines
No EOL
6.1 KiB
Bash
Executable file
159 lines
No EOL
6.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# tools/cmd/forgejo-backup — Forgejo Option A backup (RAIL-HO-WP-0005 T04/T09)
|
|
#
|
|
# Daily: forgejo dump (blob state) + pg_dump (forgejo-db) → age → Nextcloud forgejo/
|
|
# Retention: 14 daily + 4 weekly uploads (Sunday); local cache keep 7 per type.
|
|
#
|
|
# Env:
|
|
# KUBECONFIG (default ~/.kube/config-hosteurope)
|
|
# RAILIANCE_BACKUP_NC_TOKEN Nextcloud WebDAV token
|
|
# RAILIANCE_BACKUP_DRY_RUN=1 skip upload and remote side effects
|
|
set -euo pipefail
|
|
|
|
: "${RAILIANCE_BACKUP_NC_TOKEN:=MfTBEjcJTGS8Ywo}"
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
# shellcheck source=lib/railiance-print.sh
|
|
source "${ROOT}/lib/railiance-print.sh"
|
|
# shellcheck source=lib/railiance-backup-common.sh
|
|
source "${ROOT}/lib/railiance-backup-common.sh"
|
|
|
|
KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config-hosteurope}"
|
|
export KUBECONFIG
|
|
DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-0}"
|
|
|
|
FORGEJO_NAMESPACE="${FORGEJO_NAMESPACE:-forgejo}"
|
|
FORGEJO_RELEASE="${FORGEJO_RELEASE:-forgejo}"
|
|
FORGEJO_DB_NAMESPACE="${FORGEJO_DB_NAMESPACE:-databases}"
|
|
FORGEJO_DB_CLUSTER="${FORGEJO_DB_CLUSTER:-forgejo-db}"
|
|
FORGEJO_DB_POD="${FORGEJO_DB_POD:-$(kubectl get pods -n "${FORGEJO_DB_NAMESPACE}" \
|
|
-l "cnpg.io/cluster=${FORGEJO_DB_CLUSTER}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)}"
|
|
|
|
BACKUP_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/forgejo"
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
DOW="$(date -u +%u)" # 7 = Sunday
|
|
|
|
DUMP_PLAIN="${BACKUP_DIR}/forgejo-dump-${TS}.zip"
|
|
DUMP_AGE="${DUMP_PLAIN}.age"
|
|
DB_PLAIN="${BACKUP_DIR}/forgejo-db-${TS}.sql"
|
|
DB_AGE="${DB_PLAIN}.age"
|
|
|
|
cleanup_plain() {
|
|
rm -f "${DUMP_PLAIN}" "${DB_PLAIN}"
|
|
}
|
|
trap cleanup_plain EXIT
|
|
|
|
railiance_backup_require_tools
|
|
railiance_backup_require_openbao_lane
|
|
|
|
if [[ -z "${FORGEJO_DB_POD}" ]]; then
|
|
bad "preflight" "forgejo-db pod not found (cluster ${FORGEJO_DB_CLUSTER})"
|
|
exit 1
|
|
fi
|
|
|
|
if ! kubectl wait --for=condition=Ready "pod/${FORGEJO_DB_POD}" \
|
|
-n "${FORGEJO_DB_NAMESPACE}" --timeout=180s >/dev/null 2>&1; then
|
|
bad "preflight" "forgejo-db pod not Ready (check: make forgejo-db-status)"
|
|
exit 1
|
|
fi
|
|
|
|
PROD_POD="$(kubectl get pods -n "${FORGEJO_NAMESPACE}" \
|
|
-l "app.kubernetes.io/instance=${FORGEJO_RELEASE}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
if [[ -z "${PROD_POD}" ]]; then
|
|
bad "preflight" "Forgejo app pod not found in ${FORGEJO_NAMESPACE}"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${BACKUP_DIR}"
|
|
print_hdr "Forgejo backup — ${TS}"
|
|
ok "kubeconfig" "${KUBECONFIG}"
|
|
ok "forgejo pod" "${FORGEJO_NAMESPACE}/${PROD_POD}"
|
|
ok "forgejo-db" "${FORGEJO_DB_NAMESPACE}/${FORGEJO_DB_POD}"
|
|
|
|
# 1. Application dump (repos, packages, attachments, LFS, avatars)
|
|
ok "forgejo dump" "running in pod…"
|
|
DUMP_REMOTE="/tmp/forgejo-backup-${TS}.zip"
|
|
kubectl exec --request-timeout=0 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
forgejo dump -f "${DUMP_REMOTE}"
|
|
# Large dumps (~700M) fail via stdout stream or single kubectl cp — use chunks.
|
|
dump_size="$(kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
stat -c%s "${DUMP_REMOTE}" 2>/dev/null || echo 0)"
|
|
copy_from_pod() {
|
|
local remote_path="$1" local_path="$2" attempt
|
|
for attempt in 1 2 3; do
|
|
if kubectl cp -n "${FORGEJO_NAMESPACE}" -c gitea \
|
|
"${FORGEJO_NAMESPACE}/${PROD_POD}:${remote_path}" "${local_path}" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep $((attempt * 3))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
if [[ "${dump_size}" -lt 52428800 ]]; then
|
|
copy_from_pod "${DUMP_REMOTE}" "${DUMP_PLAIN}" || exit 1
|
|
else
|
|
ok "forgejo dump" "chunked copy (${dump_size} bytes)…"
|
|
chunk_dir="$(mktemp -d)"
|
|
parts="$(kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
sh -c "split -b 16m '${DUMP_REMOTE}' /tmp/forgejo-backup-part- && ls /tmp/forgejo-backup-part-*")"
|
|
for part in ${parts}; do
|
|
base="$(basename "${part}")"
|
|
copy_from_pod "${part}" "${chunk_dir}/${base}" || { bad "forgejo dump" "chunk copy failed: ${base}"; exit 1; }
|
|
done
|
|
cat "${chunk_dir}"/forgejo-backup-part-* > "${DUMP_PLAIN}"
|
|
rm -rf "${chunk_dir}"
|
|
kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
rm -f /tmp/forgejo-backup-part-* "${DUMP_REMOTE}" || true
|
|
fi
|
|
if [[ ! -f "${DUMP_PLAIN}" || ! -s "${DUMP_PLAIN}" ]]; then
|
|
bad "forgejo dump" "failed to copy dump from pod"
|
|
exit 1
|
|
fi
|
|
ok "forgejo dump" "$(du -h "${DUMP_PLAIN}" | awk '{print $1}') $(basename "${DUMP_PLAIN}")"
|
|
|
|
# 2. PostgreSQL logical dump (plain SQL to stdout; CNPG root FS is read-only)
|
|
ok "pg_dump" "database forgejo…"
|
|
kubectl exec -n "${FORGEJO_DB_NAMESPACE}" "${FORGEJO_DB_POD}" -c postgres -- \
|
|
pg_dump -U postgres -d forgejo --no-owner --no-acl > "${DB_PLAIN}"
|
|
ok "pg_dump" "$(du -h "${DB_PLAIN}" | awk '{print $1}') $(basename "${DB_PLAIN}")"
|
|
|
|
# 3. Encrypt
|
|
railiance_backup_encrypt "${DUMP_PLAIN}" "${DUMP_AGE}"
|
|
railiance_backup_encrypt "${DB_PLAIN}" "${DB_AGE}"
|
|
ok "age" "encrypted dump + db"
|
|
|
|
if [[ "${DRY_RUN}" == "1" ]]; then
|
|
warn "dry-run" "skipping Nextcloud upload"
|
|
else
|
|
railiance_backup_nc_upload "${DUMP_AGE}" "forgejo-dump-${TS}.zip.age"
|
|
ok "upload" "forgejo-dump-${TS}.zip.age"
|
|
railiance_backup_nc_upload "${DB_AGE}" "forgejo-db-${TS}.sql.age"
|
|
ok "upload" "forgejo-db-${TS}.sql.age"
|
|
|
|
if [[ "${DOW}" == "7" ]]; then
|
|
railiance_backup_nc_upload "${DUMP_AGE}" "forgejo-dump-weekly-${TS}.zip.age"
|
|
railiance_backup_nc_upload "${DB_AGE}" "forgejo-db-weekly-${TS}.sql.age"
|
|
ok "weekly" "Sunday copies uploaded"
|
|
fi
|
|
|
|
railiance_backup_record_success "${BACKUP_DIR}" "${TS}"
|
|
fi
|
|
|
|
# 4. Local cache prune (keep 7 per type)
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-dump-*.zip.age' 7
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-db-*.sql.age' 7
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-dump-*.zip' 3
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-db-*.sql' 3
|
|
ok "cache" "local copies pruned"
|
|
|
|
echo
|
|
ok "done" "Forgejo backup complete — ${TS}"
|
|
if [[ "${DRY_RUN}" != "1" ]]; then
|
|
echo " Dump: ${RAILIANCE_BACKUP_NC_PREFIX}/forgejo-dump-${TS}.zip.age"
|
|
echo " DB: ${RAILIANCE_BACKUP_NC_PREFIX}/forgejo-db-${TS}.sql.age"
|
|
echo
|
|
echo " Decrypt with ~/.config/age/railiance-backup.key"
|
|
echo " Restore drill: railiance-infra tools/forgejo-restore-drill.sh"
|
|
fi |