tegwick
dd6dc38e1e
Use 4m split chunks and up to 8 retries per chunk to survive intermittent kubectl cp/exec websocket EOF on large dumps.
214 lines
No EOL
8 KiB
Bash
Executable file
214 lines
No EOL
8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# tools/cmd/forgejo-backup — Forgejo Option A backup (RAIL-HO-WP-0005 T04/T09)
|
|
#
|
|
# Daily: forgejo dump (blob state) + pg_dump (forgejo-db) → age → Nextcloud forgejo/
|
|
# Retention: 14 daily + 4 weekly uploads (Sunday); local cache keep 7 per type.
|
|
#
|
|
# Env:
|
|
# KUBECONFIG (default ~/.kube/config-hosteurope)
|
|
# RAILIANCE_BACKUP_NC_TOKEN Nextcloud WebDAV token
|
|
# RAILIANCE_BACKUP_DRY_RUN=1 skip upload and remote side effects
|
|
set -euo pipefail
|
|
|
|
: "${RAILIANCE_BACKUP_NC_TOKEN:=MfTBEjcJTGS8Ywo}"
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
# shellcheck source=lib/railiance-print.sh
|
|
source "${ROOT}/lib/railiance-print.sh"
|
|
# shellcheck source=lib/railiance-backup-common.sh
|
|
source "${ROOT}/lib/railiance-backup-common.sh"
|
|
|
|
KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config-hosteurope}"
|
|
export KUBECONFIG
|
|
DRY_RUN="${RAILIANCE_BACKUP_DRY_RUN:-0}"
|
|
|
|
FORGEJO_NAMESPACE="${FORGEJO_NAMESPACE:-forgejo}"
|
|
FORGEJO_RELEASE="${FORGEJO_RELEASE:-forgejo}"
|
|
FORGEJO_DB_NAMESPACE="${FORGEJO_DB_NAMESPACE:-databases}"
|
|
FORGEJO_DB_CLUSTER="${FORGEJO_DB_CLUSTER:-forgejo-db}"
|
|
FORGEJO_DB_POD="${FORGEJO_DB_POD:-$(kubectl get pods -n "${FORGEJO_DB_NAMESPACE}" \
|
|
-l "cnpg.io/cluster=${FORGEJO_DB_CLUSTER}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)}"
|
|
|
|
BACKUP_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/forgejo"
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
DOW="$(date -u +%u)" # 7 = Sunday
|
|
|
|
DUMP_PLAIN="${BACKUP_DIR}/forgejo-dump-${TS}.zip"
|
|
DUMP_AGE="${DUMP_PLAIN}.age"
|
|
DB_PLAIN="${BACKUP_DIR}/forgejo-db-${TS}.sql"
|
|
DB_AGE="${DB_PLAIN}.age"
|
|
|
|
cleanup_plain() {
|
|
rm -f "${DUMP_PLAIN}" "${DB_PLAIN}"
|
|
}
|
|
trap cleanup_plain EXIT
|
|
|
|
railiance_backup_require_tools
|
|
railiance_backup_require_openbao_lane
|
|
|
|
if [[ -z "${FORGEJO_DB_POD}" ]]; then
|
|
bad "preflight" "forgejo-db pod not found (cluster ${FORGEJO_DB_CLUSTER})"
|
|
exit 1
|
|
fi
|
|
|
|
if ! kubectl wait --for=condition=Ready "pod/${FORGEJO_DB_POD}" \
|
|
-n "${FORGEJO_DB_NAMESPACE}" --timeout=180s >/dev/null 2>&1; then
|
|
bad "preflight" "forgejo-db pod not Ready (check: make forgejo-db-status)"
|
|
exit 1
|
|
fi
|
|
|
|
PROD_POD="$(kubectl get pods -n "${FORGEJO_NAMESPACE}" \
|
|
-l "app.kubernetes.io/instance=${FORGEJO_RELEASE}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
if [[ -z "${PROD_POD}" ]]; then
|
|
bad "preflight" "Forgejo app pod not found in ${FORGEJO_NAMESPACE}"
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${BACKUP_DIR}"
|
|
print_hdr "Forgejo backup — ${TS}"
|
|
ok "kubeconfig" "${KUBECONFIG}"
|
|
ok "forgejo pod" "${FORGEJO_NAMESPACE}/${PROD_POD}"
|
|
ok "forgejo-db" "${FORGEJO_DB_NAMESPACE}/${FORGEJO_DB_POD}"
|
|
|
|
# 1. Application dump (repos, packages, attachments, LFS, avatars)
|
|
ok "forgejo dump" "running in pod…"
|
|
DUMP_REMOTE="/tmp/forgejo-backup-${TS}.zip"
|
|
# Long dumps drop the kubectl exec websocket if attached to the process; run in-pod
|
|
# background and poll for the archive file instead.
|
|
kubectl exec --request-timeout=0 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
sh -eu -c "rm -f '${DUMP_REMOTE}'; nohup forgejo dump -f '${DUMP_REMOTE}' >/tmp/forgejo-dump.log 2>&1 &"
|
|
dump_wait=0
|
|
while [[ "${dump_wait}" -lt 1800 ]]; do
|
|
if kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
test -s "${DUMP_REMOTE}" 2>/dev/null; then
|
|
if kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
sh -c "pgrep -f 'forgejo dump' >/dev/null"; then
|
|
sleep 10
|
|
dump_wait=$((dump_wait + 10))
|
|
continue
|
|
fi
|
|
break
|
|
fi
|
|
sleep 10
|
|
dump_wait=$((dump_wait + 10))
|
|
done
|
|
if ! kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
test -s "${DUMP_REMOTE}" 2>/dev/null; then
|
|
bad "forgejo dump" "timed out or empty archive (see /tmp/forgejo-dump.log in pod)"
|
|
exit 1
|
|
fi
|
|
# Large dumps (~700M) fail via stdout stream or single kubectl cp — use chunks.
|
|
dump_size="$(kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
stat -c%s "${DUMP_REMOTE}" 2>/dev/null || echo 0)"
|
|
refresh_prod_pod() {
|
|
PROD_POD="$(kubectl get pods -n "${FORGEJO_NAMESPACE}" \
|
|
-l "app.kubernetes.io/instance=${FORGEJO_RELEASE}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
}
|
|
|
|
copy_from_pod() {
|
|
local remote_path="$1" local_path="$2" attempt
|
|
for attempt in 1 2 3 4 5; do
|
|
refresh_prod_pod
|
|
if [[ -z "${PROD_POD}" ]]; then
|
|
sleep $((attempt * 5))
|
|
continue
|
|
fi
|
|
if kubectl cp -n "${FORGEJO_NAMESPACE}" -c gitea \
|
|
"${FORGEJO_NAMESPACE}/${PROD_POD}:${remote_path}" "${local_path}" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
if kubectl exec --request-timeout=120 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
cat "${remote_path}" > "${local_path}" 2>/dev/null && [[ -s "${local_path}" ]]; then
|
|
return 0
|
|
fi
|
|
sleep $((attempt * 5))
|
|
done
|
|
return 1
|
|
}
|
|
|
|
if [[ "${dump_size}" -lt 52428800 ]]; then
|
|
copy_from_pod "${DUMP_REMOTE}" "${DUMP_PLAIN}" || exit 1
|
|
else
|
|
ok "forgejo dump" "chunked copy (${dump_size} bytes)…"
|
|
chunk_dir="$(mktemp -d)"
|
|
refresh_prod_pod
|
|
parts="$(kubectl exec --request-timeout=120 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
sh -c "rm -f /tmp/forgejo-backup-part-*; split -b 4m '${DUMP_REMOTE}' /tmp/forgejo-backup-part- && ls /tmp/forgejo-backup-part-*")"
|
|
for part in ${parts}; do
|
|
base="$(basename "${part}")"
|
|
local_chunk="${chunk_dir}/${base}"
|
|
if [[ -f "${local_chunk}" && -s "${local_chunk}" ]]; then
|
|
ok "forgejo dump" "reuse cached chunk ${base}"
|
|
continue
|
|
fi
|
|
chunk_ok=0
|
|
for chunk_try in 1 2 3 4 5 6 7 8; do
|
|
if copy_from_pod "${part}" "${local_chunk}"; then
|
|
chunk_ok=1
|
|
ok "forgejo dump" "chunk ${base} (${chunk_try})"
|
|
break
|
|
fi
|
|
sleep $((chunk_try * 3))
|
|
done
|
|
if [[ "${chunk_ok}" -ne 1 ]]; then
|
|
bad "forgejo dump" "chunk copy failed: ${base}"
|
|
exit 1
|
|
fi
|
|
done
|
|
cat "${chunk_dir}"/forgejo-backup-part-* > "${DUMP_PLAIN}"
|
|
rm -rf "${chunk_dir}"
|
|
kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
|
|
rm -f /tmp/forgejo-backup-part-* "${DUMP_REMOTE}" || true
|
|
fi
|
|
if [[ ! -f "${DUMP_PLAIN}" || ! -s "${DUMP_PLAIN}" ]]; then
|
|
bad "forgejo dump" "failed to copy dump from pod"
|
|
exit 1
|
|
fi
|
|
ok "forgejo dump" "$(du -h "${DUMP_PLAIN}" | awk '{print $1}') $(basename "${DUMP_PLAIN}")"
|
|
|
|
# 2. PostgreSQL logical dump (plain SQL to stdout; CNPG root FS is read-only)
|
|
ok "pg_dump" "database forgejo…"
|
|
kubectl exec -n "${FORGEJO_DB_NAMESPACE}" "${FORGEJO_DB_POD}" -c postgres -- \
|
|
pg_dump -U postgres -d forgejo --no-owner --no-acl > "${DB_PLAIN}"
|
|
ok "pg_dump" "$(du -h "${DB_PLAIN}" | awk '{print $1}') $(basename "${DB_PLAIN}")"
|
|
|
|
# 3. Encrypt
|
|
railiance_backup_encrypt "${DUMP_PLAIN}" "${DUMP_AGE}"
|
|
railiance_backup_encrypt "${DB_PLAIN}" "${DB_AGE}"
|
|
ok "age" "encrypted dump + db"
|
|
|
|
if [[ "${DRY_RUN}" == "1" ]]; then
|
|
warn "dry-run" "skipping Nextcloud upload"
|
|
else
|
|
railiance_backup_nc_upload "${DUMP_AGE}" "forgejo-dump-${TS}.zip.age"
|
|
ok "upload" "forgejo-dump-${TS}.zip.age"
|
|
railiance_backup_nc_upload "${DB_AGE}" "forgejo-db-${TS}.sql.age"
|
|
ok "upload" "forgejo-db-${TS}.sql.age"
|
|
|
|
if [[ "${DOW}" == "7" ]]; then
|
|
railiance_backup_nc_upload "${DUMP_AGE}" "forgejo-dump-weekly-${TS}.zip.age"
|
|
railiance_backup_nc_upload "${DB_AGE}" "forgejo-db-weekly-${TS}.sql.age"
|
|
ok "weekly" "Sunday copies uploaded"
|
|
fi
|
|
|
|
railiance_backup_record_success "${BACKUP_DIR}" "${TS}"
|
|
fi
|
|
|
|
# 4. Local cache prune (keep 7 per type)
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-dump-*.zip.age' 7
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-db-*.sql.age' 7
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-dump-*.zip' 3
|
|
railiance_backup_prune_local "${BACKUP_DIR}" 'forgejo-db-*.sql' 3
|
|
ok "cache" "local copies pruned"
|
|
|
|
echo
|
|
ok "done" "Forgejo backup complete — ${TS}"
|
|
if [[ "${DRY_RUN}" != "1" ]]; then
|
|
echo " Dump: ${RAILIANCE_BACKUP_NC_PREFIX}/forgejo-dump-${TS}.zip.age"
|
|
echo " DB: ${RAILIANCE_BACKUP_NC_PREFIX}/forgejo-db-${TS}.sql.age"
|
|
echo
|
|
echo " Decrypt with ~/.config/age/railiance-backup.key"
|
|
echo " Restore drill: railiance-infra tools/forgejo-restore-drill.sh"
|
|
fi |