fix(backup): retry kubectl preflight when railiance01 API is flaky
Poll up to 10 times for forgejo and forgejo-db pods before failing preflight when the k3s API returns ServiceUnavailable.
This commit is contained in:
parent
dd6dc38e1e
commit
2806f2cdda
1 changed files with 18 additions and 5 deletions
|
|
@ -26,8 +26,20 @@ FORGEJO_NAMESPACE="${FORGEJO_NAMESPACE:-forgejo}"
|
||||||
FORGEJO_RELEASE="${FORGEJO_RELEASE:-forgejo}"
|
FORGEJO_RELEASE="${FORGEJO_RELEASE:-forgejo}"
|
||||||
FORGEJO_DB_NAMESPACE="${FORGEJO_DB_NAMESPACE:-databases}"
|
FORGEJO_DB_NAMESPACE="${FORGEJO_DB_NAMESPACE:-databases}"
|
||||||
FORGEJO_DB_CLUSTER="${FORGEJO_DB_CLUSTER:-forgejo-db}"
|
FORGEJO_DB_CLUSTER="${FORGEJO_DB_CLUSTER:-forgejo-db}"
|
||||||
FORGEJO_DB_POD="${FORGEJO_DB_POD:-$(kubectl get pods -n "${FORGEJO_DB_NAMESPACE}" \
|
|
||||||
-l "cnpg.io/cluster=${FORGEJO_DB_CLUSTER}" -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)}"
|
kubectl_get_pod() {
|
||||||
|
local ns="$1" selector="$2" attempt out
|
||||||
|
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
||||||
|
out="$(kubectl get pods -n "${ns}" -l "${selector}" \
|
||||||
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
||||||
|
if [[ -n "${out}" ]]; then
|
||||||
|
echo "${out}"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
sleep $((attempt * 3))
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
BACKUP_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/forgejo"
|
BACKUP_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/railiance/backups/forgejo"
|
||||||
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||||
|
|
@ -46,6 +58,8 @@ trap cleanup_plain EXIT
|
||||||
railiance_backup_require_tools
|
railiance_backup_require_tools
|
||||||
railiance_backup_require_openbao_lane
|
railiance_backup_require_openbao_lane
|
||||||
|
|
||||||
|
FORGEJO_DB_POD="${FORGEJO_DB_POD:-$(kubectl_get_pod "${FORGEJO_DB_NAMESPACE}" \
|
||||||
|
"cnpg.io/cluster=${FORGEJO_DB_CLUSTER}" || true)}"
|
||||||
if [[ -z "${FORGEJO_DB_POD}" ]]; then
|
if [[ -z "${FORGEJO_DB_POD}" ]]; then
|
||||||
bad "preflight" "forgejo-db pod not found (cluster ${FORGEJO_DB_CLUSTER})"
|
bad "preflight" "forgejo-db pod not found (cluster ${FORGEJO_DB_CLUSTER})"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
@ -57,9 +71,8 @@ if ! kubectl wait --for=condition=Ready "pod/${FORGEJO_DB_POD}" \
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
PROD_POD="$(kubectl get pods -n "${FORGEJO_NAMESPACE}" \
|
PROD_POD="$(kubectl_get_pod "${FORGEJO_NAMESPACE}" \
|
||||||
-l "app.kubernetes.io/instance=${FORGEJO_RELEASE}" \
|
"app.kubernetes.io/instance=${FORGEJO_RELEASE}" || true)"
|
||||||
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
||||||
if [[ -z "${PROD_POD}" ]]; then
|
if [[ -z "${PROD_POD}" ]]; then
|
||||||
bad "preflight" "Forgejo app pod not found in ${FORGEJO_NAMESPACE}"
|
bad "preflight" "Forgejo app pod not found in ${FORGEJO_NAMESPACE}"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue