Reject unfinished or truncated Forgejo backup archives before encryption

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883

scripts/capture_forgejo_archive.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Capture only a completed, byte-identical and CRC-verified Forgejo ZIP."""
+import argparse
+import hashlib
+from pathlib import Path
+import secrets
+import subprocess
+import time
+import zipfile
+
+
+def validate_archive(path):
+    with zipfile.ZipFile(path) as archive:
+        names = set(archive.namelist())
+        if 'forgejo-db.sql' not in names or not any(n.startswith('repos/') for n in names):
+            raise ValueError('required archive content missing')
+        if archive.testzip() is not None:
+            raise ValueError('archive checksum failure')
+
+
+def capture(namespace, pod, destination):
+    k = ['kubectl', 'exec', '--request-timeout=120s', '-n', namespace, pod, '-c', 'gitea', '--']
+    def call(args, timeout=150):
+        r = subprocess.run(k + args, capture_output=True, timeout=timeout)
+        if r.returncode: raise ValueError('capture command failed')
+        return r.stdout
+    remote = '/tmp/wp0029-backup-' + secrets.token_hex(12)
+    completed = False
+    try:
+        # Completion belongs to this exact process, not a global pgrep or file existence.
+        command = f'umask 077; forgejo dump -f {remote}.zip >{remote}.log 2>&1; result=$?; printf "%s" "$result" >{remote}.exit'
+        call(['sh','-c', 'nohup sh -c "$1" >/dev/null 2>&1 </dev/null &', 'sh', command])
+        for _ in range(180):
+            r = subprocess.run(k + ['cat',remote+'.exit'], capture_output=True, timeout=30)
+            if r.returncode == 0:
+                completed = True
+                if r.stdout.strip() != b'0': raise ValueError('dump process failed')
+                break
+            time.sleep(10)
+        else: raise ValueError('dump completion timeout')
+        size = int(call(['stat','-c%s',remote+'.zip']).strip())
+        expected = call(['sha256sum',remote+'.zip']).split()[0].decode()
+        if size <= 0: raise ValueError('empty archive')
+        chunk = 4*1024*1024
+        with destination.open('xb') as out:
+            destination.chmod(0o600)
+            for index in range((size+chunk-1)//chunk):
+                piece = call(['dd','if='+remote+'.zip','bs='+str(chunk),'skip='+str(index),'count=1'])
+                if len(piece) != min(chunk,size-index*chunk): raise ValueError('short archive chunk')
+                out.write(piece)
+        actual = hashlib.file_digest(destination.open('rb'),'sha256').hexdigest()
+        if actual != expected: raise ValueError('archive transfer mismatch')
+        validate_archive(destination)
+    finally:
+        # Do not remove a file while a timed-out producer may still be writing it.
+        if completed:
+            call(['rm','-f',remote+'.zip',remote+'.log',remote+'.exit'])
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--namespace',required=True); p.add_argument('--pod',required=True)
+    p.add_argument('--output',required=True,type=Path)
+    a=p.parse_args()
+    try: capture(a.namespace,a.pod,a.output)
+    except Exception:
+        print('ERROR: Forgejo archive capture or integrity validation failed')
+        return 1
+    print('Forgejo archive capture and integrity verified')
+    return 0
+
+if __name__=='__main__': raise SystemExit(main())

tests/test_forgejo_archive_integrity.py

@@ -0,0 +1,24 @@
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from zipfile import ZipFile, BadZipFile
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
+from capture_forgejo_archive import validate_archive
+
+class ArchiveIntegrity(unittest.TestCase):
+    def test_complete_archive(self):
+        with tempfile.TemporaryDirectory() as d:
+            p=Path(d)/'backup.zip'
+            with ZipFile(p,'w') as z:
+                z.writestr('forgejo-db.sql','SELECT 1;')
+                z.writestr('repos/example.git/HEAD','ref: refs/heads/main')
+            validate_archive(p)
+            p.write_bytes(p.read_bytes()[:-22])
+            with self.assertRaises(BadZipFile): validate_archive(p)
+
+    def test_missing_database_rejected(self):
+        with tempfile.TemporaryDirectory() as d:
+            p=Path(d)/'backup.zip'
+            with ZipFile(p,'w') as z: z.writestr('repos/example.git/HEAD','x')
+            with self.assertRaises(ValueError): validate_archive(p)

tools/cmd/forgejo-backup

@@ -87,99 +87,8 @@ ok "forgejo-db" "${FORGEJO_DB_NAMESPACE}/${FORGEJO_DB_POD}"
 
 # 1. Application dump (repos, packages, attachments, LFS, avatars)
 ok "forgejo dump" "running in pod…"
-DUMP_REMOTE="/tmp/forgejo-backup-${TS}.zip"
-# Long dumps drop the kubectl exec websocket if attached to the process; run in-pod
-# background and poll for the archive file instead.
-kubectl exec --request-timeout=0 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-  sh -eu -c "rm -f '${DUMP_REMOTE}'; nohup forgejo dump -f '${DUMP_REMOTE}' >/tmp/forgejo-dump.log 2>&1 &"
-dump_wait=0
-while [[ "${dump_wait}" -lt 1800 ]]; do
-  if kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-    test -s "${DUMP_REMOTE}" 2>/dev/null; then
-    if kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-      sh -c "pgrep -f 'forgejo dump' >/dev/null"; then
-      sleep 10
-      dump_wait=$((dump_wait + 10))
-      continue
-    fi
-    break
-  fi
-  sleep 10
-  dump_wait=$((dump_wait + 10))
-done
-if ! kubectl exec --request-timeout=30 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-  test -s "${DUMP_REMOTE}" 2>/dev/null; then
-  bad "forgejo dump" "timed out or empty archive (see /tmp/forgejo-dump.log in pod)"
-  exit 1
-fi
-# Large dumps (~700M) fail via stdout stream or single kubectl cp — use chunks.
-dump_size="$(kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-  stat -c%s "${DUMP_REMOTE}" 2>/dev/null || echo 0)"
-refresh_prod_pod() {
-  PROD_POD="$(kubectl get pods -n "${FORGEJO_NAMESPACE}" \
-    -l "app.kubernetes.io/instance=${FORGEJO_RELEASE}" \
-    -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
-}
-
-copy_from_pod() {
-  local remote_path="$1" local_path="$2" attempt
-  for attempt in 1 2 3 4 5; do
-    refresh_prod_pod
-    if [[ -z "${PROD_POD}" ]]; then
-      sleep $((attempt * 5))
-      continue
-    fi
-    if kubectl cp -n "${FORGEJO_NAMESPACE}" -c gitea \
-      "${FORGEJO_NAMESPACE}/${PROD_POD}:${remote_path}" "${local_path}" 2>/dev/null; then
-      return 0
-    fi
-    if kubectl exec --request-timeout=120 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-      cat "${remote_path}" > "${local_path}" 2>/dev/null && [[ -s "${local_path}" ]]; then
-      return 0
-    fi
-    sleep $((attempt * 5))
-  done
-  return 1
-}
-
-if [[ "${dump_size}" -lt 52428800 ]]; then
-  copy_from_pod "${DUMP_REMOTE}" "${DUMP_PLAIN}" || exit 1
-else
-  ok "forgejo dump" "chunked copy (${dump_size} bytes)…"
-  chunk_dir="$(mktemp -d)"
-  refresh_prod_pod
-  parts="$(kubectl exec --request-timeout=120 -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-    sh -c "rm -f /tmp/forgejo-backup-part-*; split -b 4m '${DUMP_REMOTE}' /tmp/forgejo-backup-part- && ls /tmp/forgejo-backup-part-*")"
-  for part in ${parts}; do
-    base="$(basename "${part}")"
-    local_chunk="${chunk_dir}/${base}"
-    if [[ -f "${local_chunk}" && -s "${local_chunk}" ]]; then
-      ok "forgejo dump" "reuse cached chunk ${base}"
-      continue
-    fi
-    chunk_ok=0
-    for chunk_try in 1 2 3 4 5 6 7 8; do
-      if copy_from_pod "${part}" "${local_chunk}"; then
-        chunk_ok=1
-        ok "forgejo dump" "chunk ${base} (${chunk_try})"
-        break
-      fi
-      sleep $((chunk_try * 3))
-    done
-    if [[ "${chunk_ok}" -ne 1 ]]; then
-      bad "forgejo dump" "chunk copy failed: ${base}"
-      exit 1
-    fi
-  done
-  cat "${chunk_dir}"/forgejo-backup-part-* > "${DUMP_PLAIN}"
-  rm -rf "${chunk_dir}"
-  kubectl exec -n "${FORGEJO_NAMESPACE}" "${PROD_POD}" -c gitea -- \
-    rm -f /tmp/forgejo-backup-part-* "${DUMP_REMOTE}" || true
-fi
-if [[ ! -f "${DUMP_PLAIN}" || ! -s "${DUMP_PLAIN}" ]]; then
-  bad "forgejo dump" "failed to copy dump from pod"
-  exit 1
-fi
+python3 "${ROOT}/scripts/capture_forgejo_archive.py" \
+  --namespace "${FORGEJO_NAMESPACE}" --pod "${PROD_POD}" --output "${DUMP_PLAIN}"
 ok "forgejo dump" "$(du -h "${DUMP_PLAIN}" | awk '{print $1}') $(basename "${DUMP_PLAIN}")"
 
 # 2. PostgreSQL logical dump (plain SQL to stdout; CNPG root FS is read-only)