Prepare isolated offsite restore and stream large backup transfers

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883

scripts/restore_forgejo_offsite_locally.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Restore a fetched Forgejo archive on a disposable, internal Docker network."""
+import argparse
+import configparser
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import secrets
+import shutil
+import stat
+import subprocess
+import time
+import zipfile
+from capture_forgejo_archive import validate_archive
+
+FORGE='code.forgejo.org/forgejo/forgejo@sha256:e2684fd8707d486329084a695ed91999a4072a798e5409d45c1eb8a2911ff4b9'
+POSTGRES='postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70'
+
+
+def file_digest(path):
+    with path.open('rb') as f: return hashlib.file_digest(f,'sha256').hexdigest()
+
+
+def run(archive, receipt):
+    validate_archive(archive)
+    prefix='wp0029-'+secrets.token_hex(5)
+    network=prefix+'-net'; db=prefix+'-db'; app=prefix+'-app'
+    staging=archive.parent/(prefix+'-staging')
+    staging.mkdir(mode=0o700)
+    resources=[]
+    def command(args, payload=None, env=None, check=True):
+        r=subprocess.run(args,input=payload,capture_output=True,env=env,timeout=180)
+        if check and r.returncode: raise ValueError('isolated_command_failed')
+        return r
+    def docker(*args, **kw): return command(['docker',*args],**kw)
+    try:
+        receipt['stage']='archive_validation'
+        with zipfile.ZipFile(archive) as z:
+            if sum(i.file_size for i in z.infolist()) > 20*1024**3: raise ValueError('archive_too_large')
+            for info in z.infolist():
+                p=PurePosixPath(info.filename)
+                if p.is_absolute() or '..' in p.parts or stat.S_ISLNK(info.external_attr>>16): raise ValueError('unsafe_archive_member')
+            z.extractall(staging)
+        # Build an independent configuration; never reuse production app.ini.
+        password=secrets.token_urlsafe(32)
+        config=f'''APP_NAME = Isolated WP-0029 recovery
+RUN_USER = git
+RUN_MODE = prod
+WORK_PATH = /data/gitea
+[database]
+DB_TYPE = postgres
+HOST = {db}:5432
+NAME = forgejo
+USER = forgejo
+PASSWD = {password}
+SSL_MODE = disable
+[repository]
+ROOT = /data/git/repositories
+[server]
+APP_DATA_PATH = /data/gitea
+DOMAIN = localhost
+ROOT_URL = http://localhost:3000/
+HTTP_PORT = 3000
+DISABLE_SSH = true
+LFS_START_SERVER = false
+OFFLINE_MODE = true
+[security]
+INSTALL_LOCK = true
+SECRET_KEY = {secrets.token_urlsafe(32)}
+[service]
+DISABLE_REGISTRATION = true
+REQUIRE_SIGNIN_VIEW = false
+[mailer]
+ENABLED = false
+[actions]
+ENABLED = false
+[webhook]
+ALLOWED_HOST_LIST = loopback
+[cron]
+ENABLED = false
+[log]
+MODE = console
+LEVEL = Error
+'''
+        ini=staging/'isolated.ini'; ini.write_text(config);ini.chmod(0o600)
+        receipt['stage']='isolated_database'
+        docker('network','create','--internal',network);resources.append(('network',network))
+        env=os.environ.copy();env['POSTGRES_PASSWORD']=password
+        docker('run','-d','--name',db,'--network',network,'--log-driver','none','--memory','1g',
+               '-e','POSTGRES_PASSWORD','-e','POSTGRES_USER=forgejo','-e','POSTGRES_DB=forgejo',POSTGRES,env=env)
+        resources.append(('container',db))
+        for _ in range(30):
+            if docker('exec',db,'pg_isready','-U','forgejo',check=False).returncode==0: break
+            time.sleep(2)
+        else: raise ValueError('database_not_ready')
+        sql=(staging/'forgejo-db.sql').read_bytes()
+        docker('exec','-i',db,'psql','-U','forgejo','-d','forgejo','-v','ON_ERROR_STOP=1',payload=sql)
+        receipt['database_import']=True
+        counts=docker('exec',db,'psql','-U','forgejo','-d','forgejo','-Atqc',
+            'SELECT (SELECT count(*) FROM repository),(SELECT count(*) FROM "user"),(SELECT count(*) FROM package_blob);').stdout.decode().strip()
+        numbers=[int(n) for n in counts.split('|')]
+        if min(numbers[:2]) <= 0: raise ValueError('empty_database')
+        receipt['database_counts']=dict(zip(['repositories','users','package_blobs'],numbers))
+        receipt['stage']='isolated_application'
+        docker('run','-d','--name',app,'--network',network,'--log-driver','none','--memory','1g',
+               '--entrypoint','sleep',FORGE,'7200');resources.append(('container',app))
+        docker('exec',app,'mkdir','-p','/data/gitea','/data/git/repositories')
+        docker('cp',str(staging/'data')+'/.',app+':/data/gitea/')
+        docker('cp',str(staging/'repos')+'/.',app+':/data/git/repositories/')
+        docker('cp',str(ini),app+':/data/isolated.ini')
+        docker('exec',app,'chown','-R','1000:1000','/data')
+        docker('exec','-d','--user','1000:1000',app,'forgejo','--config','/data/isolated.ini','web')
+        for _ in range(60):
+            r=docker('exec',app,'wget','-qO-','http://127.0.0.1:3000/api/healthz',check=False)
+            if r.returncode==0: break
+            time.sleep(2)
+        else: raise ValueError('application_not_ready')
+        receipt['application_health']=True
+        receipt['stage']='repository_recovery'
+        # Select restored public repositories only, keeping payloads out of evidence.
+        rows=docker('exec',db,'psql','-U','forgejo','-d','forgejo','-Atqc',
+            'SELECT u.lower_name || \'/\' || r.lower_name FROM repository r JOIN "user" u ON u.id=r.owner_id WHERE NOT r.is_private AND NOT r.is_empty LIMIT 2;').stdout.decode().splitlines()
+        if not rows: raise ValueError('public_repository_fixture_missing')
+        receipt['repositories_verified']=[]
+        for index,repo in enumerate(rows):
+            if any(c not in 'abcdefghijklmnopqrstuvwxyz0123456789-_./' for c in repo): raise ValueError('unsafe_repository_name')
+            response=docker('exec',app,'wget','-qO-','http://127.0.0.1:3000/api/v1/repos/'+repo).stdout
+            if json.loads(response)['full_name'].lower()!=repo: raise ValueError('repository_metadata_mismatch')
+            target='/tmp/restore-clone-'+str(index)
+            docker('exec','--user','1000:1000',app,'git','clone','--quiet','http://127.0.0.1:3000/'+repo+'.git',target)
+            docker('exec','--user','1000:1000',app,'git','-C',target,'fsck','--full')
+            receipt['repositories_verified'].append(repo)
+        receipt['stage']='package_blob_recovery'
+        # Each stored blob must survive extraction and match its database digest.
+        rows=docker('exec',db,'psql','-U','forgejo','-d','forgejo','-Atqc','SELECT hash_sha256 FROM package_blob;').stdout.decode().splitlines()
+        files=list((staging/'data'/'packages').rglob('*')) if (staging/'data'/'packages').exists() else []
+        available={p.name:p for p in files if p.is_file()}
+        verified=0
+        for digest in rows:
+            matches=[p for name,p in available.items() if digest in name]
+            if len(matches)!=1 or file_digest(matches[0])!=digest: raise ValueError('package_blob_missing_or_corrupt')
+            verified+=1
+        receipt['package_blobs_verified']=verified
+        receipt['status']='restored'
+    finally:
+        receipt['cleanup']=True
+        for kind,name in reversed(resources):
+            args=('rm','-f','-v',name) if kind=='container' else ('network','rm',name)
+            if docker(*args,check=False).returncode: receipt['cleanup']=False
+        shutil.rmtree(staging)
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--archive',required=True,type=Path);p.add_argument('--receipt',required=True)
+    a=p.parse_args();receipt={'schema':'platform.forgejo-isolated-restore.v1','status':'failed','images':[FORGE,POSTGRES]}
+    fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
+    try: run(a.archive,receipt)
+    except Exception: receipt['error']='isolated_restore_failed'
+    finally:
+        with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2)
+    return int(receipt['status']!='restored' or not receipt.get('cleanup'))
+
+if __name__=='__main__': raise SystemExit(main())

scripts/verify_nextcloud_offsite_restore.py

@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
 """Silent attended transfer of a real encrypted backup for isolated recovery."""
 import argparse
+import base64
+import urllib.request
+import shutil
 import hashlib
 import json
 import os
 from pathlib import Path
 import subprocess
-from migrate_nextcloud_backup_account import HOST, OPERATOR_PATH, LANE, request, quota, require
+from migrate_nextcloud_backup_account import HOST, OPERATOR_PATH, LANE, NoRedirect, quota, require
 from state_hub_preflight_lane import bao, data, LaneError
 
 
@@ -21,24 +24,37 @@ def run(source, directory, receipt):
     require(source.stat().st_size < receipt['quota_before']['available_bytes'], 'insufficient_quota')
     name = 'wp0029-recovery-' + source.name
     receipt.update(artifact=name, source_backup=source.name, kv_version=lane['metadata']['version'])
-    payload = source.read_bytes()
-    code, _ = request(values['NC_WEBDAV_URL']+'/forgejo/'+name, 'PUT', payload,
-                      (values['NC_WEBDAV_TOKEN'], ''), {'Content-Type':'application/octet-stream','If-None-Match':'*'})
-    require(code == 201, 'new_offsite_upload_required')
-    receipt['upload_http_status'] = code
-    # Recovery must fetch from the owner path; never decrypt the local cache.
-    code, fetched = request(HOST+'/remote.php/dav/files/Backup/railiance-backups/forgejo/'+name, auth=auth)
-    require(code == 200 and fetched == payload, 'offsite_download_mismatch')
-    receipt.update(download_http_status=code, ciphertext_bytes=len(fetched), ciphertext_sha256=hashlib.sha256(fetched).hexdigest())
     directory.mkdir(mode=0o700)
     encrypted = directory/'fetched.zip.age'
-    encrypted.write_bytes(fetched)
-    encrypted.chmod(0o600)
+    def stream(url, method, credentials, source_file=None, target=None):
+        require(url.startswith(HOST+'/'), 'unapproved_provider_origin')
+        headers={'Authorization':'Basic '+base64.b64encode((credentials[0]+':'+credentials[1]).encode()).decode()}
+        if source_file:
+            headers.update({'Content-Length':str(source.stat().st_size),'Content-Type':'application/octet-stream','If-None-Match':'*'})
+        req=urllib.request.Request(url,data=source_file,method=method,headers=headers)
+        with urllib.request.build_opener(NoRedirect()).open(req,timeout=1200) as response:
+            if target:
+                with target.open('xb') as out:
+                    target.chmod(0o600)
+                    shutil.copyfileobj(response,out,1024*1024)
+            else: response.read()
+            return response.status
+    with source.open('rb') as payload:
+        code=stream(values['NC_WEBDAV_URL']+'/forgejo/'+name,'PUT',
+                    (values['NC_WEBDAV_TOKEN'],''),source_file=payload)
+    require(code == 201, 'new_offsite_upload_required')
+    receipt['upload_http_status']=code
+    code=stream(HOST+'/remote.php/dav/files/Backup/railiance-backups/forgejo/'+name,'GET',auth,target=encrypted)
+    def sha(path):
+        with path.open('rb') as f: return hashlib.file_digest(f,'sha256').hexdigest()
+    digest=sha(encrypted)
+    require(code==200 and digest==sha(source), 'offsite_download_mismatch')
+    receipt.update(download_http_status=code,ciphertext_bytes=encrypted.stat().st_size,ciphertext_sha256=digest)
     plain = directory/'fetched.zip'
     with plain.open('xb') as output:
         plain.chmod(0o600)
         result = subprocess.run(['age','-d','-i','/dev/stdin',str(encrypted)],
-            input=(values['AGE_PRIVATE_KEY'].strip()+'\n').encode(), stdout=output, stderr=subprocess.PIPE, timeout=180)
+            input=(values['AGE_PRIVATE_KEY'].strip()+'\n').encode(), stdout=output, stderr=subprocess.PIPE, timeout=1200)
     require(result.returncode == 0, 'offsite_decryption_failed')
     receipt.update(decrypted=True, quota_after=quota(auth), status='offsite_fetched_pending_isolated_restore')
 

tests/test_offsite_restore_boundaries.py

@@ -0,0 +1,21 @@
+import importlib.util
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+import sys
+sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
+import verify_nextcloud_offsite_restore as transfer
+import restore_forgejo_offsite_locally as restore
+
+class RestoreBoundary(unittest.TestCase):
+    def test_missing_backup_never_fetches_credentials(self):
+        with tempfile.TemporaryDirectory() as d, patch.object(transfer,'bao') as bao:
+            with self.assertRaises(transfer.LaneError): transfer.run(Path(d)/'absent.zip.age',Path(d)/'stage',{})
+            bao.assert_not_called()
+
+    def test_truncated_backup_never_starts_docker(self):
+        with tempfile.TemporaryDirectory() as d, patch.object(restore.subprocess,'run') as command:
+            p=Path(d)/'fetched.zip';p.write_bytes(b'PK\x03\x04truncated')
+            with self.assertRaises(Exception): restore.run(p,{})
+            command.assert_not_called()

workplans/RPF-WP-0029-backup-credential-default-removal.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Remove backup credential default and verify governed replacement"
 domain: financials
 repo: railiance-platform
-status: blocked
+status: active
 owner: codex
 created: "2026-09-05"
 updated: "2026-09-05"
@@ -36,7 +36,7 @@ redirects/non-success status. Added transport containment and failure tests.
 
 ```task
 id: RPF-WP-0029-T02
-status: wait
+status: progress
 priority: high
 state_hub_task_id: "b3f3402f-890b-5781-9b3e-1c9c0d28cea8"
 ```