diff --git a/scripts/capture_forgejo_archive.py b/scripts/capture_forgejo_archive.py index 742cb51..cf83bc3 100644 --- a/scripts/capture_forgejo_archive.py +++ b/scripts/capture_forgejo_archive.py @@ -9,18 +9,24 @@ import time import zipfile -def validate_archive(path): +ESSENTIALS_FLAGS = ['--skip-package-data', '--skip-log', '--skip-index', '--skip-repo-archives'] + + +def validate_archive(path, profile="full"): with zipfile.ZipFile(path) as archive: if sum(i.file_size for i in archive.infolist()) > 20*1024**3: raise ValueError('archive exceeds recovery size bound') names = set(archive.namelist()) if 'forgejo-db.sql' not in names or not any(n.startswith('repos/') for n in names): raise ValueError('required archive content missing') + if profile == 'essentials' and any(n.startswith(('data/packages/', 'data/repo-archive/', 'data/indexers/')) and not n.endswith('/') for n in names): + raise ValueError('excluded data present in essentials archive') if archive.testzip() is not None: raise ValueError('archive checksum failure') -def capture(namespace, pod, destination): +def capture(namespace, pod, destination, profile="full"): + if profile not in ("full", "essentials"): raise ValueError("unknown archive profile") k = ['kubectl', 'exec', '--request-timeout=120s', '-n', namespace, pod, '-c', 'gitea', '--'] def call(args, timeout=150): r = subprocess.run(k + args, capture_output=True, timeout=timeout) @@ -30,7 +36,8 @@ def capture(namespace, pod, destination): completed = False try: # Completion belongs to this exact process, not a global pgrep or file existence. - command = f'umask 077; forgejo dump -f {remote}.zip >{remote}.log 2>&1; result=$?; printf "%s" "$result" >{remote}.exit' + flags = ' '.join(ESSENTIALS_FLAGS) if profile == 'essentials' else '' + command = f'umask 077; forgejo dump {flags} -f {remote}.zip >{remote}.log 2>&1; result=$?; printf "%s" "$result" >{remote}.exit' call(['sh','-c', 'nohup sh -c "$1" >/dev/null 2>&1 600*1024**2: + raise ValueError("essentials archive exceeds budget") finally: # Do not remove a file while a timed-out producer may still be writing it. if completed: @@ -64,8 +73,9 @@ def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--namespace',required=True); p.add_argument('--pod',required=True) p.add_argument('--output',required=True,type=Path) + p.add_argument("--profile",choices=["full","essentials"],default="full") a=p.parse_args() - try: capture(a.namespace,a.pod,a.output) + try: capture(a.namespace,a.pod,a.output,a.profile) except Exception: print('ERROR: Forgejo archive capture or integrity validation failed') return 1 diff --git a/scripts/scaleway_forgejo_archive.py b/scripts/scaleway_forgejo_archive.py new file mode 100644 index 0000000..e0b3d8a --- /dev/null +++ b/scripts/scaleway_forgejo_archive.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Bounded primary archive multipart PUT/GET using existing backup custody.""" +import argparse +import base64 +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import secrets +import subprocess +from state_hub_preflight_lane import assert_cluster + +ENDPOINT='https://s3.nl-ams.scw.cloud' +BUCKET='railiance-platform-pg-backup' +PREFIX='platform-pg/application-archives/forgejo/' +PART_SIZE=64*1024**2 + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream,'sha256').hexdigest() + + +def transfer(client, source, output, receipt, checkpoint=lambda: None): + expected=digest(source) + key=PREFIX+datetime.now(timezone.utc).strftime('%Y/%m/%d/%H%M%S-')+secrets.token_hex(12)+'.zip.age' + receipt.update(destination='s3://'+BUCKET+'/'+key, ciphertext_sha256=expected, + ciphertext_bytes=source.stat().st_size, stage='multipart_upload', uploaded_bytes=0) + checkpoint() + upload=None + try: + upload=client.create_multipart_upload(Bucket=BUCKET,Key=key,ContentType='application/octet-stream',Metadata={'sha256':expected})['UploadId'] + parts=[] + with source.open('rb') as stream: + while block:=stream.read(PART_SIZE): + number=len(parts)+1 + result=client.upload_part(Bucket=BUCKET,Key=key,UploadId=upload,PartNumber=number,Body=block) + parts.append({'PartNumber':number,'ETag':result['ETag']}) + receipt['uploaded_bytes']+=len(block);checkpoint() + completed=client.complete_multipart_upload(Bucket=BUCKET,Key=key,UploadId=upload,MultipartUpload={'Parts':parts}) + upload=None + version=completed.get('VersionId') + receipt.update(stage='primary_download',multipart_completed=True,version_pinned=bool(version),downloaded_bytes=0) + checkpoint() + args={'Bucket':BUCKET,'Key':key} + if version: args['VersionId']=version + response=client.get_object(**args) + if response['ContentLength']!=source.stat().st_size: raise ValueError('primary_length_mismatch') + with response['Body'] as body, output.open('xb') as target: + output.chmod(0o600) + while block:=body.read(8*1024**2): + target.write(block);receipt['downloaded_bytes']+=len(block);checkpoint() + if digest(output)!=expected: raise ValueError('primary_digest_mismatch') + receipt.update(status='primary_fetched_pending_application_restore',stage='transfer_verified',download_hash_matches=True) + checkpoint() + finally: + if upload: + try: + client.abort_multipart_upload(Bucket=BUCKET,Key=key,UploadId=upload) + receipt['multipart_aborted']=True + except Exception: + receipt['multipart_aborted']=False + checkpoint() + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--source',required=True,type=Path);p.add_argument('--source-receipt',required=True,type=Path) + p.add_argument('--output',required=True,type=Path);p.add_argument('--receipt',required=True,type=Path) + p.add_argument('--kubeconfig',required=True) + a=p.parse_args() + receipt={'schema':'platform.forgejo-primary-archive.v1','status':'running'} + fd=os.open(a.receipt,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600);os.close(fd) + def checkpoint(): a.receipt.write_text(json.dumps(receipt,indent=2)+'\n') + try: + source=json.loads(a.source_receipt.read_text()) + if (source.get('status')!='offsite_fetched_pending_isolated_restore' + or not a.source.name.endswith('.zip.age') or a.output.exists() + or not 0