diff --git a/scripts/capture_forgejo_archive.py b/scripts/capture_forgejo_archive.py index cf83bc3..02f6e77 100644 --- a/scripts/capture_forgejo_archive.py +++ b/scripts/capture_forgejo_archive.py @@ -61,8 +61,10 @@ def capture(namespace, pod, destination, profile="full"): actual = hashlib.file_digest(source,'sha256').hexdigest() if actual != expected: raise ValueError('archive transfer mismatch') validate_archive(destination, profile) - if profile == "essentials" and destination.stat().st_size > 600*1024**2: - raise ValueError("essentials archive exceeds budget") + if profile == "essentials": + from forgejo_essentials_profile import seal + seal(destination) + validate_archive(destination, profile) finally: # Do not remove a file while a timed-out producer may still be writing it. if completed: @@ -76,8 +78,8 @@ def main(): p.add_argument("--profile",choices=["full","essentials"],default="full") a=p.parse_args() try: capture(a.namespace,a.pod,a.output,a.profile) - except Exception: - print('ERROR: Forgejo archive capture or integrity validation failed') + except Exception as error: + print('ERROR: Forgejo archive capture or integrity validation failed ('+type(error).__name__+')') return 1 print('Forgejo archive capture and integrity verified') return 0 diff --git a/scripts/decrypt_primary_forgejo_archive.py b/scripts/decrypt_primary_forgejo_archive.py new file mode 100644 index 0000000..3682bff --- /dev/null +++ b/scripts/decrypt_primary_forgejo_archive.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Silent attended decryption of the hash-verified primary archive download.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +from state_hub_preflight_lane import bao, data + + +def digest(path): + with path.open('rb') as source: return hashlib.file_digest(source,'sha256').hexdigest() + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + for name in ['source','output','transfer-receipt','receipt']: p.add_argument('--'+name,required=True,type=Path) + a=p.parse_args();receipt={'schema':'platform.forgejo-primary-decryption.v1','status':'failed'} + fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + try: + transfer=json.loads(a.transfer_receipt.read_text()) + if (transfer.get('status')!='primary_fetched_pending_application_restore' + or not transfer.get('download_hash_matches') + or digest(a.source)!=transfer.get('ciphertext_sha256')): + raise ValueError('verified_primary_download_required') + value=data(bao(['read','-format=json','platform/data/workloads/railiance/backup/offsite-lane']))['data']['data']['AGE_PRIVATE_KEY'] + with a.output.open('xb') as target: + a.output.chmod(0o600) + result=subprocess.run(['age','-d','-i','/dev/stdin',str(a.source)],input=(value.strip()+'\n').encode(),stdout=target,stderr=subprocess.PIPE,timeout=1200) + if result.returncode: raise ValueError('primary_decryption_failed') + receipt.update(transfer,decrypted=True,plaintext_sha256=digest(a.output)) + except Exception: + receipt['error']='bounded_primary_decryption_failed' + finally: + with os.fdopen(fd,'w') as output: json.dump(receipt,output,indent=2) + return int(receipt['status']=='failed') +if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/forgejo_essentials_profile.py b/scripts/forgejo_essentials_profile.py new file mode 100644 index 0000000..e190d5b --- /dev/null +++ b/scripts/forgejo_essentials_profile.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Seal a measured essentials archive with an explicit recovery contract.""" +from datetime import datetime, timezone +import json +import os +from pathlib import PurePosixPath +import shutil +import stat +import zipfile + +MANIFEST='backup-manifest.json' +OMITTED=('data/packages/','data/repo-archive/','data/indexers/','data/actions_log/','log/') +BUDGET=600*1024**2 + + +def seal(path): + temporary=path.with_name(path.name+'.sealing') + fd=os.open(temporary,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600);os.close(fd) + try: + with zipfile.ZipFile(path) as source, zipfile.ZipFile(temporary,'w') as target: + kept=0 + for member in source.infolist(): + name=PurePosixPath(member.filename) + if name.is_absolute() or '..' in name.parts or stat.S_ISLNK(member.external_attr>>16): + raise ValueError('unsafe_archive_member') + if member.filename==MANIFEST: raise ValueError('archive_already_sealed') + if member.filename.startswith(OMITTED): continue + with source.open(member) as incoming, target.open(member,'w') as outgoing: + shutil.copyfileobj(incoming,outgoing,1024*1024) + kept+=1 + target.writestr(MANIFEST,json.dumps({'schema':'platform.forgejo-backup-profile.v1', + 'profile':'essentials','created_at':datetime.now(timezone.utc).isoformat(), + 'complete_application_backup':False,'omitted_prefixes':list(OMITTED), + 'kept_members':kept,'package_registry_available':False, + 'recovery_contract':'Git and collaboration recovery; packages require full primary or an independent artifact source.', + 'unique_data_policy':'Keep repositories, database, configuration, attachments, LFS and Actions artifacts when present.'},sort_keys=True),compress_type=zipfile.ZIP_DEFLATED) + if temporary.stat().st_size>BUDGET: raise ValueError('essentials_budget_exceeded') + os.replace(temporary,path) + finally: + temporary.unlink(missing_ok=True) diff --git a/scripts/restore_forgejo_offsite_locally.py b/scripts/restore_forgejo_offsite_locally.py index b27a2ee..fe733bc 100644 --- a/scripts/restore_forgejo_offsite_locally.py +++ b/scripts/restore_forgejo_offsite_locally.py @@ -22,8 +22,14 @@ def file_digest(path): with path.open('rb') as f: return hashlib.file_digest(f,'sha256').hexdigest() -def run(archive, receipt): - validate_archive(archive) +def run(archive, receipt, profile="full"): + validate_archive(archive,profile) + if profile == "essentials": + with zipfile.ZipFile(archive) as bundle: + manifest=json.loads(bundle.read("backup-manifest.json")) + if manifest.get("profile")!="essentials" or manifest.get("package_registry_available") is not False: + raise ValueError("essentials_contract_required") + receipt["archive_profile"]=profile prefix='wp0029-'+secrets.token_hex(5) network=prefix+'-net'; db=prefix+'-db'; app=prefix+'-app' staging=archive.parent/(prefix+'-staging') @@ -90,6 +96,8 @@ DISABLE_REGISTRATION = true REQUIRE_SIGNIN_VIEW = false [mailer] ENABLED = false +[packages] +ENABLED = {"false" if profile == "essentials" else "true"} [actions] ENABLED = false [webhook] @@ -150,6 +158,12 @@ LEVEL = Error docker('exec','--user','1000:1000',app,'git','clone','--quiet','http://127.0.0.1:3000/'+repo+'.git',target) docker('exec','--user','1000:1000',app,'git','-C',target,'fsck','--full') receipt['repositories_verified'].append(repo) + if profile == 'essentials': + if any(p.is_file() for p in (staging/'data'/'packages').rglob('*')): + raise ValueError('unexpected_package_payload') + receipt.update(status='restored_essentials',package_registry_available=False, + primary_storage_accessed=False) + return receipt['stage']='package_blob_recovery' # Each stored blob must survive extraction and match its database digest. rows=docker('exec',db,'psql','-U','forgejo','-d','forgejo','-Atqc','SELECT hash_sha256 FROM package_blob;').stdout.decode().splitlines() @@ -173,21 +187,29 @@ LEVEL = Error def main(): p=argparse.ArgumentParser(description=__doc__) p.add_argument('--archive',required=True,type=Path);p.add_argument('--receipt',required=True) + p.add_argument('--profile',choices=['full','essentials'],default='full') p.add_argument('--transfer-receipt',required=True,type=Path) a=p.parse_args();receipt={'schema':'platform.forgejo-isolated-restore.v1','status':'failed','images':[FORGE,POSTGRES]} fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) try: transfer=json.loads(a.transfer_receipt.read_text()) - if (transfer.get('status') != 'offsite_fetched_pending_isolated_restore' - or transfer.get('download_http_status') != 200 + nextcloud = (transfer.get('status') == 'offsite_fetched_pending_isolated_restore' + and transfer.get('download_http_status') == 200) + primary = (transfer.get('status') == 'primary_fetched_pending_application_restore' + and transfer.get('download_hash_matches') is True and transfer.get('decrypted') is True) + if (not (nextcloud or primary) or transfer.get('plaintext_sha256') != file_digest(a.archive)): raise ValueError('verified_offsite_provenance_required') - receipt['offsite_artifact']=transfer['artifact'] + receipt['source_provider']='Scaleway' if primary else 'Nextcloud' + receipt['offsite_artifact']=transfer['destination'] if primary else transfer['artifact'] receipt['ciphertext_sha256']=transfer['ciphertext_sha256'] - run(a.archive,receipt) - except Exception: receipt['error']='isolated_restore_failed' + if transfer.get('archive_profile','full')!=a.profile: raise ValueError('profile_provenance_mismatch') + run(a.archive,receipt,a.profile) + except Exception: + receipt['status']='failed' + receipt['error']='isolated_restore_failed' finally: with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2) - return int(receipt['status']!='restored' or not receipt.get('cleanup')) + return int(receipt['status'] not in ('restored','restored_essentials') or not receipt.get('cleanup')) if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/scaleway_forgejo_archive.py b/scripts/scaleway_forgejo_archive.py index e0b3d8a..6f88f4d 100644 --- a/scripts/scaleway_forgejo_archive.py +++ b/scripts/scaleway_forgejo_archive.py @@ -74,16 +74,20 @@ def main(): fd=os.open(a.receipt,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600);os.close(fd) def checkpoint(): a.receipt.write_text(json.dumps(receipt,indent=2)+'\n') try: + receipt['stage']='source_validation';checkpoint() source=json.loads(a.source_receipt.read_text()) if (source.get('status')!='offsite_fetched_pending_isolated_restore' or not a.source.name.endswith('.zip.age') or a.output.exists() or not 0