From 9269d9d8f49e6d28bc0513543c727476770a5c7c Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 6 Sep 2026 00:29:35 +0200 Subject: [PATCH] Prepare bounded recovery verification from the Scaleway primary Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883 --- scripts/verify_scaleway_primary_restore.py | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 scripts/verify_scaleway_primary_restore.py diff --git a/scripts/verify_scaleway_primary_restore.py b/scripts/verify_scaleway_primary_restore.py new file mode 100644 index 0000000..d094629 --- /dev/null +++ b/scripts/verify_scaleway_primary_restore.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Bounded apps-pg recovery from the existing Scaleway primary; no in-place restore.""" +import argparse +import copy +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import secrets +import subprocess +import time +from state_hub_preflight_lane import assert_cluster + + +def run(k, receipt): + def cmd(args,payload=None,check=True): + r=subprocess.run(k+args,input=None if payload is None else json.dumps(payload).encode(),capture_output=True,timeout=180) + if check and r.returncode: raise RuntimeError('bounded_restore_command_failed') + return r + def get(args): return json.loads(cmd(args).stdout) + assert_cluster(k) + live=get(['-n','databases','get','cluster','apps-pg','-o','json']) + store=copy.deepcopy(live['spec']['backup']['barmanObjectStore']) + if store['destinationPath']!='s3://railiance-platform-pg-backup/platform-pg/apps-pg/' or store['endpointURL']!='https://s3.nl-ams.scw.cloud': raise RuntimeError('unexpected_primary_destination') + recent=datetime.fromisoformat(live['status']['lastSuccessfulBackup'].replace('Z','+00:00')) + if not 0 <= (datetime.now(timezone.utc)-recent).total_seconds() < 129600: raise RuntimeError('primary_backup_not_fresh') + name='apps-pg-recovery-'+secrets.token_hex(4) + receipt.update(namespace=name,primary_destination=store['destinationPath'],last_successful_backup=live['status']['lastSuccessfulBackup'],source='Scaleway Barman base backup and WAL',started_at=datetime.now(timezone.utc).isoformat()) + created=False + try: + cmd(['create','namespace',name]);created=True + receipt['stage']='credential_projection' + # Exact existing S3 fields only; capture JSON in memory and never print it. + for secret_name in {store['s3Credentials'][key]['name'] for key in ('accessKeyId','secretAccessKey')}: + source=get(['-n','databases','get','secret',secret_name,'-o','json']) + fields={store['s3Credentials'][key]['key'] for key in ('accessKeyId','secretAccessKey') if store['s3Credentials'][key]['name']==secret_name} + doc={'apiVersion':'v1','kind':'Secret','metadata':{'name':secret_name,'namespace':name},'type':'Opaque','data':{f:source['data'][f] for f in fields}} + cmd(['create','-f','-'],doc) + policy={'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy','metadata':{'name':'isolated-recovery','namespace':name},'spec':{'podSelector':{},'policyTypes':['Ingress','Egress'],'ingress':[{'from':[{'namespaceSelector':{'matchLabels':{'kubernetes.io/metadata.name':'cnpg-system'}}}],'ports':[{'port':8000,'protocol':'TCP'}]}],'egress':[{'ports':[{'port':p,'protocol':proto} for p,proto in [(443,'TCP'),(6443,'TCP'),(53,'TCP'),(53,'UDP')]]}]}} + cmd(['create','-f','-'],policy) + store['serverName']='apps-pg' + cluster={'apiVersion':'postgresql.cnpg.io/v1','kind':'Cluster','metadata':{'name':name,'namespace':name},'spec':{'instances':1,'imageName':live['spec']['imageName'],'enableSuperuserAccess':False,'storage':{'size':'10Gi'},'resources':{'requests':{'cpu':'100m','memory':'256Mi'},'limits':{'cpu':'1','memory':'1Gi'}},'bootstrap':{'recovery':{'source':'apps-pg'}},'externalClusters':[{'name':'apps-pg','barmanObjectStore':store}]}} + receipt['stage']='physical_restore' + cmd(['create','--dry-run=server','-f','-'],cluster) + started=time.monotonic();cmd(['create','-f','-'],cluster) + for _ in range(60): + state=get(['-n',name,'get','cluster',name,'-o','json']) + if any(c['type']=='Ready' and c['status']=='True' for c in state.get('status',{}).get('conditions',[])): break + time.sleep(5) + else: raise RuntimeError('scratch_restore_not_ready') + receipt['restore_seconds']=round(time.monotonic()-started,2) + pod=state['status']['currentPrimary'] + def sql(database,query): return cmd(['-n',name,'exec',pod,'-c','postgres','--','psql','-U','postgres','-d',database,'-Atqc',query]).stdout.decode().strip() + receipt['stage']='database_acceptance' + databases=sql('postgres',"SELECT datname FROM pg_database WHERE NOT datistemplate ORDER BY datname;").splitlines() + if not {'apps_meta','coulomb_social_db','vergabe_db'}.issubset(databases): raise RuntimeError('restored_database_missing') + receipt['databases']=databases + receipt['public_table_counts']={db:int(sql(db,"SELECT count(*) FROM pg_tables WHERE schemaname='public';")) for db in ['coulomb_social_db','vergabe_db']} + roles=sql('postgres',"SELECT rolname || ':' || rolconnlimit FROM pg_roles WHERE rolname IN ('coulomb_social','vergabe') ORDER BY rolname;").splitlines() + if roles!=['coulomb_social:20','vergabe:20']: raise RuntimeError('restored_role_boundary_mismatch') + receipt['consumer_connection_limits_preserved']=True + live_after=get(['-n','databases','get','cluster','apps-pg','-o','json']) + receipt['production_ready']=any(c['type']=='Ready' and c['status']=='True' for c in live_after['status']['conditions']) + if not receipt['production_ready']: raise RuntimeError('production_not_ready') + receipt['status']='verified' + finally: + if created: + receipt['cleanup']=cmd(['delete','namespace',name,'--wait=true','--timeout=120s'],check=False).returncode==0 + receipt['finished_at']=datetime.now(timezone.utc).isoformat() + + +def main(): + p=argparse.ArgumentParser(description=__doc__);p.add_argument('--kubeconfig',required=True);p.add_argument('--receipt',required=True);a=p.parse_args() + receipt={'schema':'platform.scaleway-primary-restore.v1','status':'failed'} + fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) + try: run(['kubectl','--kubeconfig',a.kubeconfig],receipt) + except Exception: receipt['error']='primary_restore_failed' + finally: + with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2) + return int(receipt['status']!='verified' or not receipt.get('cleanup')) +if __name__=='__main__': raise SystemExit(main())