coulomb/railiance-platform
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Extend isolated Scaleway recovery checks to Forgejo database metadata
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
Details
CI Smoke / container-smoke (push) Successful in 1s
Details

Browse source 
Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883
This commit is contained in:
codex 2026-09-06 00:51:55 +02:00
parent 54819fb3f4
commit be45c4e3fd
1 changed files with 23 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

38
scripts/verify_scaleway_primary_restore.py
View file

@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Bounded apps-pg recovery from the existing Scaleway primary; no in-place restore."""
"""Bounded shared/Forgejo database recovery from the existing Scaleway primary; no in-place restore."""
import argparse
import copy
from datetime import datetime, timezone
@ -12,19 +12,20 @@ import time
from state_hub_preflight_lane import assert_cluster
def run(k, receipt):
def run(k, receipt, source_cluster="apps-pg"):
if source_cluster not in ("apps-pg", "forgejo-db"): raise ValueError("unsupported source cluster")
def cmd(args,payload=None,check=True):
r=subprocess.run(k+args,input=None if payload is None else json.dumps(payload).encode(),capture_output=True,timeout=180)
if check and r.returncode: raise RuntimeError('bounded_restore_command_failed')
return r
def get(args): return json.loads(cmd(args).stdout)
assert_cluster(k)
live=get(['-n','databases','get','cluster','apps-pg','-o','json'])
live=get(['-n','databases','get','cluster',source_cluster,'-o','json'])
store=copy.deepcopy(live['spec']['backup']['barmanObjectStore'])
if store['destinationPath']!='s3://railiance-platform-pg-backup/platform-pg/apps-pg/' or store['endpointURL']!='https://s3.nl-ams.scw.cloud': raise RuntimeError('unexpected_primary_destination')
if store['destinationPath']!='s3://railiance-platform-pg-backup/platform-pg/'+source_cluster+'/' or store['endpointURL']!='https://s3.nl-ams.scw.cloud': raise RuntimeError('unexpected_primary_destination')
recent=datetime.fromisoformat(live['status']['lastSuccessfulBackup'].replace('Z','+00:00'))
if not 0 <= (datetime.now(timezone.utc)-recent).total_seconds() < 129600: raise RuntimeError('primary_backup_not_fresh')
name='apps-pg-recovery-'+secrets.token_hex(4)
name=source_cluster+'-recovery-'+secrets.token_hex(4)
receipt.update(namespace=name,primary_destination=store['destinationPath'],last_successful_backup=live['status']['lastSuccessfulBackup'],source='Scaleway Barman base backup and WAL',started_at=datetime.now(timezone.utc).isoformat())
created=False
try:
@ -38,8 +39,8 @@ def run(k, receipt):
cmd(['create','-f','-'],doc)
policy={'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy','metadata':{'name':'isolated-recovery','namespace':name},'spec':{'podSelector':{},'policyTypes':['Ingress','Egress'],'ingress':[{'from':[{'namespaceSelector':{'matchLabels':{'kubernetes.io/metadata.name':'cnpg-system'}}}],'ports':[{'port':8000,'protocol':'TCP'}]}],'egress':[{'ports':[{'port':p,'protocol':proto} for p,proto in [(443,'TCP'),(6443,'TCP'),(53,'TCP'),(53,'UDP')]]}]}}
cmd(['create','-f','-'],policy)
store['serverName']='apps-pg'
cluster={'apiVersion':'postgresql.cnpg.io/v1','kind':'Cluster','metadata':{'name':name,'namespace':name},'spec':{'instances':1,'imageName':live['spec']['imageName'],'enableSuperuserAccess':False,'storage':{'size':'10Gi'},'resources':{'requests':{'cpu':'100m','memory':'256Mi'},'limits':{'cpu':'1','memory':'1Gi'}},'bootstrap':{'recovery':{'source':'apps-pg'}},'externalClusters':[{'name':'apps-pg','barmanObjectStore':store}]}}
store['serverName']=source_cluster
cluster={'apiVersion':'postgresql.cnpg.io/v1','kind':'Cluster','metadata':{'name':name,'namespace':name},'spec':{'instances':1,'imageName':live['spec']['imageName'],'enableSuperuserAccess':False,'storage':{'size':'10Gi'},'resources':{'requests':{'cpu':'100m','memory':'256Mi'},'limits':{'cpu':'1','memory':'1Gi'}},'bootstrap':{'recovery':{'source':source_cluster}},'externalClusters':[{'name':source_cluster,'barmanObjectStore':store}]}}
receipt['stage']='physical_restore'
cmd(['create','--dry-run=server','-f','-'],cluster)
started=time.monotonic();cmd(['create','-f','-'],cluster)
@ -53,13 +54,20 @@ def run(k, receipt):
def sql(database,query): return cmd(['-n',name,'exec',pod,'-c','postgres','--','psql','-U','postgres','-d',database,'-Atqc',query]).stdout.decode().strip()
receipt['stage']='database_acceptance'
databases=sql('postgres',"SELECT datname FROM pg_database WHERE NOT datistemplate ORDER BY datname;").splitlines()
if not {'apps_meta','coulomb_social_db','vergabe_db'}.issubset(databases): raise RuntimeError('restored_database_missing')
receipt['databases']=databases
receipt['public_table_counts']={db:int(sql(db,"SELECT count(*) FROM pg_tables WHERE schemaname='public';")) for db in ['coulomb_social_db','vergabe_db']}
roles=sql('postgres',"SELECT rolname || ':' || rolconnlimit FROM pg_roles WHERE rolname IN ('coulomb_social','vergabe') ORDER BY rolname;").splitlines()
if roles!=['coulomb_social:20','vergabe:20']: raise RuntimeError('restored_role_boundary_mismatch')
receipt['consumer_connection_limits_preserved']=True
live_after=get(['-n','databases','get','cluster','apps-pg','-o','json'])
if source_cluster=='apps-pg':
if not {'apps_meta','coulomb_social_db','vergabe_db'}.issubset(databases): raise RuntimeError('restored_database_missing')
receipt['public_table_counts']={db:int(sql(db,"SELECT count(*) FROM pg_tables WHERE schemaname='public';")) for db in ['coulomb_social_db','vergabe_db']}
roles=sql('postgres',"SELECT rolname || ':' || rolconnlimit FROM pg_roles WHERE rolname IN ('coulomb_social','vergabe') ORDER BY rolname;").splitlines()
if roles!=['coulomb_social:20','vergabe:20']: raise RuntimeError('restored_role_boundary_mismatch')
receipt['consumer_connection_limits_preserved']=True
else:
if 'forgejo' not in databases: raise RuntimeError('restored_database_missing')
receipt['repository_count']=int(sql('forgejo','SELECT count(*) FROM repository;'))
receipt['user_count']=int(sql('forgejo','SELECT count(*) FROM "user";'))
receipt['package_blob_count']=int(sql('forgejo','SELECT count(*) FROM package_blob;'))
if min(receipt['repository_count'],receipt['user_count'])<=0: raise RuntimeError('restored_forgejo_metadata_empty')
live_after=get(['-n','databases','get','cluster',source_cluster,'-o','json'])
receipt['production_ready']=any(c['type']=='Ready' and c['status']=='True' for c in live_after['status']['conditions'])
if not receipt['production_ready']: raise RuntimeError('production_not_ready')
receipt['status']='verified'
@ -70,10 +78,10 @@ def run(k, receipt):
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--kubeconfig',required=True);p.add_argument('--receipt',required=True);a=p.parse_args()
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--kubeconfig',required=True);p.add_argument('--receipt',required=True);p.add_argument('--source-cluster',choices=['apps-pg','forgejo-db'],default='apps-pg');a=p.parse_args()
receipt={'schema':'platform.scaleway-primary-restore.v1','status':'failed'}
fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
try: run(['kubectl','--kubeconfig',a.kubeconfig],receipt)
try: run(['kubectl','--kubeconfig',a.kubeconfig],receipt,a.source_cluster)
except Exception: receipt['error']='primary_restore_failed'
finally:
with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2)