Verify full and essentials recovery and implement bounded retention tooling

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883

scripts/backup_retention_plan.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Plan bounded essentials retention; never delete provider objects."""
+import argparse
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import re
+
+NAME=re.compile(r'^forgejo-essentials-(\d{8}T\d{6}Z)\.zip\.age$')
+DIGEST=re.compile(r'^[0-9a-f]{64}$')
+RESERVE=2*1024**3
+MAX_INCOMING=600*1024**2
+
+
+def plan(inventory, protected, incoming_bytes, available_bytes, now=None):
+    now=now or datetime.now(timezone.utc)
+    if not protected or not 0<incoming_bytes<=MAX_INCOMING or available_bytes<0:
+        raise ValueError('invalid_retention_contract')
+    items={};ignored=[]
+    for item in inventory:
+        name=item['name'];match=NAME.fullmatch(name)
+        if not match or item.get('verified') is not True:
+            ignored.append(name);continue
+        stamp=datetime.strptime(match[1],'%Y%m%dT%H%M%SZ').replace(tzinfo=timezone.utc)
+        etag=item.get('etag','')
+        if (name in items or stamp>now or not DIGEST.fullmatch(item.get('sha256',''))
+                or not etag.startswith('"') or not etag.endswith('"')
+                or any(c in etag for c in '\r\n') or not 0<item.get('bytes',0)<=MAX_INCOMING):
+            raise ValueError('unverified_object_identity')
+        items[name]=(stamp,item)
+    if not set(protected)<=items.keys(): raise ValueError('protected_recovery_point_missing')
+    ordered=sorted(items,key=lambda name:items[name][0],reverse=True)
+    daily={};weekly={}
+    for name in ordered:
+        stamp=items[name][0]
+        daily.setdefault(stamp.date(),name)
+        weekly.setdefault(stamp.isocalendar()[:2],name)
+    keep=set(protected)|set(list(daily.values())[:7])|set(list(weekly.values())[:2])
+    keep.add(ordered[0])
+    candidates=[{'name':name,'etag':items[name][1]['etag'],'bytes':items[name][1]['bytes']}
+                for name in ordered if name not in keep]
+    return {'schema':'platform.essentials-retention-plan.v1','status':'ready' if available_bytes>=incoming_bytes+RESERVE else 'insufficient_upload_headroom',
+        'keep':sorted(keep),'protected':sorted(protected),'delete_candidates':candidates,'ignored':sorted(ignored),
+        'incoming_bytes':incoming_bytes,'available_bytes':available_bytes,'reserved_bytes':RESERVE,
+        'upload_before_expiration':True,'provider_mutations':False,
+        'execution_gate':'Separate owner executor must revalidate ETags, protected recovery receipts and completed replacement before expiration.'}
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__);p.add_argument('--inventory',required=True,type=Path)
+    p.add_argument('--protected',required=True,action='append');p.add_argument('--incoming-bytes',required=True,type=int)
+    p.add_argument('--available-bytes',required=True,type=int);a=p.parse_args()
+    result=plan(json.loads(a.inventory.read_text()),a.protected,a.incoming_bytes,a.available_bytes)
+    print(json.dumps(result,indent=2));return int(result['status']!='ready')
+if __name__=='__main__': raise SystemExit(main())

scripts/execute_nextcloud_retention.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Attended owner expiration of reviewed essentials candidates; no workload deletion grant."""
+import argparse
+import base64
+import json
+import hashlib
+import fcntl
+import os
+from pathlib import Path
+import urllib.request
+from backup_retention_plan import NAME, RESERVE
+from migrate_nextcloud_backup_account import HOST, OPERATOR_PATH, NoRedirect, quota, require
+from state_hub_preflight_lane import bao,data
+
+
+def validate(plan, restore):
+    require(plan.get('schema')=='platform.essentials-retention-plan.v1' and plan.get('status')=='ready','ready_plan_required')
+    keep=set(plan['keep']); protected=set(plan['protected'])
+    require(bool(protected) and protected<=keep,'protected_anchor_required')
+    require(restore.get('status')=='restored_essentials' and restore.get('cleanup') is True
+            and restore.get('offsite_artifact') in protected,'verified_recovery_anchor_required')
+    candidates=plan['delete_candidates']; names=[item['name'] for item in candidates]
+    require(len(names)==len(set(names)) and not keep.intersection(names),'ambiguous_expiration_plan')
+    for name in list(keep)+names: require(NAME.fullmatch(name) is not None,'managed_essentials_name_required')
+    for item in candidates:
+        etag=item['etag']
+        require(etag.startswith('"') and etag.endswith('"') and not any(c in etag for c in '\r\n'),'strong_etag_required')
+    return candidates
+
+
+def execute(plan,restore,receipt):
+    candidates=validate(plan,restore)
+    if not candidates:
+        receipt['status']='no_expiration_needed';return
+    entry=data(bao(['read','-format=json',OPERATOR_PATH]))['data']['data']
+    require(entry['BACKUP_USERNAME']=='Backup','backup_owner_required')
+    auth=(entry['BACKUP_USERNAME'],entry['BACKUP_PASSWORD'])
+    headers={'Authorization':'Basic '+base64.b64encode((auth[0]+':'+auth[1]).encode()).decode()}
+    opener=urllib.request.build_opener(NoRedirect())
+    root=HOST+'/remote.php/dav/files/Backup/railiance-backups/forgejo/'
+    receipt['quota_before']=quota(auth)
+    require(receipt['quota_before']['available_bytes']>=RESERVE,'headroom_recheck_failed')
+    # Verify every retained recovery point is present before any expiration.
+    for name in plan['keep']:
+        with opener.open(urllib.request.Request(root+name,method='HEAD',headers=headers),timeout=60) as response:
+            require(response.status==200 and int(response.headers['Content-Length'])>0,'retained_point_missing')
+    # Revalidate the exact recovered ciphertext, not merely a nonempty name.
+    with opener.open(urllib.request.Request(root+restore['offsite_artifact'],headers=headers),timeout=60) as response:
+        require(0<int(response.headers['Content-Length'])<=600*1024**2,'anchor_size_outside_budget')
+        digest=hashlib.sha256()
+        while block:=response.read(1024*1024): digest.update(block)
+    require(digest.hexdigest()==restore['ciphertext_sha256'],'recovery_anchor_changed')
+    receipt['removed']=[]
+    for item in candidates:
+        with opener.open(urllib.request.Request(root+item['name'],method='HEAD',headers=headers),timeout=60) as response:
+            require(response.headers.get('ETag')==item['etag'] and int(response.headers['Content-Length'])==item['bytes'],'candidate_changed')
+        conditional=dict(headers,**{'If-Match':item['etag']})
+        with opener.open(urllib.request.Request(root+item['name'],method='DELETE',headers=conditional),timeout=60) as response:
+            require(response.status==204,'conditional_expiration_failed')
+        receipt['removed'].append(item['name'])
+    receipt.update(status='expired',quota_after=quota(auth))
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--plan',required=True,type=Path);p.add_argument('--restore-receipt',required=True,type=Path)
+    p.add_argument('--receipt',required=True);p.add_argument('--apply',action='store_true')
+    args=p.parse_args();result={'schema':'platform.essentials-retention-execution.v1','status':'failed'}
+    fd=os.open(args.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
+    try:
+        plan=json.loads(args.plan.read_text());restore=json.loads(args.restore_receipt.read_text())
+        candidates=validate(plan,restore)
+        if args.apply:
+            lock=os.open('/tmp/railiance-platform-essentials-retention-'+str(os.getuid())+'.lock',
+                         os.O_WRONLY|os.O_CREAT|os.O_NOFOLLOW,0o600)
+            try:
+                require(os.fstat(lock).st_uid==os.getuid(),'retention_lock_owner_mismatch')
+                fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+                execute(plan,restore,result)
+            finally: os.close(lock)
+        else: result.update(status='preview',candidate_count=len(candidates),provider_mutations=False)
+    except Exception: result['error']='bounded_essentials_expiration_refused'
+    finally:
+        with os.fdopen(fd,'w') as output: json.dump(result,output,indent=2)
+    return int(result['status']=='failed')
+if __name__=='__main__': raise SystemExit(main())