net-kingdom/tools/bootstrap-recovery/identity-runtime.py

#!/usr/bin/env python3
"""Root-owned Railiance backup job. No operator password or age private key.

Publisher and retention are separate processes with separate scoped delivery.
Scheduled execution is gated by an independently recovered archive receipt.
"""
import argparse
import base64
from datetime import datetime,timezone
import fcntl
import importlib.util
import json
import os
from pathlib import Path
import signal
import shutil
import socket
import tempfile
import urllib.request

HERE=Path(__file__).resolve().parent


def load(name):
    spec=importlib.util.spec_from_file_location(name,HERE/(name+'.py'))
    m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m);return m


capture=load('capture-identity');offsite=load('identity-offsite');retention=load('identity-retention')


def write(path,value):
    fd,name=tempfile.mkstemp(prefix='.identity-receipt-',dir=path.parent)
    try:
        with os.fdopen(fd,'w') as out:json.dump(value,out,indent=2);out.write('\n');out.flush();os.fsync(out.fileno())
        os.replace(name,path)
    finally:Path(name).unlink(missing_ok=True)


def token(action):
    name='net-kingdom-backup-'+action
    es=capture.get('sso','externalsecret',name)
    entries=[]
    for entry in es['spec'].get('data',[]):
        item=dict(entry);ref=dict(item.get('remoteRef',{}))
        for key,default in [('conversionStrategy','Default'),('decodingStrategy','None'),('metadataPolicy','None')]:
            if ref.pop(key,default)!=default:raise offsite.Refused()
        item['remoteRef']=ref;entries.append(item)
    if (es['spec']['target']['name']!=name or entries!=[{'secretKey':'token','remoteRef':{
        'key':'workloads/net-kingdom/identity-backup/'+action,'property':'NC_WEBDAV_TOKEN'}}]
        or es['spec'].get('dataFrom') or es['spec'].get('secretStoreRef')!={'kind':'SecretStore','name':name}
        or not any(c['type']=='Ready' and c['status']=='True' for c in es['status'].get('conditions',[]))):raise offsite.Refused()
    secret=capture.get('sso','secret',name)
    if (set(secret['data'])!={'token'} or not any(o['uid']==es['metadata']['uid'] for o in secret['metadata'].get('ownerReferences',[]))):raise offsite.Refused()
    value=base64.b64decode(secret['data']['token'],validate=True).decode();offsite.headers(value)
    return value


def accepted(path):
    capture.bundle.private(path)
    r=json.loads(path.read_text())
    if (r.get('schema')!='netkingdom.identity-recovery.v1' or r.get('status')!='identity_archive_verified'
        or r.get('all_manifest_hashes_verified') is not True or r.get('container_cleanup_verified') is not True
        or r.get('temporary_plaintext_cleanup_verified') is not True or not offsite.NAME.fullmatch(r.get('artifact',''))):raise offsite.Refused()
    return r


def post_event(state,summary):
    # Non-secret durable reporting: replay retained notes at the next run.
    directory=state/'event-outbox';directory.mkdir(mode=0o700,exist_ok=True)
    import uuid
    path=directory/(uuid.uuid4().hex+'.json')
    write(path,{'summary':summary,'event_type':'note','author':'net-kingdom-identity-backup',
        'workplan_id':'17d315f0-829e-57c7-85cc-3a7914152804','task_id':'d68cd4d3-1ba3-530e-88f9-26ebb8dd6eac'})
    for pending in sorted(directory.glob('*.json'))[:64]:
        request=urllib.request.Request('http://10.43.68.154:8000/progress/',data=pending.read_bytes(),
            headers={'Content-Type':'application/json'},method='POST')
        try:
            with urllib.request.build_opener(urllib.request.ProxyHandler({})).open(request,timeout=10) as response:
                if response.status!=201:break
            pending.unlink()
        except Exception:break


def backup(state,decision,first=False):
    if not first:accepted(state/'accepted-recovery.json')
    if (state/'capture-in-progress.json').exists():raise offsite.Refused()
    publisher=token('publisher')
    client=offsite.opener()
    if offsite.quota(client,publisher)<offsite.HEADROOM+offsite.LIMIT:raise offsite.Refused()
    if any(shutil.disk_usage(path).free<2*offsite.LIMIT+offsite.HEADROOM for path in [state,Path('/tmp')]):raise offsite.Refused()
    p=capture.plan();stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
    name='net-kingdom-identity-'+stamp+'-'+p['run_id']+'.tar.age'
    directory=state/'runs'/name;directory.mkdir(mode=0o700,parents=True)
    plan_path=directory/'plan.json';write(plan_path,p)
    import hashlib
    capture.checked_plan(plan_path,hashlib.sha256(plan_path.read_bytes()).hexdigest())
    source=directory/name;r={'started_at':datetime.now(timezone.utc).isoformat(),'status':'capture_started'}
    record=directory/'capture.json';write(record,r)
    write(state/'capture-in-progress.json',{'receipt':str(record),'artifact':name})
    try:capture.capture(p,source,r,decision,lambda:write(record,r))
    except Exception:
        if r.get('temporary_plaintext_directory') and not Path(r['temporary_plaintext_directory']).exists():r['temporary_plaintext_cleanup_verified']=True
        r['status']='failed';write(record,r);raise
    finally:
        if r.get('helper_cleanup_verified') and r.get('temporary_plaintext_cleanup_verified'):
            (state/'capture-in-progress.json').unlink()
    write(record,r)
    r['finished_at']=datetime.now(timezone.utc).isoformat();write(record,r)
    write(directory/'transfer-attempt.json',{'status':'pending','artifact':name,'capture_receipt':'capture.json',
        'ciphertext_sha256':r['ciphertext_sha256'],'ciphertext_bytes':r['ciphertext_bytes']})
    remote=offsite.publish(client,publisher,source,r,directory/'fetched.tar.age')
    write(directory/'offsite.json',remote)
    # The original private local ciphertext and safe receipts remain. Every
    # successful fetched duplicate is unnecessary after first acceptance and
    # may be removed by the owner under a separate local-cache policy.
    write(state/'last-success.json',{'finished_at':datetime.now(timezone.utc).isoformat(),
        'artifact':name,'directory':str(directory),'ciphertext_sha256':remote['ciphertext_sha256'],
        'offsite_verified':True})
    post_event(state,'Identity backup completed: encrypted full coverage captured, Storage Share upload and downloaded digest verified. Native service authentication remains separate evidence.')


def expire(state,execute=False):
    if (state/'retention-blocked.json').exists():raise offsite.Refused()
    proof=accepted(state/'accepted-recovery.json')
    r={'status':'pending','started_at':datetime.now(timezone.utc).isoformat()}
    stamp=datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%S%fZ')
    record=state/('retention-'+stamp+'.json')
    write(record,r)
    try:
        retention.run(offsite.opener(),token('retention'),proof,r,execute,lambda:write(record,r))
        if execute:r['local_generations_removed']=prune_local(state,proof)
    except Exception:
        r['status']='failed';write(record,r)
        # Stop future automatic expiration after an unknown/partial outcome.
        # Owner review must explicitly clear this marker; no blind retries.
        if execute:write(state/'retention-blocked.json',{'receipt':str(record),'requires_owner_reconciliation':True})
        raise
    write(record,r)


def prune_local(state,proof):
    root=state/'runs';rows=[];known={}
    if not root.exists():return []
    capture.bundle.private(root,True)
    for directory in root.iterdir():
        if directory.is_symlink() or not directory.is_dir() or not offsite.NAME.fullmatch(directory.name):continue
        capture.bundle.private(directory,True)
        allowed={'plan.json','capture.json','transfer-attempt.json','offsite.json','fetched.tar.age',directory.name}
        if {p.name for p in directory.iterdir()}-allowed:continue
        try:
            remote=json.loads((directory/'offsite.json').read_text());local=json.loads((directory/'capture.json').read_text())
            if (remote.get('status')!='readback_verified' or remote.get('verified_index_published') is not True
                or remote.get('artifact')!=directory.name or remote.get('ciphertext_sha256')!=local.get('ciphertext_sha256')
                or local.get('helper_cleanup_verified') is not True or local.get('temporary_plaintext_cleanup_verified') is not True):continue
            for path in directory.iterdir():capture.bundle.private(path,max_bytes=offsite.LIMIT)
            if offsite.digest(directory/directory.name)!=remote['ciphertext_sha256']:continue
            stamp=datetime.strptime(directory.name.removeprefix('net-kingdom-identity-')[:16],'%Y%m%dT%H%M%SZ').replace(tzinfo=timezone.utc)
            rows.append({'artifact':directory.name,'stamp':stamp});known[directory.name]=directory
        except Exception:continue
    rows.sort(key=lambda r:r['stamp'],reverse=True)
    if not rows:return []
    protected=proof['artifact'] if proof['artifact'] in known else rows[0]['artifact']
    candidates,_=retention.select(rows,protected)
    removed=[]
    for row in candidates:
        directory=known[row['artifact']]
        for path in directory.iterdir():path.unlink()
        directory.rmdir();removed.append(row['artifact'])
    return removed


def main():
    p=argparse.ArgumentParser(description=__doc__)
    p.add_argument('action',choices=['first-capture','scheduled','retention-preview','retention','status'])
    p.add_argument('--kubeconfig',required=True);p.add_argument('--state',default='/var/lib/net-kingdom-backup')
    p.add_argument('--decision-id');a=p.parse_args();os.umask(0o077)
    state=Path(a.state).absolute()
    ready=False
    try:
        capture.bundle.require_runtime_host()
        capture.KUBE=['kubectl','--kubeconfig',a.kubeconfig]
        uid=capture.get(None,'namespace','kube-system')['metadata']['uid']
        if uid!=capture.prepare.CLUSTER_UID:raise offsite.Refused()
        state.mkdir(mode=0o700,parents=True,exist_ok=True);capture.bundle.private(state,True);ready=True
        fd=None
        try:
            if a.action!='status':
                fd=os.open(state/'.lock',os.O_RDWR|os.O_CREAT|os.O_NOFOLLOW,0o600)
                fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)
            if a.action in {'first-capture','scheduled'}:
                import uuid
                uuid.UUID(a.decision_id or '')
                backup(state,a.decision_id,a.action=='first-capture')
            elif a.action.startswith('retention'):expire(state,a.action=='retention')
            else:
                last=json.loads((state/'last-success.json').read_text())
                age=(datetime.now(timezone.utc)-datetime.fromisoformat(last['finished_at'])).total_seconds()
                if not 0<=age<=25*3600:raise offsite.Refused()
        finally:
            if fd is not None:os.close(fd)
    except Exception:
        if ready:
            try:post_event(state,'Identity backup '+a.action+' failed or refused. Private diagnostics suppressed; inspect root-owned receipts and reconcile partial state before retry.')
            except Exception:pass
        print('ERROR: identity backup operation failed; inspect private owner receipts. No secret diagnostics emitted.');return 1
    print('OK: identity backup '+a.action+' completed.');return 0


def interrupted(*args):raise offsite.Refused()


if __name__=='__main__':
    for sig in (signal.SIGHUP,signal.SIGTERM,signal.SIGINT):signal.signal(sig,interrupted)
    raise SystemExit(main())