net-kingdom/tools/bootstrap-recovery/capture-identity.py

#!/usr/bin/env python3
"""Owner-run identity capture: metadata plan first, exact bounded execution later.

Use the admitted Railiance caller. Flags bind reviewed source/targets; they do
not grant authority. Private child output and provider diagnostics never print.
"""
import argparse
from datetime import datetime,timezone
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import signal
import subprocess
import tempfile
import uuid

HERE=Path(__file__).resolve().parent


def load(name):
    spec=importlib.util.spec_from_file_location(name,HERE/(name+'.py'))
    module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module);return module


bundle=load('identity-bundle');prepare=load('prepare-identity-coverage')
IMAGE='ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473'
SOURCES=['capture-identity.py','identity-files.py','identity-bundle.py','prepare-identity-coverage.py','identity-backup-profile.json']
KUBE=['kubectl']


class Refused(Exception):pass


def command(args,payload=None,timeout=60):
    result=subprocess.run(KUBE+['--request-timeout=30s']+args,input=payload,
                          stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=timeout)
    if result.returncode:raise Refused()
    return result.stdout


def get(ns,kind,name):
    scope=['-n',ns] if ns else []
    return json.loads(command(scope+['get',kind,name,'-o','json']))


def selects(selector,labels):
    if any(labels.get(k)!=v for k,v in selector.get('matchLabels',{}).items()):return False
    for expression in selector.get('matchExpressions',[]):
        key=expression['key'];op=expression['operator'];values=expression.get('values',[])
        if op=='In' and labels.get(key) not in values:return False
        if op=='NotIn' and key in labels and labels[key] in values:return False
        if op=='Exists' and key not in labels:return False
        if op=='DoesNotExist' and key in labels:return False
        if op not in {'In','NotIn','Exists','DoesNotExist'}:raise Refused()
    return True


def network_review(run_id,actual_labels=None):
    rows=[]
    for ns in ('sso','mfa'):
        policies=json.loads(command(['-n',ns,'get','networkpolicies','-o','json']))['items']
        for obj in policies:
            m=obj['metadata'];s=obj['spec']
            labels=(actual_labels or {}).get(ns,{'net-kingdom/capture':run_id})
            if selects(s['podSelector'],labels) and (s.get('ingress') or s.get('egress')):
                rows.append({'namespace':ns,'name':m['name'],'uid':m['uid'],
                    'spec_sha256':hashlib.sha256(json.dumps(s,sort_keys=True).encode()).hexdigest()})
    return rows


def validate_helper(actual,wanted):
    a=actual['spec'];w=wanted['spec']
    if (any(a.get(k) for k in ('hostNetwork','hostPID','hostIPC','initContainers','ephemeralContainers'))
        or a.get('automountServiceAccountToken') is not False or a.get('activeDeadlineSeconds')!=900
        or len(a['containers'])!=1 or a['volumes']!=w['volumes']):raise Refused()
    c=a['containers'][0];expected=w['containers'][0]
    if any(c.get(k)!=expected[k] for k in ('name','image','command','volumeMounts')):raise Refused()
    if c.get('env') or c.get('envFrom') or c.get('securityContext',{}).get('privileged'):raise Refused()
    if any(c.get('securityContext',{}).get(k)!=v for k,v in expected['securityContext'].items()):raise Refused()
    if any(a.get('securityContext',{}).get(k)!=v for k,v in w['securityContext'].items()):raise Refused()


def manifests(run_id):
    result=[]
    for ns,claims in [('sso',[('lldap','lldap-data'),('authelia','authelia-data'),('keycape','keycape-authentication-policy')]),
                      ('mfa',[('privacyidea','privacyidea-data')])]:
        label={'net-kingdom/capture':run_id};name='nk49-capture-'+run_id
        result.append({'apiVersion':'networking.k8s.io/v1','kind':'NetworkPolicy',
            'metadata':{'name':name,'namespace':ns},
            'spec':{'podSelector':{'matchLabels':label},'policyTypes':['Ingress','Egress'],'ingress':[],'egress':[]}})
        result.append({'apiVersion':'v1','kind':'Pod','metadata':{'name':name,'namespace':ns,'labels':label},
            'spec':{'restartPolicy':'Never','activeDeadlineSeconds':900,'automountServiceAccountToken':False,
                'securityContext':{'runAsUser':0,'runAsGroup':0},
                'containers':[{'name':'capture','image':IMAGE,'imagePullPolicy':'IfNotPresent',
                    'command':['python3','-c','import time; time.sleep(900)'],
                    'securityContext':{'allowPrivilegeEscalation':False,'readOnlyRootFilesystem':True,
                        'capabilities':{'drop':['ALL'],'add':['DAC_OVERRIDE']}},
                    'resources':{'requests':{'cpu':'50m','memory':'128Mi'},'limits':{'cpu':'1','memory':'2Gi'}},
                    'volumeMounts':[{'name':key,'mountPath':'/source/'+key,'readOnly':True} for key,_ in claims]+
                        [{'name':'scratch','mountPath':'/scratch'}]}],
                'volumes':[{'name':key,'persistentVolumeClaim':{'claimName':claim,'readOnly':True}} for key,claim in claims]+
                    [{'name':'scratch','emptyDir':{'medium':'Memory','sizeLimit':'512Mi'}}]}})
    return result


def plan():
    p=bundle.profile();binding=prepare.prepare(p,get=get)
    cluster=get('databases','cluster','net-kingdom-pg')
    primary=cluster['status']['currentPrimary'];pod=get('databases','pod',primary)
    if (not any(o.get('uid')==cluster['metadata']['uid'] for o in pod['metadata'].get('ownerReferences',[]))
        or not any(c['type']=='Ready' and c['status']=='True' for c in pod['status'].get('conditions',[]))):raise Refused()
    run_id=uuid.uuid4().hex[:12]
    return {'schema':'netkingdom.identity-capture-plan.v1','status':'requires_owner_execution_admission',
            'created_at':datetime.now(timezone.utc).isoformat(),'run_id':run_id,'binding':binding,'kubeconfig':KUBE[2],
            'sources':[{'path':name,'sha256':hashlib.sha256((HERE/name).read_bytes()).hexdigest()} for name in SOURCES],
            'database':{'namespace':'databases','cluster':'net-kingdom-pg','cluster_uid':cluster['metadata']['uid'],
                'pod':primary,'pod_uid':pod['metadata']['uid'],'database':'privacyidea_db'},
            'create_resources':manifests(run_id),'service_changes':False,'credential_grants':False,
            'matching_network_allow_policies':network_review(run_id),
            'network_policy_enforcement_requires_owner_confirmation':True,
            'offsite_delivery_included':False,'global_atomic_snapshot_claimed':False}


def checked_plan(path,expected):
    raw=path.read_bytes()
    if hashlib.sha256(raw).hexdigest()!=expected:raise Refused()
    p=json.loads(raw)
    if p.get('schema')!='netkingdom.identity-capture-plan.v1':raise Refused()
    if p.get('kubeconfig')!=KUBE[2]:raise Refused()
    if network_review(p['run_id']):raise Refused()
    if p['sources']!=[{'path':name,'sha256':hashlib.sha256((HERE/name).read_bytes()).hexdigest()} for name in SOURCES]:raise Refused()
    if p['create_resources']!=manifests(p['run_id']):raise Refused()
    now=datetime.now(timezone.utc)
    if not 0<=(now-datetime.fromisoformat(p['created_at'])).total_seconds()<3600:raise Refused()
    # Recheck the exact immutable source/target bindings before private reads.
    current=prepare.prepare(bundle.profile(),get=get)
    if current['bindings']!=p['binding']['bindings'] or current['profile_sha256']!=p['binding']['profile_sha256']:raise Refused()
    return p


def delete_created(obj):
    m=obj['metadata'];ns=m['namespace'];name=m['name'];uid=m['uid']
    if obj['kind']=='Pod':path=f'/api/v1/namespaces/{ns}/pods/{name}'
    elif obj['kind']=='NetworkPolicy':path=f'/apis/networking.k8s.io/v1/namespaces/{ns}/networkpolicies/{name}'
    else:raise Refused()
    payload=json.dumps({'apiVersion':'v1','kind':'DeleteOptions','preconditions':{'uid':uid},'gracePeriodSeconds':1}).encode()
    command(['delete','--raw',path,'-f','-'],payload)
    if obj['kind']=='Pod':command(['-n',ns,'wait','--for=delete','pod/'+name,'--timeout=90s'],timeout=100)


def bounded_file(args,path,limit):
    # Stream binary dump to private staging, with a byte bound and suppressed
    # diagnostics. API command argv contains only public resource/database names.
    proc=subprocess.Popen(KUBE+['--request-timeout=600s']+args,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL)
    try:
        count=0
        with path.open('xb') as out:
            while block:=proc.stdout.read(1024*1024):
                count+=len(block)
                if count>limit:raise Refused()
                out.write(block)
        if proc.wait(timeout=600):raise Refused()
    finally:
        if proc.poll() is None:proc.kill();proc.wait()


def capture(p,output,receipt,decision,checkpoint=lambda:None):
    created=[];receipt.update(status='capture_started',decision_ref=decision,plan_run_id=p['run_id'],
        created_resources=[],unverified_create_outcomes=[],helper_cleanup_verified=False,temporary_plaintext_cleanup_verified=False)
    checkpoint()
    with tempfile.TemporaryDirectory(prefix='nk49-identity-',dir='/tmp') as temporary:
        receipt['temporary_plaintext_directory']=temporary
        checkpoint()
        root=Path(temporary)/'payload';root.mkdir(mode=0o700)
        held={}
        try:
            for c in bundle.profile()['components']:
                for kind,names in [('deployment',[c['deployment']]),('secret',c['secret_refs']),('configmap',c['configmap_refs'])]:
                    for name in names:
                        obj=get(c['namespace'],kind,name)
                        # Keep reconstructable deployed resources inside encryption,
                        # without unrelated stale last-applied annotations.
                        m=obj['metadata'];obj['metadata']={k:m[k] for k in ['name','namespace','uid','resourceVersion']}
                        obj.pop('status',None)
                        raw=json.dumps(obj,sort_keys=True).encode();slot=f"resources/{c['namespace']}/{kind}/{name}.json"
                        held[(c['namespace'],kind,name)]=(m['uid'],m['resourceVersion'])
                        bundle.save(root,slot,raw)
            for proposed in p['create_resources']:
                # create, never adopt/apply an existing resource. Unknown create
                # outcomes stop; only returned exact UIDs are eligible for cleanup.
                pending={'kind':proposed['kind'],**proposed['metadata']}
                receipt['unverified_create_outcomes'].append(pending)
                checkpoint()
                obj=json.loads(command(['create','-f','-','-o','json'],json.dumps(proposed).encode()))
                if (obj['kind']!=proposed['kind'] or obj['metadata']['name']!=proposed['metadata']['name']
                    or obj['metadata']['namespace']!=proposed['metadata']['namespace'] or not obj['metadata'].get('uid')):raise Refused()
                created.append(obj)
                receipt['unverified_create_outcomes'].remove(pending)
                receipt['created_resources'].append({'namespace':obj['metadata']['namespace'],
                    'kind':obj['kind'],'name':obj['metadata']['name'],'uid':obj['metadata']['uid']})
                checkpoint()
            for ns in ('sso','mfa'):
                name='nk49-capture-'+p['run_id']
                command(['-n',ns,'wait','--for=condition=Ready','pod/'+name,'--timeout=180s'],timeout=190)
                live=get(ns,'pod',name)
                expected=next(o for o in created if o['kind']=='Pod' and o['metadata']['namespace']==ns)
                if live['metadata']['uid']!=expected['metadata']['uid']:raise Refused()
                wanted=next(o for o in p['create_resources'] if o['kind']=='Pod' and o['metadata']['namespace']==ns)
                validate_helper(live,wanted)
                if network_review(p['run_id'],{ns:live['metadata']['labels']}):raise Refused()
                raw=command(['-n',ns,'exec','-i',name,'-c','capture','--','python3','-c',
                    (HERE/'identity-files.py').read_text(),ns],timeout=600)
                if len(raw)>bundle.LIMIT+2*1024*1024:raise Refused()
                bundle.import_files(raw,root,ns)
                if get(ns,'pod',name)['metadata']['uid']!=expected['metadata']['uid']:raise Refused()
            db=p['database'];current=get('databases','cluster','net-kingdom-pg')
            if current['metadata']['uid']!=db['cluster_uid'] or current['status']['currentPrimary']!=db['pod']:raise Refused()
            if get('databases','pod',db['pod'])['metadata']['uid']!=db['pod_uid']:raise Refused()
            (root/'databases').mkdir(mode=0o700)
            bounded_file(['-n','databases','exec',db['pod'],'-c','postgres','--','pg_dump','-U','postgres',
                '-d','privacyidea_db','-Fc','--no-owner','--no-acl'],root/'databases/privacyidea.dump',bundle.FILE_LIMIT)
            if get('databases','pod',db['pod'])['metadata']['uid']!=db['pod_uid']:raise Refused()
            for (ns,kind,name),binding in held.items():
                m=get(ns,kind,name)['metadata']
                if (m['uid'],m['resourceVersion'])!=binding:raise Refused()
            # Re-read original keys after the database dump. No rotation is
            # inferred from file presence; a changed key set refuses publication.
            raw=command(['-n','mfa','exec','-i','nk49-capture-'+p['run_id'],'-c','capture','--','python3','-c',
                (HERE/'identity-files.py').read_text(),'mfa'],timeout=60)
            with tempfile.TemporaryDirectory(prefix='nk49-key-recheck-',dir='/tmp') as keytemp:
                again=Path(keytemp);bundle.import_files(raw,again,'mfa')
                for name in ('enckey','private.pem','public.pem'):
                    if (root/'data/privacyidea'/name).read_bytes()!=(again/'data/privacyidea'/name).read_bytes():raise Refused()
            receipt.update(bundle.seal(root,output,{'started_at':receipt['started_at'],
                'finished_at':datetime.now(timezone.utc).isoformat(),'database':db,'decision_ref':decision,
                'cluster_uid':prepare.CLUSTER_UID,'global_atomic_snapshot_claimed':False,
                'omissions':['historical local backups','SQLite WAL/SHM/journal represented by native snapshots',
                    'Authelia notification.txt','privacyIDEA service logs; audit data retained in database',
                    'OpenBao root/unseal/age private key custody remains independently held']}))
        finally:
            failures=[]
            for obj in reversed(created):
                try:delete_created(obj)
                except Exception:failures.append(obj['metadata']['uid'])
            receipt['helper_cleanup_verified']=not failures and not receipt['unverified_create_outcomes']
            checkpoint()
            if not receipt['helper_cleanup_verified']:raise Refused()
    receipt['temporary_plaintext_cleanup_verified']=not Path(temporary).exists()
    checkpoint()
    if not receipt['temporary_plaintext_cleanup_verified']:raise Refused()


def main():
    global KUBE
    parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('action',choices=['plan','capture'])
    parser.add_argument('--kubeconfig',required=True)
    parser.add_argument('--receipt',required=True);parser.add_argument('--plan');parser.add_argument('--plan-sha256')
    parser.add_argument('--output');parser.add_argument('--decision-id');a=parser.parse_args();os.umask(0o077)
    KUBE=['kubectl','--kubeconfig',str(Path(a.kubeconfig).absolute())]
    fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
    os.close(fd)
    r={'status':'failed','started_at':datetime.now(timezone.utc).isoformat(),'secret_values_exported':False}
    def checkpoint():
        fd,temp=tempfile.mkstemp(prefix='.capture-receipt-',dir=Path(a.receipt).absolute().parent)
        try:
            with os.fdopen(fd,'w') as out:json.dump(r,out,indent=2);out.write('\n');out.flush();os.fsync(out.fileno())
            os.replace(temp,a.receipt)
        finally:Path(temp).unlink(missing_ok=True)
    code=1
    try:
        if a.action=='plan':
            if any([a.plan,a.plan_sha256,a.output,a.decision_id]):raise Refused()
            r=plan();code=0
        else:
            if not all([a.plan,a.plan_sha256,a.output,a.decision_id]):raise Refused()
            uuid.UUID(a.decision_id)
            p=checked_plan(Path(a.plan),a.plan_sha256)
            capture(p,Path(a.output),r,a.decision_id,checkpoint);code=0
    except Exception:
        r.update(status='failed',error='capture_refused_or_failed_private_diagnostics_suppressed')
        if r.get('temporary_plaintext_directory') and not Path(r['temporary_plaintext_directory']).exists():r['temporary_plaintext_cleanup_verified']=True
    finally:
        r['finished_at']=datetime.now(timezone.utc).isoformat()
        checkpoint()
    print('OK: identity '+a.action+' complete; see safe receipt.' if code==0 else 'ERROR: identity operation failed; inspect safe receipt before retry.')
    return code


def interrupted(*args):raise Refused()


if __name__=='__main__':
    for sig in (signal.SIGHUP,signal.SIGTERM,signal.SIGINT):signal.signal(sig,interrupted)
    raise SystemExit(main())