473 lines
30 KiB
Diff
473 lines
30 KiB
Diff
--- r1/execute-install.py
|
|
+++ r2/execute-install.py
|
|
@@ -14,13 +14,13 @@
|
|
|
|
HERE=Path(__file__).resolve().parent
|
|
DIGEST='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
|
-STAGE='/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017'
|
|
+STAGE='/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017'
|
|
STORE='/home/tegwick/.helixforge-factory/runtimes'
|
|
SOURCE=Path('/tmp/netkingdom-runtime-canonical-20261016/runtime')
|
|
|
|
-READ_TARGET=r'''import json,os,pwd,stat,subprocess,shutil,time
|
|
+READ_TARGET=r'''import json,os,pwd,stat,subprocess,shutil,time,ctypes,gzip
|
|
from pathlib import Path
|
|
-store=Path('/home/tegwick/.helixforge-factory/runtimes'); stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');digest='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
|
+store=Path('/home/tegwick/.helixforge-factory/runtimes'); stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017');digest='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
|
def meta(p):
|
|
try:
|
|
s=p.lstat();return {'uid':s.st_uid,'gid':s.st_gid,'mode':oct(stat.S_IMODE(s.st_mode)),'device':s.st_dev,'inode':s.st_ino,'directory':stat.S_ISDIR(s.st_mode),'symlink':stat.S_ISLNK(s.st_mode)}
|
|
@@ -31,7 +31,11 @@
|
|
mem=dict(x.split(':',1) for x in Path('/proc/meminfo').read_text().splitlines())
|
|
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
|
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];busy=1-(d[3]+d[4])/sum(d)
|
|
-print(json.dumps({'account':pwd.getpwuid(os.getuid()).pw_name,'uid':os.getuid(),'gid':os.getgid(),'store':meta(store),'destination':meta(store/digest),'stage':meta(stage),'memory_available_bytes':int(mem['MemAvailable'].split()[0])*1024,'disk_free_bytes':shutil.disk_usage(store).free,'load_average':list(os.getloadavg()),'cpu_count':os.cpu_count(),'cpu_busy_fraction':busy,'cpu_sample_seconds':1,'node_addresses':nodes[0]['status']['addresses'],'node_ready':any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions'])}))'''
|
|
+help_result=subprocess.run(['/usr/bin/bwrap','--help'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=10)
|
|
+version=subprocess.run(['/usr/bin/bwrap','--version'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=10)
|
|
+manual=gzip.decompress(Path('/usr/share/man/man1/bwrap.1.gz').read_bytes()).decode();i=manual.find('die\\-with\\-parent')
|
|
+fd=os.pidfd_open(os.getpid());os.close(fd);value=ctypes.c_int();subreaper_get=ctypes.CDLL(None).prctl(37,ctypes.byref(value),0,0,0)
|
|
+print(json.dumps({'bwrap_version':version.stdout.decode().strip(),'bwrap_die_with_parent_supported':help_result.returncode==0 and b'--die-with-parent' in help_result.stdout,'primary_manual_contract_excerpt':manual[max(0,i-20):i+430] if i>=0 else 'absent','pidfd_available':True,'subreaper_get_supported':subreaper_get==0,'account':pwd.getpwuid(os.getuid()).pw_name,'uid':os.getuid(),'gid':os.getgid(),'store':meta(store),'destination':meta(store/digest),'stage':meta(stage),'memory_available_bytes':int(mem['MemAvailable'].split()[0])*1024,'disk_free_bytes':shutil.disk_usage(store).free,'load_average':list(os.getloadavg()),'cpu_count':os.cpu_count(),'cpu_busy_fraction':busy,'cpu_sample_seconds':1,'node_addresses':nodes[0]['status']['addresses'],'node_ready':any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions'])}))'''
|
|
|
|
|
|
def sha(path):
|
|
@@ -65,6 +69,8 @@
|
|
store=observation['store']
|
|
expected=json.loads((HERE/'target-readonly-observation.json').read_text())['paths'][STORE]
|
|
checks={
|
|
+ 'bwrap_parent_death_contract':observation['bwrap_die_with_parent_supported'] and observation['bwrap_version']=='bubblewrap 0.9.0' and 'PR_SET_PDEATHSIG' in observation['primary_manual_contract_excerpt'],
|
|
+ 'supervision_primitives':observation['pidfd_available'] and observation['subreaper_get_supported'],
|
|
'account':observation['account']=='tegwick' and observation['uid']==1000 and observation['gid']==1000,
|
|
'protected_store':store['uid']==1000 and store['gid']==1000 and store['mode']=='0o700' and store['directory'] and not store['symlink'] and store['device']==expected['device'] and store['inode']==expected['inode'],
|
|
'destination_absent':observation['destination'] is None,
|
|
@@ -91,7 +97,7 @@
|
|
if args.apply and args.dry_run:raise ValueError('conflicting_modes')
|
|
inputs,observation=preflight()
|
|
if not args.apply:
|
|
- report={'status':'read_only_dry_run_passed' if all(observation['checks'].values()) else 'read_only_dry_run_blocked','proposal_id':inputs['proposal_id'],'revision':1,
|
|
+ report={'status':'read_only_dry_run_passed' if all(observation['checks'].values()) else 'read_only_dry_run_blocked','proposal_id':inputs['proposal_id'],'revision':2,
|
|
'observed_at':datetime.now(timezone.utc).isoformat(),'target':observation,
|
|
'source_runtime_complete_digest_verified':True,'archive':inputs['archive'],
|
|
'remote_mutations':0,'provider_calls':0,'private_configuration_reads':False,
|
|
@@ -100,10 +106,10 @@
|
|
if not all(observation['checks'].values()):raise ValueError('target_headroom_or_identity_refused')
|
|
receipt=HERE/'execution-receipt.json'
|
|
if receipt.exists():raise ValueError('prior_attempt_requires_reconciliation')
|
|
- save(receipt,{'status':'intent_recorded','proposal_id':inputs['proposal_id'],'revision':1,
|
|
+ save(receipt,{'status':'intent_recorded','proposal_id':inputs['proposal_id'],'revision':2,
|
|
'automatic_retry':False,'observed_at':datetime.now(timezone.utc).isoformat()})
|
|
# No native attempt is retried by this helper. Intent survives an SSH loss.
|
|
- with tempfile.TemporaryDirectory(prefix='nk-runtime-install-r1-') as temporary:
|
|
+ with tempfile.TemporaryDirectory(prefix='nk-runtime-install-r2-') as temporary:
|
|
archive=Path(temporary)/'runtime.tar.gz'
|
|
with archive.open('xb') as stream:built=module('reviewed_transfer',HERE/'build-transfer.py').build(stream)
|
|
if built!=inputs['archive']:raise ValueError('archive_changed')
|
|
@@ -116,14 +122,14 @@
|
|
assert not (store/'f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d').exists()
|
|
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
|
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];assert 1-(d[3]+d[4])/sum(d)<=0.85
|
|
-stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');stage.mkdir(mode=0o700)
|
|
+stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017');stage.mkdir(mode=0o700)
|
|
s=stage.lstat();print(json.dumps({'inode':s.st_ino,'device':s.st_dev}))'''
|
|
created=ssh(create)
|
|
- receipt.write_text(json.dumps({'status':'staging_created','phase':'before_transfer','proposal_id':inputs['proposal_id'],'revision':1,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
- command(['scp','-q',str(archive),str(HERE/'runtime.py'),str(HERE/'install-verified.py'),'railiance01:'+STAGE+'/'],120)
|
|
+ receipt.write_text(json.dumps({'status':'staging_created','phase':'before_transfer','proposal_id':inputs['proposal_id'],'revision':2,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
+ command(['scp','-q',str(archive),str(HERE/'runtime.py'),str(HERE/'install-verified.py'),str(HERE/'process_supervision.py'),'railiance01:'+STAGE+'/'],120)
|
|
execute=r'''import hashlib,json,os,stat,sys
|
|
from pathlib import Path
|
|
-stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');s=stage.lstat()
|
|
+stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017');s=stage.lstat()
|
|
assert stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_ino==int(sys.argv[1]) and s.st_dev==2049
|
|
expected=json.loads(sys.argv[2]);assert {p.name for p in stage.iterdir()}==set(expected)
|
|
for name,want in expected.items():
|
|
@@ -133,10 +139,11 @@
|
|
os.execve('/usr/bin/python3',['python3','-I','-B',str(stage/'install-verified.py'),'--stage-inode',sys.argv[1]],{'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'})'''
|
|
hashes={'runtime.tar.gz':inputs['archive']['sha256'].removeprefix('sha256:'),
|
|
'runtime.py':inputs['helper_hashes']['runtime.py'].removeprefix('sha256:'),
|
|
- 'install-verified.py':inputs['helper_hashes']['install-verified.py'].removeprefix('sha256:')}
|
|
+ 'install-verified.py':inputs['helper_hashes']['install-verified.py'].removeprefix('sha256:'),
|
|
+ 'process_supervision.py':inputs['helper_hashes']['process_supervision.py'].removeprefix('sha256:')}
|
|
# Parse the structured final helper return even when its action failed.
|
|
remote='/usr/bin/python3 -I -B -c '+shlex.quote(execute)+' '+shlex.quote(str(created['inode']))+' '+shlex.quote(json.dumps(hashes,separators=(',',':')))
|
|
- receipt.write_text(json.dumps({'status':'remote_execution_started','proposal_id':inputs['proposal_id'],'revision':1,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
+ receipt.write_text(json.dumps({'status':'remote_execution_started','proposal_id':inputs['proposal_id'],'revision':2,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
cp=subprocess.run(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01',remote],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=600)
|
|
try:result=json.loads(cp.stdout)
|
|
except Exception:raise RuntimeError('remote_outcome_uncertain') from None
|
|
--- r1/install-verified.py
|
|
+++ r2/install-verified.py
|
|
@@ -20,7 +20,7 @@
|
|
RUNTIME_MODULE = '19b5fb687288bbc836978b0251599dc1e4d1593e8b5efee85a6abb6a7f388412'
|
|
STORE_MODULE = 'c3d6ec09d08881611d3d02b8f55d68deed6ecfedaa1a809b6b11c57cc4ca97cf'
|
|
REHEARSAL_MODULE = '192f654cad4a5f065c7faa5e35b03493297a472e8d7fe641be2c42c5fb00dcc3'
|
|
-STAGE = Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017')
|
|
+STAGE = Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017')
|
|
STORE = Path('/home/tegwick/.helixforge-factory/runtimes')
|
|
DESTINATION = STORE/DIGEST
|
|
LOGICAL_BYTES = 260668241
|
|
@@ -46,14 +46,14 @@
|
|
return s
|
|
|
|
|
|
-def identify():
|
|
+def identify(supervisor):
|
|
require(os.getuid()==1000 and os.getgid()==1000 and pwd.getpwuid(1000).pw_name=='tegwick','account_identity')
|
|
for p in (Path('/home/tegwick'),Path('/home/tegwick/.helixforge-factory'),STORE):
|
|
protected(p,directory=True,mode=0o700 if p!=Path('/home/tegwick') else 0o750)
|
|
protected(STORE/'.install.lock',mode=0o600)
|
|
- cp=subprocess.run(['/usr/local/bin/kubectl','get','nodes','-o','json'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=20,env=ENV)
|
|
- require(cp.returncode==0,'node_readback')
|
|
- nodes=json.loads(cp.stdout)['items']
|
|
+ code,output=supervisor.run(['/usr/local/bin/kubectl','get','nodes','-o','json'],20,capture=True)
|
|
+ require(code==0,'node_readback')
|
|
+ nodes=json.loads(output)['items']
|
|
require(len(nodes)==1 and any(a['address']=='92.205.62.239' for a in nodes[0]['status']['addresses'])
|
|
and any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions']),'node_identity')
|
|
require(shutil.disk_usage(STORE).free >= 2*1024**3,'disk_headroom')
|
|
@@ -133,12 +133,18 @@
|
|
print(json.dumps({'status':'installed','result':result},sort_keys=True))
|
|
|
|
|
|
-def rehearse_phase():
|
|
+def rehearse_phase(supervision):
|
|
from rein_aharness import owner_rehearsal
|
|
require(Path(owner_rehearsal.__file__).resolve().is_relative_to(DESTINATION)
|
|
and file_digest(Path(owner_rehearsal.__file__))==REHEARSAL_MODULE,'rehearsal_source_pin')
|
|
+ require(owner_rehearsal.SYSTEM_RO_BINDS==('/usr','/bin','/lib','/lib64','/etc/resolv.conf','/etc/ssl/certs')
|
|
+ and all(Path(p).exists() for p in owner_rehearsal.SYSTEM_RO_BINDS),'frozen_system_bind_contract')
|
|
signal.signal(signal.SIGALRM,expired);signal.alarm(240)
|
|
- try:steps=owner_rehearsal.rehearse(DESTINATION)
|
|
+ # Keep the frozen owner launch contract; use its supported Runner seam.
|
|
+ def bounded_run(argv,timeout):
|
|
+ require(argv[0]=='/usr/bin/bwrap' and '--die-with-parent' not in argv,'bwrap_runner_contract')
|
|
+ return supervision_request(supervision,[argv[0],'--die-with-parent',*argv[1:]],timeout)[0]
|
|
+ try:steps=owner_rehearsal.rehearse(DESTINATION,run=bounded_run,bwrap_bin='/usr/bin/bwrap')
|
|
finally:signal.alarm(0)
|
|
require([x.name for x in steps]==['rein-aharness','glas-harness','claude'],'rehearsal_steps')
|
|
report={x.name:x.exit_code for x in steps}
|
|
@@ -147,43 +153,55 @@
|
|
return 0 if all(x.ok for x in steps) else 1
|
|
|
|
|
|
-def phase(argv, timeout):
|
|
- p=subprocess.run(argv,env=ENV,stdin=subprocess.DEVNULL,stdout=subprocess.PIPE,
|
|
- stderr=subprocess.DEVNULL,timeout=timeout)
|
|
- require(len(p.stdout)<=65536,'phase_output_bound')
|
|
- try:result=json.loads(p.stdout)
|
|
+def supervision_module():
|
|
+ spec=importlib.util.spec_from_file_location('approved_process_supervision',STAGE/'process_supervision.py')
|
|
+ module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
|
|
+ return module
|
|
+
|
|
+
|
|
+def supervision_request(directory,argv,timeout):
|
|
+ return supervision_module().request(directory,argv,timeout)
|
|
+
|
|
+
|
|
+def phase(supervisor, argv, timeout):
|
|
+ code,output=supervisor.run(argv,timeout,capture=True)
|
|
+ try:result=json.loads(output)
|
|
except Exception:raise ValueError('phase_receipt_unavailable') from None
|
|
- return p.returncode,result
|
|
+ return code,result
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--stage-inode',type=int)
|
|
parser.add_argument('--phase',choices=('install','rehearse'))
|
|
+ parser.add_argument('--supervision-dir')
|
|
args=parser.parse_args()
|
|
if args.phase=='install':install_phase();return 0
|
|
- if args.phase=='rehearse':return rehearse_phase()
|
|
- receipt={'proposal_id':'REINAH-WP-0008-T07-install-f599-r1','revision':1,'status':'failed',
|
|
+ if args.phase=='rehearse':
|
|
+ require(args.supervision_dir==str(STAGE/'supervision'),'supervision_path')
|
|
+ return rehearse_phase(args.supervision_dir)
|
|
+ receipt={'proposal_id':'REINAH-WP-0008-T07-install-f599-r1','revision':2,'status':'failed',
|
|
'model_calls':0,'credential_reads':False,'owner_configuration_read':False,
|
|
'renewal_or_catalog_or_queue_mutation':False,'destination':str(DESTINATION)}
|
|
- identity=None
|
|
+ identity=None;supervisor=None;quiescent=False
|
|
try:
|
|
observed=protected(STAGE,directory=True,mode=0o700)
|
|
require(observed.st_ino==args.stage_inode and observed.st_dev==2049,'stage_identity')
|
|
identity=observed
|
|
- identify()
|
|
+ os.nice(19)
|
|
+ supervisor=supervision_module().Supervisor(STAGE/'supervision')
|
|
+ identify(supervisor)
|
|
require(not DESTINATION.exists() and not DESTINATION.is_symlink(),'destination_not_absent')
|
|
- require({p.name for p in STAGE.iterdir()}=={'runtime.tar.gz','runtime.py','install-verified.py'},'stage_inventory')
|
|
- os.nice(19)
|
|
+ require({p.name for p in STAGE.iterdir()}=={'runtime.tar.gz','runtime.py','install-verified.py','process_supervision.py','supervision'},'stage_inventory')
|
|
receipt['phase']='extract_and_verify';source=extract()
|
|
receipt['extracted_complete_digest_verified']=True
|
|
receipt['phase']='install'
|
|
- code,result=phase([str(source/'bin/python3'),'-I','-B',str(STAGE/'install-verified.py'),'--phase','install'],200)
|
|
+ code,result=phase(supervisor,[str(source/'bin/python3'),'-I','-B',str(STAGE/'install-verified.py'),'--phase','install'],200)
|
|
receipt['installation']=result;require(code==0 and result['status']=='installed','installation_failed')
|
|
require(runtime_reader().runtime_digest(DESTINATION)==DIGEST,'installed_complete_digest')
|
|
receipt['installed_complete_digest_verified']=True
|
|
receipt['phase']='offline_rehearsal'
|
|
- code,result=phase([str(DESTINATION/'bin/python3'),'-I','-B',str(STAGE/'install-verified.py'),'--phase','rehearse'],260)
|
|
+ code,result=phase(supervisor,[str(DESTINATION/'bin/python3'),'-I','-B',str(STAGE/'install-verified.py'),'--phase','rehearse','--supervision-dir',str(STAGE/'supervision')],260)
|
|
receipt['rehearsal']=result;require(code==0 and result['status']=='rehearsed','rehearsal_failed')
|
|
require(runtime_reader().runtime_digest(DESTINATION)==DIGEST,'post_rehearsal_complete_digest')
|
|
receipt['post_rehearsal_complete_digest_verified']=True
|
|
@@ -193,15 +211,23 @@
|
|
if isinstance(exc,ValueError) and str(exc).replace('_','').isalnum():receipt['failure_code']=str(exc)
|
|
finally:
|
|
receipt['destination_exists']=DESTINATION.exists() or DESTINATION.is_symlink()
|
|
- if identity is not None:
|
|
+ if supervisor is not None:
|
|
+ try:
|
|
+ receipt['process_supervision']=supervisor.close();quiescent=True
|
|
+ except Exception:
|
|
+ receipt['process_supervision']={'quiescent':False,'status':'cleanup_unverified'}
|
|
+ receipt['status']='failed'
|
|
+ if identity is not None and quiescent:
|
|
try:
|
|
current=protected(STAGE,directory=True,mode=0o700)
|
|
require((current.st_dev,current.st_ino)==(identity.st_dev,identity.st_ino),'cleanup_stage_identity')
|
|
- require({p.name for p in STAGE.iterdir()} <= {'runtime.tar.gz','runtime.py','install-verified.py','runtime'},'cleanup_unexpected_stage_member')
|
|
+ require({p.name for p in STAGE.iterdir()} <= {'runtime.tar.gz','runtime.py','install-verified.py','process_supervision.py','runtime','supervision'},'cleanup_unexpected_stage_member')
|
|
shutil.rmtree(STAGE)
|
|
receipt['owned_staging_removed']=not STAGE.exists() and not STAGE.is_symlink()
|
|
except Exception:receipt['owned_staging_removed']=False;receipt['status']='failed'
|
|
- else:receipt['owned_staging_removed']=False
|
|
+ else:
|
|
+ receipt['owned_staging_removed']=False
|
|
+ receipt['staging_retained_for_quiescence_review']=True
|
|
print(json.dumps(receipt,sort_keys=True))
|
|
return 0 if receipt['status']=='installed_and_offline_rehearsal_verified' else 1
|
|
|
|
--- r1/cleanup-upload-staging.py
|
|
+++ r2/cleanup-upload-staging.py
|
|
@@ -8,16 +8,16 @@
|
|
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--apply',action='store_true');args=p.parse_args()
|
|
receipt=json.loads((HERE/'execution-receipt.json').read_text())
|
|
if not (receipt.get('status')=='staging_created' and receipt.get('phase')=='before_transfer'
|
|
- and receipt.get('proposal_id')=='REINAH-WP-0008-T07-install-f599-r1'):
|
|
+ and receipt.get('revision')==2 and receipt.get('proposal_id')=='REINAH-WP-0008-T07-install-f599-r1'):
|
|
raise ValueError('not_a_preexecution_upload_failure')
|
|
inode=receipt['staging_identity']['inode']
|
|
if not args.apply:
|
|
print(json.dumps({'status':'conditional_cleanup_review','inode':inode,'native_mutations':0}));return 0
|
|
code=r'''import json,os,shutil,stat,sys
|
|
from pathlib import Path
|
|
-p=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');s=p.lstat()
|
|
+p=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r2-20261017');s=p.lstat()
|
|
assert os.getuid()==1000 and os.getgid()==1000 and stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and s.st_gid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_dev==2049 and s.st_ino==int(sys.argv[1])
|
|
-assert {f.name for f in p.iterdir()} <= {'runtime.tar.gz','runtime.py','install-verified.py'}
|
|
+assert {f.name for f in p.iterdir()} <= {'runtime.tar.gz','runtime.py','install-verified.py','process_supervision.py'}
|
|
for f in p.iterdir():
|
|
s=f.lstat();assert stat.S_ISREG(s.st_mode) and s.st_uid==1000 and not stat.S_ISLNK(s.st_mode)
|
|
shutil.rmtree(p);print(json.dumps({'status':'owned_upload_staging_removed','absent':not p.exists()}))'''
|
|
--- /dev/null
|
|
+++ r2/process_supervision.py
|
|
@@ -0,0 +1,204 @@
|
|
+#!/usr/bin/env python3
|
|
+"""One-action private process supervisor; no credentials, network or runtime policy."""
|
|
+import argparse
|
|
+import base64
|
|
+import ctypes
|
|
+import json
|
|
+import os
|
|
+from pathlib import Path
|
|
+import select
|
|
+import signal
|
|
+import stat
|
|
+import subprocess
|
|
+import sys
|
|
+import time
|
|
+import uuid
|
|
+
|
|
+MAX_OUTPUT=65536
|
|
+ENV={'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'}
|
|
+
|
|
+
|
|
+def facts(pid):
|
|
+ try:
|
|
+ path=Path('/proc')/str(pid);raw=(path/'stat').read_text();v=raw[raw.rfind(')')+2:].split()
|
|
+ return {'pid':pid,'ppid':int(v[1]),'pgrp':int(v[2]),'sid':int(v[3]),'start':int(v[19]),'uid':path.stat().st_uid,'state':v[0]}
|
|
+ except FileNotFoundError:return None
|
|
+
|
|
+
|
|
+def write(path,value):
|
|
+ temp=path.with_name('.'+path.name+'.'+uuid.uuid4().hex)
|
|
+ fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
+ with os.fdopen(fd,'w') as f:json.dump(value,f);f.flush();os.fsync(f.fileno())
|
|
+ temp.replace(path)
|
|
+
|
|
+
|
|
+def dead(fd):
|
|
+ p=select.poll();p.register(fd,select.POLLIN);return bool(p.poll(0))
|
|
+
|
|
+
|
|
+def descendants(parent):
|
|
+ rows={}
|
|
+ for p in Path('/proc').iterdir():
|
|
+ if p.name.isdigit():
|
|
+ row=facts(int(p.name))
|
|
+ if row:rows[row['pid']]=row
|
|
+ selected={parent};changed=True
|
|
+ while changed:
|
|
+ before=len(selected);selected.update(r['pid'] for r in rows.values() if r['ppid'] in selected)
|
|
+ changed=len(selected)!=before
|
|
+ return {p:rows[p] for p in selected-{parent} if p in rows}
|
|
+
|
|
+
|
|
+def quiesce(parent,children):
|
|
+ """Signal only pidfd-bound actual descendants; subreaper adopts escaped sessions."""
|
|
+ uncertain=False
|
|
+ for sig,limit in ((signal.SIGTERM,0.5),(signal.SIGKILL,3.0)):
|
|
+ end=time.monotonic()+limit
|
|
+ while time.monotonic()<end:
|
|
+ try:rows=descendants(parent)
|
|
+ except Exception:return False
|
|
+ for pid,row in rows.items():
|
|
+ try:
|
|
+ if row['uid']!=os.getuid():uncertain=True;continue
|
|
+ fd=os.pidfd_open(pid)
|
|
+ current=facts(pid)
|
|
+ if current is not None and (current['start'],current['uid'])==(row['start'],row['uid']):
|
|
+ signal.pidfd_send_signal(fd,sig)
|
|
+ elif current is not None:uncertain=True
|
|
+ os.close(fd)
|
|
+ except ProcessLookupError:pass
|
|
+ except Exception:uncertain=True
|
|
+ for child in children:child.poll()
|
|
+ while True:
|
|
+ try:
|
|
+ pid,_=os.waitpid(-1,os.WNOHANG)
|
|
+ if pid==0:break
|
|
+ except ChildProcessError:break
|
|
+ try:remaining=descendants(parent)
|
|
+ except Exception:return False
|
|
+ if not remaining:return not uncertain
|
|
+ time.sleep(0.02)
|
|
+ return False
|
|
+
|
|
+
|
|
+def parent_death(expected):
|
|
+ # Called before exec in a single-threaded supervisor. No application code runs
|
|
+ # if the supervisor disappeared between fork and PR_SET_PDEATHSIG.
|
|
+ if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:os._exit(125)
|
|
+ if os.getppid()!=expected:os._exit(125)
|
|
+
|
|
+
|
|
+def watchdog(directory,parent_fd):
|
|
+ directory=Path(directory)
|
|
+ if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:raise RuntimeError('subreaper_unavailable')
|
|
+ root=json.loads((directory/'root.json').read_text())
|
|
+ current=facts(root['pid'])
|
|
+ if current is None or current['start']!=root['start'] or current['uid']!=os.getuid() or dead(parent_fd):return 1
|
|
+ write(directory/'ready.json',facts(os.getpid()))
|
|
+ jobs={};abort=None;uncertain=False
|
|
+ try:
|
|
+ while True:
|
|
+ if dead(parent_fd):abort='controller_lost'
|
|
+ if (directory/'shutdown').exists():abort=abort or 'shutdown'
|
|
+ for path in sorted(directory.glob('request-*.json')):
|
|
+ key=path.stem.removeprefix('request-')
|
|
+ if key in jobs:continue
|
|
+ try:
|
|
+ r=json.loads(path.read_text());owner=facts(r['owner']['pid'])
|
|
+ if owner is None or owner['start']!=r['owner']['start'] or owner['uid']!=os.getuid():raise ValueError()
|
|
+ if owner['pid']!=root['pid'] and owner['pid'] not in descendants(os.getpid()):raise ValueError()
|
|
+ if type(r['timeout']) not in (int,float) or not 0<r['timeout']<=300:raise ValueError()
|
|
+ if not isinstance(r['argv'],list) or not r['argv'] or any(not isinstance(x,str) for x in r['argv']):raise ValueError()
|
|
+ if abort:raise ValueError()
|
|
+ owner_fd=os.pidfd_open(owner['pid'])
|
|
+ supervisor_pid=os.getpid()
|
|
+ child=subprocess.Popen(r['argv'],stdin=subprocess.DEVNULL,
|
|
+ stdout=subprocess.PIPE if r['capture'] else subprocess.DEVNULL,
|
|
+ stderr=subprocess.DEVNULL,env=ENV,start_new_session=True,
|
|
+ preexec_fn=lambda parent=supervisor_pid:parent_death(parent))
|
|
+ jobs[key]={'p':child,'request':r,'owner_fd':owner_fd,'deadline':time.monotonic()+r['timeout'],
|
|
+ 'output':bytearray(),'eof':not r['capture'],'done':False}
|
|
+ if child.stdout:os.set_blocking(child.stdout.fileno(),False)
|
|
+ except Exception:
|
|
+ abort=abort or 'request_mapping_refused';uncertain=True
|
|
+ write(directory/('result-'+key+'.json'),{'returncode':125,'quiescent':False,'status':'mapping_refused'})
|
|
+ for key,job in list(jobs.items()):
|
|
+ if job['done']:continue
|
|
+ p=job['p']
|
|
+ if p.stdout and not job['eof']:
|
|
+ try:
|
|
+ raw=os.read(p.stdout.fileno(),MAX_OUTPUT+1)
|
|
+ if not raw:job['eof']=True
|
|
+ else:
|
|
+ job['output'].extend(raw)
|
|
+ if len(job['output'])>MAX_OUTPUT:abort=abort or 'output_bound'
|
|
+ except BlockingIOError:pass
|
|
+ if dead(job['owner_fd']):abort=abort or 'caller_lost'
|
|
+ if time.monotonic()>=job['deadline']:abort=abort or 'timeout'
|
|
+ if p.poll() is not None and job['eof'] and not abort:
|
|
+ # A returned leader is not evidence that its children exited.
|
|
+ own=descendants(os.getpid())
|
|
+ leaders={x['p'].pid for x in jobs.values() if not x['done']}
|
|
+ if any(row['sid']==p.pid or (row['ppid']==os.getpid() and row['pid'] not in leaders) for row in own.values()):abort='leader_left_descendants'
|
|
+ else:
|
|
+ write(directory/('result-'+key+'.json'),{'returncode':p.returncode,'quiescent':True,'status':'completed',
|
|
+ 'output_b64':base64.b64encode(job['output']).decode() if job['request']['capture'] else ''})
|
|
+ job['done']=True;os.close(job['owner_fd'])
|
|
+ if abort:
|
|
+ ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()]) and not uncertain
|
|
+ for key,job in jobs.items():
|
|
+ if not job['done']:
|
|
+ write(directory/('result-'+key+'.json'),{'returncode':124 if abort=='timeout' else 125,
|
|
+ 'quiescent':ok,'status':abort,'output_b64':''});job['done']=True
|
|
+ write(directory/'quiescent.json',{'quiescent':ok,'reason':abort,'no_live_or_waitable_descendants':ok,
|
|
+ 'stderr_suppressed':True,'process_values_emitted':False})
|
|
+ return 0 if ok else 1
|
|
+ time.sleep(0.01)
|
|
+ except BaseException:
|
|
+ ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()])
|
|
+ write(directory/'quiescent.json',{'quiescent':False,'reason':'supervisor_exception','best_effort_stopped':ok})
|
|
+ return 1
|
|
+
|
|
+
|
|
+class Supervisor:
|
|
+ def __init__(self,directory):
|
|
+ self.directory=Path(directory);self.directory.mkdir(mode=0o700)
|
|
+ write(self.directory/'root.json',facts(os.getpid()))
|
|
+ fd=os.pidfd_open(os.getpid())
|
|
+ try:self.process=subprocess.Popen([sys.executable,'-I','-B',str(Path(__file__).resolve()),'--watchdog',str(self.directory),'--parent-fd',str(fd)],
|
|
+ stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,pass_fds=(fd,),start_new_session=True,env=ENV)
|
|
+ finally:os.close(fd)
|
|
+ end=time.monotonic()+5
|
|
+ while not (self.directory/'ready.json').exists():
|
|
+ if self.process.poll() is not None or time.monotonic()>end:raise RuntimeError('supervisor_start_failed')
|
|
+ time.sleep(0.01)
|
|
+ def run(self,argv,timeout,*,capture=False):return request(self.directory,argv,timeout,capture=capture)
|
|
+ def close(self):
|
|
+ write(self.directory/'shutdown',{})
|
|
+ try:self.process.wait(timeout=8)
|
|
+ except subprocess.TimeoutExpired:raise RuntimeError('supervisor_quiescence_unknown') from None
|
|
+ report=json.loads((self.directory/'quiescent.json').read_text())
|
|
+ if self.process.returncode or report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
|
+ return report
|
|
+
|
|
+
|
|
+def request(directory,argv,timeout,*,capture=False):
|
|
+ directory=Path(directory);key=uuid.uuid4().hex
|
|
+ write(directory/('request-'+key+'.json'),{'owner':facts(os.getpid()),'argv':list(argv),'timeout':timeout,'capture':capture})
|
|
+ end=time.monotonic()+timeout+10
|
|
+ result=directory/('result-'+key+'.json')
|
|
+ while not result.exists():
|
|
+ if time.monotonic()>end:raise RuntimeError('supervision_reply_unknown')
|
|
+ time.sleep(0.01)
|
|
+ report=json.loads(result.read_text())
|
|
+ if report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
|
+ return report['returncode'],base64.b64decode(report.get('output_b64',''),validate=True)
|
|
+
|
|
+
|
|
+def main():
|
|
+ p=argparse.ArgumentParser();p.add_argument('--watchdog');p.add_argument('--parent-fd',type=int);a=p.parse_args()
|
|
+ return watchdog(a.watchdog,a.parent_fd)
|
|
+
|
|
+if __name__=='__main__':
|
|
+ try:raise SystemExit(main())
|
|
+ except Exception:raise SystemExit(1) from None
|