Record failed runtime trial and reviewed verification proposal (3/4)
This commit is contained in:
codex
parent
08870d65de
commit
4016f3bdee
16 changed files with 1426 additions and 0 deletions
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Conditional cleanup only when the durable receipt proves exec never started."""
|
||||
import argparse,json,shlex,subprocess
|
||||
from pathlib import Path
|
||||
HERE=Path(__file__).resolve().parent
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser(description=__doc__);p.add_argument('--apply',action='store_true');args=p.parse_args()
|
||||
receipt=json.loads((HERE/'execution-receipt.json').read_text())
|
||||
if not (receipt.get('status')=='staging_created' and receipt.get('phase')=='before_transfer'
|
||||
and receipt.get('revision')==3 and receipt.get('proposal_id')=='REINAH-WP-0008-T07-install-f599-r1'):
|
||||
raise ValueError('not_a_preexecution_upload_failure')
|
||||
inode=receipt['staging_identity']['inode']
|
||||
if not args.apply:
|
||||
print(json.dumps({'status':'conditional_cleanup_review','inode':inode,'native_mutations':0}));return 0
|
||||
code=r'''import json,os,shutil,stat,sys
|
||||
from pathlib import Path
|
||||
p=Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017');s=p.lstat()
|
||||
assert os.getuid()==1000 and os.getgid()==1000 and stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and s.st_gid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_dev==2049 and s.st_ino==int(sys.argv[1])
|
||||
assert {f.name for f in p.iterdir()} <= {'runtime.py','verify-installed.py','process_supervision.py'}
|
||||
for f in p.iterdir():
|
||||
s=f.lstat();assert stat.S_ISREG(s.st_mode) and s.st_uid==1000 and not stat.S_ISLNK(s.st_mode)
|
||||
shutil.rmtree(p);print(json.dumps({'status':'owned_upload_staging_removed','absent':not p.exists()}))'''
|
||||
cp=subprocess.run(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01','/usr/bin/python3 -I -B -c '+shlex.quote(code)+' '+shlex.quote(str(inode))],capture_output=True,text=True,timeout=30)
|
||||
if cp.returncode:raise RuntimeError('conditional_cleanup_refused_or_uncertain')
|
||||
result=json.loads(cp.stdout);(HERE/'upload-cleanup-receipt.json').write_text(json.dumps(result,indent=2)+'\n');print(json.dumps(result));return 0
|
||||
|
||||
if __name__=='__main__':
|
||||
try:raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(json.dumps({'status':'cleanup_refused_or_uncertain','failure_type':type(exc).__name__}));raise SystemExit(1) from None
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Verification-only orchestration. Fresh exact revision-3 founder approval required."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
HERE=Path(__file__).resolve().parent
|
||||
DIGEST='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
||||
STAGE='/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017'
|
||||
STORE='/home/tegwick/.helixforge-factory/runtimes'
|
||||
DURABLE_RECEIPT=STAGE+'.receipt.json'
|
||||
|
||||
READ_TARGET=r'''import json,os,pwd,stat,subprocess,shutil,time,ctypes,gzip,sys
|
||||
from pathlib import Path
|
||||
store=Path('/home/tegwick/.helixforge-factory/runtimes'); stage=Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017');digest='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
||||
def meta(p):
|
||||
try:
|
||||
s=p.lstat();return {'uid':s.st_uid,'gid':s.st_gid,'mode':oct(stat.S_IMODE(s.st_mode)),'device':s.st_dev,'inode':s.st_ino,'directory':stat.S_ISDIR(s.st_mode),'symlink':stat.S_ISLNK(s.st_mode)}
|
||||
except FileNotFoundError:return None
|
||||
cp=subprocess.run(['/usr/local/bin/kubectl','get','nodes','-o','json'],capture_output=True,timeout=20)
|
||||
assert cp.returncode==0
|
||||
nodes=json.loads(cp.stdout)['items'];assert len(nodes)==1
|
||||
mem=dict(x.split(':',1) for x in Path('/proc/meminfo').read_text().splitlines())
|
||||
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
||||
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];busy=1-(d[3]+d[4])/sum(d)
|
||||
help_result=subprocess.run(['/usr/bin/bwrap','--help'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=10)
|
||||
version=subprocess.run(['/usr/bin/bwrap','--version'],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=10)
|
||||
manual=gzip.decompress(Path('/usr/share/man/man1/bwrap.1.gz').read_bytes()).decode();i=manual.find('die\\-with\\-parent')
|
||||
fd=os.pidfd_open(os.getpid());os.close(fd);value=ctypes.c_int();subreaper_get=ctypes.CDLL(None).prctl(37,ctypes.byref(value),0,0,0)
|
||||
reader={};exec(compile(sys.argv[1],'reviewed_public_digest_reader','exec'),reader)
|
||||
native_digest=reader['runtime_digest'](store/digest)
|
||||
print(json.dumps({'installed_complete_digest':native_digest,'durable_receipt':meta(Path(str(stage)+'.receipt.json')),'bwrap_version':version.stdout.decode().strip(),'bwrap_die_with_parent_supported':help_result.returncode==0 and b'--die-with-parent' in help_result.stdout,'primary_manual_contract_excerpt':manual[max(0,i-20):i+430] if i>=0 else 'absent','pidfd_available':True,'subreaper_get_supported':subreaper_get==0,'account':pwd.getpwuid(os.getuid()).pw_name,'uid':os.getuid(),'gid':os.getgid(),'store':meta(store),'destination':meta(store/digest),'stage':meta(stage),'memory_available_bytes':int(mem['MemAvailable'].split()[0])*1024,'disk_free_bytes':shutil.disk_usage(store).free,'load_average':list(os.getloadavg()),'cpu_count':os.cpu_count(),'cpu_busy_fraction':busy,'cpu_sample_seconds':1,'node_addresses':nodes[0]['status']['addresses'],'node_ready':any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions'])}))'''
|
||||
|
||||
|
||||
def sha(path):
|
||||
with path.open('rb') as stream:return 'sha256:'+hashlib.file_digest(stream,'sha256').hexdigest()
|
||||
|
||||
|
||||
def command(argv,timeout=60):
|
||||
cp=subprocess.run(argv,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=timeout)
|
||||
if cp.returncode:raise RuntimeError('command_failed')
|
||||
return cp.stdout
|
||||
|
||||
|
||||
def ssh(code,args=(),timeout=60):
|
||||
remote='/usr/bin/python3 -I -B -c '+shlex.quote(code)
|
||||
if args:remote+=' '+' '.join(shlex.quote(str(x)) for x in args)
|
||||
return json.loads(command(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01',remote],timeout))
|
||||
|
||||
|
||||
def preflight():
|
||||
inputs=json.loads((HERE/'action-inputs.json').read_text())
|
||||
for name,expected in inputs['helper_hashes'].items():
|
||||
if sha(HERE/name)!=expected:raise ValueError('helper_changed')
|
||||
observation=ssh(READ_TARGET,args=((HERE/'runtime.py').read_text(),))
|
||||
store=observation['store']
|
||||
expected={'device':2049,'inode':4751222}
|
||||
checks={
|
||||
'bwrap_parent_death_contract':observation['bwrap_die_with_parent_supported'] and observation['bwrap_version']=='bubblewrap 0.9.0' and 'PR_SET_PDEATHSIG' in observation['primary_manual_contract_excerpt'],
|
||||
'supervision_primitives':observation['pidfd_available'] and observation['subreaper_get_supported'],
|
||||
'account':observation['account']=='tegwick' and observation['uid']==1000 and observation['gid']==1000,
|
||||
'protected_store':store['uid']==1000 and store['gid']==1000 and store['mode']=='0o700' and store['directory'] and not store['symlink'] and store['device']==expected['device'] and store['inode']==expected['inode'],
|
||||
'exact_installed_runtime':observation['destination'] is not None and observation['destination']['directory'] and not observation['destination']['symlink'] and observation['destination']['uid']==1000 and observation['destination']['gid']==1000 and observation['installed_complete_digest']==DIGEST,
|
||||
'durable_receipt_absent':observation['durable_receipt'] is None,
|
||||
'staging_absent':observation['stage'] is None,
|
||||
'exact_ready_node':observation['node_ready'] and any(x['address']=='92.205.62.239' for x in observation['node_addresses']),
|
||||
'disk_headroom':observation['disk_free_bytes']>=2*1024**3,
|
||||
'memory_headroom':observation['memory_available_bytes']>=1024**3,
|
||||
'load_headroom':observation['load_average'][0]<=2*observation['cpu_count'],
|
||||
'cpu_headroom':observation['cpu_busy_fraction']<=0.85,
|
||||
}
|
||||
observation['checks']=checks
|
||||
return inputs,observation
|
||||
|
||||
|
||||
def save(path,value):
|
||||
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as stream:json.dump(value,stream,indent=2);stream.write('\n')
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--apply',action='store_true');parser.add_argument('--dry-run',action='store_true')
|
||||
args=parser.parse_args()
|
||||
if args.apply and args.dry_run:raise ValueError('conflicting_modes')
|
||||
inputs,observation=preflight()
|
||||
if not args.apply:
|
||||
report={'status':'read_only_dry_run_passed' if all(observation['checks'].values()) else 'read_only_dry_run_blocked','proposal_id':inputs['proposal_id'],'revision':3,
|
||||
'observed_at':datetime.now(timezone.utc).isoformat(),'target':observation,
|
||||
'installed_runtime_complete_digest_verified':observation['installed_complete_digest']==DIGEST,
|
||||
'remote_mutations':0,'provider_calls':0,'private_configuration_reads':False,
|
||||
'next_exact_command':'python3 -I -B '+str(HERE/'execute-verify.py')+' --apply'}
|
||||
print(json.dumps(report,sort_keys=True));return 0 if all(observation['checks'].values()) else 2
|
||||
if not all(observation['checks'].values()):raise ValueError('target_headroom_or_identity_refused')
|
||||
receipt=HERE/'execution-receipt.json'
|
||||
if receipt.exists():raise ValueError('prior_attempt_requires_reconciliation')
|
||||
save(receipt,{'status':'intent_recorded','proposal_id':inputs['proposal_id'],'revision':3,
|
||||
'automatic_retry':False,'observed_at':datetime.now(timezone.utc).isoformat()})
|
||||
# No native attempt is retried by this helper. Intent survives an SSH loss.
|
||||
create=r'''import json,os,stat,time
|
||||
from pathlib import Path
|
||||
assert os.getuid()==1000 and os.getgid()==1000
|
||||
store=Path('/home/tegwick/.helixforge-factory/runtimes');s=store.lstat()
|
||||
assert stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_ino==4751222 and s.st_dev==2049
|
||||
assert (store/'f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d').is_dir()
|
||||
assert not Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017.receipt.json').exists()
|
||||
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
||||
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];assert 1-(d[3]+d[4])/sum(d)<=0.85
|
||||
stage=Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017');stage.mkdir(mode=0o700)
|
||||
s=stage.lstat();print(json.dumps({'inode':s.st_ino,'device':s.st_dev}))'''
|
||||
created=ssh(create)
|
||||
receipt.write_text(json.dumps({'status':'staging_created','phase':'before_transfer','proposal_id':inputs['proposal_id'],'revision':3,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
||||
command(['scp','-q',str(HERE/'runtime.py'),str(HERE/'verify-installed.py'),str(HERE/'process_supervision.py'),'railiance01:'+STAGE+'/'],120)
|
||||
execute=r'''import hashlib,json,os,stat,sys
|
||||
from pathlib import Path
|
||||
stage=Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017');s=stage.lstat()
|
||||
assert stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_ino==int(sys.argv[1]) and s.st_dev==2049
|
||||
expected=json.loads(sys.argv[2]);assert {p.name for p in stage.iterdir()}==set(expected)
|
||||
for name,want in expected.items():
|
||||
p=stage/name;s=p.lstat();assert stat.S_ISREG(s.st_mode) and s.st_uid==1000 and not stat.S_ISLNK(s.st_mode)
|
||||
with p.open('rb') as stream:assert hashlib.file_digest(stream,'sha256').hexdigest()==want
|
||||
p.chmod(0o600)
|
||||
os.execve('/usr/bin/python3',['python3','-I','-B',str(stage/'verify-installed.py'),'--stage-inode',sys.argv[1]],{'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'})'''
|
||||
hashes={'runtime.py':inputs['helper_hashes']['runtime.py'].removeprefix('sha256:'),
|
||||
'verify-installed.py':inputs['helper_hashes']['verify-installed.py'].removeprefix('sha256:'),
|
||||
'process_supervision.py':inputs['helper_hashes']['process_supervision.py'].removeprefix('sha256:')}
|
||||
# Parse the structured final helper return even when its action failed.
|
||||
remote='/usr/bin/python3 -I -B -c '+shlex.quote(execute)+' '+shlex.quote(str(created['inode']))+' '+shlex.quote(json.dumps(hashes,separators=(',',':')))
|
||||
receipt.write_text(json.dumps({'status':'remote_execution_started','proposal_id':inputs['proposal_id'],'revision':3,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
||||
cp=subprocess.run(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01',remote],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=600)
|
||||
try:result=json.loads(cp.stdout)
|
||||
except Exception:raise RuntimeError('remote_outcome_uncertain') from None
|
||||
result['ssh_exit_code']=cp.returncode;result['staging_identity']=created
|
||||
# Replace only our exact local intent receipt, never a previous attempt.
|
||||
receipt.write_text(json.dumps(result,indent=2)+'\n');receipt.chmod(0o600)
|
||||
print(json.dumps(result,sort_keys=True))
|
||||
return 0 if cp.returncode==0 and result.get('status')=='installed_runtime_offline_rehearsal_verified' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(json.dumps({'status':'failed_or_uncertain','failure_type':type(exc).__name__,
|
||||
'automatic_retry':False,'private_diagnostics_suppressed':True}))
|
||||
raise SystemExit(1) from None
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"installed_runtime_complete_digest_verified": true, "next_exact_command": "python3 -I -B /home/worsch/the-custodian/docs/changes/netkingdom-target-2026-10-17/native-runtime-install-r3/execute-verify.py --apply", "observed_at": "2026-10-17T08:41:54.503313+00:00", "private_configuration_reads": false, "proposal_id": "REINAH-WP-0008-T07-install-f599-r1", "provider_calls": 0, "remote_mutations": 0, "revision": 3, "status": "read_only_dry_run_passed", "target": {"account": "tegwick", "bwrap_die_with_parent_supported": true, "bwrap_version": "bubblewrap 0.9.0", "checks": {"account": true, "bwrap_parent_death_contract": true, "cpu_headroom": true, "disk_headroom": true, "durable_receipt_absent": true, "exact_installed_runtime": true, "exact_ready_node": true, "load_headroom": true, "memory_headroom": true, "protected_store": true, "staging_absent": true, "supervision_primitives": true}, "cpu_busy_fraction": 0.22164948453608246, "cpu_count": 4, "cpu_sample_seconds": 1, "destination": {"device": 2049, "directory": true, "gid": 1000, "inode": 4743998, "mode": "0o755", "symlink": false, "uid": 1000}, "disk_free_bytes": 123064221696, "durable_receipt": null, "gid": 1000, "installed_complete_digest": "f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d", "load_average": [0.4267578125, 1.38232421875, 1.736328125], "memory_available_bytes": 7668154368, "node_addresses": [{"address": "92.205.62.239", "type": "InternalIP"}, {"address": "2a00:1169:111:6d70::", "type": "InternalIP"}, {"address": "239.62.205.92.host.secureserver.net", "type": "Hostname"}], "node_ready": true, "pidfd_available": true, "primary_manual_contract_excerpt": ")\\&.\n.RE\n.PP\n\\fB\\-\\-die\\-with\\-parent\\fR\n.RS 4\nEnsures child process (COMMAND) dies when bwrap\\*(Aqs parent dies\\&. Kills (SIGKILL) all bwrap sandbox processes in sequence from parent to child including COMMAND process when bwrap or bwrap\\*(Aqs parent dies\\&. See prctl, PR_SET_PDEATHSIG\\&.\n.RE\n.PP\n\\fB\\-\\-as\\-pid\\-1\\fR\n.RS 4\nDo not create a process with PID=1 in the sandbox to reap child processes\\&.\n.RE\n.PP\n\\fB\\-\\-cap\\-add \\fR\\fBCAP\\fR\n.RS 4\nAdd ", "stage": null, "store": {"device": 2049, "directory": true, "gid": 1000, "inode": 4751222, "mode": "0o700", "symlink": false, "uid": 1000}, "subreaper_get_supported": true, "uid": 1000}}
|
||||
|
|
@ -0,0 +1,181 @@
|
|||
{
|
||||
"source_api": "owner_rehearsal.rehearse(runtime, run=bounded_run, bwrap_bin=/usr/bin/bwrap)",
|
||||
"source_sha256": "sha256:192f654cad4a5f065c7faa5e35b03493297a472e8d7fe641be2c42c5fb00dcc3",
|
||||
"captured_using_existing_runner_seam": true,
|
||||
"launcher_executions": 0,
|
||||
"steps": [
|
||||
{
|
||||
"argv": [
|
||||
"/usr/bin/bwrap",
|
||||
"--die-with-parent",
|
||||
"--unshare-user",
|
||||
"--unshare-pid",
|
||||
"--unshare-ipc",
|
||||
"--unshare-uts",
|
||||
"--unshare-cgroup",
|
||||
"--unshare-net",
|
||||
"--tmpfs",
|
||||
"/",
|
||||
"--proc",
|
||||
"/proc",
|
||||
"--dev",
|
||||
"/dev",
|
||||
"--clearenv",
|
||||
"--ro-bind",
|
||||
"/usr",
|
||||
"/usr",
|
||||
"--ro-bind",
|
||||
"/bin",
|
||||
"/bin",
|
||||
"--ro-bind",
|
||||
"/lib",
|
||||
"/lib",
|
||||
"--ro-bind",
|
||||
"/lib64",
|
||||
"/lib64",
|
||||
"--ro-bind",
|
||||
"/etc/resolv.conf",
|
||||
"/etc/resolv.conf",
|
||||
"--ro-bind",
|
||||
"/etc/ssl/certs",
|
||||
"/etc/ssl/certs",
|
||||
"--dir",
|
||||
"/opt",
|
||||
"--dir",
|
||||
"/opt/sandboxer",
|
||||
"--ro-bind",
|
||||
"/home/tegwick/.helixforge-factory/runtimes/f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d",
|
||||
"/opt/sandboxer/runtime",
|
||||
"--dir",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"HOME",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"PATH",
|
||||
"/opt/sandboxer/runtime/bin:/usr/bin:/bin",
|
||||
"--chdir",
|
||||
"/tmp",
|
||||
"/opt/sandboxer/runtime/bin/rein-aharness",
|
||||
"--help"
|
||||
],
|
||||
"timeout_seconds": 60
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"/usr/bin/bwrap",
|
||||
"--die-with-parent",
|
||||
"--unshare-user",
|
||||
"--unshare-pid",
|
||||
"--unshare-ipc",
|
||||
"--unshare-uts",
|
||||
"--unshare-cgroup",
|
||||
"--unshare-net",
|
||||
"--tmpfs",
|
||||
"/",
|
||||
"--proc",
|
||||
"/proc",
|
||||
"--dev",
|
||||
"/dev",
|
||||
"--clearenv",
|
||||
"--ro-bind",
|
||||
"/usr",
|
||||
"/usr",
|
||||
"--ro-bind",
|
||||
"/bin",
|
||||
"/bin",
|
||||
"--ro-bind",
|
||||
"/lib",
|
||||
"/lib",
|
||||
"--ro-bind",
|
||||
"/lib64",
|
||||
"/lib64",
|
||||
"--ro-bind",
|
||||
"/etc/resolv.conf",
|
||||
"/etc/resolv.conf",
|
||||
"--ro-bind",
|
||||
"/etc/ssl/certs",
|
||||
"/etc/ssl/certs",
|
||||
"--dir",
|
||||
"/opt",
|
||||
"--dir",
|
||||
"/opt/sandboxer",
|
||||
"--ro-bind",
|
||||
"/home/tegwick/.helixforge-factory/runtimes/f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d",
|
||||
"/opt/sandboxer/runtime",
|
||||
"--dir",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"HOME",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"PATH",
|
||||
"/opt/sandboxer/runtime/bin:/usr/bin:/bin",
|
||||
"--chdir",
|
||||
"/tmp",
|
||||
"/opt/sandboxer/runtime/bin/glas-harness",
|
||||
"--help"
|
||||
],
|
||||
"timeout_seconds": 60
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"/usr/bin/bwrap",
|
||||
"--die-with-parent",
|
||||
"--unshare-user",
|
||||
"--unshare-pid",
|
||||
"--unshare-ipc",
|
||||
"--unshare-uts",
|
||||
"--unshare-cgroup",
|
||||
"--unshare-net",
|
||||
"--tmpfs",
|
||||
"/",
|
||||
"--proc",
|
||||
"/proc",
|
||||
"--dev",
|
||||
"/dev",
|
||||
"--clearenv",
|
||||
"--ro-bind",
|
||||
"/usr",
|
||||
"/usr",
|
||||
"--ro-bind",
|
||||
"/bin",
|
||||
"/bin",
|
||||
"--ro-bind",
|
||||
"/lib",
|
||||
"/lib",
|
||||
"--ro-bind",
|
||||
"/lib64",
|
||||
"/lib64",
|
||||
"--ro-bind",
|
||||
"/etc/resolv.conf",
|
||||
"/etc/resolv.conf",
|
||||
"--ro-bind",
|
||||
"/etc/ssl/certs",
|
||||
"/etc/ssl/certs",
|
||||
"--dir",
|
||||
"/opt",
|
||||
"--dir",
|
||||
"/opt/sandboxer",
|
||||
"--ro-bind",
|
||||
"/home/tegwick/.helixforge-factory/runtimes/f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d",
|
||||
"/opt/sandboxer/runtime",
|
||||
"--dir",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"HOME",
|
||||
"/tmp",
|
||||
"--setenv",
|
||||
"PATH",
|
||||
"/opt/sandboxer/runtime/bin:/usr/bin:/bin",
|
||||
"--chdir",
|
||||
"/tmp",
|
||||
"/opt/sandboxer/runtime/bin/claude",
|
||||
"--version"
|
||||
],
|
||||
"timeout_seconds": 60
|
||||
}
|
||||
],
|
||||
"runtime_source_file_unchanged": true,
|
||||
"complete_runtime_digest_verified_after_bytecode_refinement": true
|
||||
}
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
{
|
||||
"results": [
|
||||
{
|
||||
"argv": [
|
||||
"/usr/bin/python3",
|
||||
"-B",
|
||||
"docs/changes/netkingdom-target-2026-10-17/native-runtime-install-r3/test_process_supervision.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "",
|
||||
"stderr_test_framework_only": "........\n----------------------------------------------------------------------\nRan 8 tests in 3.212s\n\nOK\n"
|
||||
},
|
||||
{
|
||||
"argv": [
|
||||
"/usr/bin/python3",
|
||||
"-B",
|
||||
"docs/changes/netkingdom-target-2026-10-17/native-runtime-install-r3/test_receipt_cleanup.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"stdout": "",
|
||||
"stderr_test_framework_only": "..\n----------------------------------------------------------------------\nRan 2 tests in 0.024s\n\nOK\n"
|
||||
}
|
||||
],
|
||||
"total_cases": 10,
|
||||
"native_runtime_or_namespace_launches": 0,
|
||||
"provider_calls": 0,
|
||||
"application_stderr": "discarded",
|
||||
"earlier_fixture_expectation_refinement": "retained-fixture-refinement.json",
|
||||
"observed_at": "2026-10-17T08:42:30.244972+00:00"
|
||||
}
|
||||
|
|
@ -0,0 +1,239 @@
|
|||
#!/usr/bin/env python3
|
||||
"""One-action private process supervisor; no credentials, network or runtime policy."""
|
||||
import argparse
|
||||
import base64
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import select
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
MAX_OUTPUT=65536
|
||||
ENV={'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'}
|
||||
|
||||
|
||||
def facts(pid):
|
||||
try:
|
||||
path=Path('/proc')/str(pid);raw=(path/'stat').read_text();v=raw[raw.rfind(')')+2:].split()
|
||||
return {'pid':pid,'ppid':int(v[1]),'pgrp':int(v[2]),'sid':int(v[3]),'start':int(v[19]),'uid':path.stat().st_uid,'state':v[0]}
|
||||
except (FileNotFoundError,ProcessLookupError):return None
|
||||
|
||||
|
||||
def write(path,value):
|
||||
temp=path.with_name('.'+path.name+'.'+uuid.uuid4().hex)
|
||||
fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
with os.fdopen(fd,'w') as f:json.dump(value,f);f.flush();os.fsync(f.fileno())
|
||||
temp.replace(path)
|
||||
|
||||
|
||||
def dead(fd):
|
||||
p=select.poll();p.register(fd,select.POLLIN);return bool(p.poll(0))
|
||||
|
||||
|
||||
def descendants(parent):
|
||||
rows={}
|
||||
for p in Path('/proc').iterdir():
|
||||
if p.name.isdigit():
|
||||
row=facts(int(p.name))
|
||||
if row:rows[row['pid']]=row
|
||||
selected={parent};changed=True
|
||||
while changed:
|
||||
before=len(selected);selected.update(r['pid'] for r in rows.values() if r['ppid'] in selected)
|
||||
changed=len(selected)!=before
|
||||
return {p:rows[p] for p in selected-{parent} if p in rows}
|
||||
|
||||
|
||||
def quiesce(parent,children):
|
||||
"""Signal only pidfd-bound actual descendants; subreaper adopts escaped sessions."""
|
||||
uncertain=False
|
||||
for sig,limit in ((signal.SIGTERM,0.5),(signal.SIGKILL,3.0)):
|
||||
end=time.monotonic()+limit
|
||||
while time.monotonic()<end:
|
||||
try:rows=descendants(parent)
|
||||
except Exception:return False
|
||||
for pid,row in rows.items():
|
||||
try:
|
||||
if row['uid']!=os.getuid():uncertain=True;continue
|
||||
fd=os.pidfd_open(pid)
|
||||
current=facts(pid)
|
||||
if current is not None and (current['start'],current['uid'])==(row['start'],row['uid']):
|
||||
signal.pidfd_send_signal(fd,sig)
|
||||
elif current is not None:uncertain=True
|
||||
os.close(fd)
|
||||
except ProcessLookupError:pass
|
||||
except Exception:uncertain=True
|
||||
for child in children:child.poll()
|
||||
while True:
|
||||
try:
|
||||
pid,_=os.waitpid(-1,os.WNOHANG)
|
||||
if pid==0:break
|
||||
except ChildProcessError:break
|
||||
try:remaining=descendants(parent)
|
||||
except Exception:return False
|
||||
if not remaining:return not uncertain
|
||||
time.sleep(0.02)
|
||||
return False
|
||||
|
||||
|
||||
def parent_death(expected):
|
||||
# Called before exec in a single-threaded supervisor. No application code runs
|
||||
# if the supervisor disappeared between fork and PR_SET_PDEATHSIG.
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:os._exit(125)
|
||||
if os.getppid()!=expected:os._exit(125)
|
||||
|
||||
|
||||
def reap_orphan_zombies(rows,leaders):
|
||||
"""Reap only dead adopted direct children; preserve every Popen wait status."""
|
||||
reaped=[]
|
||||
for pid,row in rows.items():
|
||||
if row['ppid']!=os.getpid() or row['state']!='Z' or pid in leaders:continue
|
||||
current=facts(pid)
|
||||
if current is None:continue
|
||||
if current!=row or row['uid']!=os.getuid():raise RuntimeError('orphan_mapping_unknown')
|
||||
waited,status=os.waitpid(pid,os.WNOHANG)
|
||||
if waited:reaped.append({'facts':row,'exit_code':os.waitstatus_to_exitcode(status)})
|
||||
return reaped
|
||||
|
||||
|
||||
def job_report(jobs):
|
||||
return [{'label':j['request']['label'],'leader':j['leader'],'returncode':j['p'].returncode,
|
||||
'status':j.get('status','running')} for j in jobs.values()][:16]
|
||||
|
||||
|
||||
def watchdog(directory,parent_fd):
|
||||
directory=Path(directory)
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:raise RuntimeError('subreaper_unavailable')
|
||||
root=json.loads((directory/'root.json').read_text())
|
||||
current=facts(root['pid'])
|
||||
if current is None or current['start']!=root['start'] or current['uid']!=os.getuid() or dead(parent_fd):return 1
|
||||
write(directory/'ready.json',facts(os.getpid()))
|
||||
jobs={};abort=None;uncertain=False;reaped=[];leftover=[]
|
||||
try:
|
||||
while True:
|
||||
if dead(parent_fd):abort='controller_lost'
|
||||
if (directory/'shutdown').exists():abort=abort or 'shutdown'
|
||||
for path in sorted(directory.glob('request-*.json')):
|
||||
key=path.stem.removeprefix('request-')
|
||||
if key in jobs:continue
|
||||
try:
|
||||
r=json.loads(path.read_text());owner=facts(r['owner']['pid'])
|
||||
if owner is None or owner['start']!=r['owner']['start'] or owner['uid']!=os.getuid():raise ValueError()
|
||||
if owner['pid']!=root['pid'] and owner['pid'] not in descendants(os.getpid()):raise ValueError()
|
||||
if type(r['timeout']) not in (int,float) or not 0<r['timeout']<=300:raise ValueError()
|
||||
if not isinstance(r['argv'],list) or not r['argv'] or any(not isinstance(x,str) for x in r['argv']):raise ValueError()
|
||||
if r.get('label') not in ('job','node-read','rehearsal-phase','rein-aharness','glas-harness','claude'):raise ValueError()
|
||||
if type(r.get('capture')) is not bool:raise ValueError()
|
||||
if len(jobs)>=16 or abort:raise ValueError()
|
||||
owner_fd=os.pidfd_open(owner['pid'])
|
||||
bound=facts(owner['pid'])
|
||||
if bound is None or bound['start']!=owner['start'] or bound['uid']!=owner['uid']:
|
||||
os.close(owner_fd);raise ValueError()
|
||||
supervisor_pid=os.getpid()
|
||||
child=subprocess.Popen(r['argv'],stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE if r['capture'] else subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,env=ENV,start_new_session=True,
|
||||
preexec_fn=lambda parent=supervisor_pid:parent_death(parent))
|
||||
jobs[key]={'p':child,'request':r,'owner_fd':owner_fd,'deadline':time.monotonic()+r['timeout'],
|
||||
'output':bytearray(),'eof':not r['capture'],'done':False,'leader':facts(child.pid)}
|
||||
write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
||||
if child.stdout:os.set_blocking(child.stdout.fileno(),False)
|
||||
except Exception:
|
||||
abort=abort or 'request_mapping_refused';uncertain=True
|
||||
write(directory/('result-'+key+'.json'),{'returncode':125,'quiescent':False,'status':'mapping_refused'})
|
||||
for key,job in list(jobs.items()):
|
||||
if job['done']:continue
|
||||
p=job['p']
|
||||
if p.stdout and not job['eof']:
|
||||
try:
|
||||
raw=os.read(p.stdout.fileno(),MAX_OUTPUT+1)
|
||||
if not raw:job['eof']=True
|
||||
else:
|
||||
job['output'].extend(raw)
|
||||
if len(job['output'])>MAX_OUTPUT:abort=abort or 'output_bound'
|
||||
except BlockingIOError:pass
|
||||
if dead(job['owner_fd']):abort=abort or 'caller_lost'
|
||||
if time.monotonic()>=job['deadline']:abort=abort or 'timeout'
|
||||
if p.poll() is not None and job['eof'] and not abort:
|
||||
# An adopted zombie is already dead, but must be waited
|
||||
# before claiming quiescence. Do not steal a Popen leader's
|
||||
# exit status or dismiss a genuinely live leftover.
|
||||
own=descendants(os.getpid())
|
||||
leaders={x['p'].pid for x in jobs.values()}
|
||||
reaped.extend(reap_orphan_zombies(own,leaders))
|
||||
own=descendants(os.getpid())
|
||||
active={x['p'].pid for x in jobs.values() if not x['done']}
|
||||
leftover=[row for row in own.values() if row['sid']==p.pid or
|
||||
(row['ppid']==os.getpid() and row['pid'] not in active)]
|
||||
if leftover:abort='leader_left_descendants'
|
||||
else:
|
||||
write(directory/('result-'+key+'.json'),{'returncode':p.returncode,'quiescent':True,'status':'completed',
|
||||
'output_b64':base64.b64encode(job['output']).decode() if job['request']['capture'] else ''})
|
||||
job['done']=True;job['status']='completed';os.close(job['owner_fd'])
|
||||
write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
||||
if abort:
|
||||
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()]) and not uncertain
|
||||
for key,job in jobs.items():
|
||||
if not job['done']:
|
||||
write(directory/('result-'+key+'.json'),{'returncode':124 if abort=='timeout' else 125,
|
||||
'quiescent':ok,'status':abort,'output_b64':''});job['done']=True;job['status']=abort
|
||||
write(directory/'quiescent.json',{'quiescent':ok,'reason':abort,'no_live_or_waitable_descendants':ok,
|
||||
'stderr_suppressed':True,'private_values_emitted':False,'jobs':job_report(jobs),
|
||||
'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
||||
return 0 if ok else 1
|
||||
time.sleep(0.01)
|
||||
except BaseException:
|
||||
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()])
|
||||
write(directory/'quiescent.json',{'quiescent':False,'reason':'supervisor_exception','best_effort_stopped':ok,
|
||||
'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
||||
return 1
|
||||
|
||||
|
||||
class Supervisor:
|
||||
def __init__(self,directory):
|
||||
self.directory=Path(directory);self.directory.mkdir(mode=0o700)
|
||||
write(self.directory/'root.json',facts(os.getpid()))
|
||||
fd=os.pidfd_open(os.getpid())
|
||||
try:self.process=subprocess.Popen([sys.executable,'-I','-B',str(Path(__file__).resolve()),'--watchdog',str(self.directory),'--parent-fd',str(fd)],
|
||||
stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,pass_fds=(fd,),start_new_session=True,env=ENV)
|
||||
finally:os.close(fd)
|
||||
end=time.monotonic()+5
|
||||
while not (self.directory/'ready.json').exists():
|
||||
if self.process.poll() is not None or time.monotonic()>end:raise RuntimeError('supervisor_start_failed')
|
||||
time.sleep(0.01)
|
||||
def run(self,argv,timeout,*,capture=False,label='job'):return request(self.directory,argv,timeout,capture=capture,label=label)
|
||||
def close(self):
|
||||
write(self.directory/'shutdown',{})
|
||||
try:self.process.wait(timeout=8)
|
||||
except subprocess.TimeoutExpired:raise RuntimeError('supervisor_quiescence_unknown') from None
|
||||
report=json.loads((self.directory/'quiescent.json').read_text())
|
||||
if self.process.returncode or report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
||||
return report
|
||||
|
||||
|
||||
def request(directory,argv,timeout,*,capture=False,label='job'):
|
||||
directory=Path(directory);key=uuid.uuid4().hex
|
||||
if (directory/'quiescent.json').exists():raise RuntimeError('supervision_already_stopped')
|
||||
write(directory/('request-'+key+'.json'),{'owner':facts(os.getpid()),'argv':list(argv),'timeout':timeout,'capture':capture,'label':label})
|
||||
end=time.monotonic()+timeout+10
|
||||
result=directory/('result-'+key+'.json')
|
||||
while not result.exists():
|
||||
if time.monotonic()>end:raise RuntimeError('supervision_reply_unknown')
|
||||
time.sleep(0.01)
|
||||
report=json.loads(result.read_text())
|
||||
if report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
||||
return report['returncode'],base64.b64decode(report.get('output_b64',''),validate=True)
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--watchdog');p.add_argument('--parent-fd',type=int);a=p.parse_args()
|
||||
return watchdog(a.watchdog,a.parent_fd)
|
||||
|
||||
if __name__=='__main__':
|
||||
try:raise SystemExit(main())
|
||||
except Exception:raise SystemExit(1) from None
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
{
|
||||
"proposal_id": "REINAH-WP-0008-T07-install-f599-r1",
|
||||
"revision": 2,
|
||||
"actual_status": "failed/partial installed; launcher verification unproven",
|
||||
"files": [
|
||||
{
|
||||
"path": "FINAL-PROPOSAL.md",
|
||||
"sha256": "sha256:44b17589c323e1a966caf6c1b1def123092ee289b929c8ea014bc84109103b1d"
|
||||
},
|
||||
{
|
||||
"path": "PROCESS-CONTRACT.md",
|
||||
"sha256": "sha256:ed79d14bde467e8893c341419c37fa2992184597fc9015c2e7d81fb4ece63fd0"
|
||||
},
|
||||
{
|
||||
"path": "action-inputs.json",
|
||||
"sha256": "sha256:ce7f5a25be440561f5179ae9980d85eb693a11b58d2012c8aea24b8bae04ab6f"
|
||||
},
|
||||
{
|
||||
"path": "artifact-manifest.json",
|
||||
"sha256": "sha256:aea5dbfe0742fe0bb6c36a41ca7b0d70473683b2731f92cc279d43929e5c35c9"
|
||||
},
|
||||
{
|
||||
"path": "build-transfer.py",
|
||||
"sha256": "sha256:99914629f354d4b6d5a4750850b32ec2cb64ac5ce18e86702d9157ea6f6b0124"
|
||||
},
|
||||
{
|
||||
"path": "cleanup-upload-staging.py",
|
||||
"sha256": "sha256:cec7eab2b25e4a0077ec45c055d7ba9542bc8e8724b53f2117be61c913269188"
|
||||
},
|
||||
{
|
||||
"path": "cpu-readonly-observation.json",
|
||||
"sha256": "sha256:c424ecb281baf128d6c79f1e6c6f678093f9b5e9c84a3454f9891fe6197b2e95"
|
||||
},
|
||||
{
|
||||
"path": "executable-readonly-observation.json",
|
||||
"sha256": "sha256:3ee73cdeeb4f82a55b31d19c6a24aacc4199e2e8647335d65b4e8f2e7d079c18"
|
||||
},
|
||||
{
|
||||
"path": "execute-install.py",
|
||||
"sha256": "sha256:44d93406ce62cc1f08ca9c7deb3eed6f45c74d61efd978f7b0acec31ba04cdc5"
|
||||
},
|
||||
{
|
||||
"path": "execution-receipt.json",
|
||||
"sha256": "sha256:59969ad9992ac289a553e1076209ba0d04b6fa7cd171bfad83dcc1f96d80a1c6"
|
||||
},
|
||||
{
|
||||
"path": "final-readonly-dry-run.json",
|
||||
"sha256": "sha256:17144cdb22f975212ea02a5109c94740c430d79272ef9d919ce57dec68c2d880"
|
||||
},
|
||||
{
|
||||
"path": "frozen-native-argv.json",
|
||||
"sha256": "sha256:0efb13aa725111cc227453bc7fc40eaf039c83cbd291ca7cb0409daf6b1e6c55"
|
||||
},
|
||||
{
|
||||
"path": "install-verified.py",
|
||||
"sha256": "sha256:c6895881b83b3f5d5408c646040368cc5c59a99e1495fa0914b9183e087a6831"
|
||||
},
|
||||
{
|
||||
"path": "local-bytecode-refinement.json",
|
||||
"sha256": "sha256:08c92506d6abefe15aff4a9b7908ab8bee425e7792abc6095e1dd04101a2a540"
|
||||
},
|
||||
{
|
||||
"path": "local-transfer-check.py",
|
||||
"sha256": "sha256:97fbec826d517b1746a2ec5ef67e5aab053777690977abdddc8b33ec5804e22e"
|
||||
},
|
||||
{
|
||||
"path": "local-transfer-verification.json",
|
||||
"sha256": "sha256:2331b854389b4c15f5adce1f62aa0846e9f47dfda351a7ef166c90b5e9b2737c"
|
||||
},
|
||||
{
|
||||
"path": "pre-absolute-path-dry-run.json",
|
||||
"sha256": "sha256:9ea47b71b8636e4beb7dadf595f1d08bde261c9986986a1335d0c00d581a6d0c"
|
||||
},
|
||||
{
|
||||
"path": "process-fixture-results.json",
|
||||
"sha256": "sha256:fc72ce36758ea40010aeb7c4ce97c167d09784fde63eeb9b2ccaa3428dc9a831"
|
||||
},
|
||||
{
|
||||
"path": "process_supervision.py",
|
||||
"sha256": "sha256:4d990ee05003040e74bfdac7b2e49a3ef1512d4a054b3ff2317647b3670dd1bf"
|
||||
},
|
||||
{
|
||||
"path": "r1-preservation.json",
|
||||
"sha256": "sha256:50848ad45fda68f43e11710bb04b0793717a5d0659e3fbcf8003708ee5ccb542"
|
||||
},
|
||||
{
|
||||
"path": "r1-to-r2.patch",
|
||||
"sha256": "sha256:63f35421b67576188ff92d920319a23aa0a6606b9a2137d1061687c77126b541"
|
||||
},
|
||||
{
|
||||
"path": "r2-initial-readonly-dry-run.json",
|
||||
"sha256": "sha256:631c6f0d8c0f66f51509751094937fd909821d4f45a042fd77f9d10f237115b8"
|
||||
},
|
||||
{
|
||||
"path": "retained-headroom-refusal.json",
|
||||
"sha256": "sha256:89c847b4a75f138ff2464fc8291cf36939d293a777fd41ffd41d72f96ba9a2f8"
|
||||
},
|
||||
{
|
||||
"path": "retained-local-source-refusal.json",
|
||||
"sha256": "sha256:fdd3b584da6ebedc173ccc5f8879c42dbd821d4b98338337fbef809a403485ba"
|
||||
},
|
||||
{
|
||||
"path": "runtime.py",
|
||||
"sha256": "sha256:19b5fb687288bbc836978b0251599dc1e4d1593e8b5efee85a6abb6a7f388412"
|
||||
},
|
||||
{
|
||||
"path": "target-readonly-addendum.json",
|
||||
"sha256": "sha256:9588f51b353093d178b8c52892ca627c2317fab761591715a0765603a4182a8f"
|
||||
},
|
||||
{
|
||||
"path": "target-readonly-observation.json",
|
||||
"sha256": "sha256:ba4012303e5634686208a818b221113a1a670c31ed29269d8a4cc98308616275"
|
||||
},
|
||||
{
|
||||
"path": "test_process_supervision.py",
|
||||
"sha256": "sha256:7ea89d9dee17d6ac4b554e25c083ac43d9b94039a4bcbe411d0e2ccbfb55a180"
|
||||
},
|
||||
{
|
||||
"path": "transfer-archive.json",
|
||||
"sha256": "sha256:3222ccd39ea968353ea5e58ec36f0e3ca35b608020444243b053074a36abfe0b"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
--- r2/process_supervision.py
|
||||
+++ r3/process_supervision.py
|
||||
@@ -22,7 +22,7 @@
|
||||
try:
|
||||
path=Path('/proc')/str(pid);raw=(path/'stat').read_text();v=raw[raw.rfind(')')+2:].split()
|
||||
return {'pid':pid,'ppid':int(v[1]),'pgrp':int(v[2]),'sid':int(v[3]),'start':int(v[19]),'uid':path.stat().st_uid,'state':v[0]}
|
||||
- except FileNotFoundError:return None
|
||||
+ except (FileNotFoundError,ProcessLookupError):return None
|
||||
|
||||
|
||||
def write(path,value):
|
||||
@@ -88,6 +88,24 @@
|
||||
if os.getppid()!=expected:os._exit(125)
|
||||
|
||||
|
||||
+def reap_orphan_zombies(rows,leaders):
|
||||
+ """Reap only dead adopted direct children; preserve every Popen wait status."""
|
||||
+ reaped=[]
|
||||
+ for pid,row in rows.items():
|
||||
+ if row['ppid']!=os.getpid() or row['state']!='Z' or pid in leaders:continue
|
||||
+ current=facts(pid)
|
||||
+ if current is None:continue
|
||||
+ if current!=row or row['uid']!=os.getuid():raise RuntimeError('orphan_mapping_unknown')
|
||||
+ waited,status=os.waitpid(pid,os.WNOHANG)
|
||||
+ if waited:reaped.append({'facts':row,'exit_code':os.waitstatus_to_exitcode(status)})
|
||||
+ return reaped
|
||||
+
|
||||
+
|
||||
+def job_report(jobs):
|
||||
+ return [{'label':j['request']['label'],'leader':j['leader'],'returncode':j['p'].returncode,
|
||||
+ 'status':j.get('status','running')} for j in jobs.values()][:16]
|
||||
+
|
||||
+
|
||||
def watchdog(directory,parent_fd):
|
||||
directory=Path(directory)
|
||||
if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:raise RuntimeError('subreaper_unavailable')
|
||||
@@ -95,7 +113,7 @@
|
||||
current=facts(root['pid'])
|
||||
if current is None or current['start']!=root['start'] or current['uid']!=os.getuid() or dead(parent_fd):return 1
|
||||
write(directory/'ready.json',facts(os.getpid()))
|
||||
- jobs={};abort=None;uncertain=False
|
||||
+ jobs={};abort=None;uncertain=False;reaped=[];leftover=[]
|
||||
try:
|
||||
while True:
|
||||
if dead(parent_fd):abort='controller_lost'
|
||||
@@ -109,15 +127,21 @@
|
||||
if owner['pid']!=root['pid'] and owner['pid'] not in descendants(os.getpid()):raise ValueError()
|
||||
if type(r['timeout']) not in (int,float) or not 0<r['timeout']<=300:raise ValueError()
|
||||
if not isinstance(r['argv'],list) or not r['argv'] or any(not isinstance(x,str) for x in r['argv']):raise ValueError()
|
||||
- if abort:raise ValueError()
|
||||
+ if r.get('label') not in ('job','node-read','rehearsal-phase','rein-aharness','glas-harness','claude'):raise ValueError()
|
||||
+ if type(r.get('capture')) is not bool:raise ValueError()
|
||||
+ if len(jobs)>=16 or abort:raise ValueError()
|
||||
owner_fd=os.pidfd_open(owner['pid'])
|
||||
+ bound=facts(owner['pid'])
|
||||
+ if bound is None or bound['start']!=owner['start'] or bound['uid']!=owner['uid']:
|
||||
+ os.close(owner_fd);raise ValueError()
|
||||
supervisor_pid=os.getpid()
|
||||
child=subprocess.Popen(r['argv'],stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.PIPE if r['capture'] else subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,env=ENV,start_new_session=True,
|
||||
preexec_fn=lambda parent=supervisor_pid:parent_death(parent))
|
||||
jobs[key]={'p':child,'request':r,'owner_fd':owner_fd,'deadline':time.monotonic()+r['timeout'],
|
||||
- 'output':bytearray(),'eof':not r['capture'],'done':False}
|
||||
+ 'output':bytearray(),'eof':not r['capture'],'done':False,'leader':facts(child.pid)}
|
||||
+ write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
||||
if child.stdout:os.set_blocking(child.stdout.fileno(),False)
|
||||
except Exception:
|
||||
abort=abort or 'request_mapping_refused';uncertain=True
|
||||
@@ -136,27 +160,37 @@
|
||||
if dead(job['owner_fd']):abort=abort or 'caller_lost'
|
||||
if time.monotonic()>=job['deadline']:abort=abort or 'timeout'
|
||||
if p.poll() is not None and job['eof'] and not abort:
|
||||
- # A returned leader is not evidence that its children exited.
|
||||
+ # An adopted zombie is already dead, but must be waited
|
||||
+ # before claiming quiescence. Do not steal a Popen leader's
|
||||
+ # exit status or dismiss a genuinely live leftover.
|
||||
own=descendants(os.getpid())
|
||||
- leaders={x['p'].pid for x in jobs.values() if not x['done']}
|
||||
- if any(row['sid']==p.pid or (row['ppid']==os.getpid() and row['pid'] not in leaders) for row in own.values()):abort='leader_left_descendants'
|
||||
+ leaders={x['p'].pid for x in jobs.values()}
|
||||
+ reaped.extend(reap_orphan_zombies(own,leaders))
|
||||
+ own=descendants(os.getpid())
|
||||
+ active={x['p'].pid for x in jobs.values() if not x['done']}
|
||||
+ leftover=[row for row in own.values() if row['sid']==p.pid or
|
||||
+ (row['ppid']==os.getpid() and row['pid'] not in active)]
|
||||
+ if leftover:abort='leader_left_descendants'
|
||||
else:
|
||||
write(directory/('result-'+key+'.json'),{'returncode':p.returncode,'quiescent':True,'status':'completed',
|
||||
'output_b64':base64.b64encode(job['output']).decode() if job['request']['capture'] else ''})
|
||||
- job['done']=True;os.close(job['owner_fd'])
|
||||
+ job['done']=True;job['status']='completed';os.close(job['owner_fd'])
|
||||
+ write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
||||
if abort:
|
||||
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()]) and not uncertain
|
||||
for key,job in jobs.items():
|
||||
if not job['done']:
|
||||
write(directory/('result-'+key+'.json'),{'returncode':124 if abort=='timeout' else 125,
|
||||
- 'quiescent':ok,'status':abort,'output_b64':''});job['done']=True
|
||||
+ 'quiescent':ok,'status':abort,'output_b64':''});job['done']=True;job['status']=abort
|
||||
write(directory/'quiescent.json',{'quiescent':ok,'reason':abort,'no_live_or_waitable_descendants':ok,
|
||||
- 'stderr_suppressed':True,'process_values_emitted':False})
|
||||
+ 'stderr_suppressed':True,'private_values_emitted':False,'jobs':job_report(jobs),
|
||||
+ 'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
||||
return 0 if ok else 1
|
||||
time.sleep(0.01)
|
||||
except BaseException:
|
||||
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()])
|
||||
- write(directory/'quiescent.json',{'quiescent':False,'reason':'supervisor_exception','best_effort_stopped':ok})
|
||||
+ write(directory/'quiescent.json',{'quiescent':False,'reason':'supervisor_exception','best_effort_stopped':ok,
|
||||
+ 'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
||||
return 1
|
||||
|
||||
|
||||
@@ -172,7 +206,7 @@
|
||||
while not (self.directory/'ready.json').exists():
|
||||
if self.process.poll() is not None or time.monotonic()>end:raise RuntimeError('supervisor_start_failed')
|
||||
time.sleep(0.01)
|
||||
- def run(self,argv,timeout,*,capture=False):return request(self.directory,argv,timeout,capture=capture)
|
||||
+ def run(self,argv,timeout,*,capture=False,label='job'):return request(self.directory,argv,timeout,capture=capture,label=label)
|
||||
def close(self):
|
||||
write(self.directory/'shutdown',{})
|
||||
try:self.process.wait(timeout=8)
|
||||
@@ -182,9 +216,10 @@
|
||||
return report
|
||||
|
||||
|
||||
-def request(directory,argv,timeout,*,capture=False):
|
||||
+def request(directory,argv,timeout,*,capture=False,label='job'):
|
||||
directory=Path(directory);key=uuid.uuid4().hex
|
||||
- write(directory/('request-'+key+'.json'),{'owner':facts(os.getpid()),'argv':list(argv),'timeout':timeout,'capture':capture})
|
||||
+ if (directory/'quiescent.json').exists():raise RuntimeError('supervision_already_stopped')
|
||||
+ write(directory/('request-'+key+'.json'),{'owner':facts(os.getpid()),'argv':list(argv),'timeout':timeout,'capture':capture,'label':label})
|
||||
end=time.monotonic()+timeout+10
|
||||
result=directory/('result-'+key+'.json')
|
||||
while not result.exists():
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"actual_public_process_trace": {"leader_pid": 6, "pid": 7, "ppid": 6, "start": 13166679, "state": "Z", "uid": 1000}, "actual_quiescence": {"no_live_or_waitable_descendants": true, "process_values_emitted": false, "quiescent": true, "reason": "leader_left_descendants", "stderr_suppressed": true}, "actual_returncode": 125, "causal_limit": "proves R2 source zombie false positive; native R2 trigger remains unproven because trace was deleted", "expected_after_dead_orphan_reap": 0, "fixture": "leader exits zero after proving its only child is already-dead zombie", "native_or_namespace_or_provider_calls": 0, "subject": "exact frozen R2 supervisor", "subject_sha256": "sha256:4d990ee05003040e74bfdac7b2e49a3ef1512d4a054b3ff2317647b3670dd1bf"}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Exact frozen R2 zombie counterexample; disposable local processes only."""
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
sys.dont_write_bytecode=True
|
||||
|
||||
HERE=Path(__file__).resolve().parent
|
||||
SUBJECT=HERE.with_name('native-runtime-install-r2')/'process_supervision.py'
|
||||
SPEC=importlib.util.spec_from_file_location('exact_r2',SUBJECT)
|
||||
S=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(S)
|
||||
CODE='''import os,time,json,sys
|
||||
from pathlib import Path
|
||||
pid=os.fork()
|
||||
if pid==0:os._exit(0)
|
||||
for _ in range(100):
|
||||
text=(Path('/proc')/str(pid)/'stat').read_text();v=text[text.rfind(')')+2:].split()
|
||||
if v[0]=='Z':break
|
||||
time.sleep(.005)
|
||||
assert v[0]=='Z'
|
||||
Path(sys.argv[1]).write_text(json.dumps({'pid':pid,'ppid':int(v[1]),'state':v[0],'start':int(v[19]),'uid':os.getuid(),'leader_pid':os.getpid()}))
|
||||
'''
|
||||
with tempfile.TemporaryDirectory(prefix='nk-exact-r2-zombie-') as d:
|
||||
base=Path(d);trace=base/'public-process-trace.json';supervisor=S.Supervisor(base/'state')
|
||||
code,output=supervisor.run([sys.executable,'-B','-c',CODE,str(trace)],2,capture=True)
|
||||
report=supervisor.close()
|
||||
result={'subject':'exact frozen R2 supervisor','subject_sha256':'sha256:'+hashlib.sha256(SUBJECT.read_bytes()).hexdigest(),
|
||||
'fixture':'leader exits zero after proving its only child is already-dead zombie',
|
||||
'actual_public_process_trace':json.loads(trace.read_text()),'actual_returncode':code,'actual_quiescence':report,
|
||||
'expected_after_dead_orphan_reap':0,'native_or_namespace_or_provider_calls':0,
|
||||
'causal_limit':'proves R2 source zombie false positive; native R2 trigger remains unproven because trace was deleted'}
|
||||
assert code==125 and report['reason']=='leader_left_descendants' and report['quiescent']
|
||||
print(json.dumps(result,sort_keys=True))
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
{
|
||||
"first_run": "7 passed, 1 fixture expectation failed",
|
||||
"case": "true live orphan",
|
||||
"observed": 124,
|
||||
"initial_expected": 125,
|
||||
"cause": "live child inherited captured stdout and held EOF open, so existing whole-job timeout correctly stopped it before normal-completion orphan classification",
|
||||
"refinement": "disable capture for this case to exercise immediate leader_left_descendants classification; actual runtime launcher capture remains false",
|
||||
"native_actions": 0
|
||||
}
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
"""Digest-pinned, owner-configured Python runtimes for bwrap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
|
||||
RUNTIME_MOUNT = "/opt/sandboxer/runtime"
|
||||
|
||||
|
||||
def runtime_digest(root: Path) -> str:
|
||||
"""Hash the complete artifact, including modes and internal symlink targets."""
|
||||
if not root.is_absolute() or root.resolve(strict=True) != root or not root.is_dir():
|
||||
raise ValueError("runtime path must be a canonical absolute directory")
|
||||
records = []
|
||||
for path in sorted(root.rglob("*")):
|
||||
mode = path.lstat().st_mode
|
||||
if stat.S_ISLNK(mode):
|
||||
target = os.readlink(path)
|
||||
if Path(target).is_absolute() or not path.resolve(strict=True).is_relative_to(root):
|
||||
raise ValueError("runtime symlink escapes the artifact")
|
||||
value = ["symlink", target]
|
||||
elif stat.S_ISREG(mode):
|
||||
with path.open("rb") as stream:
|
||||
value = ["file", hashlib.file_digest(stream, "sha256").hexdigest()]
|
||||
elif stat.S_ISDIR(mode):
|
||||
value = ["directory"]
|
||||
else:
|
||||
raise ValueError("runtime contains an unsupported special file")
|
||||
records.append([path.relative_to(root).as_posix(), stat.S_IMODE(mode), *value])
|
||||
return hashlib.sha256(json.dumps(records, separators=(",", ":")).encode()).hexdigest()
|
||||
|
||||
|
||||
def verified_runtime(config: dict) -> Path | None:
|
||||
"""Only owner extension configuration can select a runtime artifact."""
|
||||
runtime = config.get("runtime")
|
||||
if runtime is None:
|
||||
return None
|
||||
if not isinstance(runtime, dict) or set(runtime) != {"path", "sha256"}:
|
||||
raise ValueError("runtime requires exactly path and sha256")
|
||||
if not isinstance(runtime["path"], str) or not isinstance(runtime["sha256"], str):
|
||||
raise ValueError("runtime path and sha256 must be strings")
|
||||
root = Path(runtime["path"])
|
||||
if (not (root / "pyvenv.cfg").is_file() or not (root / "bin/python3").is_file()
|
||||
or (root / ".git").exists()):
|
||||
raise ValueError("runtime must be a standalone Python environment, not a checkout")
|
||||
if runtime_digest(root) != runtime["sha256"]:
|
||||
raise ValueError("runtime artifact digest does not match owner configuration")
|
||||
return root
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"owner_task": "HARNESS-WP-0008-T07",
|
||||
"canonical_owner_source_edited": false,
|
||||
"source_refs": [
|
||||
{
|
||||
"path": "/home/worsch/rein-aharness/AGENTS.md",
|
||||
"sha256": "sha256:da71c6e3d76d7c867b2db6e67cd919fba594acc3804d3784dd744438103560c2"
|
||||
},
|
||||
{
|
||||
"path": "/home/worsch/rein-aharness/workplans/REINAH-WP-0008-simpler-metered-retry.md",
|
||||
"sha256": "sha256:1118924bb159f8a52a37ace8b8663c75274b89928f7a8b957638428af224b054"
|
||||
},
|
||||
{
|
||||
"path": "/home/worsch/sand-boxer/docs/bwrap-runtime.md",
|
||||
"sha256": "sha256:3b45a7f9a8e7f496f6b4102cd2508826c17a7d61b7578ede1a4c63215c467134"
|
||||
},
|
||||
{
|
||||
"path": "/tmp/netkingdom-runtime-canonical-20261016/runtime/lib/python3.12/site-packages/rein_aharness/owner_rehearsal.py",
|
||||
"sha256": "sha256:192f654cad4a5f065c7faa5e35b03493297a472e8d7fe641be2c42c5fb00dcc3"
|
||||
}
|
||||
],
|
||||
"r2_existing_bytes_unchanged": true,
|
||||
"r2_native_cause": "unproven; exact source zombie bug demonstrated only",
|
||||
"r3_no_execution_receipt": true
|
||||
}
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Disposable actual process-tree oracles; never invoke runtime, SSH or bwrap."""
|
||||
import ctypes
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import select
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE=Path(__file__).resolve().parent
|
||||
SPEC=importlib.util.spec_from_file_location('supervised',HERE/'process_supervision.py')
|
||||
S=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(S)
|
||||
CANARY='fixture-private-stderr-must-not-appear'
|
||||
# A session-escaping grandchild remains owned by the subreaper even after the
|
||||
# intermediate process exits. No namespace or application launcher is involved.
|
||||
LEAF="import os,time,sys;from pathlib import Path;Path(sys.argv[1]).write_text(str(os.getpid()));print(sys.argv[2],file=sys.stderr,flush=True);time.sleep(30)"
|
||||
LEADER="import subprocess,sys,time;subprocess.Popen([sys.executable,'-c',sys.argv[1],sys.argv[2],sys.argv[3]],start_new_session=True);time.sleep(30)"
|
||||
|
||||
|
||||
def wait_until(predicate,timeout=8):
|
||||
end=time.monotonic()+timeout
|
||||
while time.monotonic()<end:
|
||||
if predicate():return
|
||||
time.sleep(.02)
|
||||
raise AssertionError('bounded fixture condition failed')
|
||||
|
||||
|
||||
def not_alive(pid):
|
||||
f=S.facts(pid)
|
||||
return f is None or f['state']=='Z'
|
||||
|
||||
|
||||
class ProcessSupervisionTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# Reap only the known detached fixture watchdog after killed-controller
|
||||
# tests, so this test does not depend on the workstation PID1 reaper.
|
||||
assert ctypes.CDLL(None).prctl(36,1,0,0,0)==0
|
||||
|
||||
def setUp(self):
|
||||
self.temp=tempfile.TemporaryDirectory(prefix='nk-supervision-fixture-')
|
||||
self.base=Path(self.temp.name);self.sentinel=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'])
|
||||
self.addCleanup(self.cleanup)
|
||||
|
||||
def cleanup(self):
|
||||
self.sentinel.terminate();self.sentinel.wait(timeout=3);self.temp.cleanup()
|
||||
|
||||
def argv(self,marker):return [sys.executable,'-c',LEADER,LEAF,str(marker),CANARY]
|
||||
|
||||
def assert_quiet(self,state,pid):
|
||||
wait_until(lambda:(state/'quiescent.json').exists())
|
||||
r=json.loads((state/'quiescent.json').read_text())
|
||||
self.assertTrue(r['quiescent']);self.assertTrue(r['no_live_or_waitable_descendants'])
|
||||
wait_until(lambda:not_alive(pid));self.assertIsNone(self.sentinel.poll())
|
||||
for p in state.glob('*.json'):
|
||||
if p.name.startswith('request-'):continue # test input deliberately contains canary
|
||||
self.assertNotIn(CANARY,p.read_text())
|
||||
return r
|
||||
|
||||
def test_success_and_bounded_stdout(self):
|
||||
s=S.Supervisor(self.base/'state')
|
||||
self.assertEqual(s.run([sys.executable,'-c','print("receipt")'],2,capture=True),(0,b'receipt\n'))
|
||||
self.assertTrue(s.close()['quiescent'])
|
||||
|
||||
def test_timeout_reaps_session_escaped_descendant_and_retains_sibling(self):
|
||||
state=self.base/'state';marker=self.base/'child';s=S.Supervisor(state)
|
||||
code,output=s.run(self.argv(marker),.6,capture=True)
|
||||
self.assertEqual(code,124);self.assertEqual(output,b'')
|
||||
self.assertTrue(marker.exists());r=self.assert_quiet(state,int(marker.read_text()))
|
||||
self.assertEqual(r['reason'],'timeout');self.assertTrue(s.close()['quiescent'])
|
||||
|
||||
def test_hard_controller_loss_reaps_descendant_and_retains_stage(self):
|
||||
state=self.base/'state';marker=self.base/'child'
|
||||
controller="import importlib.util,sys;from pathlib import Path;s=importlib.util.spec_from_file_location('s',sys.argv[1]);m=importlib.util.module_from_spec(s);s.loader.exec_module(m);x=m.Supervisor(Path(sys.argv[2]));x.run(sys.argv[3:],20,capture=True)"
|
||||
p=subprocess.Popen([sys.executable,'-I','-B','-c',controller,str(HERE/'process_supervision.py'),str(state),*self.argv(marker)],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
self.addCleanup(lambda:p.poll() is None and p.kill())
|
||||
wait_until(marker.exists);watchdog=json.loads((state/'ready.json').read_text())['pid']
|
||||
os.kill(p.pid,signal.SIGKILL);p.wait(timeout=3)
|
||||
r=self.assert_quiet(state,int(marker.read_text()));self.assertEqual(r['reason'],'controller_lost')
|
||||
self.assertTrue(state.exists()) # no controller remains to delete its stage
|
||||
wait_until(lambda:not_alive(watchdog));os.waitpid(watchdog,0)
|
||||
|
||||
def test_outer_phase_timeout_stops_nested_runner_sibling(self):
|
||||
state=self.base/'state';marker=self.base/'child';s=S.Supervisor(state)
|
||||
phase="import importlib.util,sys;from pathlib import Path;s=importlib.util.spec_from_file_location('s',sys.argv[1]);m=importlib.util.module_from_spec(s);s.loader.exec_module(m);m.request(Path(sys.argv[2]),sys.argv[3:],20)"
|
||||
code,output=s.run([sys.executable,'-I','-B','-c',phase,str(HERE/'process_supervision.py'),str(state),*self.argv(marker)],.7,capture=True)
|
||||
self.assertEqual(code,124);self.assertTrue(marker.exists())
|
||||
self.assert_quiet(state,int(marker.read_text()));self.assertTrue(s.close()['quiescent'])
|
||||
|
||||
def test_dead_adopted_zombie_is_reaped_without_false_live_failure(self):
|
||||
state=self.base/'state';s=S.Supervisor(state)
|
||||
code="import os,time;pid=os.fork();os._exit(0) if pid==0 else None;time.sleep(.1)"
|
||||
rc,out=s.run([sys.executable,'-B','-c',code],2,capture=True)
|
||||
self.assertEqual(rc,0);report=s.close();self.assertTrue(report['quiescent'])
|
||||
self.assertTrue(report['reaped_orphan_zombies'])
|
||||
self.assertTrue(all(x['facts']['state']=='Z' and x['exit_code']==0 for x in report['reaped_orphan_zombies']))
|
||||
|
||||
def test_true_live_orphan_still_refuses_and_reports_exact_public_state(self):
|
||||
state=self.base/'state';marker=self.base/'child';s=S.Supervisor(state)
|
||||
code="import subprocess,sys,time;subprocess.Popen([sys.executable,'-c',sys.argv[1],sys.argv[2],sys.argv[3]],start_new_session=True);time.sleep(.1)"
|
||||
rc,out=s.run([sys.executable,'-B','-c',code,LEAF,str(marker),CANARY],2,capture=False)
|
||||
self.assertEqual(rc,125);report=self.assert_quiet(state,int(marker.read_text()))
|
||||
self.assertEqual(report['reason'],'leader_left_descendants');self.assertTrue(report['leftover_at_failure'])
|
||||
self.assertTrue(any(x['state']!='Z' for x in report['leftover_at_failure']))
|
||||
self.assertTrue(s.close()['quiescent'])
|
||||
|
||||
def test_three_nested_launcher_receipts_survive_normal_zombie_reaping(self):
|
||||
state=self.base/'state';s=S.Supervisor(state)
|
||||
phase="import importlib.util,sys,json;s=importlib.util.spec_from_file_location('s',sys.argv[1]);m=importlib.util.module_from_spec(s);s.loader.exec_module(m);code='import os,time;pid=os.fork();os._exit(0) if pid==0 else None;time.sleep(.08)';results={name:m.request(sys.argv[2],[sys.executable,'-B','-c',code],2,label=name)[0] for name in ('rein-aharness','glas-harness','claude')};print(json.dumps(results))"
|
||||
rc,out=s.run([sys.executable,'-I','-B','-c',phase,str(HERE/'process_supervision.py'),str(state)],5,capture=True,label='rehearsal-phase')
|
||||
self.assertEqual(rc,0);self.assertEqual(json.loads(out),{'rein-aharness':0,'glas-harness':0,'claude':0})
|
||||
report=s.close();self.assertEqual(len(report['jobs']),4);self.assertEqual(len(report['reaped_orphan_zombies']),3)
|
||||
self.assertTrue(all(x['status']=='completed' and x['returncode']==0 for x in report['jobs']))
|
||||
|
||||
def test_invalid_mapping_retains_failed_quiescence_evidence(self):
|
||||
state=self.base/'state';s=S.Supervisor(state);owner=S.facts(os.getpid());owner['start']+=1
|
||||
S.write(state/'request-invalid.json',{'owner':owner,'argv':[sys.executable,'-c','pass'],'timeout':1,'capture':False})
|
||||
wait_until(lambda:(state/'quiescent.json').exists())
|
||||
self.assertFalse(json.loads((state/'quiescent.json').read_text())['quiescent'])
|
||||
with self.assertRaises(RuntimeError):s.close()
|
||||
self.assertTrue(state.exists());self.assertIsNone(self.sentinel.poll())
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Exercise actual controller receipt/cleanup ordering with local disposable IO.
|
||||
Only native admission/launch inputs are stubbed; no runtime/native calls.
|
||||
"""
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
HERE=Path(__file__).resolve().parent
|
||||
|
||||
|
||||
class ReceiptCleanupTests(unittest.TestCase):
|
||||
def exercise(self,fail_persist):
|
||||
spec=importlib.util.spec_from_file_location('controller',HERE/'verify-installed.py')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
with tempfile.TemporaryDirectory(prefix='nk-receipt-order-') as d:
|
||||
base=Path(d);stage=base/'stage';stage.mkdir(mode=0o700)
|
||||
receipt=base/'receipt.json';destination=base/'installed';destination.mkdir()
|
||||
for name in ('runtime.py','verify-installed.py','process_supervision.py'):(stage/name).touch(mode=0o600)
|
||||
jobs=[{'label':name,'returncode':0,'status':'completed'} for name in ('rein-aharness','glas-harness','claude')]
|
||||
class FakeSupervisor:
|
||||
def __init__(self,state):
|
||||
state.mkdir(mode=0o700);p=state/'public-journal.json';p.write_text(json.dumps({'jobs':jobs}));p.chmod(0o600)
|
||||
def close(self):return {'quiescent':True,'no_live_or_waitable_descendants':True,'jobs':jobs}
|
||||
original_protected=m.protected
|
||||
def protected(path,**kwargs):
|
||||
actual=original_protected(path,**kwargs)
|
||||
if path==stage:return SimpleNamespace(st_ino=actual.st_ino,st_dev=2049)
|
||||
return actual
|
||||
removed=[];real_rmtree=shutil.rmtree
|
||||
def checked_remove(path):
|
||||
before=json.loads(receipt.read_text())
|
||||
self.assertTrue(before['durable_receipt_before_cleanup'])
|
||||
self.assertTrue(before['process_supervision']['quiescent'])
|
||||
self.assertEqual(before['cleanup_status'],'pending')
|
||||
self.assertEqual([x['exit_code'] for x in before['launcher_status']],[0,0,0])
|
||||
removed.append(True);real_rmtree(path)
|
||||
output=io.StringIO()
|
||||
with contextlib.ExitStack() as stack:
|
||||
for name,value in {'STAGE':stage,'DESTINATION':destination,'DURABLE_RECEIPT':receipt,
|
||||
'protected':protected,'identify':lambda supervisor:None,
|
||||
'supervision_module':lambda:SimpleNamespace(Supervisor=FakeSupervisor),
|
||||
'runtime_reader':lambda:SimpleNamespace(runtime_digest=lambda path:m.DIGEST),
|
||||
'phase':lambda *args:(0,{'status':'rehearsed','steps':{j['label']:0 for j in jobs}})}.items():
|
||||
stack.enter_context(patch.object(m,name,value))
|
||||
stack.enter_context(patch.object(m.os,'nice',lambda value:None))
|
||||
stack.enter_context(patch.object(m.shutil,'rmtree',checked_remove))
|
||||
stack.enter_context(patch.object(sys,'argv',['verify-installed.py','--stage-inode',str(stage.stat().st_ino)]))
|
||||
if fail_persist:stack.enter_context(patch.object(m,'persist',side_effect=OSError('fixture write failure')))
|
||||
with contextlib.redirect_stdout(output):code=m.main()
|
||||
result=json.loads(output.getvalue())
|
||||
if fail_persist:
|
||||
self.assertEqual(code,1);self.assertFalse(removed);self.assertTrue(stage.exists())
|
||||
self.assertFalse(result['durable_receipt_before_cleanup'])
|
||||
else:
|
||||
self.assertEqual(code,0);self.assertEqual(removed,[True]);self.assertFalse(stage.exists())
|
||||
checkpoint=json.loads(receipt.read_text());self.assertEqual(checkpoint['cleanup_status'],'pending')
|
||||
self.assertTrue(result['owned_staging_removed']);self.assertEqual(receipt.stat().st_mode&0o777,0o600)
|
||||
|
||||
def test_actual_cleanup_waits_for_durable_complete_public_receipt(self):self.exercise(False)
|
||||
def test_actual_cleanup_refuses_when_durable_receipt_write_fails(self):self.exercise(True)
|
||||
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Verify already-installed immutable runtime only; fresh revision-3 approval required."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import pwd
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import sys
|
||||
import time
|
||||
|
||||
DIGEST = 'f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
||||
RUNTIME_MODULE = '19b5fb687288bbc836978b0251599dc1e4d1593e8b5efee85a6abb6a7f388412'
|
||||
REHEARSAL_MODULE = '192f654cad4a5f065c7faa5e35b03493297a472e8d7fe641be2c42c5fb00dcc3'
|
||||
STAGE = Path('/home/tegwick/.helixforge-factory/runtime-verify-f599-r3-20261017')
|
||||
STORE = Path('/home/tegwick/.helixforge-factory/runtimes')
|
||||
DESTINATION = STORE/DIGEST
|
||||
DURABLE_RECEIPT = Path(str(STAGE)+'.receipt.json')
|
||||
ENV = {'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'}
|
||||
|
||||
|
||||
def require(value, code):
|
||||
if not value: raise ValueError(code)
|
||||
|
||||
|
||||
def file_digest(path):
|
||||
with path.open('rb') as source:return hashlib.file_digest(source,'sha256').hexdigest()
|
||||
|
||||
|
||||
def protected(path, *, directory=False, mode=None):
|
||||
s=path.lstat()
|
||||
require(not stat.S_ISLNK(s.st_mode) and s.st_uid==1000 and s.st_gid==1000,'path_ownership')
|
||||
require(stat.S_ISDIR(s.st_mode) if directory else stat.S_ISREG(s.st_mode),'path_type')
|
||||
require(not stat.S_IMODE(s.st_mode)&0o022,'path_write_mode')
|
||||
if mode is not None:require(stat.S_IMODE(s.st_mode)==mode,'path_exact_mode')
|
||||
return s
|
||||
|
||||
|
||||
def identify(supervisor):
|
||||
require(os.getuid()==1000 and os.getgid()==1000 and pwd.getpwuid(1000).pw_name=='tegwick','account_identity')
|
||||
for p in (Path('/home/tegwick'),Path('/home/tegwick/.helixforge-factory'),STORE):
|
||||
protected(p,directory=True,mode=0o700 if p!=Path('/home/tegwick') else 0o750)
|
||||
protected(STORE/'.install.lock',mode=0o600)
|
||||
code,output=supervisor.run(['/usr/local/bin/kubectl','get','nodes','-o','json'],20,capture=True,label='node-read')
|
||||
require(code==0,'node_readback')
|
||||
nodes=json.loads(output)['items']
|
||||
require(len(nodes)==1 and any(a['address']=='92.205.62.239' for a in nodes[0]['status']['addresses'])
|
||||
and any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions']),'node_identity')
|
||||
require(shutil.disk_usage(STORE).free >= 2*1024**3,'disk_headroom')
|
||||
mem=dict(x.split(':',1) for x in Path('/proc/meminfo').read_text().splitlines())
|
||||
require(int(mem['MemAvailable'].split()[0])*1024>=1024**3,'memory_headroom')
|
||||
require(os.getloadavg()[0]<=2*(os.cpu_count() or 1),'load_headroom')
|
||||
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
||||
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)]
|
||||
require(1-(d[3]+d[4])/sum(d)<=0.85,'cpu_headroom')
|
||||
|
||||
|
||||
def runtime_reader():
|
||||
p=STAGE/'runtime.py';protected(p,mode=0o600)
|
||||
require(file_digest(p)==RUNTIME_MODULE,'runtime_reader_pin')
|
||||
spec=importlib.util.spec_from_file_location('approved_runtime_reader',p)
|
||||
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def expired(signum, frame):
|
||||
raise TimeoutError('bounded_stage_deadline')
|
||||
|
||||
|
||||
def rehearse_phase(supervision):
|
||||
from rein_aharness import owner_rehearsal
|
||||
require(Path(owner_rehearsal.__file__).resolve().is_relative_to(DESTINATION)
|
||||
and file_digest(Path(owner_rehearsal.__file__))==REHEARSAL_MODULE,'rehearsal_source_pin')
|
||||
require(owner_rehearsal.SYSTEM_RO_BINDS==('/usr','/bin','/lib','/lib64','/etc/resolv.conf','/etc/ssl/certs')
|
||||
and all(Path(p).exists() for p in owner_rehearsal.SYSTEM_RO_BINDS),'frozen_system_bind_contract')
|
||||
signal.signal(signal.SIGALRM,expired);signal.alarm(240)
|
||||
# Keep the frozen owner launch contract; use its supported Runner seam.
|
||||
def bounded_run(argv,timeout):
|
||||
require(argv[0]=='/usr/bin/bwrap' and '--die-with-parent' not in argv,'bwrap_runner_contract')
|
||||
return supervision_request(supervision,[argv[0],'--die-with-parent',*argv[1:]],timeout,label=Path(argv[-2]).name)[0]
|
||||
try:steps=owner_rehearsal.rehearse(DESTINATION,run=bounded_run,bwrap_bin='/usr/bin/bwrap')
|
||||
finally:signal.alarm(0)
|
||||
require([x.name for x in steps]==['rein-aharness','glas-harness','claude'],'rehearsal_steps')
|
||||
report={x.name:x.exit_code for x in steps}
|
||||
print(json.dumps({'status':'rehearsed' if all(x.ok for x in steps) else 'rehearsal_failed','steps':report,
|
||||
'model_calls':0,'credential_reads':False,'owner_configuration_read':False},sort_keys=True))
|
||||
return 0 if all(x.ok for x in steps) else 1
|
||||
|
||||
|
||||
def supervision_module():
|
||||
spec=importlib.util.spec_from_file_location('approved_process_supervision',STAGE/'process_supervision.py')
|
||||
module=importlib.util.module_from_spec(spec);spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def supervision_request(directory,argv,timeout,*,label):
|
||||
return supervision_module().request(directory,argv,timeout,label=label)
|
||||
|
||||
|
||||
def phase(supervisor, argv, timeout):
|
||||
code,output=supervisor.run(argv,timeout,capture=True,label='rehearsal-phase')
|
||||
try:result=json.loads(output)
|
||||
except Exception:raise ValueError('phase_receipt_unavailable') from None
|
||||
return code,result
|
||||
|
||||
|
||||
def public_snapshot():
|
||||
p=STAGE/'supervision'/'public-journal.json'
|
||||
if not p.exists():return {'status':'unavailable'}
|
||||
protected(p,mode=0o600);require(p.stat().st_size<=65536,'journal_bound')
|
||||
return json.loads(p.read_text())
|
||||
|
||||
|
||||
def persist(fd,receipt):
|
||||
# The descriptor remains bound to the one exclusively created public receipt.
|
||||
data=(json.dumps(receipt,sort_keys=True)+'\n').encode()
|
||||
require(len(data)<=65536,'durable_receipt_bound')
|
||||
require(os.fstat(fd).st_size==0,'durable_receipt_already_written')
|
||||
with os.fdopen(os.dup(fd),'wb') as stream:stream.write(data);stream.flush();os.fsync(stream.fileno())
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--stage-inode',type=int)
|
||||
parser.add_argument('--phase',choices=('rehearse',))
|
||||
parser.add_argument('--supervision-dir')
|
||||
args=parser.parse_args()
|
||||
if args.phase:
|
||||
require(args.supervision_dir==str(STAGE/'supervision'),'supervision_path')
|
||||
return rehearse_phase(args.supervision_dir)
|
||||
receipt={'proposal_id':'REINAH-WP-0008-T07-install-f599-r1','revision':3,'status':'failed',
|
||||
'model_calls':0,'credential_reads':False,'owner_configuration_read':False,
|
||||
'renewal_or_catalog_or_queue_mutation':False,'install_calls':0,'destination':str(DESTINATION),
|
||||
'durable_public_receipt':str(DURABLE_RECEIPT)}
|
||||
identity=None;supervisor=None;quiescent=False;receipt_fd=None;durable=False
|
||||
try:
|
||||
observed=protected(STAGE,directory=True,mode=0o700)
|
||||
require(observed.st_ino==args.stage_inode and observed.st_dev==2049,'stage_identity')
|
||||
identity=observed
|
||||
require(not DURABLE_RECEIPT.exists() and not DURABLE_RECEIPT.is_symlink(),'receipt_already_present')
|
||||
os.nice(19)
|
||||
supervisor=supervision_module().Supervisor(STAGE/'supervision')
|
||||
identify(supervisor);protected(DESTINATION,directory=True)
|
||||
require({p.name for p in STAGE.iterdir()}=={'runtime.py','verify-installed.py','process_supervision.py','supervision'},'stage_inventory')
|
||||
require(runtime_reader().runtime_digest(DESTINATION)==DIGEST,'installed_complete_digest')
|
||||
receipt['installed_complete_digest_verified']=True
|
||||
receipt['phase']='offline_rehearsal'
|
||||
code,result=phase(supervisor,[str(DESTINATION/'bin/python3'),'-I','-B',str(STAGE/'verify-installed.py'),'--phase','rehearse','--supervision-dir',str(STAGE/'supervision')],260)
|
||||
receipt['rehearsal']=result;require(code==0 and result['status']=='rehearsed','rehearsal_failed')
|
||||
require(runtime_reader().runtime_digest(DESTINATION)==DIGEST,'post_rehearsal_complete_digest')
|
||||
receipt['post_rehearsal_complete_digest_verified']=True
|
||||
receipt['status']='installed_runtime_offline_rehearsal_verified'
|
||||
except Exception as exc:
|
||||
receipt['failure_type']=type(exc).__name__
|
||||
if isinstance(exc,ValueError) and str(exc).replace('_','').isalnum():receipt['failure_code']=str(exc)
|
||||
finally:
|
||||
if supervisor is not None:
|
||||
try:
|
||||
receipt['process_supervision']=supervisor.close();quiescent=True
|
||||
except Exception:
|
||||
receipt['process_supervision']={'quiescent':False,'status':'cleanup_unverified'}
|
||||
receipt['status']='failed'
|
||||
if identity is not None:
|
||||
try:receipt['public_job_journal']=public_snapshot()
|
||||
except Exception:receipt['public_job_journal']={'status':'unavailable_or_invalid'};receipt['status']='failed'
|
||||
receipt['launcher_status']=[]
|
||||
jobs=receipt.get('process_supervision',{}).get('jobs',receipt.get('public_job_journal',{}).get('jobs',[]))
|
||||
for name in ('rein-aharness','glas-harness','claude'):
|
||||
found=[j for j in jobs if j['label']==name]
|
||||
receipt['launcher_status'].append({'name':name,'status':found[0]['status'] if len(found)==1 else 'not_started_or_unverified',
|
||||
'exit_code':found[0]['returncode'] if len(found)==1 else None})
|
||||
receipt['owned_staging_removed']=False;receipt['cleanup_status']='pending'
|
||||
receipt['native_receipt_scope']='immutable precleanup checkpoint; final cleanup outcome in local execution receipt'
|
||||
receipt['durable_receipt_before_cleanup']=True
|
||||
try:
|
||||
receipt_fd=os.open(DURABLE_RECEIPT,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
||||
persist(receipt_fd,receipt)
|
||||
directory_fd=os.open(DURABLE_RECEIPT.parent,os.O_RDONLY|os.O_DIRECTORY)
|
||||
try:os.fsync(directory_fd)
|
||||
finally:os.close(directory_fd)
|
||||
durable=True
|
||||
except Exception:
|
||||
receipt['status']='failed';receipt['durable_receipt_before_cleanup']=False
|
||||
if quiescent and durable:
|
||||
try:
|
||||
current=protected(STAGE,directory=True,mode=0o700)
|
||||
require((current.st_dev,current.st_ino)==(identity.st_dev,identity.st_ino),'cleanup_stage_identity')
|
||||
require({p.name for p in STAGE.iterdir()} <= {'runtime.py','verify-installed.py','process_supervision.py','supervision'},'cleanup_unexpected_stage_member')
|
||||
shutil.rmtree(STAGE)
|
||||
receipt['owned_staging_removed']=not STAGE.exists() and not STAGE.is_symlink()
|
||||
receipt['cleanup_status']='removed_after_durable_receipt_and_quiescence'
|
||||
except Exception:receipt['cleanup_status']='failed';receipt['status']='failed'
|
||||
else:
|
||||
receipt['cleanup_status']='retained_for_quiescence_or_receipt_review'
|
||||
if receipt_fd is not None:os.close(receipt_fd)
|
||||
print(json.dumps(receipt,sort_keys=True))
|
||||
return 0 if receipt['status']=='installed_runtime_offline_rehearsal_verified' else 1
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
try:raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(json.dumps({'status':'failed','failure_type':type(exc).__name__,'private_diagnostics_suppressed':True}))
|
||||
raise SystemExit(1) from None
|
||||
Loading…
Add table
Add a link
Reference in a new issue