239 lines
12 KiB
Python
239 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""One-action private process supervisor; no credentials, network or runtime policy."""
|
|
import argparse
|
|
import base64
|
|
import ctypes
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import select
|
|
import signal
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
MAX_OUTPUT=65536
|
|
ENV={'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'}
|
|
|
|
|
|
def facts(pid):
|
|
try:
|
|
path=Path('/proc')/str(pid);raw=(path/'stat').read_text();v=raw[raw.rfind(')')+2:].split()
|
|
return {'pid':pid,'ppid':int(v[1]),'pgrp':int(v[2]),'sid':int(v[3]),'start':int(v[19]),'uid':path.stat().st_uid,'state':v[0]}
|
|
except (FileNotFoundError,ProcessLookupError):return None
|
|
|
|
|
|
def write(path,value):
|
|
temp=path.with_name('.'+path.name+'.'+uuid.uuid4().hex)
|
|
fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'w') as f:json.dump(value,f);f.flush();os.fsync(f.fileno())
|
|
temp.replace(path)
|
|
|
|
|
|
def dead(fd):
|
|
p=select.poll();p.register(fd,select.POLLIN);return bool(p.poll(0))
|
|
|
|
|
|
def descendants(parent):
|
|
rows={}
|
|
for p in Path('/proc').iterdir():
|
|
if p.name.isdigit():
|
|
row=facts(int(p.name))
|
|
if row:rows[row['pid']]=row
|
|
selected={parent};changed=True
|
|
while changed:
|
|
before=len(selected);selected.update(r['pid'] for r in rows.values() if r['ppid'] in selected)
|
|
changed=len(selected)!=before
|
|
return {p:rows[p] for p in selected-{parent} if p in rows}
|
|
|
|
|
|
def quiesce(parent,children):
|
|
"""Signal only pidfd-bound actual descendants; subreaper adopts escaped sessions."""
|
|
uncertain=False
|
|
for sig,limit in ((signal.SIGTERM,0.5),(signal.SIGKILL,3.0)):
|
|
end=time.monotonic()+limit
|
|
while time.monotonic()<end:
|
|
try:rows=descendants(parent)
|
|
except Exception:return False
|
|
for pid,row in rows.items():
|
|
try:
|
|
if row['uid']!=os.getuid():uncertain=True;continue
|
|
fd=os.pidfd_open(pid)
|
|
current=facts(pid)
|
|
if current is not None and (current['start'],current['uid'])==(row['start'],row['uid']):
|
|
signal.pidfd_send_signal(fd,sig)
|
|
elif current is not None:uncertain=True
|
|
os.close(fd)
|
|
except ProcessLookupError:pass
|
|
except Exception:uncertain=True
|
|
for child in children:child.poll()
|
|
while True:
|
|
try:
|
|
pid,_=os.waitpid(-1,os.WNOHANG)
|
|
if pid==0:break
|
|
except ChildProcessError:break
|
|
try:remaining=descendants(parent)
|
|
except Exception:return False
|
|
if not remaining:return not uncertain
|
|
time.sleep(0.02)
|
|
return False
|
|
|
|
|
|
def parent_death(expected):
|
|
# Called before exec in a single-threaded supervisor. No application code runs
|
|
# if the supervisor disappeared between fork and PR_SET_PDEATHSIG.
|
|
if ctypes.CDLL(None,use_errno=True).prctl(1,signal.SIGKILL,0,0,0)!=0:os._exit(125)
|
|
if os.getppid()!=expected:os._exit(125)
|
|
|
|
|
|
def reap_orphan_zombies(rows,leaders):
|
|
"""Reap only dead adopted direct children; preserve every Popen wait status."""
|
|
reaped=[]
|
|
for pid,row in rows.items():
|
|
if row['ppid']!=os.getpid() or row['state']!='Z' or pid in leaders:continue
|
|
current=facts(pid)
|
|
if current is None:continue
|
|
if current!=row or row['uid']!=os.getuid():raise RuntimeError('orphan_mapping_unknown')
|
|
waited,status=os.waitpid(pid,os.WNOHANG)
|
|
if waited:reaped.append({'facts':row,'exit_code':os.waitstatus_to_exitcode(status)})
|
|
return reaped
|
|
|
|
|
|
def job_report(jobs):
|
|
return [{'label':j['request']['label'],'leader':j['leader'],'returncode':j['p'].returncode,
|
|
'status':j.get('status','running')} for j in jobs.values()][:16]
|
|
|
|
|
|
def watchdog(directory,parent_fd):
|
|
directory=Path(directory)
|
|
if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:raise RuntimeError('subreaper_unavailable')
|
|
root=json.loads((directory/'root.json').read_text())
|
|
current=facts(root['pid'])
|
|
if current is None or current['start']!=root['start'] or current['uid']!=os.getuid() or dead(parent_fd):return 1
|
|
write(directory/'ready.json',facts(os.getpid()))
|
|
jobs={};abort=None;uncertain=False;reaped=[];leftover=[]
|
|
try:
|
|
while True:
|
|
if dead(parent_fd):abort='controller_lost'
|
|
if (directory/'shutdown').exists():abort=abort or 'shutdown'
|
|
for path in sorted(directory.glob('request-*.json')):
|
|
key=path.stem.removeprefix('request-')
|
|
if key in jobs:continue
|
|
try:
|
|
r=json.loads(path.read_text());owner=facts(r['owner']['pid'])
|
|
if owner is None or owner['start']!=r['owner']['start'] or owner['uid']!=os.getuid():raise ValueError()
|
|
if owner['pid']!=root['pid'] and owner['pid'] not in descendants(os.getpid()):raise ValueError()
|
|
if type(r['timeout']) not in (int,float) or not 0<r['timeout']<=300:raise ValueError()
|
|
if not isinstance(r['argv'],list) or not r['argv'] or any(not isinstance(x,str) for x in r['argv']):raise ValueError()
|
|
if r.get('label') not in ('job','node-read','rehearsal-phase','rein-aharness','glas-harness','claude'):raise ValueError()
|
|
if type(r.get('capture')) is not bool:raise ValueError()
|
|
if len(jobs)>=16 or abort:raise ValueError()
|
|
owner_fd=os.pidfd_open(owner['pid'])
|
|
bound=facts(owner['pid'])
|
|
if bound is None or bound['start']!=owner['start'] or bound['uid']!=owner['uid']:
|
|
os.close(owner_fd);raise ValueError()
|
|
supervisor_pid=os.getpid()
|
|
child=subprocess.Popen(r['argv'],stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.PIPE if r['capture'] else subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,env=ENV,start_new_session=True,
|
|
preexec_fn=lambda parent=supervisor_pid:parent_death(parent))
|
|
jobs[key]={'p':child,'request':r,'owner_fd':owner_fd,'deadline':time.monotonic()+r['timeout'],
|
|
'output':bytearray(),'eof':not r['capture'],'done':False,'leader':facts(child.pid)}
|
|
write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
|
if child.stdout:os.set_blocking(child.stdout.fileno(),False)
|
|
except Exception:
|
|
abort=abort or 'request_mapping_refused';uncertain=True
|
|
write(directory/('result-'+key+'.json'),{'returncode':125,'quiescent':False,'status':'mapping_refused'})
|
|
for key,job in list(jobs.items()):
|
|
if job['done']:continue
|
|
p=job['p']
|
|
if p.stdout and not job['eof']:
|
|
try:
|
|
raw=os.read(p.stdout.fileno(),MAX_OUTPUT+1)
|
|
if not raw:job['eof']=True
|
|
else:
|
|
job['output'].extend(raw)
|
|
if len(job['output'])>MAX_OUTPUT:abort=abort or 'output_bound'
|
|
except BlockingIOError:pass
|
|
if dead(job['owner_fd']):abort=abort or 'caller_lost'
|
|
if time.monotonic()>=job['deadline']:abort=abort or 'timeout'
|
|
if p.poll() is not None and job['eof'] and not abort:
|
|
# An adopted zombie is already dead, but must be waited
|
|
# before claiming quiescence. Do not steal a Popen leader's
|
|
# exit status or dismiss a genuinely live leftover.
|
|
own=descendants(os.getpid())
|
|
leaders={x['p'].pid for x in jobs.values()}
|
|
reaped.extend(reap_orphan_zombies(own,leaders))
|
|
own=descendants(os.getpid())
|
|
active={x['p'].pid for x in jobs.values() if not x['done']}
|
|
leftover=[row for row in own.values() if row['sid']==p.pid or
|
|
(row['ppid']==os.getpid() and row['pid'] not in active)]
|
|
if leftover:abort='leader_left_descendants'
|
|
else:
|
|
write(directory/('result-'+key+'.json'),{'returncode':p.returncode,'quiescent':True,'status':'completed',
|
|
'output_b64':base64.b64encode(job['output']).decode() if job['request']['capture'] else ''})
|
|
job['done']=True;job['status']='completed';os.close(job['owner_fd'])
|
|
write(directory/'public-journal.json',{'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:]})
|
|
if abort:
|
|
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()]) and not uncertain
|
|
for key,job in jobs.items():
|
|
if not job['done']:
|
|
write(directory/('result-'+key+'.json'),{'returncode':124 if abort=='timeout' else 125,
|
|
'quiescent':ok,'status':abort,'output_b64':''});job['done']=True;job['status']=abort
|
|
write(directory/'quiescent.json',{'quiescent':ok,'reason':abort,'no_live_or_waitable_descendants':ok,
|
|
'stderr_suppressed':True,'private_values_emitted':False,'jobs':job_report(jobs),
|
|
'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
|
return 0 if ok else 1
|
|
time.sleep(0.01)
|
|
except BaseException:
|
|
ok=quiesce(os.getpid(),[x['p'] for x in jobs.values()])
|
|
write(directory/'quiescent.json',{'quiescent':False,'reason':'supervisor_exception','best_effort_stopped':ok,
|
|
'jobs':job_report(jobs),'reaped_orphan_zombies':reaped[-16:],'leftover_at_failure':leftover[:16]})
|
|
return 1
|
|
|
|
|
|
class Supervisor:
|
|
def __init__(self,directory):
|
|
self.directory=Path(directory);self.directory.mkdir(mode=0o700)
|
|
write(self.directory/'root.json',facts(os.getpid()))
|
|
fd=os.pidfd_open(os.getpid())
|
|
try:self.process=subprocess.Popen([sys.executable,'-I','-B',str(Path(__file__).resolve()),'--watchdog',str(self.directory),'--parent-fd',str(fd)],
|
|
stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,pass_fds=(fd,),start_new_session=True,env=ENV)
|
|
finally:os.close(fd)
|
|
end=time.monotonic()+5
|
|
while not (self.directory/'ready.json').exists():
|
|
if self.process.poll() is not None or time.monotonic()>end:raise RuntimeError('supervisor_start_failed')
|
|
time.sleep(0.01)
|
|
def run(self,argv,timeout,*,capture=False,label='job'):return request(self.directory,argv,timeout,capture=capture,label=label)
|
|
def close(self):
|
|
write(self.directory/'shutdown',{})
|
|
try:self.process.wait(timeout=8)
|
|
except subprocess.TimeoutExpired:raise RuntimeError('supervisor_quiescence_unknown') from None
|
|
report=json.loads((self.directory/'quiescent.json').read_text())
|
|
if self.process.returncode or report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
|
return report
|
|
|
|
|
|
def request(directory,argv,timeout,*,capture=False,label='job'):
|
|
directory=Path(directory);key=uuid.uuid4().hex
|
|
if (directory/'quiescent.json').exists():raise RuntimeError('supervision_already_stopped')
|
|
write(directory/('request-'+key+'.json'),{'owner':facts(os.getpid()),'argv':list(argv),'timeout':timeout,'capture':capture,'label':label})
|
|
end=time.monotonic()+timeout+10
|
|
result=directory/('result-'+key+'.json')
|
|
while not result.exists():
|
|
if time.monotonic()>end:raise RuntimeError('supervision_reply_unknown')
|
|
time.sleep(0.01)
|
|
report=json.loads(result.read_text())
|
|
if report.get('quiescent') is not True:raise RuntimeError('descendant_quiescence_unverified')
|
|
return report['returncode'],base64.b64decode(report.get('output_b64',''),validate=True)
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--watchdog');p.add_argument('--parent-fd',type=int);a=p.parse_args()
|
|
return watchdog(a.watchdog,a.parent_fd)
|
|
|
|
if __name__=='__main__':
|
|
try:raise SystemExit(main())
|
|
except Exception:raise SystemExit(1) from None
|