155 lines
10 KiB
Python
155 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Install-only orchestration. Default is read-only; --apply requires founder go."""
|
|
import argparse
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE=Path(__file__).resolve().parent
|
|
DIGEST='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
|
STAGE='/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017'
|
|
STORE='/home/tegwick/.helixforge-factory/runtimes'
|
|
SOURCE=Path('/tmp/netkingdom-runtime-canonical-20261016/runtime')
|
|
|
|
READ_TARGET=r'''import json,os,pwd,stat,subprocess,shutil,time
|
|
from pathlib import Path
|
|
store=Path('/home/tegwick/.helixforge-factory/runtimes'); stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');digest='f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d'
|
|
def meta(p):
|
|
try:
|
|
s=p.lstat();return {'uid':s.st_uid,'gid':s.st_gid,'mode':oct(stat.S_IMODE(s.st_mode)),'device':s.st_dev,'inode':s.st_ino,'directory':stat.S_ISDIR(s.st_mode),'symlink':stat.S_ISLNK(s.st_mode)}
|
|
except FileNotFoundError:return None
|
|
cp=subprocess.run(['/usr/local/bin/kubectl','get','nodes','-o','json'],capture_output=True,timeout=20)
|
|
assert cp.returncode==0
|
|
nodes=json.loads(cp.stdout)['items'];assert len(nodes)==1
|
|
mem=dict(x.split(':',1) for x in Path('/proc/meminfo').read_text().splitlines())
|
|
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
|
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];busy=1-(d[3]+d[4])/sum(d)
|
|
print(json.dumps({'account':pwd.getpwuid(os.getuid()).pw_name,'uid':os.getuid(),'gid':os.getgid(),'store':meta(store),'destination':meta(store/digest),'stage':meta(stage),'memory_available_bytes':int(mem['MemAvailable'].split()[0])*1024,'disk_free_bytes':shutil.disk_usage(store).free,'load_average':list(os.getloadavg()),'cpu_count':os.cpu_count(),'cpu_busy_fraction':busy,'cpu_sample_seconds':1,'node_addresses':nodes[0]['status']['addresses'],'node_ready':any(c['type']=='Ready' and c['status']=='True' for c in nodes[0]['status']['conditions'])}))'''
|
|
|
|
|
|
def sha(path):
|
|
with path.open('rb') as stream:return 'sha256:'+hashlib.file_digest(stream,'sha256').hexdigest()
|
|
|
|
|
|
def command(argv,timeout=60):
|
|
cp=subprocess.run(argv,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=timeout)
|
|
if cp.returncode:raise RuntimeError('command_failed')
|
|
return cp.stdout
|
|
|
|
|
|
def ssh(code,args=(),timeout=60):
|
|
remote='/usr/bin/python3 -I -B -c '+shlex.quote(code)
|
|
if args:remote+=' '+' '.join(shlex.quote(str(x)) for x in args)
|
|
return json.loads(command(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01',remote],timeout))
|
|
|
|
|
|
def module(name,path):
|
|
spec=importlib.util.spec_from_file_location(name,path)
|
|
result=importlib.util.module_from_spec(spec);spec.loader.exec_module(result);return result
|
|
|
|
|
|
def preflight():
|
|
inputs=json.loads((HERE/'action-inputs.json').read_text())
|
|
for name,expected in inputs['helper_hashes'].items():
|
|
if sha(HERE/name)!=expected:raise ValueError('helper_changed')
|
|
if module('reviewed_runtime_reader',HERE/'runtime.py').runtime_digest(SOURCE)!=DIGEST:
|
|
raise ValueError('source_changed')
|
|
observation=ssh(READ_TARGET)
|
|
store=observation['store']
|
|
expected=json.loads((HERE/'target-readonly-observation.json').read_text())['paths'][STORE]
|
|
checks={
|
|
'account':observation['account']=='tegwick' and observation['uid']==1000 and observation['gid']==1000,
|
|
'protected_store':store['uid']==1000 and store['gid']==1000 and store['mode']=='0o700' and store['directory'] and not store['symlink'] and store['device']==expected['device'] and store['inode']==expected['inode'],
|
|
'destination_absent':observation['destination'] is None,
|
|
'staging_absent':observation['stage'] is None,
|
|
'exact_ready_node':observation['node_ready'] and any(x['address']=='92.205.62.239' for x in observation['node_addresses']),
|
|
'disk_headroom':observation['disk_free_bytes']>=2*1024**3,
|
|
'memory_headroom':observation['memory_available_bytes']>=1024**3,
|
|
'load_headroom':observation['load_average'][0]<=2*observation['cpu_count'],
|
|
'cpu_headroom':observation['cpu_busy_fraction']<=0.85,
|
|
}
|
|
observation['checks']=checks
|
|
return inputs,observation
|
|
|
|
|
|
def save(path,value):
|
|
fd=os.open(path,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600)
|
|
with os.fdopen(fd,'w') as stream:json.dump(value,stream,indent=2);stream.write('\n')
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--apply',action='store_true');parser.add_argument('--dry-run',action='store_true')
|
|
args=parser.parse_args()
|
|
if args.apply and args.dry_run:raise ValueError('conflicting_modes')
|
|
inputs,observation=preflight()
|
|
if not args.apply:
|
|
report={'status':'read_only_dry_run_passed' if all(observation['checks'].values()) else 'read_only_dry_run_blocked','proposal_id':inputs['proposal_id'],'revision':1,
|
|
'observed_at':datetime.now(timezone.utc).isoformat(),'target':observation,
|
|
'source_runtime_complete_digest_verified':True,'archive':inputs['archive'],
|
|
'remote_mutations':0,'provider_calls':0,'private_configuration_reads':False,
|
|
'next_exact_command':'python3 -I -B '+str(HERE/'execute-install.py')+' --apply'}
|
|
print(json.dumps(report,sort_keys=True));return 0 if all(observation['checks'].values()) else 2
|
|
if not all(observation['checks'].values()):raise ValueError('target_headroom_or_identity_refused')
|
|
receipt=HERE/'execution-receipt.json'
|
|
if receipt.exists():raise ValueError('prior_attempt_requires_reconciliation')
|
|
save(receipt,{'status':'intent_recorded','proposal_id':inputs['proposal_id'],'revision':1,
|
|
'automatic_retry':False,'observed_at':datetime.now(timezone.utc).isoformat()})
|
|
# No native attempt is retried by this helper. Intent survives an SSH loss.
|
|
with tempfile.TemporaryDirectory(prefix='nk-runtime-install-r1-') as temporary:
|
|
archive=Path(temporary)/'runtime.tar.gz'
|
|
with archive.open('xb') as stream:built=module('reviewed_transfer',HERE/'build-transfer.py').build(stream)
|
|
if built!=inputs['archive']:raise ValueError('archive_changed')
|
|
archive.chmod(0o600)
|
|
create=r'''import json,os,stat,time
|
|
from pathlib import Path
|
|
assert os.getuid()==1000 and os.getgid()==1000
|
|
store=Path('/home/tegwick/.helixforge-factory/runtimes');s=store.lstat()
|
|
assert stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_ino==4751222 and s.st_dev==2049
|
|
assert not (store/'f59959b605a51abd2901becde14e7abd852c2bd3f848004b51d250d1cb81af6d').exists()
|
|
read_cpu=lambda:[int(x) for x in Path('/proc/stat').read_text().splitlines()[0].split()[1:9]]
|
|
a=read_cpu();time.sleep(1);b=read_cpu();d=[y-x for x,y in zip(a,b)];assert 1-(d[3]+d[4])/sum(d)<=0.85
|
|
stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');stage.mkdir(mode=0o700)
|
|
s=stage.lstat();print(json.dumps({'inode':s.st_ino,'device':s.st_dev}))'''
|
|
created=ssh(create)
|
|
receipt.write_text(json.dumps({'status':'staging_created','phase':'before_transfer','proposal_id':inputs['proposal_id'],'revision':1,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
command(['scp','-q',str(archive),str(HERE/'runtime.py'),str(HERE/'install-verified.py'),'railiance01:'+STAGE+'/'],120)
|
|
execute=r'''import hashlib,json,os,stat,sys
|
|
from pathlib import Path
|
|
stage=Path('/home/tegwick/.helixforge-factory/runtime-install-f599-r1-20261017');s=stage.lstat()
|
|
assert stat.S_ISDIR(s.st_mode) and s.st_uid==1000 and stat.S_IMODE(s.st_mode)==0o700 and s.st_ino==int(sys.argv[1]) and s.st_dev==2049
|
|
expected=json.loads(sys.argv[2]);assert {p.name for p in stage.iterdir()}==set(expected)
|
|
for name,want in expected.items():
|
|
p=stage/name;s=p.lstat();assert stat.S_ISREG(s.st_mode) and s.st_uid==1000 and not stat.S_ISLNK(s.st_mode)
|
|
with p.open('rb') as stream:assert hashlib.file_digest(stream,'sha256').hexdigest()==want
|
|
p.chmod(0o600)
|
|
os.execve('/usr/bin/python3',['python3','-I','-B',str(stage/'install-verified.py'),'--stage-inode',sys.argv[1]],{'HOME':'/home/tegwick','PATH':'/usr/bin:/bin','LANG':'C.UTF-8','PYTHONDONTWRITEBYTECODE':'1'})'''
|
|
hashes={'runtime.tar.gz':inputs['archive']['sha256'].removeprefix('sha256:'),
|
|
'runtime.py':inputs['helper_hashes']['runtime.py'].removeprefix('sha256:'),
|
|
'install-verified.py':inputs['helper_hashes']['install-verified.py'].removeprefix('sha256:')}
|
|
# Parse the structured final helper return even when its action failed.
|
|
remote='/usr/bin/python3 -I -B -c '+shlex.quote(execute)+' '+shlex.quote(str(created['inode']))+' '+shlex.quote(json.dumps(hashes,separators=(',',':')))
|
|
receipt.write_text(json.dumps({'status':'remote_execution_started','proposal_id':inputs['proposal_id'],'revision':1,'automatic_retry':False,'staging_identity':created,'staging_path':STAGE},indent=2)+'\n')
|
|
cp=subprocess.run(['ssh','-o','BatchMode=yes','-o','ConnectTimeout=15','railiance01',remote],stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,timeout=600)
|
|
try:result=json.loads(cp.stdout)
|
|
except Exception:raise RuntimeError('remote_outcome_uncertain') from None
|
|
result['ssh_exit_code']=cp.returncode;result['staging_identity']=created
|
|
# Replace only our exact local intent receipt, never a previous attempt.
|
|
receipt.write_text(json.dumps(result,indent=2)+'\n');receipt.chmod(0o600)
|
|
print(json.dumps(result,sort_keys=True))
|
|
return 0 if cp.returncode==0 and result.get('status')=='installed_and_offline_rehearsal_verified' else 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:raise SystemExit(main())
|
|
except Exception as exc:
|
|
print(json.dumps({'status':'failed_or_uncertain','failure_type':type(exc).__name__,
|
|
'automatic_retry':False,'private_diagnostics_suppressed':True}))
|
|
raise SystemExit(1) from None
|