321 lines
17 KiB
Python
321 lines
17 KiB
Python
|
"""Actual executor process loss against independent disposable OpenBao."""
|
||
|
|
from dataclasses import replace
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import secrets
|
||
|
|
from pathlib import Path
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||
|
|
from test_task_execution_plan import compiler
|
||
|
|
from test_task_execution_native_fixture import bao_dev, owner, admit
|
||
|
|
from task_execution_journal import Journal, Refused
|
||
|
|
from task_execution_native_fixture import NativeFixtureAdapter
|
||
|
|
from task_execution_recovery import FixtureRecoveryCustody, recover_fixture_cleanup
|
||
|
|
|
||
|
|
|
||
|
|
def protected(owner,tmp_path):
|
||
|
|
native,journal,binding,call,compiler,descriptor=owner
|
||
|
|
directory=tmp_path/'separate-owner-custody';directory.mkdir(mode=0o700)
|
||
|
|
custody=FixtureRecoveryCustody(directory,integrity_key=secrets.token_bytes(32))
|
||
|
|
native=NativeFixtureAdapter(compiler,native.plan,journal,binding,native.admission,recovery_custody=custody)
|
||
|
|
return native,journal,binding,call,custody
|
||
|
|
|
||
|
|
|
||
|
|
def crash(native,journal,binding,stage='issued'):
|
||
|
|
pid=os.fork()
|
||
|
|
if pid==0:
|
||
|
|
try:
|
||
|
|
# The backend belongs to the independent parent, so this child cannot
|
||
|
|
# waitpid it. Parent retains the actual Popen and checks it on recovery.
|
||
|
|
native.admission.server_process.poll=lambda:None
|
||
|
|
native.journal=Journal(journal.directory)
|
||
|
|
with native.journal.locked():
|
||
|
|
admit(native,native.journal,binding)
|
||
|
|
if stage=='mint-response-lost':
|
||
|
|
original=native._http
|
||
|
|
def lost(method,path,token,body=b''):
|
||
|
|
result=original(method,path,token,body)
|
||
|
|
if path=='auth/token/create':os._exit(73)
|
||
|
|
return result
|
||
|
|
native._http=lost
|
||
|
|
native.materialize_grant()
|
||
|
|
native.journal.transition(binding,'grant-issued','applying')
|
||
|
|
if stage=='business-response-lost':
|
||
|
|
native.metadata_call(0)
|
||
|
|
original=native._http
|
||
|
|
def lost(method,path,token,body=b''):
|
||
|
|
result=original(method,path,token,body)
|
||
|
|
if method=='PUT':os._exit(73)
|
||
|
|
return result
|
||
|
|
native._http=lost
|
||
|
|
native.metadata_call(1)
|
||
|
|
if stage=='cleanup-response-lost':
|
||
|
|
from task_execution_cleanup import run_cleanup
|
||
|
|
original=native._http
|
||
|
|
def lost(method,path,token,body=b''):
|
||
|
|
result=original(method,path,token,body)
|
||
|
|
if path=='auth/token/revoke-self':os._exit(73)
|
||
|
|
return result
|
||
|
|
native._http=lost
|
||
|
|
run_cleanup(native.journal,binding,native)
|
||
|
|
os._exit(73)
|
||
|
|
except BaseException:os._exit(74)
|
||
|
|
_,status=os.waitpid(pid,0)
|
||
|
|
assert os.waitstatus_to_exitcode(status)==73
|
||
|
|
# Original admission never appears in the independent parent or reopened DB.
|
||
|
|
assert native.task is None
|
||
|
|
assert journal._admission is None and journal._consumed_admission is None
|
||
|
|
|
||
|
|
|
||
|
|
def permit(native,custody,binding,**kwargs):
|
||
|
|
return custody.admit(binding,owner_token=kwargs.pop('owner_token',native.admission.owner_token),
|
||
|
|
server_process=native.admission.server_process,**kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('stage',['issued','business-response-lost','cleanup-response-lost'])
|
||
|
|
def test_process_loss_then_independent_cleanup_never_resumes_business(owner,tmp_path,stage,monkeypatch,request):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path)
|
||
|
|
crash(native,journal,binding,stage)
|
||
|
|
journal=Journal(journal.directory)
|
||
|
|
request.addfinalizer(journal.close)
|
||
|
|
native.journal=journal
|
||
|
|
private=custody.read(binding)
|
||
|
|
assert private['phase']=='issued' and private['task_accessor']
|
||
|
|
target=native.compiler.before_call(native.plan,1).path
|
||
|
|
before=call('GET',target)
|
||
|
|
with journal.locked():
|
||
|
|
with pytest.raises(Refused):journal.reserve(binding,native.cleanup)
|
||
|
|
with pytest.raises(Refused):native.materialize_grant()
|
||
|
|
with pytest.raises(Refused):native.metadata_call(0)
|
||
|
|
with pytest.raises(Refused):native.revoke_task_token(native.cleanup[0].call_digest)
|
||
|
|
admitted=permit(native,custody,binding)
|
||
|
|
import task_execution_recovery as recovery_module
|
||
|
|
sent=[]
|
||
|
|
original_request=recovery_module.http.client.HTTPConnection.request
|
||
|
|
def tracked(connection,method,url,*args,**kwargs):
|
||
|
|
sent.append((method,url)) # metadata only; never capture body/headers
|
||
|
|
return original_request(connection,method,url,*args,**kwargs)
|
||
|
|
with monkeypatch.context() as patch:
|
||
|
|
patch.setattr(recovery_module.http.client.HTTPConnection,'request',tracked)
|
||
|
|
result=recover_fixture_cleanup(journal,admitted)
|
||
|
|
assert all((method,path) in {
|
||
|
|
('POST','/v1/auth/token/revoke-accessor'),('POST','/v1/auth/token/lookup-accessor'),
|
||
|
|
('GET','/v1/sys/policies/acl/'+native.policy_name),
|
||
|
|
('DELETE','/v1/sys/policies/acl/'+native.policy_name)} for method,path in sent)
|
||
|
|
assert result['confirmed']
|
||
|
|
for accessor in [private['task_accessor'],private['identity_accessor']]:
|
||
|
|
assert call('POST','auth/token/lookup-accessor',{'accessor':accessor})[0]==400
|
||
|
|
assert call('GET','sys/policies/acl/'+native.policy_name)[0]==404
|
||
|
|
after=call('GET',target)
|
||
|
|
assert after[0]==before[0] and after[1].get('data')==before[1].get('data') # request IDs vary
|
||
|
|
assert journal.observe(binding)['dispatch_allowed'] is False
|
||
|
|
if stage!='issued':assert journal.observe(binding)['external_outcome_unknown']
|
||
|
|
if stage=='cleanup-response-lost':
|
||
|
|
original=json.loads(journal.db.execute('SELECT original FROM recovery_attempts').fetchone()[0])
|
||
|
|
assert original[0]['outcome']=='pending'
|
||
|
|
assert journal.db.execute("SELECT COUNT(*) FROM cleanup_calls WHERE outcome='pending'").fetchone()[0]==1
|
||
|
|
with pytest.raises(Refused):recover_fixture_cleanup(journal,admitted)
|
||
|
|
public=(journal.directory/'journal.sqlite').read_bytes()
|
||
|
|
for value in [private['task_accessor'],private['identity_accessor'],native.admission.owner_token]:
|
||
|
|
assert value.encode() not in public
|
||
|
|
assert 'client_token' not in json.dumps(private)
|
||
|
|
assert 'token' not in repr(admitted) and private['task_accessor'] not in repr(admitted)
|
||
|
|
|
||
|
|
|
||
|
|
def test_process_loss_before_handle_commit_preserves_unknown_mint_debt(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path)
|
||
|
|
crash(native,journal,binding,'mint-response-lost')
|
||
|
|
assert custody.read(binding)['phase']=='mint-intent'
|
||
|
|
result=recover_fixture_cleanup(journal,permit(native,custody,binding))
|
||
|
|
assert not result['confirmed']
|
||
|
|
assert [x['outcome'] for x in result['obligations']]==['owed','confirmed','confirmed']
|
||
|
|
assert journal.observe(binding)['external_outcome_unknown']
|
||
|
|
assert journal.db.execute('SELECT confirmed FROM recovery_results ORDER BY obligation').fetchall()==[(0,),(1,),(1,)]
|
||
|
|
|
||
|
|
|
||
|
|
def test_revoked_expired_wrong_revision_and_public_flag_refuse_before_calls(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path);crash(native,journal,binding)
|
||
|
|
p=permit(native,custody,binding)
|
||
|
|
custody.revoke(p)
|
||
|
|
for bad in [p,replace(permit(native,custody,binding),deadline=time.monotonic()-1),
|
||
|
|
replace(permit(native,custody,binding),private_revision_digest='0'*64),{'verified':True}]:
|
||
|
|
with pytest.raises(Refused):recover_fixture_cleanup(journal,bad)
|
||
|
|
assert call('GET','sys/policies/acl/'+native.policy_name)[0]==200
|
||
|
|
assert journal.cleanup_status(binding)['obligations'][0]['outcome']=='owed'
|
||
|
|
|
||
|
|
|
||
|
|
def test_revoked_recovery_credential_cannot_fake_cleanup_success(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path);crash(native,journal,binding)
|
||
|
|
status,data=call('POST','auth/token/create',{'policies':['root'],'ttl':'30s'})
|
||
|
|
assert status==200
|
||
|
|
token=data['auth']['client_token']
|
||
|
|
assert call('POST','auth/token/revoke',{'token':token})[0] in {200,204}
|
||
|
|
result=recover_fixture_cleanup(journal,permit(native,custody,binding,owner_token=token))
|
||
|
|
assert not result['confirmed'] and all(x['outcome']=='owed' for x in result['obligations'])
|
||
|
|
assert call('GET','sys/policies/acl/'+native.policy_name)[0]==200
|
||
|
|
# A separately admitted owner attempt can recover after the failed attempt;
|
||
|
|
# neither the failed attempt nor its receipt grants an automatic retry.
|
||
|
|
assert recover_fixture_cleanup(journal,permit(native,custody,binding))['confirmed']
|
||
|
|
assert journal.db.execute('SELECT COUNT(*) FROM recovery_attempts').fetchone()[0]==2
|
||
|
|
|
||
|
|
|
||
|
|
def test_policy_name_reuse_never_deletes_different_policy(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path);crash(native,journal,binding)
|
||
|
|
replacement='path "unrelated" { capabilities = ["read"] }'
|
||
|
|
assert call('PUT','sys/policies/acl/'+native.policy_name,{'policy':replacement})[0] in {200,204}
|
||
|
|
result=recover_fixture_cleanup(journal,permit(native,custody,binding))
|
||
|
|
assert not result['confirmed']
|
||
|
|
assert [x['outcome'] for x in result['obligations']]==['confirmed','confirmed','owed']
|
||
|
|
assert call('GET','sys/policies/acl/'+native.policy_name)[1]['data']['policy']==replacement
|
||
|
|
|
||
|
|
|
||
|
|
def test_private_custody_permissions_and_identity_binding_fail_closed(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path)
|
||
|
|
path=custody._path(binding);path.chmod(0o644)
|
||
|
|
with pytest.raises(Refused):custody.read(binding)
|
||
|
|
path.chmod(0o600)
|
||
|
|
data=json.loads(path.read_bytes());data['payload']['binding']['operation_id']='other'
|
||
|
|
path.write_text(json.dumps(data))
|
||
|
|
with pytest.raises(Refused):custody.read(binding)
|
||
|
|
|
||
|
|
|
||
|
|
def test_lost_recovery_response_stays_unknown_until_new_independent_attempt(owner,tmp_path,monkeypatch):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path);crash(native,journal,binding)
|
||
|
|
import task_execution_recovery as recovery_module
|
||
|
|
original=recovery_module.http.client.HTTPConnection.getresponse
|
||
|
|
once=[True]
|
||
|
|
def lost(connection):
|
||
|
|
response=original(connection)
|
||
|
|
if once[0]:
|
||
|
|
once[0]=False
|
||
|
|
response.close()
|
||
|
|
raise Refused('synthetic lost recovery response after real revoke')
|
||
|
|
return response
|
||
|
|
first=permit(native,custody,binding)
|
||
|
|
with monkeypatch.context() as patch:
|
||
|
|
patch.setattr(recovery_module.http.client.HTTPConnection,'getresponse',lost)
|
||
|
|
assert not recover_fixture_cleanup(journal,first)['confirmed']
|
||
|
|
assert journal.observe(binding)['external_outcome_unknown']
|
||
|
|
with pytest.raises(Refused):recover_fixture_cleanup(journal,first)
|
||
|
|
assert recover_fixture_cleanup(journal,permit(native,custody,binding))['confirmed']
|
||
|
|
# Reconciliation does not erase the historic uncertain invocation.
|
||
|
|
assert journal.observe(binding)['external_outcome_unknown']
|
||
|
|
assert journal.db.execute("SELECT COUNT(*) FROM recovery_calls WHERE outcome='pending'").fetchone()[0]==1
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def capsule(tmp_path):
|
||
|
|
from task_execution_journal import Binding,Cleanup
|
||
|
|
from task_execution_plan import cleanup_specs
|
||
|
|
binding=Binding('SYNTHETIC','sha256:'+'a'*64,'12345678-1234-1234-1234-123456789abc','b'*64)
|
||
|
|
directory=tmp_path/'private-capsule';directory.mkdir(mode=0o700)
|
||
|
|
custody=FixtureRecoveryCustody(directory,integrity_key=secrets.token_bytes(32))
|
||
|
|
cleanup=tuple(Cleanup(x['kind'],x['call_digest']) for x in cleanup_specs(binding.operation_id))
|
||
|
|
custody.prepare(binding,cleanup,12345,'path "fixture" {}\n','synthetic-private-accessor')
|
||
|
|
return custody,binding
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('mutate',[
|
||
|
|
lambda v:v.update(extra=True),lambda v:v.pop('phase'),lambda v:v.update(binding=True),
|
||
|
|
lambda v:v['binding'].update(approval_id=True),lambda v:v['binding'].update(extra='x'),
|
||
|
|
lambda v:v['binding'].update(request_digest='0'*64),lambda v:v.update(port=True),
|
||
|
|
lambda v:v.update(port=12345.0),lambda v:v.update(port=0),lambda v:v.update(phase=[]),
|
||
|
|
lambda v:v.update(phase='issued'),lambda v:v.update(revision=False),
|
||
|
|
lambda v:v.update(revision=0.0),lambda v:v.update(identity_accessor=False),
|
||
|
|
lambda v:v.update(task_accessor='unexpected-before-issue'),lambda v:v.update(policy_hcl=[]),
|
||
|
|
lambda v:v.update(cleanup={}),lambda v:v['cleanup'].pop(),
|
||
|
|
lambda v:v['cleanup'].__setitem__(0,False),lambda v:v['cleanup'][0].update(extra=True),
|
||
|
|
lambda v:v['cleanup'][0].update(kind=True),lambda v:v['cleanup'][0].update(call_digest=0),
|
||
|
|
lambda v:v['cleanup'][0].update(call_digest='f'*64),lambda v:v['cleanup'].reverse(),
|
||
|
|
])
|
||
|
|
def test_authenticated_but_damaged_capsule_schema_refuses(capsule,mutate):
|
||
|
|
custody,binding=capsule
|
||
|
|
value=custody.read(binding);mutate(value)
|
||
|
|
# Trusted test owner re-signs malformed payload to exercise schema defenses
|
||
|
|
# independently of MAC validation; untrusted corruption cannot do this.
|
||
|
|
custody._write(binding,value)
|
||
|
|
with pytest.raises(Refused):custody.read(binding)
|
||
|
|
|
||
|
|
|
||
|
|
def test_capsule_corruption_wrong_key_duplicate_and_nonfinite_refuse(capsule):
|
||
|
|
custody,binding=capsule;path=custody._path(binding);original=path.read_bytes()
|
||
|
|
value=json.loads(original);value['payload']['identity_accessor']='different-synthetic-accessor'
|
||
|
|
path.write_text(json.dumps(value))
|
||
|
|
with pytest.raises(Refused):custody.read(binding)
|
||
|
|
path.write_bytes(original)
|
||
|
|
other=FixtureRecoveryCustody(custody.directory,integrity_key=b'X'*32)
|
||
|
|
with pytest.raises(Refused):other.read(binding)
|
||
|
|
for raw in [b'{',original[:-1]+b',"mac":"duplicate"}',original.replace(b'12345',b'NaN'),b'[]',b'{}',b'{"payload":{},"mac":"'+b'\xc3\xa9'*64+b'"}']:
|
||
|
|
path.write_bytes(raw)
|
||
|
|
with pytest.raises(Refused):custody.read(binding)
|
||
|
|
path.write_bytes(original)
|
||
|
|
assert custody.read(binding)['identity_accessor']=='synthetic-private-accessor'
|
||
|
|
|
||
|
|
|
||
|
|
def test_exact_absence_and_single_terminal_lf_contract():
|
||
|
|
from task_execution_recovery import _policy_absent,_same_policy
|
||
|
|
assert _policy_absent(404,{'errors':[]})
|
||
|
|
for status,body in [(403,{'errors':[]}),(404,{}),(404,{'errors':['denied']}),
|
||
|
|
(404,{'errors':None}),(404,{'errors':[]} | {'data':{}}),
|
||
|
|
(404,{'errors':False}),(404,{'errors':0}),(404,{'errors':{}})]:
|
||
|
|
assert not _policy_absent(status,body)
|
||
|
|
assert _same_policy('fixture','fixture\n')
|
||
|
|
assert _same_policy('fixture\n','fixture\n')
|
||
|
|
for text in ['fixture \n','fixture\t','fixture\r\n','fixture\n\n',False,None]:
|
||
|
|
assert not _same_policy(text,'fixture\n')
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('after_delete',[False,True])
|
||
|
|
def test_recovery_policy_absence_rejects_misleading_native_http_bodies(owner,tmp_path,monkeypatch,after_delete):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path);crash(native,journal,binding)
|
||
|
|
import task_execution_recovery as module
|
||
|
|
malformed=[{}, {'errors':['permission denied']}, {'errors':None}, {'errors':False},
|
||
|
|
{'errors':0}, {'errors':[],'data':{}}, {'errors':['not really absent']},
|
||
|
|
b'{"errors":["permission denied"],"errors":[]}',b'{"errors":NaN}',b'not JSON']
|
||
|
|
path='sys/policies/acl/'+native.policy_name
|
||
|
|
for body in malformed:
|
||
|
|
sent=[];gets=[0]
|
||
|
|
class Response:
|
||
|
|
def __init__(self,status,data):self.status=status;self.data=data
|
||
|
|
def read(self,limit):return self.data if type(self.data) is bytes else json.dumps(self.data).encode()
|
||
|
|
class Connection:
|
||
|
|
def __init__(self,*args,**kwargs):pass
|
||
|
|
def request(self,method,url,**kwargs):sent.append((method,url))
|
||
|
|
def getresponse(self):
|
||
|
|
method,url=sent[-1]
|
||
|
|
if url=='/v1/'+path:
|
||
|
|
if method=='DELETE':return Response(204,{})
|
||
|
|
gets[0]+=1
|
||
|
|
if after_delete and gets[0]==1:
|
||
|
|
return Response(200,{'data':{'policy':native.hcl[:-1]}})
|
||
|
|
return Response(404,body)
|
||
|
|
return Response(403,{'errors':['synthetic refusal']})
|
||
|
|
def close(self):pass
|
||
|
|
with monkeypatch.context() as patch:
|
||
|
|
patch.setattr(module.http.client,'HTTPConnection',Connection)
|
||
|
|
result=recover_fixture_cleanup(journal,permit(native,custody,binding))
|
||
|
|
assert not result['confirmed'] and result['obligations'][2]['outcome']=='owed'
|
||
|
|
assert ('DELETE','/v1/'+path) in sent if after_delete else ('DELETE','/v1/'+path) not in sent
|
||
|
|
# The denied/malformed probes above emitted no real native calls.
|
||
|
|
assert call('GET',path)[0]==200
|
||
|
|
assert recover_fixture_cleanup(journal,permit(native,custody,binding))['confirmed']
|
||
|
|
|
||
|
|
|
||
|
|
def test_authenticated_prepared_capsule_rollback_cannot_hide_later_mint(owner,tmp_path):
|
||
|
|
native,journal,binding,call,custody=protected(owner,tmp_path)
|
||
|
|
old=custody._path(binding).read_bytes()
|
||
|
|
crash(native,journal,binding)
|
||
|
|
current=custody.read(binding)
|
||
|
|
custody._path(binding).write_bytes(old)
|
||
|
|
# Still authentic under the owner key, but the durable journal proves the
|
||
|
|
# operation reached issuance. The stale prepared record cannot prove absence.
|
||
|
|
result=recover_fixture_cleanup(journal,permit(native,custody,binding))
|
||
|
|
assert not result['confirmed'] and result['obligations'][0]['outcome']=='owed'
|
||
|
|
assert call('POST','auth/token/lookup-accessor',{'accessor':current['task_accessor']})[0]==200
|
||
|
|
assert call('POST','auth/token/revoke-accessor',{'accessor':current['task_accessor']})[0] in {200,204}
|