From aaa4b8e686b748309ba855c517cf1209d67dc7bb Mon Sep 17 00:00:00 2001 From: codex Date: Fri, 11 Sep 2026 12:37:29 +0200 Subject: [PATCH] Retain bounded native probe failure diagnostics before cleanup Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a07ff8-19d0-7820-b4d0-1353833cb7fc --- scripts/native_factory_acceptance.py | 27 ++++++++++++++++++++++++ scripts/native_factory_probe/producer.py | 18 +++++++++++----- tests/test_factory_native_acceptance.py | 7 ++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/scripts/native_factory_acceptance.py b/scripts/native_factory_acceptance.py index a05b423..6ff4aa3 100644 --- a/scripts/native_factory_acceptance.py +++ b/scripts/native_factory_acceptance.py @@ -137,6 +137,26 @@ def operator_identity(rows): return candidates[0] +def failure_diagnostic(raw): + """Persist only known assertion codes and HTTP status numbers, never raw logs.""" + if len(raw)>8192:return {'error':'probe_diagnostic_unavailable'} + try:body=json.loads(raw) + except (ValueError,UnicodeError):return {'error':'probe_diagnostic_unavailable'} + codes={'contained_probe_failed','lost-receipt_failed','restart-retry_failed', + 'source_bundle_mismatch','fresh_probe_state_required','unexpected_approval', + 'outbox_not_preserved','unexpected_drain_result','unexpected_outbox_state', + 'native_accept_duplicate_not_observed','response_too_large', + 'source_refusal_inconclusive','tenant_refusal_inconclusive', + 'read_refusal_inconclusive','invalid_bearer_not_refused','own_reconciliation_failed', + 'probe_count_not_one','sibling_reconciliation_not_refused'} + result={k:body[k] for k in ['error','child_error'] if body.get(k) in codes} + if body.get('phase') in {'bootstrap','lost-receipt','restart-retry'}:result['phase']=body['phase'] + statuses=body.get('observed_http_statuses',[]) + if isinstance(statuses,list) and len(statuses)<=10 and all(type(x) is int and 100<=x<=599 for x in statuses): + result['observed_http_statuses']=statuses + return result or {'error':'probe_diagnostic_unavailable'} + + def jobs(packet, kube, receipt, save): """Consume only the producer's already-delivered credential inside its Job.""" receipt['receiver_before']=receiver_check(kube,RECEIVER) @@ -160,6 +180,13 @@ def jobs(packet, kube, receipt, save): job=data(command(kube+['-n',sender,'get','job',name,'-o','json'])) if job.get('status',{}).get('succeeded')==1:break if job.get('status',{}).get('failed'): + failed_pods=data(command(kube+['-n',sender,'get','pods','-l','job-name='+name,'-o','json']))['items'] + diagnostics=[] + for failed in failed_pods: + if any(o['uid']==job['metadata']['uid'] and o.get('controller') is True for o in failed['metadata'].get('ownerReferences',[])): + logs=command(kube+['-n',sender,'logs',failed['metadata']['name'],'-c','probe'],allow_failure=True) + diagnostics.append(failure_diagnostic(logs.stdout)) + receipt['failed_producer']={'sender':sender,'diagnostics':diagnostics};save() raise LaneError('native_probe_job_failed_'+sender) time.sleep(2) else:raise LaneError('native_probe_job_timeout_'+sender) diff --git a/scripts/native_factory_probe/producer.py b/scripts/native_factory_probe/producer.py index 3f61b3f..46be089 100644 --- a/scripts/native_factory_probe/producer.py +++ b/scripts/native_factory_probe/producer.py @@ -73,8 +73,8 @@ def bootstrap(config, dbpath, envelope): return envelope -def delivery_phase(config, phase, dbpath): - observed = [] +def delivery_phase(config, phase, dbpath, receipt): + observed = receipt.setdefault("observed_http_statuses", []) lose = phase == 'lost-receipt' if config['sender'] == 'approval-engine': from approval_engine.store import Engine @@ -160,7 +160,9 @@ def main(): directory = Path('/state/private'); directory.mkdir(mode=0o700,exist_ok=True) dbpath = directory/'outbox.db' if len(sys.argv)>1: - receipt = {'status':'passed','http_status':delivery_phase(config,sys.argv[1],dbpath)} + receipt['phase'] = sys.argv[1] + receipt['http_status'] = delivery_phase(config,sys.argv[1],dbpath,receipt) + receipt['status'] = 'passed' else: require(not dbpath.exists(), 'fresh_probe_state_required') envelope = {'id':config['event_id'], 'type':'factory.audit-probe.'+config['run_id'], @@ -168,12 +170,18 @@ def main(): 'tenant':'tenant:platform', 'correlation_id':config['run_id'], 'occurred_at':datetime.now(timezone.utc).isoformat(), 'data':{'synthetic':True,'no_human_action':True,'purpose':'audit-sender-acceptance'}} + receipt['phase'] = 'bootstrap' envelope = bootstrap(config,dbpath,envelope) phases=[] for phase in ['lost-receipt','restart-retry']: + receipt['phase'] = phase child = subprocess.run([sys.executable,__file__,phase],capture_output=True,timeout=30) - require(child.returncode == 0, phase+'_failed') - phases.append(json.loads(child.stdout)['http_status']) + child_receipt = json.loads(child.stdout) + if child.returncode: + receipt['child_error'] = child_receipt.get('error','contained_probe_failed') + receipt['observed_http_statuses'] = child_receipt.get('observed_http_statuses',[]) + raise Refused(phase+'_failed') + phases.append(child_receipt['http_status']) receipt.update(status='passed', sender=config['sender'], event_id=config['event_id'], run_id=config['run_id'], source_zip_sha256=config['source_zip_sha256'], http_statuses=phases, process_restart_retry=True, synthetic_outbox_only=True, diff --git a/tests/test_factory_native_acceptance.py b/tests/test_factory_native_acceptance.py index df1096e..4635f7d 100644 --- a/tests/test_factory_native_acceptance.py +++ b/tests/test_factory_native_acceptance.py @@ -17,6 +17,13 @@ import native_factory_acceptance as native class Contracts(unittest.TestCase): + def test_failure_diagnostic_discards_unrecognized_content(self): + fixture={'error':'secret-like-unrecognized-content','child_error':'native_accept_duplicate_not_observed', + 'phase':'lost-receipt','observed_http_statuses':[400],'Authorization':'must-never-be-retained'} + self.assertEqual(native.failure_diagnostic(json.dumps(fixture).encode()), + {'child_error':'native_accept_duplicate_not_observed','phase':'lost-receipt','observed_http_statuses':[400]}) + self.assertEqual(native.failure_diagnostic(b'non-json secret-like text'),{'error':'probe_diagnostic_unavailable'}) + def test_jobs_do_not_read_operator_credentials_and_refuse_additive_egress(self): result=subprocess.CompletedProcess([],0,b'{"items":[{"metadata":{"name":"existing-allow"}}]}',b'') with patch.object(native,'receiver_check',return_value={}),patch.object(native,'command',return_value=result) as cmd,patch.object(native,'bao',side_effect=AssertionError('no OpenBao call permitted')):