Implement bounded release runtime and durable native health sampler
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a126e1-76f8-7643-8cf3-e93336539ade
This commit is contained in:
parent
cc2bb9d0f7
commit
52dcc08f23
13 changed files with 937 additions and 10 deletions
124
tests/test_release_health.py
Normal file
124
tests/test_release_health.py
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import copy
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from temporalio.client import (Schedule, ScheduleActionStartWorkflow, ScheduleCalendarSpec,
|
||||
SchedulePolicy, ScheduleRange, ScheduleSpec, ScheduleState, ScheduleOverlapPolicy)
|
||||
from temporalio.converter import DataConverter
|
||||
from temporalio.api.common.v1 import Payloads, WorkflowType
|
||||
from temporalio.api.taskqueue.v1 import TaskQueue
|
||||
from temporalio.api.workflow.v1 import NewWorkflowExecutionInfo
|
||||
|
||||
from activity_core.release_health import FRONTEND, SINK, definition_matches, probe, schedule_matches
|
||||
|
||||
|
||||
def rows():
|
||||
return [SimpleNamespace(id=UUID(identity),enabled=True,trigger_type='cron',
|
||||
trigger_config={'cron_expression':cron,'timezone':'Europe/Berlin','misfire_policy':'catchup_latest','catchup_window_seconds':86400},
|
||||
instructions_json=[{'id':f'frontend-patterns-{cadence}-report','model':'deterministic','report_sinks':[dict(SINK)]}])
|
||||
for identity,(cadence,cron) in FRONTEND.items()]
|
||||
|
||||
|
||||
def description(identity,calendar=False):
|
||||
cadence,cron=FRONTEND[identity]
|
||||
action=ScheduleActionStartWorkflow('RunActivityWorkflow',args=[identity,'scheduled',None],
|
||||
id=f'activity-{identity}:${{firstScheduledTime}}',task_queue='orchestrator-tq')
|
||||
if calendar:
|
||||
cal=ScheduleCalendarSpec(hour=[ScheduleRange({'daily':2,'weekly':3,'monthly':9}[cadence])],
|
||||
day_of_week=[ScheduleRange(2)] if cadence=='weekly' else [ScheduleRange(0,6)],
|
||||
day_of_month=[ScheduleRange(1)] if cadence=='monthly' else [ScheduleRange(1,31)])
|
||||
spec=ScheduleSpec(calendars=[cal],time_zone_name='Europe/Berlin')
|
||||
else:spec=ScheduleSpec(cron_expressions=[cron],time_zone_name='Europe/Berlin')
|
||||
schedule=Schedule(action=action,spec=spec,policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.BUFFER_ONE,
|
||||
catchup_window=timedelta(days=1)),state=ScheduleState(paused=False))
|
||||
return SimpleNamespace(schedule=schedule)
|
||||
|
||||
|
||||
def sources(items=None,descriptions=None):
|
||||
data=items if items is not None else rows()
|
||||
class Session:
|
||||
async def __aenter__(self):return self
|
||||
async def __aexit__(self,*_):pass
|
||||
async def scalars(self,statement):
|
||||
assert len(statement.compile().params['id_1'])==3
|
||||
return SimpleNamespace(all=lambda:data)
|
||||
descriptions=descriptions or {identity:description(identity,calendar=True) for identity in FRONTEND}
|
||||
class Temporal:
|
||||
data_converter=DataConverter.default
|
||||
def get_schedule_handle(self,sid):
|
||||
assert sid.startswith('activity-schedule-')
|
||||
async def describe():return descriptions[sid.removeprefix('activity-schedule-')]
|
||||
return SimpleNamespace(describe=describe)
|
||||
return Session,Temporal()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('calendar',[False,True])
|
||||
async def test_real_invariants_accept_exact_cron_and_server_normalized_calendar(calendar):
|
||||
session,temporal=sources(descriptions={identity:description(identity,calendar) for identity in FRONTEND})
|
||||
result=await probe(session,temporal)
|
||||
assert result['healthy'] is True and result['definition_count']==result['schedule_count']==3
|
||||
assert set(result)=={'schema_version','observed_at','healthy','definition_count','schedule_count','definitions_and_report_sinks_match','schedules_match'}
|
||||
|
||||
|
||||
async def test_actual_describe_payload_arguments_are_decoded_with_bound_client_converter():
|
||||
descriptions={identity:description(identity,True) for identity in FRONTEND}
|
||||
for identity,value in descriptions.items():
|
||||
encoded=await DataConverter.default.encode([identity,'scheduled',None])
|
||||
raw=NewWorkflowExecutionInfo(workflow_id=f'activity-{identity}:${{firstScheduledTime}}',
|
||||
workflow_type=WorkflowType(name='RunActivityWorkflow'),task_queue=TaskQueue(name='orchestrator-tq'),input=Payloads(payloads=encoded))
|
||||
value.schedule.action=ScheduleActionStartWorkflow('<unused>',raw_info=raw)
|
||||
session,temporal=sources(descriptions=descriptions)
|
||||
assert (await probe(session,temporal))['healthy'] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value',[('enabled',False),('trigger_type','event'),
|
||||
('instructions_json',[{'model':'llm','report_sinks':[SINK]}])])
|
||||
async def test_definition_behavior_drift_refused(field,value):
|
||||
data=rows();setattr(data[0],field,value)
|
||||
assert (await probe(*sources(items=data)))['healthy'] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize('field,value',[('cron_expression','* * * * *'),('timezone','UTC'),
|
||||
('misfire_policy','catchup_all'),('catchup_window_seconds',3600),('jitter_seconds',5)])
|
||||
def test_trigger_scope_drift_refused(field,value):
|
||||
row=rows()[0];row.trigger_config[field]=value
|
||||
assert not definition_matches(row)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change',['pause','limit','extra_calendar','interval','skip','cron','queue','args','sink','policy','timezone'])
|
||||
async def test_schedule_and_report_drift_fail_closed(change):
|
||||
data=rows();descriptions={identity:description(identity,True) for identity in FRONTEND}
|
||||
identity=next(iter(FRONTEND));s=descriptions[identity].schedule
|
||||
if change=='pause':s.state.paused=True
|
||||
elif change=='limit':s.state.limited_actions=True
|
||||
elif change=='extra_calendar':s.spec.calendars.append(ScheduleCalendarSpec())
|
||||
elif change=='interval':s.spec.intervals=[object()]
|
||||
elif change=='skip':s.spec.skip=[ScheduleCalendarSpec()]
|
||||
elif change=='cron':s.spec.calendars[0].hour=[ScheduleRange(5)]
|
||||
elif change=='queue':s.action.task_queue='privileged-other-queue'
|
||||
elif change=='args':s.action.args=['other-identity','scheduled',None]
|
||||
elif change=='sink':data[0].instructions_json[0]['report_sinks'][0]['topic_id']='foreign'
|
||||
elif change=='policy':s.policy.overlap=ScheduleOverlapPolicy.ALLOW_ALL
|
||||
elif change=='timezone':s.spec.time_zone_name='UTC'
|
||||
assert (await probe(*sources(items=data,descriptions=descriptions)))['healthy'] is False
|
||||
|
||||
|
||||
async def test_missing_duplicate_rows_and_private_exceptions_are_not_healthy_or_leaked():
|
||||
for data in [[],rows()[:2],[rows()[0]]*3]:assert not (await probe(*sources(items=data)))['healthy']
|
||||
def unavailable():raise RuntimeError('PRIVATE-DATABASE-PASSWORD-CANARY')
|
||||
result=await probe(unavailable,sources()[1])
|
||||
assert not result['healthy'] and 'CANARY' not in str(result)
|
||||
|
||||
|
||||
def test_release_health_endpoint_is_read_only_and_uses_bound_dependencies(monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
from activity_core import api
|
||||
session,temporal=sources()
|
||||
monkeypatch.setattr(api,'_session_factory',session)
|
||||
monkeypatch.setattr(api,'_temporal_client',temporal)
|
||||
client=TestClient(api.app)
|
||||
result=client.get('/release/health')
|
||||
assert result.status_code==200 and result.json()['healthy']
|
||||
assert client.post('/release/health',json={'operation':'trigger'}).status_code==405
|
||||
84
tests/test_release_runtime.py
Normal file
84
tests/test_release_runtime.py
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from cryptography.hazmat.primitives.serialization import Encoding,PublicFormat
|
||||
|
||||
from activity_core.release_runtime import Admission, CLUSTER_UID, DurableAudit, verify_runtime_identity
|
||||
|
||||
|
||||
def configuration(root):
|
||||
kube=root/'kubeconfig';kube.write_text('fixture private kube configuration');kube.chmod(0o600)
|
||||
git=root/'gitconfig';git.write_text('fixture private git configuration');git.chmod(0o600)
|
||||
trust={}
|
||||
for role in ('build','review','health','retention'):
|
||||
public=Ed25519PrivateKey.generate().public_key().public_bytes(Encoding.Raw,PublicFormat.Raw)
|
||||
trust[role]={'public_key':base64.b64encode(public).decode(),'principal':role,'roles':[role]}
|
||||
value={'schema_version':'activity-core-release-runtime/v1','enabled':True,'principal':'activity-core-release-broker',
|
||||
'cluster_uid':CLUSTER_UID,'state_directory':str(root/'state'),'kubeconfig':str(kube),'git_config':str(git),'receipt_trust':trust}
|
||||
path=root/'runtime.json';path.write_text(json.dumps(value));path.chmod(0o600)
|
||||
return path,value
|
||||
|
||||
|
||||
def test_operator_private_config_drift_disable_and_custody_replacement_refused(tmp_path):
|
||||
path,value=configuration(tmp_path);admission=Admission(path);admission.require()
|
||||
assert admission.environment()['GIT_CONFIG_NOSYSTEM']=='1'
|
||||
value['enabled']=False;path.write_text(json.dumps(value))
|
||||
with pytest.raises(ValueError,match='disabled'):admission.require()
|
||||
value['enabled']=True;value['principal']='borrowed-admin';path.write_text(json.dumps(value))
|
||||
with pytest.raises(ValueError):admission.require()
|
||||
value['principal']='activity-core-release-broker';path.write_text(json.dumps(value))
|
||||
Path(value['kubeconfig']).write_text('replaced fixture custody')
|
||||
with pytest.raises(ValueError,match='custody'):admission.require()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('delta',[{'enabled':False},{'principal':'operator'},{'cluster_uid':'other'},
|
||||
{'schema_version':'other'},{'untrusted_transport_factory':'arbitrary-import'}])
|
||||
def test_disabled_widened_configuration_cannot_construct_runtime(tmp_path,delta):
|
||||
path,value=configuration(tmp_path);path.write_text(json.dumps(value|delta))
|
||||
with pytest.raises(ValueError):Admission(path)
|
||||
|
||||
|
||||
def test_public_permissions_symlink_or_incomplete_role_registry_refused(tmp_path):
|
||||
path,value=configuration(tmp_path);path.chmod(0o644)
|
||||
with pytest.raises(ValueError,match='private'):Admission(path)
|
||||
path.chmod(0o600);alias=tmp_path/'symlink.json';alias.symlink_to(path)
|
||||
with pytest.raises(ValueError,match='private'):Admission(alias)
|
||||
del value['receipt_trust']['health'];path.write_text(json.dumps(value))
|
||||
with pytest.raises(ValueError,match='independent'):Admission(path)
|
||||
|
||||
|
||||
def identity_runner(*,admin=False,extra_access=False,wrong_cluster=False):
|
||||
calls=[]
|
||||
def run(argv,**kwargs):
|
||||
calls.append(argv)
|
||||
if 'namespace' in argv:return json.dumps({'metadata':{'uid':'wrong' if wrong_cluster else CLUSTER_UID}})
|
||||
if 'whoami' in argv:return json.dumps({'status':{'userInfo':{'username':'root' if admin else 'system:serviceaccount:activity-core:activity-core-release-broker','groups':['system:masters'] if admin else ['system:serviceaccounts']}}})
|
||||
assert '--raw=/apis/authorization.k8s.io/v1/selfsubjectaccessreviews' in argv
|
||||
attrs=json.loads(kwargs['input'])['spec']['resourceAttributes']
|
||||
allowed=attrs['resource']=='applications' and attrs.get('name') in {'activity-core','railiance-apps-root'}
|
||||
if extra_access and attrs['resource']=='secrets':allowed=True
|
||||
return json.dumps({'status':{'allowed':allowed}})
|
||||
return run,calls
|
||||
|
||||
|
||||
def test_actual_subject_scope_and_denial_checks_are_required_before_dispatch():
|
||||
run,calls=identity_runner();result=verify_runtime_identity(run)
|
||||
assert result=={'principal_verified':True,'allowed_checks':4,'denied_checks':7}
|
||||
assert len(calls)==13
|
||||
for kwargs in [{'admin':True},{'extra_access':True},{'wrong_cluster':True}]:
|
||||
run,calls=identity_runner(**kwargs)
|
||||
with pytest.raises(ValueError):verify_runtime_identity(run)
|
||||
if kwargs.get('wrong_cluster'):assert len(calls)==1
|
||||
|
||||
|
||||
def test_audit_sink_durably_deduplicates_and_refuses_collision_or_private_fields(tmp_path):
|
||||
sink=DurableAudit(tmp_path/'audit.sqlite')
|
||||
event={'event_id':'release:id:1','event_type':'release_transition','release_id':'id','application':'activity-core',
|
||||
'phase':'complete','observed_at':'2026-10-11T20:00:00Z','candidate':'a'*40,'rollback':'b'*40}
|
||||
sink(event);DurableAudit(tmp_path/'audit.sqlite')(event)
|
||||
with sink.connect() as db:assert db.execute('SELECT count(*) FROM events').fetchone()[0]==1
|
||||
with pytest.raises(ValueError,match='conflict'):sink(event|{'phase':'failed'})
|
||||
with pytest.raises(ValueError):sink(event|{'credential':'secret fixture canary'})
|
||||
101
tests/test_release_sampler.py
Normal file
101
tests/test_release_sampler.py
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
from datetime import datetime,timezone,timedelta
|
||||
import pytest
|
||||
from activity_core.release_sampler import BASELINE, Sampler, invariant_matches
|
||||
|
||||
NOW=datetime(2026,10,11,20,tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_gap_sensitive_actual_sample_interval_survives_restart_without_backfill(tmp_path):
|
||||
database=tmp_path/'health.sqlite'
|
||||
sampler=Sampler(database,lambda:(BASELINE,True))
|
||||
assert sampler.sample(NOW)['continuous_24h_ready'] is False
|
||||
later=NOW+timedelta(days=1)
|
||||
assert sampler.sample(later)['healthy_since']==later.isoformat()
|
||||
for minute in range(1,1441):sampler.sample(later+timedelta(minutes=minute))
|
||||
restarted=Sampler(database,lambda:(BASELINE,True))
|
||||
result=restarted.sample(later+timedelta(days=1,seconds=60))
|
||||
assert result['continuous_24h_ready'] and result['attestation']['continuous']
|
||||
|
||||
|
||||
def test_unavailable_probe_records_failure_and_resets_interval(tmp_path):
|
||||
sampler=Sampler(tmp_path/'health.sqlite',lambda:(BASELINE,True))
|
||||
sampler.sample(NOW)
|
||||
def unavailable():raise RuntimeError('PRIVATE-TOKEN-CANARY')
|
||||
sampler.probe=unavailable
|
||||
result=sampler.sample(NOW+timedelta(minutes=1))
|
||||
assert result['healthy'] is False and result['healthy_since'] is None
|
||||
assert 'CANARY' not in str(result)
|
||||
sampler.probe=lambda:(BASELINE,True)
|
||||
result=sampler.sample(NOW+timedelta(minutes=2))
|
||||
assert result['healthy_since']==(NOW+timedelta(minutes=2)).isoformat()
|
||||
|
||||
|
||||
def test_revision_and_clock_change_refuse_old_observation(tmp_path):
|
||||
sampler=Sampler(tmp_path/'health.sqlite',lambda:(BASELINE,True));sampler.sample(NOW)
|
||||
with pytest.raises(ValueError):sampler.sample(NOW)
|
||||
sampler.probe=lambda:('b'*40,True)
|
||||
assert sampler.sample(NOW+timedelta(minutes=1))['healthy_since']==(NOW+timedelta(minutes=1)).isoformat()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('delta',[{'healthy':False},{'schedules_match':False},{'definition_count':2},
|
||||
{'observed_at':(NOW-timedelta(seconds=91)).isoformat()},{'observed_at':(NOW+timedelta(seconds=1)).isoformat()},
|
||||
{'error':'private canary'},{'schema_version':'other'}])
|
||||
def test_invariant_response_cannot_manufacture_health(delta):
|
||||
value={'schema_version':'activity-core-release-health/v1','healthy':True,'schedules_match':True,
|
||||
'definitions_and_report_sinks_match':True,'definition_count':3,'schedule_count':3,'observed_at':NOW.isoformat()}
|
||||
assert not invariant_matches(value|delta,NOW)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native_probe():
|
||||
import json
|
||||
from activity_core.release_sampler import Probe,CLUSTER_UID,RESOURCE_SET
|
||||
from activity_core.release_transport import ACTIVITY_URL
|
||||
app={'apiVersion':'argoproj.io/v1alpha1','kind':'Application',
|
||||
'metadata':{'name':'activity-core','namespace':'argocd'},
|
||||
'spec':{'project':'activity-core','source':{'repoURL':ACTIVITY_URL,'path':'k8s/gitops','targetRevision':BASELINE},
|
||||
'destination':{'namespace':'activity-core','server':'https://kubernetes.default.svc'},
|
||||
'syncPolicy':{'syncOptions':['CreateNamespace=false','ApplyOutOfSyncOnly=true','PruneLast=true','FailOnSharedResource=true']}},
|
||||
'status':{'sync':{'status':'Synced','revision':BASELINE},'health':{'status':'Healthy'},
|
||||
'resources':[{'kind':kind,'name':name,'namespace':'activity-core','status':'Synced','group':'apps' if kind=='Deployment' else ''} for kind,name in RESOURCE_SET]}}
|
||||
deployment={'metadata':{'generation':1},'spec':{'replicas':1},'status':{'observedGeneration':1,'readyReplicas':1,'updatedReplicas':1,'availableReplicas':1}}
|
||||
objects={'nodes':{'items':[{'status':{'addresses':[{'address':'92.205.62.239'}]}}]},
|
||||
'namespace':{'metadata':{'uid':CLUSTER_UID}},'application':app,'deployment':deployment,
|
||||
'service':{'spec':{'clusterIP':'10.43.1.2','ports':[{'port':8010}]}}}
|
||||
calls=[]
|
||||
def runner(argv,**kwargs):
|
||||
calls.append(argv)
|
||||
assert 'get' in argv and not any(verb in argv for verb in ['patch','apply','create','secrets'])
|
||||
return json.dumps(objects[argv[argv.index('get')+1]])
|
||||
def reader(url):
|
||||
assert url=='http://10.43.1.2:8010/release/health'
|
||||
return {'schema_version':'activity-core-release-health/v1','healthy':True,'schedules_match':True,
|
||||
'definitions_and_report_sinks_match':True,'definition_count':3,'schedule_count':3,
|
||||
'observed_at':datetime.now(timezone.utc).isoformat()}
|
||||
return Probe(runner,reader),objects,calls
|
||||
|
||||
|
||||
def test_native_probe_fixed_reads_and_invariants(native_probe):
|
||||
probe,_,calls=native_probe
|
||||
assert probe()==(BASELINE,True)
|
||||
assert len(calls)==7
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change',['estate','uid','automated','missing_resource','extra_resource','wrong_group','wrong_namespace','stale_generation','unready','revision','operation'])
|
||||
def test_native_probe_drift_fails_closed(native_probe,change):
|
||||
probe,objects,_=native_probe
|
||||
app=objects['application']
|
||||
if change=='estate':objects['nodes']['items'][0]['status']['addresses'][0]['address']='192.0.2.1'
|
||||
elif change=='uid':objects['namespace']['metadata']['uid']='foreign-cluster'
|
||||
elif change=='automated':app['spec']['syncPolicy']['automated']={}
|
||||
elif change=='missing_resource':app['status']['resources'].pop()
|
||||
elif change=='extra_resource':app['status']['resources'].append(dict(app['status']['resources'][0]))
|
||||
elif change=='wrong_group':app['status']['resources'][0]['group']='foreign'
|
||||
elif change=='wrong_namespace':app['status']['resources'][0]['namespace']='foreign'
|
||||
elif change=='stale_generation':objects['deployment']['status']['observedGeneration']=0
|
||||
elif change=='unready':objects['deployment']['status']['readyReplicas']=0
|
||||
elif change=='revision':app['status']['sync']['revision']='b'*40
|
||||
elif change=='operation':app['operation']={'sync':{}}
|
||||
try:_,healthy=probe()
|
||||
except (ValueError,KeyError):return
|
||||
assert healthy is False
|
||||
Loading…
Add table
Add a link
Reference in a new issue