192 lines
11 KiB
Python
192 lines
11 KiB
Python
|
"""Real disposable Git repositories; simulated Kubernetes boundary."""
|
||
|
|
import copy
|
||
|
|
import json
|
||
|
|
import subprocess
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import yaml
|
||
|
|
|
||
|
|
from activity_core.release_broker import fingerprint
|
||
|
|
from activity_core.release_observer import HealthObserver
|
||
|
|
from activity_core.release_operations import APPLICATION_PATH
|
||
|
|
from activity_core.release_transport import ACTIVITY_URL, PLATFORM_URL, GitArgoBackend, TransportError, run
|
||
|
|
NOW=datetime(2026,9,28,16,tzinfo=timezone.utc)
|
||
|
|
|
||
|
|
ROOT=Path(__file__).resolve().parents[1]
|
||
|
|
|
||
|
|
|
||
|
|
def git(path,*args):
|
||
|
|
return subprocess.check_output(['git','-C',str(path),*args],text=True,stderr=subprocess.DEVNULL).strip()
|
||
|
|
|
||
|
|
|
||
|
|
def init(path):
|
||
|
|
path.mkdir();git(path,'init','-b','main');git(path,'config','user.name','Fixture');git(path,'config','user.email','fixture@example.invalid')
|
||
|
|
|
||
|
|
|
||
|
|
def commit(path):
|
||
|
|
git(path,'add','.');git(path,'commit','-m','fixture');return git(path,'rev-parse','HEAD')
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def transport(tmp_path):
|
||
|
|
activity=tmp_path/'activity';init(activity)
|
||
|
|
k=activity/'k8s/gitops';k.mkdir(parents=True)
|
||
|
|
before=list(yaml.safe_load_all((ROOT/'k8s/gitops/runtime.yaml').read_text()))
|
||
|
|
(k/'runtime.yaml').write_text(yaml.safe_dump_all(before));(k/'kustomization.yaml').write_text((ROOT/'k8s/gitops/kustomization.yaml').read_text())
|
||
|
|
rollback=commit(activity)
|
||
|
|
after=copy.deepcopy(before)
|
||
|
|
next(d for d in after if d['metadata']['name']=='actcore-worker')['spec']['template']['spec']['containers'][0]['image']='forgejo.coulomb.social/coulomb/activity-core@sha256:'+'f'*64
|
||
|
|
(k/'runtime.yaml').write_text(yaml.safe_dump_all(after));candidate=commit(activity)
|
||
|
|
seed=tmp_path/'seed';init(seed);p=seed/APPLICATION_PATH;p.parent.mkdir(parents=True)
|
||
|
|
child={'apiVersion':'argoproj.io/v1alpha1','kind':'Application','metadata':{'name':'activity-core','namespace':'argocd'},'spec':{'project':'activity-core','source':{'repoURL':ACTIVITY_URL,'path':'k8s/gitops','targetRevision':rollback},'destination':{'namespace':'activity-core','server':'https://kubernetes.default.svc'},'syncPolicy':{'syncOptions':['CreateNamespace=false','ApplyOutOfSyncOnly=true','PruneLast=true','FailOnSharedResource=true']}}}
|
||
|
|
p.write_text(yaml.safe_dump(child));commit(seed)
|
||
|
|
platform=tmp_path/'platform.git';git(tmp_path,'clone','--bare',str(seed),str(platform))
|
||
|
|
plan={'repository':'coulomb/activity-core','application':'activity-core','candidate':candidate,'rollback':rollback,'before_sha256':fingerprint(before),'after_sha256':fingerprint(after)}
|
||
|
|
state={'tick':0,'calls':[],'lose_push':False,'fail_health':False}
|
||
|
|
root={'metadata':{'name':'railiance-apps-root','namespace':'argocd','resourceVersion':'1'},'spec':{'source':{'repoURL':PLATFORM_URL,'path':'argocd/railiance01/applications','targetRevision':'main'}}}
|
||
|
|
child['metadata']['resourceVersion']='1';apps={'railiance-apps-root':root,'activity-core':child}
|
||
|
|
def runner(argv,**kwargs):
|
||
|
|
if argv[0]=='git':
|
||
|
|
argv=[str(activity) if a==ACTIVITY_URL else str(platform) if a==PLATFORM_URL else 'protocol.file.allow=always' if a=='protocol.file.allow=never' else a for a in argv]
|
||
|
|
result=run(argv,**kwargs)
|
||
|
|
if 'push' in argv and state['lose_push']:
|
||
|
|
state['lose_push']=False;raise TransportError('fixture lost push response')
|
||
|
|
return result
|
||
|
|
state['calls'].append(argv)
|
||
|
|
if 'get' in argv:
|
||
|
|
i=argv.index('get');kind,name=argv[i+1:i+3]
|
||
|
|
if kind=='namespace':return 'fixture-cluster'
|
||
|
|
if kind=='application':return json.dumps(apps[name])
|
||
|
|
return json.dumps({'metadata':{'generation':1},'spec':{'replicas':1},'status':{'observedGeneration':1,'readyReplicas':1,'updatedReplicas':1,'availableReplicas':1}})
|
||
|
|
i=argv.index('patch');name=argv[i+2];patch=json.loads(kwargs['input']);obj=apps[name]
|
||
|
|
assert patch[0]=={'op':'test','path':'/metadata/resourceVersion','value':obj['metadata']['resourceVersion']}
|
||
|
|
assert patch[1]=={'op':'test','path':'/spec','value':obj['spec']}
|
||
|
|
operation=patch[2]['value'];sync=operation['sync'];assert sync['prune'] is False
|
||
|
|
if name=='railiance-apps-root':
|
||
|
|
assert sync['resources']==[{'group':'argoproj.io','kind':'Application','name':'activity-core','namespace':'argocd'}]
|
||
|
|
source=yaml.safe_load(git(platform,'show',sync['revision']+':'+APPLICATION_PATH))
|
||
|
|
apps['activity-core']['spec']=source['spec']
|
||
|
|
obj['status']={'operationState':{'phase':'Succeeded','syncResult':{'revision':sync['revision']}},'sync':{'status':'Synced','revision':sync['revision']},'health':{'status':'Healthy'}}
|
||
|
|
return '{}'
|
||
|
|
def pause(seconds):state['tick']+=seconds
|
||
|
|
backend=GitArgoBackend(plan,tmp_path/'broker',lambda target:not(state['fail_health'] and target==candidate),cluster_uid='fixture-cluster',admitted=True,runner=runner,clock=lambda:state['tick'],pause=pause,wait_seconds=2)
|
||
|
|
return backend,plan,state,platform,apps,before,after
|
||
|
|
|
||
|
|
|
||
|
|
def test_real_git_publish_recover_sync_and_rollback(transport):
|
||
|
|
b,p,state,platform,apps,*_=transport
|
||
|
|
before=git(platform,'rev-parse','main');state['lose_push']=True
|
||
|
|
with pytest.raises(TransportError):b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
published=git(platform,'rev-parse','main');assert published!=before
|
||
|
|
b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
assert git(platform,'rev-parse','main')==published
|
||
|
|
assert git(platform,'diff-tree','--no-commit-id','--name-only','-r',published)==APPLICATION_PATH
|
||
|
|
b.sync_revision(p['candidate']);assert b.healthy(p['candidate'])
|
||
|
|
state['fail_health']=True;assert not b.healthy(p['candidate'])
|
||
|
|
b.publish_revision(p['candidate'],p['rollback']);b.sync_revision(p['rollback']);assert b.healthy(p['rollback'])
|
||
|
|
assert apps['activity-core']['spec']['source']['targetRevision']==p['rollback']
|
||
|
|
|
||
|
|
|
||
|
|
def test_manifest_tampering_refused_before_push(transport):
|
||
|
|
b,p,state,platform,*_=transport;before=git(platform,'rev-parse','main');b.plan['after_sha256']='0'*64
|
||
|
|
with pytest.raises(ValueError,match='manifest binding'):b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
assert git(platform,'rev-parse','main')==before
|
||
|
|
assert not any('patch' in c for c in state['calls'])
|
||
|
|
|
||
|
|
|
||
|
|
def test_no_unbound_target_or_unadmitted_transport(transport,tmp_path):
|
||
|
|
b,p,*_=transport
|
||
|
|
with pytest.raises(ValueError):b.publish_revision(p['rollback'],'c'*40)
|
||
|
|
with pytest.raises(ValueError):b.sync_revision('c'*40)
|
||
|
|
with pytest.raises(ValueError):GitArgoBackend(p,tmp_path/'denied',lambda _:True,cluster_uid='fixture-cluster')
|
||
|
|
|
||
|
|
|
||
|
|
def test_active_foreign_operation_not_overwritten(transport):
|
||
|
|
b,p,state,platform,apps,*_=transport
|
||
|
|
b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
apps['railiance-apps-root']['operation']={'sync':{'revision':'d'*40,'prune':True}}
|
||
|
|
with pytest.raises(ValueError,match='active'):b.sync_revision(p['candidate'])
|
||
|
|
assert not any('patch' in c for c in state['calls'])
|
||
|
|
|
||
|
|
|
||
|
|
def test_health_observer_requires_samples_and_survives_restart(tmp_path):
|
||
|
|
o=HealthObserver(tmp_path/'health.sqlite');sha='a'*40
|
||
|
|
o.observe(sha,True,NOW)
|
||
|
|
o.observe(sha,True,NOW+timedelta(hours=24))
|
||
|
|
with pytest.raises(ValueError,match='incomplete'):o.attestation(sha,NOW+timedelta(hours=24))
|
||
|
|
start=NOW+timedelta(hours=24)
|
||
|
|
for n in range(1,1441):o.observe(sha,True,start+timedelta(minutes=n))
|
||
|
|
restarted=HealthObserver(tmp_path/'health.sqlite')
|
||
|
|
result=restarted.attestation(sha,start+timedelta(hours=24))
|
||
|
|
assert result['healthy_since']==start.isoformat()
|
||
|
|
with pytest.raises(ValueError,match='stale'):restarted.attestation(sha,start+timedelta(hours=24,minutes=2))
|
||
|
|
restarted.observe(sha,False,start+timedelta(hours=24,minutes=1))
|
||
|
|
with pytest.raises(ValueError,match='no healthy'):restarted.attestation(sha,start+timedelta(hours=24,minutes=1))
|
||
|
|
|
||
|
|
|
||
|
|
def test_observer_revision_change_and_clock_regression(tmp_path):
|
||
|
|
o=HealthObserver(tmp_path/'health.sqlite');o.observe('a'*40,True,NOW)
|
||
|
|
with pytest.raises(ValueError,match='non-increasing'):o.observe('a'*40,True,NOW)
|
||
|
|
t=NOW+timedelta(seconds=60)
|
||
|
|
assert o.observe('b'*40,True,t)==t.isoformat()
|
||
|
|
with pytest.raises(ValueError):o.attestation('a'*40,t)
|
||
|
|
|
||
|
|
|
||
|
|
def test_concurrent_git_writer_is_not_overwritten(transport):
|
||
|
|
b,p,state,platform,apps,*_=transport
|
||
|
|
original=b.runner
|
||
|
|
def racing(argv,**kwargs):
|
||
|
|
if argv[0]=='git' and 'push' in argv:
|
||
|
|
# Inject another committed platform update after broker's CAS read.
|
||
|
|
tree=git(platform,'rev-parse','main^{tree}')
|
||
|
|
parent=git(platform,'rev-parse','main')
|
||
|
|
env=dict(b.env)
|
||
|
|
new=run(['git','-C',str(platform),'commit-tree',tree,'-p',parent],input='concurrent writer',env=env).strip()
|
||
|
|
git(platform,'update-ref','refs/heads/main',new,parent)
|
||
|
|
return original(argv,**kwargs)
|
||
|
|
b.runner=racing
|
||
|
|
with pytest.raises(TransportError):b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
current=yaml.safe_load(git(platform,'show','main:'+APPLICATION_PATH))
|
||
|
|
assert current['spec']['source']['targetRevision']==p['rollback']
|
||
|
|
|
||
|
|
|
||
|
|
def test_broker_rolls_back_using_real_git_transport(transport,tmp_path):
|
||
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||
|
|
from cryptography.hazmat.primitives.serialization import Encoding,PublicFormat
|
||
|
|
from activity_core.release_broker import Broker,Receipts,ROLES
|
||
|
|
from tests.test_release_broker import envelopes
|
||
|
|
b,p,state,platform,apps,before,after=transport
|
||
|
|
keys={role:Ed25519PrivateKey.generate() for role in ROLES}
|
||
|
|
trust={r:{'public_key':k.public_key().public_bytes(Encoding.Raw,PublicFormat.Raw),'principal':r,'roles':{r}} for r,k in keys.items()}
|
||
|
|
broker=Broker(tmp_path/'coordinator.sqlite',Receipts(trust),admitted=True)
|
||
|
|
signed=envelopes(before,after,keys,candidate=p['candidate'],rollback=p['rollback'])
|
||
|
|
rid=broker.admit(before,after,p['candidate'],p['rollback'],signed,NOW)
|
||
|
|
state['fail_health']=True
|
||
|
|
phases=[]
|
||
|
|
for _ in range(10):
|
||
|
|
phases.append(broker.advance(rid,b,NOW))
|
||
|
|
if phases[-1]=='rolled_back':break
|
||
|
|
assert phases==['publish_pending','published','synced','rollback_planned','rollback_published','rollback_synced','rolled_back']
|
||
|
|
assert apps['activity-core']['spec']['source']['targetRevision']==p['rollback']
|
||
|
|
assert yaml.safe_load(git(platform,'show','main:'+APPLICATION_PATH))['spec']['source']['targetRevision']==p['rollback']
|
||
|
|
|
||
|
|
|
||
|
|
def test_clock_regression_invalidates_interval_until_new_sample(tmp_path):
|
||
|
|
o=HealthObserver(tmp_path/'health.sqlite');sha='a'*40
|
||
|
|
o.observe(sha,True,NOW)
|
||
|
|
with pytest.raises(ValueError):o.observe(sha,True,NOW-timedelta(seconds=1))
|
||
|
|
with pytest.raises(ValueError,match='invalidated'):HealthObserver(tmp_path/'health.sqlite').attestation(sha,NOW)
|
||
|
|
later=NOW+timedelta(seconds=30)
|
||
|
|
assert o.observe(sha,True,later)==later.isoformat()
|
||
|
|
|
||
|
|
|
||
|
|
def test_wrong_cluster_refused_before_git_publication(transport):
|
||
|
|
b,p,state,platform,*_=transport
|
||
|
|
before=git(platform,'rev-parse','main');b.cluster_uid='wrong-cluster'
|
||
|
|
with pytest.raises(ValueError,match='cluster identity'):
|
||
|
|
b.publish_revision(p['rollback'],p['candidate'])
|
||
|
|
assert git(platform,'rev-parse','main')==before
|
||
|
|
assert not any('patch' in c for c in state['calls'])
|