203 lines
9.8 KiB
Python
203 lines
9.8 KiB
Python
|
"""Isolated failure proofs: generated test keys and an in-memory deployment adapter."""
|
||
|
|
import base64
|
||
|
|
import copy
|
||
|
|
import json
|
||
|
|
import sqlite3
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import yaml
|
||
|
|
from cryptography.exceptions import InvalidSignature
|
||
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||
|
|
from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
|
||
|
|
|
||
|
|
from activity_core.release_broker import AUTHORITY, REQUIRED_CHECKS, Broker, Receipts, canonical, fingerprint
|
||
|
|
|
||
|
|
ROOT = Path(__file__).resolve().parents[1]
|
||
|
|
NOW = datetime(2026, 9, 28, 16, tzinfo=timezone.utc)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def bundle(tmp_path):
|
||
|
|
before = list(yaml.safe_load_all((ROOT / 'k8s/gitops/runtime.yaml').read_text()))
|
||
|
|
after = copy.deepcopy(before)
|
||
|
|
next(d for d in after if d['metadata']['name'] == 'actcore-worker')['spec']['template']['spec']['containers'][0]['image'] = 'forgejo.coulomb.social/coulomb/activity-core@sha256:'+'f'*64
|
||
|
|
keys = {r: Ed25519PrivateKey.generate() for r in ['build','review','health','retention']}
|
||
|
|
trusted = {r: {'public_key': k.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw),
|
||
|
|
'principal':r, 'roles':{r}} for r,k in keys.items()}
|
||
|
|
broker = Broker(tmp_path/'release.sqlite', Receipts(trusted), admitted=True)
|
||
|
|
return broker, before, after, keys
|
||
|
|
|
||
|
|
|
||
|
|
def images(docs):
|
||
|
|
return {c['image'] for d in docs if d['kind']=='Deployment' for c in d['spec']['template']['spec']['containers']}
|
||
|
|
|
||
|
|
|
||
|
|
def envelopes(before, after, keys, candidate='a'*40, rollback='b'*40):
|
||
|
|
binding = dict(repository='coulomb/activity-core',application='activity-core',
|
||
|
|
candidate=candidate,rollback=rollback,before_sha256=fingerprint(before),after_sha256=fingerprint(after))
|
||
|
|
out={}
|
||
|
|
for role,key in keys.items():
|
||
|
|
payload=dict(schema=1,role=role,authority=AUTHORITY,binding=binding,result='pass',
|
||
|
|
issued_at=NOW.isoformat(),expires_at=(NOW+timedelta(minutes=5)).isoformat())
|
||
|
|
if role=='health':payload.update(revision=rollback,continuous=True,healthy=True,synced=True,healthy_since=(NOW-timedelta(hours=25)).isoformat())
|
||
|
|
if role=='build':
|
||
|
|
payload['images']=sorted(images(after))
|
||
|
|
payload['checks']={name:'success' for name in REQUIRED_CHECKS}
|
||
|
|
if role=='retention':payload['images']=sorted(images(before)|images(after))
|
||
|
|
out[role]={'key_id':role,'payload':payload,'signature':base64.b64encode(key.sign(canonical(payload))).decode()}
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def resign(receipt,key):
|
||
|
|
receipt['signature']=base64.b64encode(key.sign(canonical(receipt['payload']))).decode()
|
||
|
|
|
||
|
|
|
||
|
|
def admit(bundle, **kwargs):
|
||
|
|
broker,before,after,keys=bundle
|
||
|
|
return broker.admit(before,after,'a'*40,'b'*40,envelopes(before,after,keys),now=NOW,**kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
class FakeBackend:
|
||
|
|
def __init__(self):
|
||
|
|
self.revision='b'*40;self.calls=[];self.fail_health=False;self.fail_rollback=False;self.lose_response=False
|
||
|
|
def publish_revision(self,expected,target):
|
||
|
|
if self.revision not in {expected,target}:raise ValueError('CAS conflict')
|
||
|
|
self.revision=target;self.calls.append(('publish',target))
|
||
|
|
if self.lose_response:
|
||
|
|
self.lose_response=False
|
||
|
|
raise ConnectionError('lost response after durable publication')
|
||
|
|
def sync_revision(self,target):
|
||
|
|
assert self.revision==target
|
||
|
|
self.calls.append(('sync',target))
|
||
|
|
def healthy(self,target):
|
||
|
|
assert self.revision==target
|
||
|
|
return not (self.fail_health if target=='a'*40 else self.fail_rollback)
|
||
|
|
|
||
|
|
|
||
|
|
def drive(broker, rid, backend, now=NOW):
|
||
|
|
phases=[]
|
||
|
|
for _ in range(9):
|
||
|
|
phase=broker.advance(rid,backend,now);phases.append(phase)
|
||
|
|
if phase in {'complete','rolled_back','cancelled'}:break
|
||
|
|
return phases
|
||
|
|
|
||
|
|
|
||
|
|
def test_normal_release_restart_and_idempotent_delivery(bundle):
|
||
|
|
broker,*_=bundle;rid=admit(bundle);backend=FakeBackend()
|
||
|
|
assert broker.advance(rid,backend,NOW)=='publish_pending'
|
||
|
|
assert not backend.calls
|
||
|
|
restarted=Broker(broker.database,broker.receipts,admitted=True)
|
||
|
|
assert drive(restarted,rid,backend)==['published','synced','complete']
|
||
|
|
assert admit(bundle)==rid
|
||
|
|
previous=list(backend.calls)
|
||
|
|
assert restarted.advance(rid,backend,NOW)=='complete'
|
||
|
|
assert backend.calls==previous
|
||
|
|
with broker.connect() as db:
|
||
|
|
assert [r[0] for r in db.execute('SELECT phase FROM transitions ORDER BY sequence')]==['planned','publish_pending','published','synced','complete']
|
||
|
|
|
||
|
|
|
||
|
|
def test_health_failure_rolls_back_through_same_adapter(bundle):
|
||
|
|
broker,*_=bundle;rid=admit(bundle);backend=FakeBackend();backend.fail_health=True
|
||
|
|
assert drive(broker,rid,backend)[-1]=='rolled_back'
|
||
|
|
assert backend.calls==[('publish','a'*40),('sync','a'*40),('publish','b'*40),('sync','b'*40)]
|
||
|
|
|
||
|
|
|
||
|
|
def test_failed_rollback_holds_exclusive_slot(bundle):
|
||
|
|
broker,before,after,keys=bundle;rid=admit(bundle);backend=FakeBackend();backend.fail_health=True;backend.fail_rollback=True
|
||
|
|
assert drive(broker,rid,backend)[-1]=='rollback_synced'
|
||
|
|
with pytest.raises(sqlite3.IntegrityError):
|
||
|
|
broker.admit(before,after,'c'*40,'b'*40,envelopes(before,after,keys,candidate='c'*40),now=NOW)
|
||
|
|
backend.fail_rollback=False
|
||
|
|
assert broker.advance(rid,backend,NOW)=='rolled_back'
|
||
|
|
|
||
|
|
|
||
|
|
def test_lost_publish_response_and_expiry_recovers_prior_revision(bundle):
|
||
|
|
broker,*_=bundle;rid=admit(bundle);backend=FakeBackend();backend.lose_response=True
|
||
|
|
assert broker.advance(rid,backend,NOW)=='publish_pending'
|
||
|
|
with pytest.raises(ConnectionError):broker.advance(rid,backend,NOW)
|
||
|
|
assert backend.revision=='a'*40
|
||
|
|
restarted=Broker(broker.database,broker.receipts,admitted=True)
|
||
|
|
assert drive(restarted,rid,backend,NOW+timedelta(minutes=6))[-1]=='rolled_back'
|
||
|
|
assert backend.revision=='b'*40
|
||
|
|
|
||
|
|
|
||
|
|
def test_expired_plan_never_publishes(bundle):
|
||
|
|
broker,*_=bundle;rid=admit(bundle);backend=FakeBackend()
|
||
|
|
assert broker.advance(rid,backend,NOW+timedelta(minutes=6))=='cancelled'
|
||
|
|
assert backend.calls==[]
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('kind',['signature','commit','stale','future','key','role','scope','retention','build','independence','health','checks'])
|
||
|
|
def test_rejects_untrusted_or_incomplete_evidence(bundle,kind):
|
||
|
|
broker,before,after,keys=bundle;e=envelopes(before,after,keys);r=e['review']
|
||
|
|
if kind=='signature':r['payload']['result']='fail'
|
||
|
|
elif kind=='commit':r['payload']['binding']['candidate']='c'*40;resign(r,keys['review'])
|
||
|
|
elif kind=='stale':r['payload']['issued_at']=(NOW-timedelta(minutes=6)).isoformat();resign(r,keys['review'])
|
||
|
|
elif kind=='future':r['payload']['issued_at']=(NOW+timedelta(seconds=1)).isoformat();resign(r,keys['review'])
|
||
|
|
elif kind=='key':r['key_id']='caller-provided'
|
||
|
|
elif kind=='role':r['key_id']='build'
|
||
|
|
elif kind=='scope':r['payload']['authority']='admin';resign(r,keys['review'])
|
||
|
|
elif kind=='retention':e['retention']['payload']['images']=[];resign(e['retention'],keys['retention'])
|
||
|
|
elif kind=='build':e['build']['payload']['images']=[];resign(e['build'],keys['build'])
|
||
|
|
elif kind=='independence':broker.receipts.trusted['review']['principal']='build'
|
||
|
|
elif kind=='checks':e['build']['payload']['checks']={};resign(e['build'],keys['build'])
|
||
|
|
elif kind=='health':e['health']['payload']['continuous']=False;resign(e['health'],keys['health'])
|
||
|
|
with pytest.raises((ValueError,InvalidSignature)):
|
||
|
|
broker.admit(before,after,'a'*40,'b'*40,e,now=NOW)
|
||
|
|
with broker.connect() as db:assert db.execute('SELECT count(*) FROM releases').fetchone()[0]==0
|
||
|
|
|
||
|
|
|
||
|
|
def test_signed_out_of_scope_manifest_still_refused(bundle):
|
||
|
|
broker,before,after,keys=bundle
|
||
|
|
next(d for d in after if d['kind']=='Deployment')['spec']['replicas']=99
|
||
|
|
with pytest.raises(ValueError,match='non-image'):
|
||
|
|
broker.admit(before,after,'a'*40,'b'*40,envelopes(before,after,keys),now=NOW)
|
||
|
|
|
||
|
|
|
||
|
|
def test_unadmitted_identity_cannot_start_or_resume(bundle):
|
||
|
|
broker,*_=bundle;rid=admit(bundle);broker.admitted=False
|
||
|
|
with pytest.raises(ValueError,match='not admitted'):admit(bundle)
|
||
|
|
with pytest.raises(ValueError,match='not admitted'):broker.advance(rid,FakeBackend(),NOW)
|
||
|
|
|
||
|
|
|
||
|
|
def test_fixed_git_and_argo_mutation_shapes():
|
||
|
|
from activity_core.release_operations import update_child, sync_operations
|
||
|
|
declaration = '''apiVersion: argoproj.io/v1alpha1
|
||
|
|
kind: Application
|
||
|
|
metadata:
|
||
|
|
name: activity-core
|
||
|
|
namespace: argocd
|
||
|
|
spec:
|
||
|
|
project: activity-core
|
||
|
|
source:
|
||
|
|
repoURL: https://forgejo.coulomb.social/coulomb/activity-core.git
|
||
|
|
targetRevision: BBBB
|
||
|
|
path: k8s/gitops
|
||
|
|
destination:
|
||
|
|
namespace: activity-core
|
||
|
|
server: https://kubernetes.default.svc
|
||
|
|
syncPolicy:
|
||
|
|
syncOptions: [CreateNamespace=false, ApplyOutOfSyncOnly=true, PruneLast=true, FailOnSharedResource=true]
|
||
|
|
'''.replace('BBBB','b'*40)
|
||
|
|
updated=update_child(declaration,'b'*40,'a'*40)
|
||
|
|
assert updated==declaration.replace('b'*40,'a'*40)
|
||
|
|
assert update_child(updated,'b'*40,'a'*40)==updated
|
||
|
|
with pytest.raises(ValueError,match='CAS'):update_child(declaration,'c'*40,'a'*40)
|
||
|
|
with pytest.raises(ValueError):update_child(declaration.replace('namespace: activity-core','namespace: foreign'),'b'*40,'a'*40)
|
||
|
|
with pytest.raises(ValueError):update_child(declaration,'b'*40,'main')
|
||
|
|
root,child=sync_operations('c'*40,'a'*40)
|
||
|
|
assert root[0]=='railiance-apps-root' and child[0]=='activity-core'
|
||
|
|
assert root[1]['operation']['sync']['resources']==[{'group':'argoproj.io','kind':'Application','name':'activity-core','namespace':'argocd'}]
|
||
|
|
assert all(op['operation']['sync']['prune'] is False for _,op in [root,child])
|
||
|
|
|
||
|
|
|
||
|
|
def test_different_principals_cannot_share_review_signing_key(bundle):
|
||
|
|
broker,before,after,keys=bundle
|
||
|
|
e=envelopes(before,after,keys)
|
||
|
|
broker.receipts.trusted['review']['public_key']=broker.receipts.trusted['build']['public_key']
|
||
|
|
resign(e['review'],keys['build'])
|
||
|
|
with pytest.raises(ValueError,match='independent'):
|
||
|
|
broker.admit(before,after,'a'*40,'b'*40,e,now=NOW)
|