Implement scoped Git Argo transport and gap-sensitive health observer
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e241-8285-7a63-8c0c-51c9cb824dc3
This commit is contained in:
parent
22935955cc
commit
0ae5e84e7b
6 changed files with 512 additions and 11 deletions
69
src/activity_core/release_observer.py
Normal file
69
src/activity_core/release_observer.py
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
"""Persistent, gap-sensitive health observations; no scheduler or signing key here."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import sqlite3
|
||||
|
||||
from .release_operations import revision
|
||||
|
||||
|
||||
class HealthObserver:
|
||||
def __init__(self, database, *, max_gap_seconds=90):
|
||||
if not 1 <= max_gap_seconds <= 90:
|
||||
raise ValueError('health gap must be at most 90 seconds')
|
||||
self.database, self.max_gap = database, timedelta(seconds=max_gap_seconds)
|
||||
with self.connect() as db:
|
||||
db.execute('CREATE TABLE IF NOT EXISTS observations (at TEXT PRIMARY KEY, revision TEXT NOT NULL, healthy INTEGER NOT NULL, since TEXT)')
|
||||
db.execute('CREATE TABLE IF NOT EXISTS observer_guard (singleton INTEGER PRIMARY KEY, invalidated INTEGER NOT NULL)')
|
||||
db.execute('INSERT OR IGNORE INTO observer_guard VALUES (1,0)')
|
||||
|
||||
def connect(self):
|
||||
db=sqlite3.connect(self.database,timeout=5,isolation_level=None)
|
||||
db.execute('PRAGMA synchronous=FULL')
|
||||
return db
|
||||
|
||||
def observe(self, commit, healthy, now=None):
|
||||
"""Called by a trusted sampler after exact-revision readiness/invariant checks.
|
||||
|
||||
Failed/unavailable probes must be recorded False. A missed sample, clock
|
||||
regression or revision change resets the interval; no timestamp backfill.
|
||||
"""
|
||||
revision(commit)
|
||||
if type(healthy) is not bool:raise ValueError('explicit boolean health required')
|
||||
now=now or datetime.now(timezone.utc)
|
||||
if now.tzinfo is None:raise ValueError('timezone required')
|
||||
db=self.connect()
|
||||
try:
|
||||
db.execute('BEGIN IMMEDIATE')
|
||||
prior=db.execute('SELECT at,revision,healthy,since FROM observations ORDER BY rowid DESC LIMIT 1').fetchone()
|
||||
invalidated=db.execute('SELECT invalidated FROM observer_guard WHERE singleton=1').fetchone()[0]
|
||||
since=now.isoformat() if healthy else None
|
||||
if prior:
|
||||
last=datetime.fromisoformat(prior[0])
|
||||
if now<=last:
|
||||
db.execute('UPDATE observer_guard SET invalidated=1 WHERE singleton=1')
|
||||
db.commit()
|
||||
raise ValueError('non-increasing observation time')
|
||||
if not invalidated and healthy and prior[2] and prior[1]==commit and now-last<=self.max_gap:
|
||||
since=prior[3]
|
||||
db.execute('INSERT INTO observations VALUES (?,?,?,?)',(now.isoformat(),commit,int(healthy),since))
|
||||
db.execute('UPDATE observer_guard SET invalidated=0 WHERE singleton=1')
|
||||
db.commit()
|
||||
return since
|
||||
except Exception:
|
||||
db.rollback();raise
|
||||
finally:db.close()
|
||||
|
||||
def attestation(self, commit, now=None):
|
||||
"""Return facts for a trusted health issuer, never fabricate a signature."""
|
||||
revision(commit);now=now or datetime.now(timezone.utc)
|
||||
db=self.connect()
|
||||
try:
|
||||
row=db.execute('SELECT at,revision,healthy,since FROM observations ORDER BY rowid DESC LIMIT 1').fetchone()
|
||||
if db.execute('SELECT invalidated FROM observer_guard WHERE singleton=1').fetchone()[0]:
|
||||
raise ValueError('observation interval invalidated by clock regression')
|
||||
finally:db.close()
|
||||
if not row or row[1]!=commit or not row[2] or not row[3]:raise ValueError('no healthy revision interval')
|
||||
measured=datetime.fromisoformat(row[0]);since=datetime.fromisoformat(row[3])
|
||||
if not timedelta(0)<=now-measured<=self.max_gap:raise ValueError('health observations stale or clock regressed')
|
||||
if measured-since<timedelta(hours=24):raise ValueError('24-hour observed interval incomplete')
|
||||
return {'revision':commit,'continuous':True,'healthy':True,'synced':True,
|
||||
'healthy_since':since.isoformat(),'observed_at':measured.isoformat()}
|
||||
183
src/activity_core/release_transport.py
Normal file
183
src/activity_core/release_transport.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
"""Fixed Git and Kubernetes transport for an admitted release broker.
|
||||
|
||||
Credentials and context are operator-provided process configuration. No bearer
|
||||
values, URLs, arbitrary commands or repository paths are accepted from a release.
|
||||
There is no service endpoint or automatic production activation in this module.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import yaml
|
||||
|
||||
from .release_broker import fingerprint
|
||||
from .release_operations import APPLICATION_PATH, revision, sync_operations, update_child
|
||||
|
||||
PLATFORM_URL='https://forgejo.coulomb.social/coulomb/railiance-platform.git'
|
||||
ACTIVITY_URL='https://forgejo.coulomb.social/coulomb/activity-core.git'
|
||||
|
||||
|
||||
class TransportError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def run(argv, *, input=None, env=None, timeout=30):
|
||||
"""Bounded, non-shell invocation; suppress outputs that could contain credentials."""
|
||||
try:
|
||||
return subprocess.run(argv,input=input,text=True,capture_output=True,check=True,
|
||||
timeout=timeout,env=env).stdout
|
||||
except (subprocess.SubprocessError,OSError):
|
||||
raise TransportError('release transport command failed or timed out') from None
|
||||
|
||||
|
||||
class GitArgoBackend:
|
||||
def __init__(self, plan, workspace, invariants, *, cluster_uid, admitted=False, runner=run,
|
||||
clock=time.monotonic, pause=time.sleep, wait_seconds=90):
|
||||
if not admitted:raise ValueError('transport identity not admitted')
|
||||
if not callable(invariants):raise ValueError('report and schedule invariant probe required')
|
||||
if not 1<=wait_seconds<=90:raise ValueError('bounded wait required')
|
||||
if not isinstance(cluster_uid,str) or not cluster_uid:raise ValueError('pinned cluster UID required')
|
||||
self.cluster_uid=cluster_uid;self.cluster_verified=False
|
||||
self.plan=dict(plan);self.workspace=Path(workspace);self.invariants=invariants
|
||||
self.runner,self.clock,self.pause,self.wait_seconds=runner,clock,pause,wait_seconds
|
||||
if plan.get('repository')!='coulomb/activity-core' or plan.get('application')!='activity-core':
|
||||
raise ValueError('transport plan outside scope')
|
||||
for name in ('candidate','rollback'):revision(plan[name])
|
||||
self.workspace.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
# Environment is trusted service configuration, not producer input. In
|
||||
# particular kubeconfig and Git's credential helper belong to this identity.
|
||||
self.env={**os.environ,'GIT_TERMINAL_PROMPT':'0','GIT_AUTHOR_NAME':'activity-core release broker',
|
||||
'GIT_AUTHOR_EMAIL':'release-broker@activity-core.invalid',
|
||||
'GIT_COMMITTER_NAME':'activity-core release broker',
|
||||
'GIT_COMMITTER_EMAIL':'release-broker@activity-core.invalid'}
|
||||
self.platform=self.workspace/'platform.git';self.activity=self.workspace/'activity.git'
|
||||
for path in (self.platform,self.activity):
|
||||
if not path.exists():self.runner(['git','init','--bare',str(path)],env=self.env,timeout=30)
|
||||
|
||||
def git(self, repo, *args, input=None):
|
||||
result=self.runner(['git','-c','core.hooksPath=/dev/null','-c','protocol.file.allow=never',
|
||||
'-C',str(repo),*args],input=input,env=self.env,timeout=30)
|
||||
return result if args[0]=='show' else result.strip()
|
||||
|
||||
def head(self):
|
||||
self.git(self.platform,'fetch','--no-tags',PLATFORM_URL,'main')
|
||||
return revision(self.git(self.platform,'rev-parse','FETCH_HEAD'))
|
||||
|
||||
def validate_sources(self):
|
||||
for key,hash_key in [('rollback','before_sha256'),('candidate','after_sha256')]:
|
||||
commit=self.plan[key]
|
||||
self.git(self.activity,'fetch','--no-tags',ACTIVITY_URL,commit)
|
||||
body=self.git(self.activity,'show',commit+':k8s/gitops/runtime.yaml')
|
||||
docs=list(yaml.safe_load_all(body))
|
||||
if fingerprint(docs)!=self.plan[hash_key]:raise ValueError('source commit manifest binding mismatch')
|
||||
# Never execute kustomize/plugins from a proposed source. Only the
|
||||
# audited single static resource file is accepted by this lane.
|
||||
kustomization=yaml.safe_load(self.git(self.activity,'show',commit+':k8s/gitops/kustomization.yaml'))
|
||||
if kustomization!={'apiVersion':'kustomize.config.k8s.io/v1beta1','kind':'Kustomization','resources':['runtime.yaml']}:
|
||||
raise ValueError('source rendering outside static projection contract')
|
||||
|
||||
def publish_revision(self, expected, target):
|
||||
self.verify_cluster()
|
||||
if {expected,target}!={self.plan['candidate'],self.plan['rollback']}:
|
||||
raise ValueError('revision outside admitted release')
|
||||
self.validate_sources()
|
||||
base=self.head()
|
||||
document=self.git(self.platform,'show',base+':'+APPLICATION_PATH)
|
||||
updated=update_child(document,expected,target)
|
||||
if updated==document:return # Recovery after a lost push response.
|
||||
blob=self.git(self.platform,'hash-object','-w','--stdin',input=updated)
|
||||
# The trusted per-broker workspace is serialized by Broker's ledger lock.
|
||||
self.git(self.platform,'read-tree',base)
|
||||
self.git(self.platform,'update-index','--add','--cacheinfo','100644',blob,APPLICATION_PATH)
|
||||
tree=self.git(self.platform,'write-tree')
|
||||
commit=self.git(self.platform,'commit-tree',tree,'-p',base,input='Promote activity-core to '+target+'\n')
|
||||
changed=self.git(self.platform,'diff-tree','--no-commit-id','--name-only','-r',commit)
|
||||
if changed!=APPLICATION_PATH:raise ValueError('publication contains unrelated paths')
|
||||
# Non-force push rejects concurrent branch advances. A retry fetches the
|
||||
# new tip and repeats the child-revision compare-and-swap.
|
||||
self.git(self.platform,'push',PLATFORM_URL,revision(commit)+':refs/heads/main')
|
||||
|
||||
def verify_cluster(self):
|
||||
if not self.cluster_verified:
|
||||
uid=self.runner(['kubectl','--request-timeout=20s','get','namespace','kube-system',
|
||||
'-o','jsonpath={.metadata.uid}'],env=self.env,timeout=25).strip()
|
||||
if uid!=self.cluster_uid:raise ValueError('cluster identity mismatch')
|
||||
self.cluster_verified=True
|
||||
|
||||
def kube(self, *args, input=None):
|
||||
self.verify_cluster()
|
||||
return self.runner(['kubectl','--request-timeout=20s',*args],input=input,env=self.env,timeout=25)
|
||||
|
||||
def app(self, name):
|
||||
if name not in {'railiance-apps-root','activity-core'}:raise ValueError('foreign application')
|
||||
obj=json.loads(self.kube('-n','argocd','get','application',name,'-o','json'))
|
||||
if obj.get('metadata',{}).get('name')!=name or obj['metadata'].get('namespace')!='argocd':
|
||||
raise ValueError('application identity mismatch')
|
||||
if name=='railiance-apps-root':
|
||||
source=obj.get('spec',{}).get('source',{})
|
||||
if (source.get('repoURL')!=PLATFORM_URL or source.get('path')!='argocd/railiance01/applications'
|
||||
or source.get('targetRevision')!='main' or obj['spec'].get('syncPolicy',{}).get('automated') is not None):
|
||||
raise ValueError('root outside release contract')
|
||||
else:
|
||||
current=obj.get('spec',{}).get('source',{}).get('targetRevision')
|
||||
# Reuse validation without changing the live object.
|
||||
update_child(yaml.safe_dump(obj),revision(current),current)
|
||||
return obj
|
||||
|
||||
def _sync(self, name, body):
|
||||
desired=body['operation']['sync'];obj=self.app(name)
|
||||
operation=obj.get('operation')
|
||||
if operation is not None:
|
||||
active=dict(operation.get('sync',{}));active.setdefault('prune',False)
|
||||
if active!=desired:raise ValueError('another ArgoCD operation is active')
|
||||
done=obj.get('status',{}).get('operationState',{})
|
||||
completed=(done.get('phase')=='Succeeded' and done.get('syncResult',{}).get('revision')==desired['revision'])
|
||||
if operation is None and not completed:
|
||||
patch=[{'op':'test','path':'/metadata/resourceVersion','value':obj['metadata']['resourceVersion']},
|
||||
{'op':'test','path':'/spec','value':obj['spec']},
|
||||
{'op':'add','path':'/operation','value':body['operation']}]
|
||||
self.kube('-n','argocd','patch','application',name,'--type=json','--patch-file=/dev/stdin',input=json.dumps(patch))
|
||||
until=self.clock()+self.wait_seconds
|
||||
while self.clock()<until:
|
||||
state=self.app(name).get('status',{}).get('operationState',{})
|
||||
result=state.get('syncResult',{}).get('revision')
|
||||
if result==desired['revision']:
|
||||
if state.get('phase')=='Succeeded':return
|
||||
if state.get('phase') in {'Failed','Error'}:raise TransportError('ArgoCD sync failed')
|
||||
self.pause(1)
|
||||
raise TransportError('ArgoCD sync wait timed out')
|
||||
|
||||
def sync_revision(self, target):
|
||||
if target not in {self.plan['candidate'],self.plan['rollback']}:raise ValueError('unbound revision')
|
||||
base=self.head()
|
||||
document=self.git(self.platform,'show',base+':'+APPLICATION_PATH)
|
||||
# Publication must remain the current source tip; never sync a stale pin.
|
||||
update_child(document,target,target)
|
||||
root,child=sync_operations(base,target)
|
||||
self._sync(*root)
|
||||
if self.app('activity-core')['spec']['source']['targetRevision']!=target:
|
||||
raise ValueError('selective root sync did not set expected child revision')
|
||||
self._sync(*child)
|
||||
|
||||
def healthy(self, target):
|
||||
if target not in {self.plan['candidate'],self.plan['rollback']}:raise ValueError('unbound revision')
|
||||
until=self.clock()+self.wait_seconds
|
||||
while self.clock()<until:
|
||||
obj=self.app('activity-core');status=obj.get('status',{})
|
||||
synced=(obj['spec']['source']['targetRevision']==target
|
||||
and status.get('sync',{}).get('revision')==target
|
||||
and status.get('sync',{}).get('status')=='Synced'
|
||||
and status.get('health',{}).get('status')=='Healthy')
|
||||
if synced:
|
||||
ready=True
|
||||
for name in ('actcore-api','actcore-worker','actcore-event-router'):
|
||||
d=json.loads(self.kube('-n','activity-core','get','deployment',name,'-o','json'))
|
||||
n=d['spec'].get('replicas',1);s=d.get('status',{})
|
||||
ready &= n>0 and s.get('observedGeneration')==d['metadata']['generation'] and all(s.get(k,0)==n for k in ('readyReplicas','updatedReplicas','availableReplicas'))
|
||||
if ready and self.invariants(target) is True:return True
|
||||
self.pause(1)
|
||||
return False
|
||||
Loading…
Add table
Add a link
Reference in a new issue