tegwick
1353267569
Assistant: codex Assistant-Model: gpt-5.6-luna Assistant-Session: 01a087b0-7b1e-7531-8ef3-6525ebaf7ee7
157 lines
16 KiB
Python
Executable file
157 lines
16 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
import copy, hashlib, importlib.util, json, sqlite3, subprocess, sys, tempfile
|
|
from pathlib import Path
|
|
import yaml, jsonschema
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
problems=[];lines=[]
|
|
def ok(s):
|
|
lines.append(s);print(s,flush=True)
|
|
def fail(s):
|
|
problems.append(s);lines.append('FAIL '+s);print('FAIL '+s,flush=True)
|
|
def run(cmd):
|
|
if cmd and cmd[0] == 'python':
|
|
cmd = [sys.executable, *cmd[1:]]
|
|
return subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True)
|
|
def load(p):return yaml.safe_load(Path(p).read_text())
|
|
def schema_ok(schema,obj):
|
|
s=json.loads((ROOT/'schemas'/schema).read_text());e=list(jsonschema.Draft202012Validator(s).iter_errors(obj));return [x.message for x in e]
|
|
def ingest(db,p,auth=False):
|
|
cmd=['python','tools/catalog_index.py','ingest-exchange',str(db),str(p)]
|
|
if auth:cmd+=['--authority-policy','policies/SNAPSHOT-AUTHORITY-v0.2.yaml']
|
|
return run(cmd)
|
|
def query(sql,db,args=()):
|
|
c=sqlite3.connect(db);c.row_factory=sqlite3.Row;r=c.execute(sql,args).fetchall();c.close();return r
|
|
|
|
def main():
|
|
# Core freeze + base semantic corpus.
|
|
want=(ROOT/'policies/CORE-FREEZE.sha256').read_text().split()[0];got=hashlib.sha256((ROOT/'schemas/FIAM-Core-v0.2.schema.json').read_bytes()).hexdigest()
|
|
if got!=want:fail('Core freeze hash changed')
|
|
else:ok('CORE-FREEZE PASS byte-identical frozen v0.2 Core')
|
|
cp=run(['python','tools/validate_fiam.py']+[str(p.relative_to(ROOT)) for p in sorted((ROOT/'specimens').rglob('*.yaml'))])
|
|
if cp.returncode or 'SUMMARY: 26 files, 0 failing, 0 errors, 0 warnings' not in cp.stdout:fail('Semantic specimen corpus regression')
|
|
else:ok('SEMANTIC-CORPUS PASS 26 specimens')
|
|
# Policy and exchange schemas.
|
|
cp=run(['python','tools/validate_authority_policy.py','policies/SNAPSHOT-AUTHORITY-v0.2.yaml'])
|
|
if cp.returncode:fail('Snapshot authority policy invalid: '+cp.stdout+cp.stderr)
|
|
else:ok('SNAPSHOT-AUTHORITY-POLICY PASS validated scoped authority grants')
|
|
for p in sorted((ROOT/'real-world/catalog-exchange').glob('*.yaml')):
|
|
if p.name=='zone-derivation-input.yaml':continue
|
|
e=schema_ok('FIAM-CatalogExchange-v0.2.schema.json',load(p))
|
|
if e:fail(f'Catalog Exchange schema {p.name}: {e[:2]}')
|
|
if not any('Catalog Exchange schema' in x for x in problems):ok('CATALOG-EXCHANGE PASS versioned exchange fixtures')
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);db=td/'estate.sqlite'
|
|
# Declared + authoritative runtime with grace / repeated absence.
|
|
if ingest(db,ROOT/'real-world/catalog-exchange/declared-2026-09-01.yaml').returncode:fail('Declared exchange ingest failed')
|
|
if ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-01.yaml',True).returncode:fail('Initial complete runtime ingest failed')
|
|
if ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-08.yaml',True).returncode:fail('First absence snapshot failed')
|
|
inv=query("SELECT active,absence_streak,closed_at FROM topology_observations_current WHERE edge_key=(SELECT edge_key FROM topology_observations_current WHERE target_ref='node.inventory' LIMIT 1)",db)
|
|
if not inv or inv[0]['active']!=1 or inv[0]['absence_streak']!=1 or inv[0]['closed_at'] is not None:fail('Grace/first absence semantics failed')
|
|
else:ok('AUTHORITY-GRACE PASS first missing complete snapshot does not close recent/once-missing edge')
|
|
if ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-09.yaml',True).returncode:fail('Second absence snapshot failed')
|
|
inv=query("SELECT active,absence_streak,closed_at,last_seen FROM topology_observations_current WHERE target_ref='node.inventory'",db)
|
|
if not inv or inv[0]['active']!=0 or inv[0]['absence_streak']<2 or inv[0]['closed_at']!='2026-09-09T00:00:00Z' or inv[0]['last_seen']!='2026-09-01T00:00:00Z':fail('Confirmed closure clocks failed')
|
|
else:ok('AUTHORITY-CLOSURE PASS last_seen evidence remains distinct from closed_at policy action')
|
|
# Temporal / boundary / policy questions before reappearance.
|
|
ch=td/'changes.yaml';bd=td/'boundary.yaml';po=td/'policy.yaml'
|
|
for cmd in [
|
|
['python','tools/architecture_changes_lens.py',str(db),'--population','prod-cluster-a','--since','2026-09-07T00:00:00Z','--until','2026-09-09T23:59:59Z','-o',str(ch)],
|
|
['python','tools/boundary_drift_lens.py',str(db),'--population','prod-cluster-a','--since','2026-09-07T00:00:00Z','--until','2026-09-09T23:59:59Z','-o',str(bd)],
|
|
['python','tools/architecture_policy_lens.py',str(db),'--population','prod-cluster-a','--since','2026-09-07T00:00:00Z','--until','2026-09-09T23:59:59Z','--policy','policies/ARCHITECTURE-GOVERNANCE-v0.2.yaml','-o',str(po)]]:
|
|
cp=run(cmd)
|
|
if cp.returncode:fail('Lens failed: '+' '.join(cmd)+' '+cp.stderr)
|
|
cv,bv,pv=load(ch),load(bd),load(po)
|
|
if cv.get('summary')!={'appeared':1,'disappeared':1,'newly_undeclared':1}:fail('Architecture changes summary unexpected '+str(cv.get('summary')))
|
|
else:ok('TEMPORAL-QUERY PASS appeared/disappeared/newly-undeclared with governed closure')
|
|
if bv.get('summary')!={'new_cross_zone':1,'new_trust_boundary':1,'new_organization_boundary':1}:fail('Boundary summary unexpected')
|
|
else:ok('BOUNDARY-DRIFT PASS trust + organization crossing')
|
|
if pv.get('summary')!={'new_dependencies':1,'material_dependencies':1,'policy_violations':1} or pv['material_dependencies'][0]['materiality']!='high':fail('Architecture policy projection unexpected')
|
|
else:ok('POLICY-INTELLIGENCE PASS undeclared partner dependency is high-materiality + policy violation under named policy')
|
|
# Reappearance opens a second interval.
|
|
if ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-10.yaml',True).returncode:fail('Reappearance snapshot failed')
|
|
n=query("SELECT COUNT(*) n FROM logical_topology_intervals WHERE target_ref='node.inventory'",db)[0]['n']
|
|
if n!=2:fail('Reappearance did not create second logical interval')
|
|
else:ok('TEMPORAL-REAPPEARANCE PASS closed history preserved and new interval opened')
|
|
# Scores are not stored in assertions table.
|
|
cols={r['name'] for r in query('PRAGMA table_info(assertions)',db)}
|
|
if {'support_score','challenge_score','net_support'} & cols:fail('Query-time evidence scores leaked into assertion table')
|
|
else:ok('QUERY-TIME-SCORING PASS index stores raw evidence, not policy scores')
|
|
# Purge only derived index.
|
|
src=ROOT/'real-world/catalog-exchange/runtime-2026-09-01.yaml';before=src.read_bytes();cp=run(['python','tools/catalog_index.py','purge',str(db),'--before','2026-09-10T00:00:00Z','--confirm-derived-index-only'])
|
|
if cp.returncode or src.read_bytes()!=before:fail('Derived-index purge mutated source artifacts')
|
|
else:ok('RETENTION PASS explicit purge affects derived index only')
|
|
# Domain-specific completeness cannot close RPC when claiming only HTTP.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);db=td/'domain.sqlite';ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-01.yaml',True)
|
|
ex=load(ROOT/'real-world/catalog-exchange/runtime-2026-09-01.yaml');ex['exchange_id']='exchange.http-only-empty';ex['created_at']=ex['observation']['snapshot_time']='2026-09-09T00:00:00Z';ex['observation']['domains']=['http'];ex['records']['topology_observations']=[];p=td/'http-only.yaml';p.write_text(yaml.safe_dump(ex,sort_keys=False))
|
|
cp=ingest(db,p,True);row=query("SELECT active,absence_streak FROM topology_observations_current WHERE domain='rpc'",db)[0]
|
|
if cp.returncode or row['active']!=1 or row['absence_streak']!=0:fail('Domain-scoped completeness closed unrelated RPC')
|
|
else:ok('AUTHORITY-DOMAIN PASS HTTP completeness cannot close RPC')
|
|
# Multi-scope union + unauthorized scope claim rejection.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);db=td/'overlap.sqlite';ingest(db,ROOT/'real-world/catalog-exchange/declared-2026-09-01.yaml');ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-01.yaml',True);ingest(db,ROOT/'real-world/catalog-exchange/mesh-runtime-2026-09-01.yaml',True);ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-08.yaml',True);ingest(db,ROOT/'real-world/catalog-exchange/runtime-2026-09-09.yaml',True)
|
|
r=query("SELECT active,active_scopes_json FROM logical_topology_current WHERE target_ref='node.inventory'",db)[0]
|
|
if r['active']!=1 or json.loads(r['active_scopes_json'])!=['prod-cluster-a-mesh']:fail('Overlapping observation scopes did not union correctly')
|
|
else:ok('MULTI-SCOPE PASS app scope closes while mesh scope keeps one logical dependency active')
|
|
bad=ingest(db,ROOT/'real-world/catalog-exchange/bad-authority-scope.yaml',True)
|
|
if bad.returncode==0:fail('Unauthorized authoritative scope claim accepted')
|
|
else:ok('AUTHORITY-OWNERSHIP PASS source cannot claim another source observation scope')
|
|
# Declaration matching fixtures.
|
|
for case in ['channel','gateway','working']:
|
|
with tempfile.TemporaryDirectory() as td:
|
|
db=Path(td)/'d.sqlite';ingest(db,ROOT/f'real-world/declaration-matching/{case}-declared.yaml');ingest(db,ROOT/f'real-world/declaration-matching/{case}-runtime.yaml');o=Path(td)/'o.yaml';cp=run(['python','tools/architecture_changes_lens.py',str(db),'--population',f'decl-{case}','--since','2026-09-08T00:00:00Z','--until','2026-09-10T00:00:00Z','-o',str(o)]);d=load(o)
|
|
if cp.returncode or d['summary']['newly_undeclared']!=0 or not d['appeared'][0]['declared']:fail(f'Declaration match failed for {case}')
|
|
if not any('Declaration match failed' in x for x in problems):ok('DECLARATION-MATCHING PASS Channel + gateway intended target + working-set preferred ids')
|
|
# Zone derivation from runtime identity tuple.
|
|
z=load(ROOT/'real-world/catalog-exchange/zone-derivation.yaml');derived=[x for x in z['records']['zone_memberships'] if x.get('derived')]
|
|
if len(derived)!=3:fail('Identity-tuple Zone derivation missing environment/cluster/namespace')
|
|
else:ok('ZONE-DERIVATION PASS deployment identity automatically yields environment/cluster/namespace memberships')
|
|
# Coverage, dormancy, security unknown.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);db=td/'sec.sqlite';ingest(db,ROOT/'real-world/catalog-exchange/security-declared.yaml');ingest(db,ROOT/'real-world/catalog-exchange/security-runtime.yaml');ingest(db,ROOT/'real-world/catalog-exchange/security-old-coverage.yaml');oc=td/'op.yaml';sc=td/'sec.yaml'
|
|
run(['python','tools/operation_coverage_lens.py',str(db),'--population','security-prod','--at','2026-09-09T23:59:59Z','--window-days','90','-o',str(oc)]);run(['python','tools/security_evidence_coverage_lens.py',str(db),'--population','security-prod','--since','2026-06-11T00:00:00Z','--until','2026-09-09T23:59:59Z','-o',str(sc)]);od,sd=load(oc),load(sc)
|
|
if od['summary']!={'dormant':1,'never-observed':2,'observed-in-window':1}:fail('Operation coverage/dormancy unexpected '+str(od['summary']))
|
|
else:ok('OPERATION-COVERAGE PASS observed + dormant + never-observed declared capabilities')
|
|
if sd['summary']['observed_operations_with_security_unknown']!=1:fail('Security unknown coverage gap not surfaced')
|
|
else:ok('SECURITY-EVIDENCE-COVERAGE PASS declared auth + observed operation remains runtime-security unknown')
|
|
# Evidence conflict/staleness query with named policy hash.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);db=td/'ev.sqlite';ingest(db,ROOT/'real-world/catalog-exchange/evidence-health.yaml');o=td/'health.yaml';cp=run(['python','tools/evidence_health_lens.py',str(db),'--at','2026-09-09T00:00:00Z','--policy','policies/EVIDENCE-SCORING-v0.2.yaml','--stale-runtime-days','30','--conflict-min-support','.2','-o',str(o)]);d=load(o)
|
|
if cp.returncode or d['summary']!={'conflicts':1,'challenged':1,'stale_runtime_only':1,'low_support':1}:fail('Evidence health unexpected '+str(d.get('summary')))
|
|
elif not d.get('policy',{}).get('sha256'):fail('Evidence-health result missing policy hash')
|
|
else:ok('EVIDENCE-HEALTH PASS conflict + challenge + stale evidence; scoring policy id/hash explicit')
|
|
# NDJSON direct index write + replay + export.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);nd=td/'stream.ndjson';db=td/'stream.sqlite';out=td/'export.yaml';cp=run(['python','tools/catalog_exchange.py','convert','real-world/catalog-exchange/runtime-2026-09-01.yaml','-o',str(nd),'--input-format','yaml','--format','ndjson']);cp2=run(['python','tools/catalog_index.py','ingest-exchange',str(db),str(nd),'--format','ndjson','--authority-policy','policies/SNAPSHOT-AUTHORITY-v0.2.yaml']);cp3=run(['python','tools/catalog_index.py','ingest-exchange',str(db),str(nd),'--format','ndjson','--authority-policy','policies/SNAPSHOT-AUTHORITY-v0.2.yaml']);run(['python','tools/catalog_index.py','export-exchange',str(db),'--observation-scope','prod-cluster-a-app-traces','-o',str(out)]);e=schema_ok('FIAM-CatalogExchange-v0.2.schema.json',load(out))
|
|
if cp.returncode or cp2.returncode or cp3.returncode or 'already-indexed' not in cp3.stdout or e:fail('NDJSON/index-native ingest/export contract failed')
|
|
else:ok('INDEX-NATIVE-INGEST PASS NDJSON direct write + replay idempotency + Catalog Exchange export')
|
|
# Set-oriented scale remains a conformance case, not a benchmark. Generate the
|
|
# Catalog Exchange directly: the operational path must not depend on rewriting a whole FIAM model.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
td=Path(td);ex=td/'scale-ex.json';db=td/'scale.sqlite';now='2026-09-09T00:00:00Z'
|
|
entities=[{'section':'nodes','entity':{'id':'node.scale.source','kind':'system-node','name':'scale-source','node_type':'fiam.node-type.service'}}]
|
|
entities += [{'section':'nodes','entity':{'id':f'node.scale.target.{i}','kind':'system-node','name':f'target-{i}','node_type':'fiam.node-type.service'}} for i in range(1000)]
|
|
topo=[{'id':f'topology.scale.{i}','domain':'rpc','kind':'direct-call','from':'node.scale.source','to':f'node.scale.target.{i}','count':1,'first_seen':now,'last_seen':now,'correlation':'scale-fixture'} for i in range(1000)]
|
|
scale_exchange={'catalog_exchange_version':'0.2','profile_id':'fiam.catalog-exchange.v0.2','exchange_id':'exchange.scale.1000','created_at':now,'source':{'source_id':'scale.fixture','source_kind':'conformance-generator','source_ref':'fixture://scale/1000','content_hash':'synthetic'},'observation':{'mode':'incremental','snapshot_time':now,'observation_scope':'scale-scope','population_id':'scale-pop','population':{},'domains':['rpc'],'aggregate_semantics':'cumulative'},'records':{'entities':entities,'assertions':[],'zone_memberships':[],'runtime_aggregates':[],'topology_observations':topo,'unresolved':[],'reconciliation_candidates':[],'declared_entity_ids':[]}}
|
|
ex.write_text(json.dumps(scale_exchange,separators=(',',':')))
|
|
cp=run(['python','tools/catalog_index.py','ingest-exchange',str(db),str(ex),'--format','json']);st=json.loads(run(['python','tools/catalog_index.py','stats',str(db)]).stdout or '{}')
|
|
if cp.returncode or st.get('active_logical_topology')!=1000:fail('1000-edge index conformance failed')
|
|
else:ok('SET-ORIENTED-INDEX PASS 1000 logical edges via direct Catalog Exchange (not a performance benchmark)')
|
|
# Existing FIAM adversarial cases still enforce semantics. Import the validator once
|
|
# rather than spawning one Python process per fixture; conformance must remain cheap enough
|
|
# to run routinely in CI and by catalog operators.
|
|
spec=importlib.util.spec_from_file_location('fiam_validate',ROOT/'tools/validate_fiam.py')
|
|
vf=importlib.util.module_from_spec(spec);spec.loader.exec_module(vf)
|
|
core=json.loads((ROOT/'schemas/FIAM-Core-v0.2.schema.json').read_text());canon=yaml.safe_load((ROOT/'canon/FIAM-Canon-v0.2.yaml').read_text())
|
|
adv_failed=False
|
|
for p in sorted((ROOT/'adversarial-cases').glob('*.yaml')):
|
|
e,w=vf.validate(p,core,canon);warning=p.name.startswith('07-legacy-operation-representation')
|
|
if warning:
|
|
if e or not w: fail(f'{p.name} warning migration behavior failed');adv_failed=True
|
|
elif not e: fail(f'{p.name} should be rejected');adv_failed=True
|
|
if not adv_failed:ok('ADVERSARIAL-FIAM PASS existing semantic rejection suite')
|
|
status='PASS' if not problems else 'FAIL';report=['FIAM v0.2-draft.6 Catalog Operations & Policy conformance: '+status]+lines
|
|
if problems:report+=['PROBLEMS:']+['- '+x for x in problems]
|
|
text='\n'.join(report)+'\n';print(text,end='');(ROOT/'CONFORMANCE.txt').write_text(text);return 0 if not problems else 1
|
|
if __name__=='__main__':raise SystemExit(main())
|