tegwick
f0eff0ac92
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e747-8f27-7242-8df8-8bc44f88c929
258 lines
12 KiB
Python
258 lines
12 KiB
Python
import asyncio
|
|
import json
|
|
import time
|
|
from dataclasses import replace
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
import sqlalchemy as sa
|
|
from fastapi.testclient import TestClient
|
|
|
|
from hub_core.runtime.app import create_app
|
|
from hub_core.runtime.config import RuntimeSettings
|
|
from hub_core.runtime.postgres_store import PostgresPortStore
|
|
from hub_core.runtime.tables import runtime_metadata, runtime_messages, runtime_audit_ledger, runtime_outcome_outbox
|
|
from hub_core.security.audit import AuditCoreSink
|
|
from hub_core.security.context import current_authorization
|
|
from test_access_boundary import Owners, HEADERS
|
|
from test_access_audit import READY
|
|
|
|
|
|
@pytest.fixture
|
|
def target(tmp_path):
|
|
url = f'sqlite+aiosqlite:///{tmp_path / "outcomes.db"}'
|
|
store = PostgresPortStore.from_url(url)
|
|
async def schema():
|
|
async with store.engine.begin() as connection:
|
|
await connection.run_sync(runtime_metadata.create_all)
|
|
asyncio.run(schema())
|
|
owners = Owners()
|
|
async def facts(actor, resource):
|
|
return replace(owners.facts, checked_at=time.time(), subject=actor.subject)
|
|
owners.resolve = facts
|
|
app = create_app(settings=RuntimeSettings(environment='test',access_mode='enforce',backend='postgresql',database_url=url),
|
|
port_store=store,access_controller=owners.controller())
|
|
with TestClient(app,raise_server_exceptions=False) as client:
|
|
yield client, store, owners, url
|
|
asyncio.run(store.aclose())
|
|
|
|
|
|
def message():
|
|
return {'schema_version':'0.1.0','correlation_id':str(uuid4()),'from_address':'agent:root',
|
|
'to_addresses':['agent:reader'],'body':'private body never archived'}
|
|
|
|
|
|
def rows(store, table):
|
|
async def read():
|
|
async with store.sessions() as session:
|
|
return [dict(r) for r in (await session.execute(sa.select(table))).mappings()]
|
|
return asyncio.run(read())
|
|
|
|
|
|
def test_committed_outcome_is_atomic_and_joins_authorization(target):
|
|
client,store,owners,_ = target
|
|
body = message()
|
|
response = client.post('/ports/messaging/messages',headers=HEADERS,json=body)
|
|
assert response.status_code == 202
|
|
ledger, = rows(store,runtime_audit_ledger)
|
|
pending, = rows(store,runtime_outcome_outbox)
|
|
envelope = pending['envelope']
|
|
authorization = envelope['data']['authorization']
|
|
assert envelope['id'] == ledger['id']
|
|
assert envelope['correlation_id'] == owners.records[0]['correlation_id'] == response.headers['x-correlation-id']
|
|
assert authorization == ledger['detail']['authorization']
|
|
assert authorization['decision_id'] == owners.records[0]['decision_id']
|
|
assert authorization['subject'] == 'immutable-root'
|
|
assert envelope['data']['business_correlation_id'] == body['correlation_id']
|
|
assert envelope['data']['subject_id'] == response.json()['id']
|
|
assert 'private body' not in json.dumps(envelope)
|
|
assert 'verified-root' not in json.dumps(envelope)
|
|
assert current_authorization.get() is None
|
|
|
|
|
|
def test_outbox_insert_failure_rolls_back_business_and_ledger(target):
|
|
client,store,owners,_ = target
|
|
def reject(connection,cursor,statement,parameters,context,many):
|
|
if statement.startswith('INSERT INTO runtime_outcome_outbox'):
|
|
raise RuntimeError('simulated storage failure')
|
|
sa.event.listen(store.engine.sync_engine,'before_cursor_execute',reject)
|
|
try:
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 500
|
|
finally:
|
|
sa.event.remove(store.engine.sync_engine,'before_cursor_execute',reject)
|
|
assert rows(store,runtime_messages) == rows(store,runtime_audit_ledger) == rows(store,runtime_outcome_outbox) == []
|
|
assert owners.records[0]['outcome'] == 'authorized' # Never claims commit.
|
|
assert current_authorization.get() is None
|
|
|
|
|
|
def test_denial_and_invalid_body_never_queue_committed_outcome(target):
|
|
client,store,owners,_ = target
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json={}).status_code == 422
|
|
owners.allow = False
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 403
|
|
assert not rows(store,runtime_outcome_outbox)
|
|
assert not rows(store,runtime_messages)
|
|
|
|
|
|
def test_lost_receipt_retry_reopen_and_duplicate_delivery(target,tmp_path):
|
|
client,store,owners,url = target
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
|
|
credential = tmp_path/'sender'
|
|
credential.write_text('fixture-only')
|
|
archived, requests = {}, []
|
|
def receiver(request):
|
|
if request.url.path == '/readyz':
|
|
return httpx.Response(200,json=READY)
|
|
envelope = json.loads(request.content)
|
|
requests.append(bytes(request.content))
|
|
assert request.headers['idempotency-key'] == envelope['id']
|
|
if envelope['id'] not in archived:
|
|
archived[envelope['id']] = envelope
|
|
raise httpx.ReadTimeout('lost receipt after custody')
|
|
assert archived[envelope['id']] == envelope
|
|
return httpx.Response(200,json={'status':'duplicate','reference':'audit:'+envelope['id']})
|
|
async def run():
|
|
async with httpx.AsyncClient(transport=httpx.MockTransport(receiver)) as http:
|
|
sink = AuditCoreSink(base_url='https://audit.example',token_file=credential,client=http)
|
|
assert await store.deliver_outcomes(sink) == 0
|
|
assert await store.deliver_outcomes(sink) == 0 # Backoff, no immediate retry.
|
|
assert len(requests) == 1
|
|
await store.aclose()
|
|
reopened = PostgresPortStore.from_url(url)
|
|
try:
|
|
async with reopened.sessions.begin() as session:
|
|
await session.execute(runtime_outcome_outbox.update().values(next_attempt=0))
|
|
assert await reopened.deliver_outcomes(sink) == 1
|
|
assert await reopened.deliver_outcomes(sink) == 0
|
|
finally:
|
|
await reopened.aclose()
|
|
asyncio.run(run())
|
|
assert len(archived) == 1 and len(requests) == 2 and requests[0] == requests[1]
|
|
pending, = rows(store,runtime_outcome_outbox)
|
|
assert pending['attempts'] == 2 and pending['delivered_at'] is not None
|
|
|
|
|
|
def test_cancellation_after_receipt_leaves_replayable_row(target):
|
|
client,store,_,_ = target
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
|
|
class Crash:
|
|
async def append_outcome(self,event):
|
|
raise asyncio.CancelledError()
|
|
async def run():
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await store.deliver_outcomes(Crash())
|
|
asyncio.run(run())
|
|
pending, = rows(store,runtime_outcome_outbox)
|
|
assert pending['delivered_at'] is None and pending['attempts'] == 0
|
|
|
|
|
|
def test_stale_backlog_is_visible_and_clears_after_receipt(target):
|
|
client,store,_,_ = target
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
|
|
class Receipt:
|
|
async def append_outcome(self,event):
|
|
pass
|
|
async def run():
|
|
assert await store.outcome_readiness() == 'ok'
|
|
async with store.sessions.begin() as session:
|
|
await session.execute(runtime_outcome_outbox.update().values(created_at=time.time()-61))
|
|
assert await store.outcome_readiness() == 'stale'
|
|
await store.deliver_outcomes(Receipt())
|
|
assert await store.outcome_readiness() == 'ok'
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_all_native_mutation_families_keep_attributed_outcomes(target):
|
|
from hub_core.conformance import ConformanceHarness
|
|
client,store,owners,_ = target
|
|
owners.facts = replace(owners.facts,producer_addresses=frozenset({'hub:ops-hub'}))
|
|
# Composition is frozen when the app is created; run the twelve business
|
|
# checks except dependency readiness, which separately reports no dispatcher.
|
|
harness = ConformanceHarness(client)
|
|
client.headers.update(HEADERS)
|
|
report = harness.run()
|
|
assert all(c.status == 'pass' for c in report.checks if c.check_id != 'C9'), report.to_dict()
|
|
assert next(c for c in report.checks if c.check_id == 'C9').status == 'fail'
|
|
assert client.get('/readyz').json()['checks']['outcome_delivery'] == 'unavailable'
|
|
pending = rows(store,runtime_outcome_outbox)
|
|
operations = {r['envelope']['data']['operation'] for r in pending}
|
|
assert {'message.accepted','event.progress.accepted','event.interaction.accepted'} <= operations
|
|
assert any(op.startswith('registry.') for op in operations)
|
|
assert all(r['envelope']['data']['authorization']['subject'] == 'immutable-root' for r in pending)
|
|
assert len(pending) == len(rows(store,runtime_audit_ledger))
|
|
|
|
|
|
def test_concurrent_requests_keep_separate_authorization_contexts(target):
|
|
from hub_core.security.identity import AccessFailure
|
|
client,store,owners,_ = target
|
|
async def authenticate(token):
|
|
if token not in {'workload:a','workload:b'}:
|
|
raise AccessFailure(401,'bad_fixture')
|
|
await asyncio.sleep(0)
|
|
return replace(owners.actor,subject=token,principal_type='service')
|
|
owners.authenticate = authenticate
|
|
async def run():
|
|
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=client.app),base_url='https://hub.example') as http:
|
|
async def send(subject):
|
|
result = await http.post('/ports/messaging/messages',headers={'Authorization':'Bearer '+subject},json=message())
|
|
assert result.status_code == 202
|
|
return result.json()['id'], subject
|
|
return dict(await asyncio.gather(send('workload:a'),send('workload:b')))
|
|
subjects = asyncio.run(run())
|
|
for row in rows(store,runtime_outcome_outbox):
|
|
data = row['envelope']['data']
|
|
assert data['authorization']['subject'] == subjects[data['subject_id']]
|
|
assert current_authorization.get() is None
|
|
|
|
|
|
def test_outcome_migration_creates_index_and_foreign_key(tmp_path):
|
|
import importlib
|
|
from alembic.migration import MigrationContext
|
|
from alembic.operations import Operations
|
|
migration = importlib.import_module('hub_core.migrations.versions.0006_outcome_outbox')
|
|
engine = sa.create_engine('sqlite:///' + str(tmp_path/'migration.db'))
|
|
try:
|
|
runtime_audit_ledger.create(engine)
|
|
with engine.begin() as connection:
|
|
with Operations.context(MigrationContext.configure(connection)):
|
|
migration.upgrade()
|
|
inspect = sa.inspect(connection)
|
|
assert {x['name'] for x in inspect.get_columns('runtime_outcome_outbox')} == set(runtime_outcome_outbox.c.keys())
|
|
assert inspect.get_foreign_keys('runtime_outcome_outbox')[0]['referred_table'] == 'runtime_audit_ledger'
|
|
assert inspect.get_indexes('runtime_outcome_outbox')[0]['name'] == 'ix_runtime_outcome_pending'
|
|
connection.execute(runtime_outcome_outbox.insert().values(id='fixture',envelope={},created_at=0,attempts=0,next_attempt=0))
|
|
with Operations.context(MigrationContext.configure(connection)):
|
|
with pytest.raises(RuntimeError,match='drained outbox'):
|
|
migration.downgrade()
|
|
connection.execute(runtime_outcome_outbox.delete())
|
|
with Operations.context(MigrationContext.configure(connection)):
|
|
migration.downgrade()
|
|
assert 'runtime_outcome_outbox' not in sa.inspect(connection).get_table_names()
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
def test_runtime_dispatcher_drains_pending_outcome_and_stops(target):
|
|
from hub_core.runtime.app import _deliver_outcomes
|
|
client,store,_,_ = target
|
|
assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
|
|
async def run():
|
|
class Receipt:
|
|
async def append_outcome(self,envelope):
|
|
pass
|
|
task = asyncio.create_task(_deliver_outcomes(store,Receipt()))
|
|
try:
|
|
async with asyncio.timeout(3):
|
|
while True:
|
|
async with store.sessions() as session:
|
|
count = (await session.execute(sa.select(sa.func.count()).select_from(runtime_outcome_outbox)
|
|
.where(runtime_outcome_outbox.c.delivered_at.is_not(None)))).scalar_one()
|
|
if count == 1:
|
|
break
|
|
await asyncio.sleep(.01)
|
|
finally:
|
|
task.cancel()
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
asyncio.run(run())
|