feat: atomically journal and deliver authorized native operation outcomes

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e747-8f27-7242-8df8-8bc44f88c929

tests/test_outcome_audit.py

@@ -0,0 +1,258 @@
+import asyncio
+import json
+import time
+from dataclasses import replace
+from uuid import uuid4
+
+import httpx
+import pytest
+import sqlalchemy as sa
+from fastapi.testclient import TestClient
+
+from hub_core.runtime.app import create_app
+from hub_core.runtime.config import RuntimeSettings
+from hub_core.runtime.postgres_store import PostgresPortStore
+from hub_core.runtime.tables import runtime_metadata, runtime_messages, runtime_audit_ledger, runtime_outcome_outbox
+from hub_core.security.audit import AuditCoreSink
+from hub_core.security.context import current_authorization
+from test_access_boundary import Owners, HEADERS
+from test_access_audit import READY
+
+
+@pytest.fixture
+def target(tmp_path):
+    url = f'sqlite+aiosqlite:///{tmp_path / "outcomes.db"}'
+    store = PostgresPortStore.from_url(url)
+    async def schema():
+        async with store.engine.begin() as connection:
+            await connection.run_sync(runtime_metadata.create_all)
+    asyncio.run(schema())
+    owners = Owners()
+    async def facts(actor, resource):
+        return replace(owners.facts, checked_at=time.time(), subject=actor.subject)
+    owners.resolve = facts
+    app = create_app(settings=RuntimeSettings(environment='test',access_mode='enforce',backend='postgresql',database_url=url),
+                     port_store=store,access_controller=owners.controller())
+    with TestClient(app,raise_server_exceptions=False) as client:
+        yield client, store, owners, url
+    asyncio.run(store.aclose())
+
+
+def message():
+    return {'schema_version':'0.1.0','correlation_id':str(uuid4()),'from_address':'agent:root',
+            'to_addresses':['agent:reader'],'body':'private body never archived'}
+
+
+def rows(store, table):
+    async def read():
+        async with store.sessions() as session:
+            return [dict(r) for r in (await session.execute(sa.select(table))).mappings()]
+    return asyncio.run(read())
+
+
+def test_committed_outcome_is_atomic_and_joins_authorization(target):
+    client,store,owners,_ = target
+    body = message()
+    response = client.post('/ports/messaging/messages',headers=HEADERS,json=body)
+    assert response.status_code == 202
+    ledger, = rows(store,runtime_audit_ledger)
+    pending, = rows(store,runtime_outcome_outbox)
+    envelope = pending['envelope']
+    authorization = envelope['data']['authorization']
+    assert envelope['id'] == ledger['id']
+    assert envelope['correlation_id'] == owners.records[0]['correlation_id'] == response.headers['x-correlation-id']
+    assert authorization == ledger['detail']['authorization']
+    assert authorization['decision_id'] == owners.records[0]['decision_id']
+    assert authorization['subject'] == 'immutable-root'
+    assert envelope['data']['business_correlation_id'] == body['correlation_id']
+    assert envelope['data']['subject_id'] == response.json()['id']
+    assert 'private body' not in json.dumps(envelope)
+    assert 'verified-root' not in json.dumps(envelope)
+    assert current_authorization.get() is None
+
+
+def test_outbox_insert_failure_rolls_back_business_and_ledger(target):
+    client,store,owners,_ = target
+    def reject(connection,cursor,statement,parameters,context,many):
+        if statement.startswith('INSERT INTO runtime_outcome_outbox'):
+            raise RuntimeError('simulated storage failure')
+    sa.event.listen(store.engine.sync_engine,'before_cursor_execute',reject)
+    try:
+        assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 500
+    finally:
+        sa.event.remove(store.engine.sync_engine,'before_cursor_execute',reject)
+    assert rows(store,runtime_messages) == rows(store,runtime_audit_ledger) == rows(store,runtime_outcome_outbox) == []
+    assert owners.records[0]['outcome'] == 'authorized'  # Never claims commit.
+    assert current_authorization.get() is None
+
+
+def test_denial_and_invalid_body_never_queue_committed_outcome(target):
+    client,store,owners,_ = target
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json={}).status_code == 422
+    owners.allow = False
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 403
+    assert not rows(store,runtime_outcome_outbox)
+    assert not rows(store,runtime_messages)
+
+
+def test_lost_receipt_retry_reopen_and_duplicate_delivery(target,tmp_path):
+    client,store,owners,url = target
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
+    credential = tmp_path/'sender'
+    credential.write_text('fixture-only')
+    archived, requests = {}, []
+    def receiver(request):
+        if request.url.path == '/readyz':
+            return httpx.Response(200,json=READY)
+        envelope = json.loads(request.content)
+        requests.append(bytes(request.content))
+        assert request.headers['idempotency-key'] == envelope['id']
+        if envelope['id'] not in archived:
+            archived[envelope['id']] = envelope
+            raise httpx.ReadTimeout('lost receipt after custody')
+        assert archived[envelope['id']] == envelope
+        return httpx.Response(200,json={'status':'duplicate','reference':'audit:'+envelope['id']})
+    async def run():
+        async with httpx.AsyncClient(transport=httpx.MockTransport(receiver)) as http:
+            sink = AuditCoreSink(base_url='https://audit.example',token_file=credential,client=http)
+            assert await store.deliver_outcomes(sink) == 0
+            assert await store.deliver_outcomes(sink) == 0  # Backoff, no immediate retry.
+            assert len(requests) == 1
+            await store.aclose()
+            reopened = PostgresPortStore.from_url(url)
+            try:
+                async with reopened.sessions.begin() as session:
+                    await session.execute(runtime_outcome_outbox.update().values(next_attempt=0))
+                assert await reopened.deliver_outcomes(sink) == 1
+                assert await reopened.deliver_outcomes(sink) == 0
+            finally:
+                await reopened.aclose()
+    asyncio.run(run())
+    assert len(archived) == 1 and len(requests) == 2 and requests[0] == requests[1]
+    pending, = rows(store,runtime_outcome_outbox)
+    assert pending['attempts'] == 2 and pending['delivered_at'] is not None
+
+
+def test_cancellation_after_receipt_leaves_replayable_row(target):
+    client,store,_,_ = target
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
+    class Crash:
+        async def append_outcome(self,event):
+            raise asyncio.CancelledError()
+    async def run():
+        with pytest.raises(asyncio.CancelledError):
+            await store.deliver_outcomes(Crash())
+    asyncio.run(run())
+    pending, = rows(store,runtime_outcome_outbox)
+    assert pending['delivered_at'] is None and pending['attempts'] == 0
+
+
+def test_stale_backlog_is_visible_and_clears_after_receipt(target):
+    client,store,_,_ = target
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
+    class Receipt:
+        async def append_outcome(self,event):
+            pass
+    async def run():
+        assert await store.outcome_readiness() == 'ok'
+        async with store.sessions.begin() as session:
+            await session.execute(runtime_outcome_outbox.update().values(created_at=time.time()-61))
+        assert await store.outcome_readiness() == 'stale'
+        await store.deliver_outcomes(Receipt())
+        assert await store.outcome_readiness() == 'ok'
+    asyncio.run(run())
+
+
+def test_all_native_mutation_families_keep_attributed_outcomes(target):
+    from hub_core.conformance import ConformanceHarness
+    client,store,owners,_ = target
+    owners.facts = replace(owners.facts,producer_addresses=frozenset({'hub:ops-hub'}))
+    # Composition is frozen when the app is created; run the twelve business
+    # checks except dependency readiness, which separately reports no dispatcher.
+    harness = ConformanceHarness(client)
+    client.headers.update(HEADERS)
+    report = harness.run()
+    assert all(c.status == 'pass' for c in report.checks if c.check_id != 'C9'), report.to_dict()
+    assert next(c for c in report.checks if c.check_id == 'C9').status == 'fail'
+    assert client.get('/readyz').json()['checks']['outcome_delivery'] == 'unavailable'
+    pending = rows(store,runtime_outcome_outbox)
+    operations = {r['envelope']['data']['operation'] for r in pending}
+    assert {'message.accepted','event.progress.accepted','event.interaction.accepted'} <= operations
+    assert any(op.startswith('registry.') for op in operations)
+    assert all(r['envelope']['data']['authorization']['subject'] == 'immutable-root' for r in pending)
+    assert len(pending) == len(rows(store,runtime_audit_ledger))
+
+
+def test_concurrent_requests_keep_separate_authorization_contexts(target):
+    from hub_core.security.identity import AccessFailure
+    client,store,owners,_ = target
+    async def authenticate(token):
+        if token not in {'workload:a','workload:b'}:
+            raise AccessFailure(401,'bad_fixture')
+        await asyncio.sleep(0)
+        return replace(owners.actor,subject=token,principal_type='service')
+    owners.authenticate = authenticate
+    async def run():
+        async with httpx.AsyncClient(transport=httpx.ASGITransport(app=client.app),base_url='https://hub.example') as http:
+            async def send(subject):
+                result = await http.post('/ports/messaging/messages',headers={'Authorization':'Bearer '+subject},json=message())
+                assert result.status_code == 202
+                return result.json()['id'], subject
+            return dict(await asyncio.gather(send('workload:a'),send('workload:b')))
+    subjects = asyncio.run(run())
+    for row in rows(store,runtime_outcome_outbox):
+        data = row['envelope']['data']
+        assert data['authorization']['subject'] == subjects[data['subject_id']]
+    assert current_authorization.get() is None
+
+
+def test_outcome_migration_creates_index_and_foreign_key(tmp_path):
+    import importlib
+    from alembic.migration import MigrationContext
+    from alembic.operations import Operations
+    migration = importlib.import_module('hub_core.migrations.versions.0006_outcome_outbox')
+    engine = sa.create_engine('sqlite:///' + str(tmp_path/'migration.db'))
+    try:
+        runtime_audit_ledger.create(engine)
+        with engine.begin() as connection:
+            with Operations.context(MigrationContext.configure(connection)):
+                migration.upgrade()
+            inspect = sa.inspect(connection)
+            assert {x['name'] for x in inspect.get_columns('runtime_outcome_outbox')} == set(runtime_outcome_outbox.c.keys())
+            assert inspect.get_foreign_keys('runtime_outcome_outbox')[0]['referred_table'] == 'runtime_audit_ledger'
+            assert inspect.get_indexes('runtime_outcome_outbox')[0]['name'] == 'ix_runtime_outcome_pending'
+            connection.execute(runtime_outcome_outbox.insert().values(id='fixture',envelope={},created_at=0,attempts=0,next_attempt=0))
+            with Operations.context(MigrationContext.configure(connection)):
+                with pytest.raises(RuntimeError,match='drained outbox'):
+                    migration.downgrade()
+            connection.execute(runtime_outcome_outbox.delete())
+            with Operations.context(MigrationContext.configure(connection)):
+                migration.downgrade()
+            assert 'runtime_outcome_outbox' not in sa.inspect(connection).get_table_names()
+    finally:
+        engine.dispose()
+
+
+def test_runtime_dispatcher_drains_pending_outcome_and_stops(target):
+    from hub_core.runtime.app import _deliver_outcomes
+    client,store,_,_ = target
+    assert client.post('/ports/messaging/messages',headers=HEADERS,json=message()).status_code == 202
+    async def run():
+        class Receipt:
+            async def append_outcome(self,envelope):
+                pass
+        task = asyncio.create_task(_deliver_outcomes(store,Receipt()))
+        try:
+            async with asyncio.timeout(3):
+                while True:
+                    async with store.sessions() as session:
+                        count = (await session.execute(sa.select(sa.func.count()).select_from(runtime_outcome_outbox)
+                            .where(runtime_outcome_outbox.c.delivered_at.is_not(None)))).scalar_one()
+                    if count == 1:
+                        break
+                    await asyncio.sleep(.01)
+        finally:
+            task.cancel()
+            with pytest.raises(asyncio.CancelledError):
+                await task
+    asyncio.run(run())