Implement approval engine production readiness

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a05e2e-805b-7042-a750-71f473bceea2

tests/conftest.py

@@ -3,6 +3,7 @@ from datetime import datetime, timezone
 import pytest
 
 from approval_engine.api import App
+from approval_engine.auth import Identity, StaticTokenAuthenticator
 from approval_engine.store import Engine
 
 FROZEN = datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)
@@ -22,7 +23,29 @@ def engine(now):
 
 @pytest.fixture
 def app(engine):
-    return App(engine)
+    identity = Identity(
+        subject="agt-secrets-engine",
+        issuer="https://keycape.example",
+        audiences=("approval-engine",),
+        principal_type="service",
+        tenant="platform",
+        roles=frozenset({"secrets-engine"}),
+        scopes=frozenset(
+            {
+                "approval:create",
+                "approval:read",
+                "approval:approve",
+                "approval:revoke",
+                "approval:supersede",
+                "approval:consume",
+                "approval:observe",
+                "approval:emit",
+            }
+        ),
+        assurance={"level": "aal1", "methods": ["test"], "source": "test"},
+        evidence_ref="test-identity:test-token",
+    )
+    return App(engine, StaticTokenAuthenticator({"test-token": identity}))
 
 
 def binding(**overrides):

tests/test_api.py

@@ -127,13 +127,29 @@ def test_claim_after_revoke(app):
 
 def test_store_unavailable_is_503():
     from approval_engine.api import App
+    from approval_engine.auth import Identity, StaticTokenAuthenticator
     from approval_engine.errors import StoreUnavailable
     from approval_engine.store import Engine
 
     class Dead(Engine):
+        def storage_status(self):
+            return {"schema_current": True, "persistent": True}
+
         def outbox_stats(self):
             raise StoreUnavailable("down")
 
-    status, body = call(App(Dead.__new__(Dead)), "GET", "/v1/readyz")
+    identity = Identity(
+        subject="test",
+        issuer="test",
+        audiences=("approval-engine",),
+        principal_type="service",
+        tenant="test",
+        roles=frozenset(),
+        scopes=frozenset(),
+        assurance={},
+        evidence_ref="test",
+    )
+    app = App(Dead.__new__(Dead), StaticTokenAuthenticator({"test-token": identity}))
+    status, body = call(app, "GET", "/v1/readyz")
     assert status == 503
     assert body["error"] == "store_unavailable"

tests/test_audit.py

@@ -0,0 +1,82 @@
+import json
+from datetime import timedelta
+
+import pytest
+
+from approval_engine.audit import AuditCoreSink, AuditDeliveryError, OutboxWorker
+from approval_engine.store import Engine
+from tests.conftest import FROZEN, approve
+
+
+class Response:
+    def __init__(self, status):
+        self.status = status
+
+    def getcode(self):
+        return self.status
+
+    def close(self):
+        pass
+
+
+def test_audit_sender_adapts_envelope_and_rereads_token(tmp_path):
+    token = tmp_path / "token"
+    token.write_text("first")
+    seen = []
+
+    def open_request(request, timeout):
+        seen.append((request, timeout))
+        return Response(202)
+
+    engine = Engine(":memory:", clock=lambda: FROZEN)
+    approve(engine)
+    sink = AuditCoreSink("http://audit-core:8080", token, opener=open_request)
+    sink(engine.undrained()[0]["payload"])
+    token.write_text("second")
+    sink(engine.undrained()[0]["payload"])
+    first, second = (item[0] for item in seen)
+    assert first.get_header("Authorization") == "Bearer first"
+    assert second.get_header("Authorization") == "Bearer second"
+    assert first.get_header("Idempotency-key") == engine.undrained()[0]["event_id"]
+    body = json.loads(first.data)
+    assert body["id"] == body["correlation_id"] or body["correlation_id"]
+    assert body["type"] == "approval.issuance"
+    assert body["source"] == "approval-engine"
+    engine.close()
+
+
+def test_nonaccepted_audit_status_remains_pending(tmp_path):
+    token = tmp_path / "token"
+    token.write_text("token")
+    engine = Engine(":memory:", clock=lambda: FROZEN)
+    approve(engine)
+    sink = AuditCoreSink(
+        "http://audit-core:8080", token, opener=lambda *_args, **_kwargs: Response(503)
+    )
+    result = engine.drain(sink)
+    assert result == {"delivered": 0, "failed": 1}
+    assert engine.undrained()[0]["last_error"] == "AuditDeliveryError"
+    engine.close()
+
+
+def test_worker_emits_due_heartbeat_and_drains():
+    now = [FROZEN]
+    engine = Engine(":memory:", clock=lambda: now[0])
+    delivered = []
+    worker = OutboxWorker(engine, delivered.append, heartbeat_interval_seconds=300)
+    first = worker.run_once()
+    assert first["delivered"] == 1
+    assert delivered[0]["action"] == "approval.heartbeat"
+    now[0] += timedelta(seconds=301)
+    second = worker.run_once()
+    assert second["delivered"] == 1
+    assert len(delivered) == 2
+    engine.close()
+
+
+def test_sender_rejects_empty_token(tmp_path):
+    token = tmp_path / "token"
+    token.write_text("")
+    sink = AuditCoreSink("http://audit-core", token)
+    with pytest.raises(AuditDeliveryError, match="credential"):
+        sink({})

tests/test_auth.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+
+import jwt
+import pytest
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+from approval_engine.api import call
+from approval_engine.auth import Identity, JWTAuthenticator, StaticTokenAuthenticator
+from approval_engine.errors import Unauthenticated
+from tests.conftest import binding, validity
+
+
+class _Key:
+    def __init__(self, key):
+        self.key = key
+
+
+class _JWKS:
+    def __init__(self, key):
+        self.key = key
+
+    def get_signing_key_from_jwt(self, _token):
+        return _Key(self.key)
+
+
+def _jwt(private_key, **overrides):
+    now = datetime.now(timezone.utc)
+    claims = {
+        "iss": "https://keycape.example",
+        "sub": "service:secrets-engine",
+        "aud": "approval-engine",
+        "iat": int(now.timestamp()),
+        "exp": int((now + timedelta(minutes=5)).timestamp()),
+        "tenant": "tenant:coulomb",
+        "principal_type": "service",
+        "roles": ["secrets-engine"],
+        "scope": "approval:read approval:consume",
+        "assurance": {"level": "aal1", "methods": ["client_secret"], "source": "key-cape"},
+    }
+    claims.update(overrides)
+    return jwt.encode(claims, private_key, algorithm="RS256", headers={"kid": "test"})
+
+
+def test_jwt_authenticator_verifies_signature_issuer_audience_and_claims():
+    private = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    auth = JWTAuthenticator(
+        issuer="https://keycape.example",
+        audience="approval-engine",
+        jwks_url="https://keycape.example/jwks",
+        jwks_client=_JWKS(private.public_key()),
+    )
+    identity = auth.authenticate("Bearer " + _jwt(private))
+    assert identity.subject == "service:secrets-engine"
+    assert identity.principal_type == "service"
+    assert identity.scopes == {"approval:read", "approval:consume"}
+    assert identity.evidence_ref.startswith("jwt-sha256:")
+
+
+@pytest.mark.parametrize(
+    "claims",
+    [
+        {"iss": "https://wrong.example"},
+        {"aud": "somewhere-else"},
+        {"exp": 1},
+    ],
+)
+def test_jwt_authenticator_fails_closed(claims):
+    private = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    auth = JWTAuthenticator(
+        issuer="https://keycape.example",
+        audience="approval-engine",
+        jwks_url="https://keycape.example/jwks",
+        jwks_client=_JWKS(private.public_key()),
+    )
+    with pytest.raises(Unauthenticated):
+        auth.authenticate("Bearer " + _jwt(private, **claims))
+
+
+def test_api_requires_scope_and_binds_create_actor(engine):
+    identity = Identity(
+        subject="service:creator",
+        issuer="test",
+        audiences=("approval-engine",),
+        principal_type="service",
+        tenant="platform",
+        roles=frozenset(),
+        scopes=frozenset({"approval:create"}),
+        assurance={"level": "aal1"},
+        evidence_ref="test",
+    )
+    from approval_engine.api import App
+
+    app = App(engine, StaticTokenAuthenticator({"creator": identity}))
+    status, body = call(
+        app,
+        "POST",
+        "/v1/approvals",
+        {"binding": binding(), "validity": validity()},
+        authorization="Bearer creator",
+    )
+    assert status == 403
+    assert body["error"] == "forbidden"
+
+    exact = binding(actor="service:creator")
+    status, created = call(
+        app,
+        "POST",
+        "/v1/approvals",
+        {"binding": exact, "validity": validity()},
+        authorization="Bearer creator",
+    )
+    assert status == 201
+    status, body = call(
+        app, "GET", f"/v1/approvals/{created['id']}/claim", authorization="Bearer creator"
+    )
+    assert status == 403
+    assert body["error"] == "forbidden"
+
+
+def test_approval_entry_uses_verified_identity_not_body(app):
+    _, created = call(
+        app,
+        "POST",
+        "/v1/approvals",
+        {"binding": binding(), "validity": validity()},
+    )
+    _, approved = call(
+        app,
+        "POST",
+        f"/v1/approvals/{created['id']}/entries",
+        {
+            "subject_id": "user:spoofed",
+            "assurance": "spoofed",
+            "evidence_ref": "spoofed",
+        },
+    )
+    assert approved["entries"][0]["subject_id"] == "agt-secrets-engine"
+    assert approved["entries"][0]["evidence_ref"] == "test-identity:test-token"
+    assert "spoofed" not in approved["entries"][0]["assurance"]
+
+
+def test_missing_token_is_unauthenticated(app):
+    status, body = call(app, "GET", "/v1/cadence", authorization=None)
+    assert status == 401
+    assert body["error"] == "unauthenticated"
+
+
+def test_wrong_tenant_is_forbidden(engine):
+    identity = Identity(
+        subject="agt-secrets-engine",
+        issuer="test",
+        audiences=("approval-engine",),
+        principal_type="service",
+        tenant="another-tenant",
+        roles=frozenset(),
+        scopes=frozenset({"approval:observe"}),
+        assurance={},
+        evidence_ref="test",
+    )
+    from approval_engine.api import App
+
+    app = App(engine, StaticTokenAuthenticator({"wrong": identity}))
+    status, body = call(app, "GET", "/v1/cadence", authorization="Bearer wrong")
+    assert status == 403
+    assert body["error"] == "forbidden"

tests/test_cli.py

@@ -0,0 +1,22 @@
+import json
+
+import pytest
+
+from approval_engine.cli import main
+
+
+def test_migrate_verify_and_backup_commands(tmp_path, capsys):
+    database = tmp_path / "approval.sqlite"
+    backup = tmp_path / "approval.backup.sqlite"
+    assert main(["migrate", "--db", str(database)]) == 0
+    migrated = json.loads(capsys.readouterr().out)
+    assert migrated["ok"] is True
+    assert main(["verify", "--db", str(database)]) == 0
+    assert json.loads(capsys.readouterr().out)["schema_current"] is True
+    assert main(["backup", "--db", str(database), "--output", str(backup)]) == 0
+    assert json.loads(capsys.readouterr().out)["integrity"] == "ok"
+
+
+def test_production_refuses_memory_store_before_serving():
+    with pytest.raises(SystemExit):
+        main(["serve", "--production", "--db", ":memory:"])

tests/test_outbox.py

@@ -82,3 +82,19 @@ def test_failed_outbox_rolls_back_consume(engine):
         pass
     assert engine.get(obj.id).status == "approved"
     assert all(item["class"] != "use" for item in engine.undrained())
+
+
+def test_drain_failure_records_bounded_attempt_state(engine):
+    approve(engine)
+
+    class SensitiveFailure(Exception):
+        pass
+
+    result = engine.drain(lambda _payload: (_ for _ in ()).throw(SensitiveFailure("secret")))
+    assert result["failed"] == 1
+    pending = engine.undrained()
+    assert pending[0]["attempts"] == 1
+    assert pending[0]["last_error"] == "SensitiveFailure"
+    stats = engine.outbox_stats()
+    assert stats["failed_pending"] == 1
+    assert stats["attempts"] == 1

tests/test_pep.py

@@ -0,0 +1,105 @@
+import json
+
+import pytest
+
+from approval_engine.pep import (
+    ApprovalHTTPClient,
+    ApprovalProtocolError,
+    ProtectedActionHarness,
+)
+
+
+class Client:
+    def __init__(self, claim=None, consume=None, failure=None):
+        self.claim_result = claim or {"valid_now": True, "consumed": False}
+        self.consume_result = consume
+        self.failure = failure
+        self.calls = []
+
+    def claim(self, approval_id):
+        self.calls.append("claim")
+        if self.failure == "claim":
+            raise ApprovalProtocolError("down")
+        return self.claim_result
+
+    def consume(self, approval_id, digest, decision_id):
+        self.calls.append("consume")
+        if self.failure == "consume":
+            raise ApprovalProtocolError("conflict")
+        return self.consume_result or {"status": "consumed", "request_digest": digest}
+
+
+DIGEST = "sha256:" + "ab" * 32
+
+
+class Response:
+    def __init__(self, body):
+        self.body = json.dumps(body).encode()
+
+    def getcode(self):
+        return 200
+
+    def read(self, _size):
+        return self.body
+
+    def close(self):
+        pass
+
+
+def test_http_client_rereads_mounted_token(tmp_path):
+    token = tmp_path / "token"
+    token.write_text("first")
+    seen = []
+
+    def opener(request, timeout):
+        seen.append((request.get_header("Authorization"), timeout))
+        return Response({"valid_now": True, "consumed": False})
+
+    client = ApprovalHTTPClient("http://approval-engine:8080", token, opener=opener)
+    client.claim("approval-1")
+    token.write_text("second")
+    client.claim("approval-1")
+    assert [item[0] for item in seen] == ["Bearer first", "Bearer second"]
+
+
+def allow(_claim):
+    return {"effect": "ALLOW", "decision_id": "decision:1", "request_digest": DIGEST}
+
+
+def test_side_effect_occurs_only_after_claim_decision_and_consume():
+    client = Client()
+    order = client.calls
+    result = ProtectedActionHarness(client).execute(
+        "approval:1",
+        DIGEST,
+        lambda claim: (order.append("decision"), allow(claim))[1],
+        lambda: (order.append("side-effect"), "dry-run-only")[1],
+    )
+    assert result == "dry-run-only"
+    assert order == ["claim", "decision", "consume", "side-effect"]
+
+
+@pytest.mark.parametrize("failure", ["claim", "consume"])
+def test_unavailable_or_conflicting_engine_prevents_side_effect(failure):
+    client = Client(failure=failure)
+    effects = []
+    with pytest.raises(ApprovalProtocolError):
+        ProtectedActionHarness(client).execute(
+            "approval:1", DIGEST, allow, lambda: effects.append("called")
+        )
+    assert effects == []
+
+
+def test_deny_or_digest_mismatch_prevents_consume_and_side_effect():
+    for decision in (
+        {"effect": "DENY", "decision_id": "decision:1", "request_digest": DIGEST},
+        {"effect": "ALLOW", "decision_id": "decision:1", "request_digest": "sha256:" + "cd" * 32},
+    ):
+        client = Client()
+        effects = []
+        with pytest.raises(ApprovalProtocolError):
+            ProtectedActionHarness(client).execute(
+                "approval:1", DIGEST, lambda _claim, value=decision: value, lambda: effects.append("called")
+            )
+        assert client.calls == ["claim"]
+        assert effects == []

tests/test_storage.py

@@ -0,0 +1,55 @@
+import os
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from approval_engine.errors import Conflict, StoreUnavailable
+from approval_engine.store import Engine, LATEST_SCHEMA_VERSION
+from tests.conftest import FROZEN, approve
+
+
+def test_production_open_refuses_unmigrated_database(tmp_path):
+    path = tmp_path / "approval.sqlite"
+    sqlite3.connect(path).close()
+    with pytest.raises(StoreUnavailable, match="run approval-engine migrate"):
+        Engine(path, clock=lambda: FROZEN, auto_migrate=False)
+
+
+def test_migrate_then_open_without_auto_migrate(tmp_path):
+    path = tmp_path / "approval.sqlite"
+    migrated = Engine(path, clock=lambda: FROZEN)
+    migrated.close()
+    production = Engine(path, clock=lambda: FROZEN, auto_migrate=False)
+    status = production.storage_status(integrity=True)
+    assert status["schema_version"] == LATEST_SCHEMA_VERSION
+    assert status["schema_current"] is True
+    assert status["persistent"] is True
+    assert status["ok"] is True
+    production.close()
+
+
+def test_online_backup_is_mode_0600_and_restorable(tmp_path):
+    source = tmp_path / "approval.sqlite"
+    backup = tmp_path / "approval.backup.sqlite"
+    engine = Engine(source, clock=lambda: FROZEN)
+    obj = approve(engine)
+    result = engine.backup(backup)
+    assert result["integrity"] == "ok"
+    assert os.stat(backup).st_mode & 0o777 == 0o600
+    restored = Engine(backup, clock=lambda: FROZEN, auto_migrate=False)
+    assert restored.get(obj.id).id == obj.id
+    assert restored.storage_status(integrity=True)["ok"] is True
+    restored.close()
+    engine.close()
+
+
+def test_backup_refuses_overwrite(tmp_path):
+    source = tmp_path / "approval.sqlite"
+    target = tmp_path / "existing.sqlite"
+    target.write_text("do not overwrite")
+    engine = Engine(source, clock=lambda: FROZEN)
+    with pytest.raises(Conflict, match="already exists"):
+        engine.backup(target)
+    assert target.read_text() == "do not overwrite"
+    engine.close()