Implement approval engine production readiness

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a05e2e-805b-7042-a750-71f473bceea2

approval_engine/__init__.py

@@ -5,8 +5,10 @@ from .errors import (
     ApprovalError,
     Conflict,
     DuplicateApprover,
+    Forbidden,
     NotFound,
     StoreUnavailable,
+    Unauthenticated,
     Unprocessable,
 )
 from .store import Engine
@@ -16,8 +18,10 @@ __all__ = [
     "ApprovalError",
     "Conflict",
     "DuplicateApprover",
+    "Forbidden",
     "NotFound",
     "StoreUnavailable",
+    "Unauthenticated",
     "Unprocessable",
     "binding_digest",
     "canonical_binding",

approval_engine/api.py

@@ -5,17 +5,25 @@ from __future__ import annotations
 import json
 from typing import Any, Callable
 
-from .errors import ApprovalError
+from .auth import Authenticator, DenyAllAuthenticator, Identity
+from .errors import ApprovalError, Forbidden
 from .store import Engine
 
 FORBIDDEN_DECISION_KEYS = frozenset({"effect", "decision", "allow", "deny"})
 
 
 def _read_json(environ: dict[str, Any]) -> dict[str, Any]:
-    length = int(environ.get("CONTENT_LENGTH") or 0)
+    try:
+        length = int(environ.get("CONTENT_LENGTH") or 0)
+    except (TypeError, ValueError) as exc:
+        raise ApprovalError("invalid content length") from exc
+    if length < 0 or length > 256 * 1024:
+        raise ApprovalError("request body is too large")
     if length == 0:
         return {}
     raw = environ["wsgi.input"].read(length)
+    if len(raw) != length:
+        raise ApprovalError("truncated request body")
     if not raw:
         return {}
     try:
@@ -40,8 +48,24 @@ def _assert_not_decision(payload: Any) -> None:
 
 
 class App:
-    def __init__(self, engine: Engine) -> None:
+    def __init__(
+        self,
+        engine: Engine,
+        authenticator: Authenticator | None = None,
+        *,
+        require_persistent: bool = False,
+    ) -> None:
         self.engine = engine
+        self.authenticator = authenticator or DenyAllAuthenticator()
+        self.require_persistent = require_persistent
+
+    def identity(self, environ: dict[str, Any], scope: str) -> Identity:
+        identity = self.authenticator.authenticate(
+            environ.get("HTTP_AUTHORIZATION")
+        ).require(scope)
+        if identity.tenant != self.engine.tenant:
+            raise Forbidden("caller tenant does not match this approval store")
+        return identity
 
     def __call__(self, environ: dict[str, Any], start_response: Callable) -> list[bytes]:
         method = environ.get("REQUEST_METHOD", "GET").upper()
@@ -64,19 +88,38 @@ class App:
         return [payload]
 
     def dispatch(self, method: str, path: str, environ: dict[str, Any]) -> tuple[int, dict[str, Any]]:
+        if path in ("/healthz", "/v1/healthz") and method == "GET":
+            return 200, {"status": "ok"}
         if path in ("/readyz", "/v1/readyz") and method == "GET":
+            storage = self.engine.storage_status()
+            ready = storage["schema_current"] and (
+                storage["persistent"] or not self.require_persistent
+            )
             self.engine.outbox_stats()
-            return 200, {"status": "ok", "store": "ok"}
+            return (200 if ready else 503), {
+                "status": "ok" if ready else "unavailable",
+                "store": "ok" if ready else "not-production-ready",
+            }
+        if path == "/v1/storage/status" and method == "GET":
+            self.identity(environ, "approval:observe")
+            return 200, self.engine.storage_status(integrity=True)
         if path == "/v1/cadence" and method == "GET":
+            self.identity(environ, "approval:observe")
             return 200, self.engine.transition_counts() | {"form": "heartbeat-or-reconciliation"}
         if path == "/v1/outbox/stats" and method == "GET":
+            self.identity(environ, "approval:observe")
             return 200, self.engine.outbox_stats()
         if path == "/v1/heartbeat" and method == "POST":
+            self.identity(environ, "approval:emit")
             return 200, self.engine.emit_heartbeat()
         if path == "/v1/approvals" and method == "POST":
+            identity = self.identity(environ, "approval:create")
             data = _read_json(environ)
+            binding = data.get("binding") or {}
+            if binding.get("actor") != identity.subject:
+                raise Forbidden("binding.actor must match the authenticated subject")
             obj = self.engine.create(
-                data.get("binding") or {},
+                binding,
                 data.get("validity") or {},
                 int(data.get("required_count") or 1),
                 pdp_digest=data.get("pdp_digest"),
@@ -88,24 +131,32 @@ class App:
             approval_id = parts[2]
             rest = parts[3:]
             if not rest and method == "GET":
+                self.identity(environ, "approval:read")
                 return 200, self.engine.get(approval_id).as_dict()
             if rest == ["claim"] and method == "GET":
+                self.identity(environ, "approval:read")
                 return 200, self.engine.claim(approval_id)
             if rest == ["entries"] and method == "POST":
-                data = _read_json(environ)
+                identity = self.identity(environ, "approval:approve")
+                _read_json(environ)
                 obj = self.engine.add_entry(
                     approval_id,
-                    data.get("subject_id") or "",
-                    assurance=data.get("assurance"),
-                    evidence_ref=data.get("evidence_ref"),
+                    identity.subject,
+                    assurance=json.dumps(identity.assurance, sort_keys=True),
+                    evidence_ref=identity.evidence_ref,
                 )
                 return 200, obj.as_dict()
             if rest == ["revoke"] and method == "POST":
+                self.identity(environ, "approval:revoke")
                 return 200, self.engine.revoke(approval_id).as_dict()
             if rest == ["supersede"] and method == "POST":
+                self.identity(environ, "approval:supersede")
                 data = _read_json(environ)
                 return 200, self.engine.supersede(approval_id, data.get("successor_id"))
             if rest == ["consume"] and method == "POST":
+                identity = self.identity(environ, "approval:consume")
+                if identity.principal_type not in {"service", "agent"}:
+                    raise Forbidden("consume requires a service or agent principal")
                 data = _read_json(environ)
                 return 200, self.engine.consume(
                     approval_id,
@@ -117,7 +168,14 @@ class App:
         return 404, {"error": "not_found", "message": path}
 
 
-def call(app: App, method: str, path: str, body: dict[str, Any] | None = None) -> tuple[int, dict[str, Any]]:
+def call(
+    app: App,
+    method: str,
+    path: str,
+    body: dict[str, Any] | None = None,
+    *,
+    authorization: str | None = "Bearer test-token",
+) -> tuple[int, dict[str, Any]]:
     """In-process WSGI helper for tests."""
     raw = json.dumps(body or {}).encode("utf-8") if body is not None else b""
     environ = {
@@ -127,6 +185,8 @@ def call(app: App, method: str, path: str, body: dict[str, Any] | None = None) -
         "CONTENT_LENGTH": str(len(raw)) if body is not None else "0",
         "QUERY_STRING": "",
     }
+    if authorization is not None:
+        environ["HTTP_AUTHORIZATION"] = authorization
     status_headers: list[tuple[str, list]] = []
 
     def start_response(status: str, headers: list[tuple[str, str]]) -> None:

approval_engine/audit.py

@@ -0,0 +1,100 @@
+"""Asynchronous delivery of the transactional outbox to audit-core."""
+
+from __future__ import annotations
+
+import json
+import threading
+from pathlib import Path
+from typing import Any, Callable
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from .store import Engine
+
+
+class AuditDeliveryError(RuntimeError):
+    """Bounded delivery failure; details deliberately exclude response bodies."""
+
+
+def audit_envelope(payload: dict[str, Any]) -> dict[str, Any]:
+    details = dict(payload.get("details") or {})
+    return {
+        "id": payload["event_id"],
+        "type": payload["action"],
+        "source": payload["source"],
+        "subject": payload["resource"],
+        "tenant": payload["tenant"],
+        "correlation_id": details.get("approval_id") or payload["event_id"],
+        "occurred_at": payload["observed_at"],
+        "data": {
+            "schema_version": payload["schema_version"],
+            "scope": payload["scope"],
+            "actor": payload.get("actor"),
+            "resource": payload["resource"],
+            "outcome": payload["outcome"],
+            "reason": payload.get("reason"),
+            "details": details,
+        },
+    }
+
+
+class AuditCoreSink:
+    def __init__(
+        self,
+        base_url: str,
+        token_file: str | Path,
+        *,
+        timeout_seconds: float = 5,
+        opener: Callable[..., Any] = urlopen,
+    ) -> None:
+        self.url = base_url.rstrip("/") + "/v1/events"
+        self.token_file = Path(token_file)
+        self.timeout_seconds = timeout_seconds
+        self.opener = opener
+
+    def __call__(self, payload: dict[str, Any]) -> None:
+        token = self.token_file.read_text(encoding="utf-8").strip()
+        if not token or any(ch.isspace() for ch in token):
+            raise AuditDeliveryError("audit credential is unavailable")
+        body = json.dumps(audit_envelope(payload), sort_keys=True).encode("utf-8")
+        request = Request(
+            self.url,
+            data=body,
+            method="POST",
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
+                "Idempotency-Key": payload["event_id"],
+            },
+        )
+        try:
+            response = self.opener(request, timeout=self.timeout_seconds)
+            status = int(response.getcode())
+            response.close()
+        except (HTTPError, URLError, OSError) as exc:
+            raise AuditDeliveryError(type(exc).__name__) from exc
+        if status not in {200, 202}:
+            raise AuditDeliveryError(f"audit ingest returned status {status}")
+
+
+class OutboxWorker:
+    def __init__(
+        self,
+        engine: Engine,
+        sink: Callable[[dict[str, Any]], None],
+        *,
+        heartbeat_interval_seconds: int = 300,
+    ) -> None:
+        self.engine = engine
+        self.sink = sink
+        self.heartbeat_interval_seconds = heartbeat_interval_seconds
+
+    def run_once(self) -> dict[str, int]:
+        if self.engine.heartbeat_due(self.heartbeat_interval_seconds):
+            self.engine.emit_heartbeat()
+        return self.engine.drain(self.sink)
+
+    def run_forever(self, stop: threading.Event, poll_seconds: float = 5) -> None:
+        while not stop.is_set():
+            self.run_once()
+            stop.wait(poll_seconds)

approval_engine/auth.py

@@ -0,0 +1,169 @@
+"""Verified caller identity for approval-engine's HTTP boundary.
+
+The engine enforces scopes issued by the identity owner; it does not decide who
+ought to hold them. Production uses KeyCape's RS256/JWKS contract. Tests use an
+explicit static verifier so no unverified claim can accidentally become a
+production fallback.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Any, Mapping, Protocol
+
+import jwt
+
+from .errors import Forbidden, Unauthenticated
+
+
+@dataclass(frozen=True)
+class Identity:
+    subject: str
+    issuer: str
+    audiences: tuple[str, ...]
+    principal_type: str
+    tenant: str
+    roles: frozenset[str]
+    scopes: frozenset[str]
+    assurance: dict[str, Any]
+    evidence_ref: str
+
+    def require(self, scope: str) -> "Identity":
+        if scope not in self.scopes:
+            raise Forbidden(f"caller lacks required scope {scope!r}")
+        return self
+
+
+class Authenticator(Protocol):
+    def authenticate(self, authorization: str | None) -> Identity: ...
+
+
+def _bearer(authorization: str | None) -> str:
+    value = str(authorization or "")
+    if not value.startswith("Bearer "):
+        raise Unauthenticated("bearer token is required")
+    token = value[7:].strip()
+    if not token or any(ch.isspace() for ch in token):
+        raise Unauthenticated("bearer token is invalid")
+    return token
+
+
+def _texts(value: object, name: str, *, allow_empty: bool = False) -> tuple[str, ...]:
+    if isinstance(value, str):
+        items = tuple(item for item in value.split() if item)
+    elif isinstance(value, (list, tuple)):
+        items = tuple(item for item in value if isinstance(item, str) and item)
+        if len(items) != len(value):
+            raise Unauthenticated(f"token claim {name!r} is invalid")
+    else:
+        raise Unauthenticated(f"token claim {name!r} is invalid")
+    if not items and not allow_empty:
+        raise Unauthenticated(f"token claim {name!r} is empty")
+    return items
+
+
+def identity_from_claims(claims: Mapping[str, Any], token: str) -> Identity:
+    try:
+        subject = str(claims["sub"])
+        issuer = str(claims["iss"])
+        principal_type = str(claims["principal_type"])
+        tenant = str(claims["tenant"])
+        assurance = claims["assurance"]
+    except (KeyError, TypeError) as exc:
+        raise Unauthenticated("token is missing required identity claims") from exc
+    if not subject or not issuer or principal_type not in {"human", "service", "agent"}:
+        raise Unauthenticated("token identity claims are invalid")
+    if not tenant or not isinstance(assurance, dict):
+        raise Unauthenticated("token tenant or assurance claim is invalid")
+    audiences = _texts(claims.get("aud"), "aud")
+    roles = frozenset(_texts(claims.get("roles"), "roles", allow_empty=True))
+    scopes = frozenset(_texts(claims.get("scope", claims.get("scp")), "scope"))
+    fingerprint = hashlib.sha256(token.encode("utf-8")).hexdigest()
+    return Identity(
+        subject=subject,
+        issuer=issuer,
+        audiences=audiences,
+        principal_type=principal_type,
+        tenant=tenant,
+        roles=roles,
+        scopes=scopes,
+        assurance=dict(assurance),
+        evidence_ref=f"jwt-sha256:{fingerprint}",
+    )
+
+
+class JWTAuthenticator:
+    """Verify KeyCape JWT signature, issuer, audience, time, and profile claims."""
+
+    def __init__(
+        self,
+        *,
+        issuer: str,
+        audience: str,
+        jwks_url: str,
+        leeway_seconds: int = 30,
+        timeout_seconds: int = 3,
+        jwks_client: Any | None = None,
+    ) -> None:
+        if not issuer or not audience or not jwks_url:
+            raise ValueError("issuer, audience, and jwks_url are required")
+        self.issuer = issuer
+        self.audience = audience
+        self.leeway_seconds = leeway_seconds
+        self.jwks_client = jwks_client or jwt.PyJWKClient(
+            jwks_url,
+            cache_keys=True,
+            cache_jwk_set=True,
+            lifespan=300,
+            timeout=timeout_seconds,
+        )
+
+    def authenticate(self, authorization: str | None) -> Identity:
+        token = _bearer(authorization)
+        try:
+            signing_key = self.jwks_client.get_signing_key_from_jwt(token)
+            claims = jwt.decode(
+                token,
+                signing_key.key,
+                algorithms=["RS256"],
+                issuer=self.issuer,
+                audience=self.audience,
+                leeway=self.leeway_seconds,
+                options={
+                    "require": [
+                        "iss",
+                        "sub",
+                        "aud",
+                        "exp",
+                        "iat",
+                        "tenant",
+                        "principal_type",
+                        "roles",
+                        "scope",
+                        "assurance",
+                    ]
+                },
+            )
+        except Exception as exc:
+            raise Unauthenticated("token verification failed") from exc
+        return identity_from_claims(claims, token)
+
+
+class StaticTokenAuthenticator:
+    """Explicit test/development verifier; never selected implicitly."""
+
+    def __init__(self, identities: Mapping[str, Identity]) -> None:
+        self.identities = dict(identities)
+
+    def authenticate(self, authorization: str | None) -> Identity:
+        token = _bearer(authorization)
+        identity = self.identities.get(token)
+        if identity is None:
+            raise Unauthenticated("token verification failed")
+        return identity
+
+
+class DenyAllAuthenticator:
+    def authenticate(self, authorization: str | None) -> Identity:
+        raise Unauthenticated("caller authentication is not configured")

approval_engine/cli.py

@@ -1,26 +1,154 @@
 from __future__ import annotations
 
 import argparse
+import json
+import threading
+from pathlib import Path
 from wsgiref.simple_server import make_server
 
 from .api import App
+from .audit import AuditCoreSink, OutboxWorker
+from .auth import Identity, JWTAuthenticator, StaticTokenAuthenticator
 from .store import Engine
 
+ALL_SCOPES = frozenset(
+    {
+        "approval:observe",
+        "approval:emit",
+        "approval:create",
+        "approval:read",
+        "approval:approve",
+        "approval:revoke",
+        "approval:supersede",
+        "approval:consume",
+    }
+)
+
+
+def _common_db(sub: argparse.ArgumentParser) -> None:
+    sub.add_argument("--db", default="approvals.sqlite")
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="approval-engine")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    serve = sub.add_parser("serve", help="serve the authenticated HTTP API")
+    _common_db(serve)
+    serve.add_argument("--host", default="127.0.0.1")
+    serve.add_argument("--port", type=int, default=8787)
+    serve.add_argument("--production", action="store_true")
+    serve.add_argument("--tenant", default="platform")
+    serve.add_argument("--jwt-issuer")
+    serve.add_argument("--jwt-audience")
+    serve.add_argument("--jwks-url")
+    serve.add_argument("--dev-token-file")
+    serve.add_argument("--audit-url")
+    serve.add_argument("--audit-token-file")
+    serve.add_argument("--outbox-poll-seconds", type=float, default=5)
+    serve.add_argument("--heartbeat-seconds", type=int, default=86400)
+    migrate = sub.add_parser("migrate", help="apply repeatable schema migrations")
+    _common_db(migrate)
+    verify = sub.add_parser("verify", help="verify schema and SQLite integrity")
+    _common_db(verify)
+    backup = sub.add_parser("backup", help="create and verify an online backup")
+    _common_db(backup)
+    backup.add_argument("--output", required=True)
+    return parser
+
+
+def _authenticator(args: argparse.Namespace, parser: argparse.ArgumentParser):
+    jwt_values = (args.jwt_issuer, args.jwt_audience, args.jwks_url)
+    if all(jwt_values):
+        return JWTAuthenticator(
+            issuer=args.jwt_issuer,
+            audience=args.jwt_audience,
+            jwks_url=args.jwks_url,
+        )
+    if any(jwt_values):
+        parser.error("--jwt-issuer, --jwt-audience, and --jwks-url are a set")
+    if args.production:
+        parser.error("production requires KeyCape JWT verifier configuration")
+    if not args.dev_token_file:
+        parser.error("configure JWT verification or explicit --dev-token-file")
+    token = Path(args.dev_token_file).read_text(encoding="utf-8").strip()
+    if not token:
+        parser.error("development token file is empty")
+    identity = Identity(
+        subject="development-agent",
+        issuer="local-development",
+        audiences=("approval-engine",),
+        principal_type="agent",
+        tenant="development",
+        roles=frozenset({"developer"}),
+        scopes=ALL_SCOPES,
+        assurance={"method": "explicit-development-token"},
+        evidence_ref="local-development-token",
+    )
+    return StaticTokenAuthenticator({token: identity})
+
+
+def _serve(args: argparse.Namespace, parser: argparse.ArgumentParser) -> int:
+    if args.production and args.db == ":memory:":
+        parser.error("production requires a persistent database")
+    if bool(args.audit_url) != bool(args.audit_token_file):
+        parser.error("--audit-url and --audit-token-file are a set")
+    if args.production and not args.audit_url:
+        parser.error("production requires authenticated audit delivery")
+    authenticator = _authenticator(args, parser)
+    engine = Engine(args.db, auto_migrate=not args.production, tenant=args.tenant)
+    storage = engine.storage_status(integrity=True)
+    if args.production and (not storage["ok"] or not storage["persistent"]):
+        parser.error("production store is not ready; run migrate and verify")
+    app = App(engine, authenticator, require_persistent=args.production)
+    stop = threading.Event()
+    if args.audit_url:
+        sink = AuditCoreSink(args.audit_url, args.audit_token_file)
+        worker = OutboxWorker(
+            engine,
+            sink,
+            heartbeat_interval_seconds=args.heartbeat_seconds,
+        )
+        threading.Thread(
+            target=worker.run_forever,
+            args=(stop, args.outbox_poll_seconds),
+            daemon=True,
+            name="approval-outbox",
+        ).start()
+    print(f"approval-engine listening on {args.host}:{args.port} db={args.db}")
+    try:
+        if args.production:
+            from waitress import serve
+
+            serve(app, host=args.host, port=args.port, threads=4)
+        else:
+            make_server(args.host, args.port, app).serve_forever()
+    finally:
+        stop.set()
+        engine.close()
+    return 0
+
 
 def main(argv: list[str] | None = None) -> int:
-    parser = argparse.ArgumentParser(prog="approval-engine")
-    sub = parser.add_subparsers(dest="cmd", required=True)
-    serve = sub.add_parser("serve", help="serve the introspection API")
-    serve.add_argument("--db", default="approvals.sqlite")
-    serve.add_argument("--host", default="127.0.0.1")
-    serve.add_argument("--port", type=int, default=8787)
+    parser = _parser()
     args = parser.parse_args(argv)
-    if args.cmd == "serve":
-        engine = Engine(args.db)
-        httpd = make_server(args.host, args.port, App(engine))
-        print(f"approval-engine on http://{args.host}:{args.port} db={args.db}")
-        httpd.serve_forever()
+    if args.cmd == "migrate":
+        engine = Engine(args.db, auto_migrate=True)
+        print(json.dumps(engine.storage_status(integrity=True), sort_keys=True))
+        engine.close()
         return 0
+    if args.cmd == "verify":
+        engine = Engine(args.db, auto_migrate=False)
+        status = engine.storage_status(integrity=True)
+        print(json.dumps(status, sort_keys=True))
+        engine.close()
+        return 0 if status["ok"] else 1
+    if args.cmd == "backup":
+        engine = Engine(args.db, auto_migrate=False)
+        print(json.dumps(engine.backup(args.output), sort_keys=True))
+        engine.close()
+        return 0
+    if args.cmd == "serve":
+        return _serve(args, parser)
     return 2
 
 

approval_engine/errors.py

@@ -3,6 +3,16 @@ class ApprovalError(Exception):
     reason_code = "error"
 
 
+class Unauthenticated(ApprovalError):
+    http_status = 401
+    reason_code = "unauthenticated"
+
+
+class Forbidden(ApprovalError):
+    http_status = 403
+    reason_code = "forbidden"
+
+
 class NotFound(ApprovalError):
     http_status = 404
     reason_code = "not_found"

approval_engine/pep.py

@@ -0,0 +1,114 @@
+"""Fail-closed PEP client and sequencing harness for GH-DEC-2026-003."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Callable
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote
+from urllib.request import Request, urlopen
+
+
+class ApprovalProtocolError(RuntimeError):
+    pass
+
+
+class ApprovalHTTPClient:
+    def __init__(
+        self,
+        base_url: str,
+        token_file: str | Path,
+        *,
+        timeout_seconds: float = 3,
+        opener: Callable[..., Any] = urlopen,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.token_file = Path(token_file)
+        self.timeout_seconds = timeout_seconds
+        self.opener = opener
+
+    def _request(
+        self, method: str, path: str, body: dict[str, Any] | None = None
+    ) -> dict[str, Any]:
+        token = self.token_file.read_text(encoding="utf-8").strip()
+        if not token or any(ch.isspace() for ch in token):
+            raise ApprovalProtocolError("approval credential is unavailable")
+        encoded = None if body is None else json.dumps(body).encode("utf-8")
+        request = Request(
+            self.base_url + path,
+            data=encoded,
+            method=method,
+            headers={
+                "Authorization": f"Bearer {token}",
+                "Content-Type": "application/json",
+            },
+        )
+        try:
+            response = self.opener(request, timeout=self.timeout_seconds)
+            status = int(response.getcode())
+            raw = response.read(256 * 1024 + 1)
+            response.close()
+        except (HTTPError, URLError, OSError) as exc:
+            raise ApprovalProtocolError(type(exc).__name__) from exc
+        if status != 200 or len(raw) > 256 * 1024:
+            raise ApprovalProtocolError(f"approval endpoint returned status {status}")
+        try:
+            result = json.loads(raw)
+        except (UnicodeDecodeError, json.JSONDecodeError) as exc:
+            raise ApprovalProtocolError("approval endpoint returned invalid JSON") from exc
+        if not isinstance(result, dict):
+            raise ApprovalProtocolError("approval endpoint returned invalid payload")
+        return result
+
+    def claim(self, approval_id: str) -> dict[str, Any]:
+        encoded_id = quote(approval_id, safe="")
+        return self._request("GET", f"/v1/approvals/{encoded_id}/claim")
+
+    def consume(
+        self, approval_id: str, request_digest: str, decision_id: str
+    ) -> dict[str, Any]:
+        encoded_id = quote(approval_id, safe="")
+        return self._request(
+            "POST",
+            f"/v1/approvals/{encoded_id}/consume",
+            {"request_digest": request_digest, "decision_id": decision_id},
+        )
+
+
+class ProtectedActionHarness:
+    """Sequence a supplied PDP decision and dry-run/protected callback.
+
+    The decision callback owns authorization. This class only enforces that a
+    fresh approval claim precedes it and a successful CAS consume precedes the
+    side effect.
+    """
+
+    def __init__(self, client: ApprovalHTTPClient) -> None:
+        self.client = client
+
+    def execute(
+        self,
+        approval_id: str,
+        request_digest: str,
+        decide: Callable[[dict[str, Any]], dict[str, Any]],
+        side_effect: Callable[[], Any],
+    ) -> Any:
+        claim = self.client.claim(approval_id)
+        if claim.get("valid_now") is not True or claim.get("consumed") is not False:
+            raise ApprovalProtocolError("approval claim is not valid for use")
+        decision = decide(claim)
+        if decision.get("effect") != "ALLOW":
+            raise ApprovalProtocolError("authorization decision did not allow")
+        decision_id = decision.get("decision_id")
+        if not isinstance(decision_id, str) or not decision_id:
+            raise ApprovalProtocolError("authorization decision lacks decision_id")
+        if decision.get("request_digest") != request_digest:
+            raise ApprovalProtocolError("authorization decision digest does not match")
+        consumed = self.client.consume(approval_id, request_digest, decision_id)
+        if (
+            consumed.get("status") != "consumed"
+            or consumed.get("request_digest") != request_digest
+        ):
+            raise ApprovalProtocolError("approval consumption was not confirmed")
+        return side_effect()

approval_engine/store.py

@@ -8,8 +8,10 @@ request digest before the protected side effect.
 from __future__ import annotations
 
 import json
+import os
 import sqlite3
 import threading
+import tempfile
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -33,6 +35,7 @@ AUDIT_SCHEMA = "audit-core.event.v1alpha1"
 SOURCE = "approval-engine"
 SCOPE = "netkingdom-approvals"
 EVENT_CLASSES = ("issuance", "use", "supersession", "revocation", "heartbeat")
+LATEST_SCHEMA_VERSION = 2
 
 SCHEMA = """
 CREATE TABLE IF NOT EXISTS approvals (
@@ -70,7 +73,10 @@ CREATE TABLE IF NOT EXISTS outbox (
   approval_id TEXT,
   payload_json TEXT NOT NULL,
   created_at TEXT NOT NULL,
-  drained_at TEXT
+  drained_at TEXT,
+  attempts INTEGER NOT NULL DEFAULT 0,
+  last_attempt_at TEXT,
+  last_error TEXT
 );
 """
 
@@ -157,12 +163,14 @@ class Engine:
         freshness_ttl: int = DEFAULT_FRESHNESS_TTL,
         fail_outbox: bool = False,
         tenant: str = "platform",
+        auto_migrate: bool = True,
     ) -> None:
         self.path = str(path)
         self.clock = clock
         self.freshness_ttl = freshness_ttl
         self.fail_outbox = fail_outbox
         self.tenant = tenant
+        self.auto_migrate = auto_migrate
         self._local = threading.local()
         self._init_schema()
 
@@ -188,13 +196,40 @@ class Engine:
     def _init_schema(self) -> None:
         conn = self._conn()
         try:
+            current = int(conn.execute("PRAGMA user_version").fetchone()[0])
+            if not self.auto_migrate:
+                tables = {
+                    row[0]
+                    for row in conn.execute(
+                        "SELECT name FROM sqlite_master WHERE type='table'"
+                    ).fetchall()
+                }
+                if current != LATEST_SCHEMA_VERSION:
+                    raise StoreUnavailable(
+                        f"database schema version {current} is not {LATEST_SCHEMA_VERSION}; "
+                        "run approval-engine migrate"
+                    )
+                if not {"approvals", "entries", "outbox"} <= tables:
+                    raise StoreUnavailable("database schema is incomplete")
+                return
             conn.executescript(SCHEMA)
-            columns = {
+            approval_columns = {
                 row["name"] for row in conn.execute("PRAGMA table_info(approvals)").fetchall()
             }
             for name in ("consumed_digest", "consumed_decision_id", "consumed_at"):
-                if name not in columns:
+                if name not in approval_columns:
                     conn.execute(f"ALTER TABLE approvals ADD COLUMN {name} TEXT")
+            outbox_columns = {
+                row["name"] for row in conn.execute("PRAGMA table_info(outbox)").fetchall()
+            }
+            if "attempts" not in outbox_columns:
+                conn.execute(
+                    "ALTER TABLE outbox ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0"
+                )
+            for name in ("last_attempt_at", "last_error"):
+                if name not in outbox_columns:
+                    conn.execute(f"ALTER TABLE outbox ADD COLUMN {name} TEXT")
+            conn.execute(f"PRAGMA user_version={LATEST_SCHEMA_VERSION}")
             conn.commit()
         except sqlite3.Error as exc:
             raise StoreUnavailable(str(exc)) from exc
@@ -208,6 +243,82 @@ class Engine:
     def now(self) -> datetime:
         return utc_now(self.clock)
 
+    def storage_status(self, *, integrity: bool = False) -> dict[str, Any]:
+        conn = self._conn()
+        try:
+            version = int(conn.execute("PRAGMA user_version").fetchone()[0])
+            result: dict[str, Any] = {
+                "schema_version": version,
+                "expected_schema_version": LATEST_SCHEMA_VERSION,
+                "schema_current": version == LATEST_SCHEMA_VERSION,
+                "persistent": self.path != ":memory:",
+            }
+            tables = {
+                row[0]
+                for row in conn.execute(
+                    "SELECT name FROM sqlite_master WHERE type='table'"
+                ).fetchall()
+            }
+            result["required_tables"] = {
+                name: name in tables for name in ("approvals", "entries", "outbox")
+            }
+            if integrity:
+                check = [
+                    row[0] for row in conn.execute("PRAGMA integrity_check").fetchall()
+                ]
+                foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall()
+                result["integrity"] = check
+                result["foreign_key_violations"] = len(foreign_keys)
+                result["ok"] = (
+                    check == ["ok"]
+                    and not foreign_keys
+                    and result["schema_current"]
+                    and all(result["required_tables"].values())
+                )
+            return result
+        except sqlite3.Error as exc:
+            raise StoreUnavailable(str(exc)) from exc
+
+    def backup(self, output: str | Path) -> dict[str, Any]:
+        if self.path == ":memory:":
+            raise Unprocessable("an in-memory database cannot be backed up")
+        target = Path(output)
+        if target.exists():
+            raise Conflict(f"backup target already exists: {target}")
+        if not target.parent.exists():
+            raise Unprocessable(f"backup parent does not exist: {target.parent}")
+        handle = tempfile.NamedTemporaryFile(
+            prefix=target.name + ".",
+            suffix=".tmp",
+            dir=target.parent,
+            delete=False,
+        )
+        temporary = Path(handle.name)
+        handle.close()
+        os.chmod(temporary, 0o600)
+        destination: sqlite3.Connection | None = None
+        try:
+            destination = sqlite3.connect(temporary)
+            self._conn().backup(destination)
+            destination.close()
+            destination = None
+            check = sqlite3.connect(temporary)
+            try:
+                integrity = [
+                    row[0] for row in check.execute("PRAGMA integrity_check")
+                ]
+            finally:
+                check.close()
+            if integrity != ["ok"]:
+                raise StoreUnavailable("backup integrity verification failed")
+            os.replace(temporary, target)
+        except Exception:
+            if destination is not None:
+                destination.close()
+            temporary.unlink(missing_ok=True)
+            raise
+        return {"path": str(target), "mode": "0600", "integrity": "ok"}
+
     def create(
         self,
         binding: dict[str, Any],
@@ -728,7 +839,8 @@ class Engine:
         now = self.now()
         try:
             rows = conn.execute(
-                "SELECT event_id, class, created_at, drained_at FROM outbox"
+                "SELECT event_id, class, created_at, drained_at, attempts, last_error "
+                "FROM outbox"
             ).fetchall()
         except sqlite3.Error as exc:
             raise StoreUnavailable(str(exc)) from exc
@@ -738,14 +850,32 @@ class Engine:
             "total": len(rows),
             "pending": len(pending),
             "max_age_seconds": max(ages) if ages else 0,
+            "attempts": sum(int(row["attempts"]) for row in rows),
+            "failed_pending": sum(
+                1 for row in pending if row["last_error"] is not None
+            ),
             "counts": self.transition_counts(),
         }
 
+    def heartbeat_due(self, interval_seconds: int) -> bool:
+        if interval_seconds < 1:
+            raise Unprocessable("heartbeat interval must be positive")
+        try:
+            row = self._conn().execute(
+                "SELECT MAX(created_at) AS created_at FROM outbox WHERE class='heartbeat'"
+            ).fetchone()
+        except sqlite3.Error as exc:
+            raise StoreUnavailable(str(exc)) from exc
+        if row["created_at"] is None:
+            return True
+        return (self.now() - parse_iso(row["created_at"])).total_seconds() >= interval_seconds
+
     def undrained(self) -> list[dict[str, Any]]:
         conn = self._conn()
         try:
             rows = conn.execute(
-                "SELECT event_id, class, approval_id, payload_json, created_at "
+                "SELECT event_id, class, approval_id, payload_json, created_at, "
+                "attempts, last_attempt_at, last_error "
                 "FROM outbox WHERE drained_at IS NULL ORDER BY created_at"
             ).fetchall()
         except sqlite3.Error as exc:
@@ -757,6 +887,9 @@ class Engine:
                 "approval_id": r["approval_id"],
                 "payload": json.loads(r["payload_json"]),
                 "created_at": r["created_at"],
+                "attempts": r["attempts"],
+                "last_attempt_at": r["last_attempt_at"],
+                "last_error": r["last_error"],
             }
             for r in rows
         ]
@@ -770,15 +903,28 @@ class Engine:
         failed = 0
         conn = self._conn()
         for item in self.undrained():
+            attempted_at = iso(self.now())
             try:
                 sink(item["payload"])
-            except Exception:
+            except Exception as exc:
                 failed += 1
+                try:
+                    conn.execute(
+                        "UPDATE outbox SET attempts=attempts+1, last_attempt_at=?, "
+                        "last_error=? WHERE event_id=? AND drained_at IS NULL",
+                        (attempted_at, type(exc).__name__, item["event_id"]),
+                    )
+                    conn.commit()
+                except sqlite3.Error as db_exc:
+                    conn.rollback()
+                    raise StoreUnavailable(str(db_exc)) from db_exc
                 continue
             try:
                 conn.execute(
-                    "UPDATE outbox SET drained_at=? WHERE event_id=? AND drained_at IS NULL",
-                    (iso(self.now()), item["event_id"]),
+                    "UPDATE outbox SET drained_at=?, attempts=attempts+1, "
+                    "last_attempt_at=?, last_error=NULL "
+                    "WHERE event_id=? AND drained_at IS NULL",
+                    (iso(self.now()), attempted_at, item["event_id"]),
                 )
                 conn.commit()
                 delivered += 1