coulomb/approval-engine
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement approval engine production readiness

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a05e2e-805b-7042-a750-71f473bceea2
This commit is contained in:
tegwick 2026-09-02 00:52:04 +02:00
parent ebce5abb27
commit 2bd2d19a98
30 changed files with 1679 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

162
approval_engine/store.py
View file

@ -8,8 +8,10 @@ request digest before the protected side effect.
from __future__ import annotations
import json
import os
import sqlite3
import threading
import tempfile
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
@ -33,6 +35,7 @@ AUDIT_SCHEMA = "audit-core.event.v1alpha1"
SOURCE = "approval-engine"
SCOPE = "netkingdom-approvals"
EVENT_CLASSES = ("issuance", "use", "supersession", "revocation", "heartbeat")
LATEST_SCHEMA_VERSION = 2
SCHEMA = """
CREATE TABLE IF NOT EXISTS approvals (
@ -70,7 +73,10 @@ CREATE TABLE IF NOT EXISTS outbox (
approval_id TEXT,
payload_json TEXT NOT NULL,
created_at TEXT NOT NULL,
drained_at TEXT
drained_at TEXT,
attempts INTEGER NOT NULL DEFAULT 0,
last_attempt_at TEXT,
last_error TEXT
);
"""
@ -157,12 +163,14 @@ class Engine:
freshness_ttl: int = DEFAULT_FRESHNESS_TTL,
fail_outbox: bool = False,
tenant: str = "platform",
auto_migrate: bool = True,
) -> None:
self.path = str(path)
self.clock = clock
self.freshness_ttl = freshness_ttl
self.fail_outbox = fail_outbox
self.tenant = tenant
self.auto_migrate = auto_migrate
self._local = threading.local()
self._init_schema()
@ -188,13 +196,40 @@ class Engine:
def _init_schema(self) -> None:
conn = self._conn()
try:
current = int(conn.execute("PRAGMA user_version").fetchone()[0])
if not self.auto_migrate:
tables = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
if current != LATEST_SCHEMA_VERSION:
raise StoreUnavailable(
f"database schema version {current} is not {LATEST_SCHEMA_VERSION}; "
"run approval-engine migrate"
)
if not {"approvals", "entries", "outbox"} <= tables:
raise StoreUnavailable("database schema is incomplete")
return
conn.executescript(SCHEMA)
columns = {
approval_columns = {
row["name"] for row in conn.execute("PRAGMA table_info(approvals)").fetchall()
}
for name in ("consumed_digest", "consumed_decision_id", "consumed_at"):
if name not in columns:
if name not in approval_columns:
conn.execute(f"ALTER TABLE approvals ADD COLUMN {name} TEXT")
outbox_columns = {
row["name"] for row in conn.execute("PRAGMA table_info(outbox)").fetchall()
}
if "attempts" not in outbox_columns:
conn.execute(
"ALTER TABLE outbox ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0"
)
for name in ("last_attempt_at", "last_error"):
if name not in outbox_columns:
conn.execute(f"ALTER TABLE outbox ADD COLUMN {name} TEXT")
conn.execute(f"PRAGMA user_version={LATEST_SCHEMA_VERSION}")
conn.commit()
except sqlite3.Error as exc:
raise StoreUnavailable(str(exc)) from exc
@ -208,6 +243,82 @@ class Engine:
def now(self) -> datetime:
return utc_now(self.clock)
def storage_status(self, *, integrity: bool = False) -> dict[str, Any]:
conn = self._conn()
try:
version = int(conn.execute("PRAGMA user_version").fetchone()[0])
result: dict[str, Any] = {
"schema_version": version,
"expected_schema_version": LATEST_SCHEMA_VERSION,
"schema_current": version == LATEST_SCHEMA_VERSION,
"persistent": self.path != ":memory:",
}
tables = {
row[0]
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'"
).fetchall()
}
result["required_tables"] = {
name: name in tables for name in ("approvals", "entries", "outbox")
}
if integrity:
check = [
row[0] for row in conn.execute("PRAGMA integrity_check").fetchall()
]
foreign_keys = conn.execute("PRAGMA foreign_key_check").fetchall()
result["integrity"] = check
result["foreign_key_violations"] = len(foreign_keys)
result["ok"] = (
check == ["ok"]
and not foreign_keys
and result["schema_current"]
and all(result["required_tables"].values())
)
return result
except sqlite3.Error as exc:
raise StoreUnavailable(str(exc)) from exc
def backup(self, output: str | Path) -> dict[str, Any]:
if self.path == ":memory:":
raise Unprocessable("an in-memory database cannot be backed up")
target = Path(output)
if target.exists():
raise Conflict(f"backup target already exists: {target}")
if not target.parent.exists():
raise Unprocessable(f"backup parent does not exist: {target.parent}")
handle = tempfile.NamedTemporaryFile(
prefix=target.name + ".",
suffix=".tmp",
dir=target.parent,
delete=False,
)
temporary = Path(handle.name)
handle.close()
os.chmod(temporary, 0o600)
destination: sqlite3.Connection | None = None
try:
destination = sqlite3.connect(temporary)
self._conn().backup(destination)
destination.close()
destination = None
check = sqlite3.connect(temporary)
try:
integrity = [
row[0] for row in check.execute("PRAGMA integrity_check")
]
finally:
check.close()
if integrity != ["ok"]:
raise StoreUnavailable("backup integrity verification failed")
os.replace(temporary, target)
except Exception:
if destination is not None:
destination.close()
temporary.unlink(missing_ok=True)
raise
return {"path": str(target), "mode": "0600", "integrity": "ok"}
def create(
self,
binding: dict[str, Any],
@ -728,7 +839,8 @@ class Engine:
now = self.now()
try:
rows = conn.execute(
"SELECT event_id, class, created_at, drained_at FROM outbox"
"SELECT event_id, class, created_at, drained_at, attempts, last_error "
"FROM outbox"
).fetchall()
except sqlite3.Error as exc:
raise StoreUnavailable(str(exc)) from exc
@ -738,14 +850,32 @@ class Engine:
"total": len(rows),
"pending": len(pending),
"max_age_seconds": max(ages) if ages else 0,
"attempts": sum(int(row["attempts"]) for row in rows),
"failed_pending": sum(
1 for row in pending if row["last_error"] is not None
),
"counts": self.transition_counts(),
}
def heartbeat_due(self, interval_seconds: int) -> bool:
if interval_seconds < 1:
raise Unprocessable("heartbeat interval must be positive")
try:
row = self._conn().execute(
"SELECT MAX(created_at) AS created_at FROM outbox WHERE class='heartbeat'"
).fetchone()
except sqlite3.Error as exc:
raise StoreUnavailable(str(exc)) from exc
if row["created_at"] is None:
return True
return (self.now() - parse_iso(row["created_at"])).total_seconds() >= interval_seconds
def undrained(self) -> list[dict[str, Any]]:
conn = self._conn()
try:
rows = conn.execute(
"SELECT event_id, class, approval_id, payload_json, created_at "
"SELECT event_id, class, approval_id, payload_json, created_at, "
"attempts, last_attempt_at, last_error "
"FROM outbox WHERE drained_at IS NULL ORDER BY created_at"
).fetchall()
except sqlite3.Error as exc:
@ -757,6 +887,9 @@ class Engine:
"approval_id": r["approval_id"],
"payload": json.loads(r["payload_json"]),
"created_at": r["created_at"],
"attempts": r["attempts"],
"last_attempt_at": r["last_attempt_at"],
"last_error": r["last_error"],
}
for r in rows
]
@ -770,15 +903,28 @@ class Engine:
failed = 0
conn = self._conn()
for item in self.undrained():
attempted_at = iso(self.now())
try:
sink(item["payload"])
except Exception:
except Exception as exc:
failed += 1
try:
conn.execute(
"UPDATE outbox SET attempts=attempts+1, last_attempt_at=?, "
"last_error=? WHERE event_id=? AND drained_at IS NULL",
(attempted_at, type(exc).__name__, item["event_id"]),
)
conn.commit()
except sqlite3.Error as db_exc:
conn.rollback()
raise StoreUnavailable(str(db_exc)) from db_exc
continue
try:
conn.execute(
"UPDATE outbox SET drained_at=? WHERE event_id=? AND drained_at IS NULL",
(iso(self.now()), item["event_id"]),
"UPDATE outbox SET drained_at=?, attempts=attempts+1, "
"last_attempt_at=?, last_error=NULL "
"WHERE event_id=? AND drained_at IS NULL",
(iso(self.now()), attempted_at, item["event_id"]),
)
conn.commit()
delivered += 1