tegwick
c97e5ea92d
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a02b6f-7db1-7222-918b-e813a6bda38d
544 lines
19 KiB
Python
544 lines
19 KiB
Python
"""Durable, idempotent close-evidence outbox core.
|
|
|
|
The outbox is intentionally not wired into the live claim loop yet. It stores
|
|
bounded Activity Core completion/failure intents outside target checkouts so a
|
|
future reconciler can retry close delivery without re-running repository work.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import uuid
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterator, TextIO
|
|
|
|
OUTBOX_VERSION = "1"
|
|
_MAX_PAYLOAD_BYTES = 65536
|
|
_MAX_STRING_LENGTH = 4000
|
|
_MAX_COLLECTION_ITEMS = 200
|
|
_MAX_JSON_DEPTH = 8
|
|
_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@+-]{0,199}")
|
|
_RECORD_KEYS = frozenset(
|
|
{
|
|
"version",
|
|
"entry_id",
|
|
"run_id",
|
|
"transaction_id",
|
|
"action",
|
|
"result",
|
|
"error",
|
|
"reopen",
|
|
"state",
|
|
"created_at",
|
|
"attempts",
|
|
"last_attempt_at",
|
|
"last_error",
|
|
}
|
|
)
|
|
|
|
|
|
class CloseOutboxError(RuntimeError):
|
|
"""Base error for invalid records or durable outbox operations."""
|
|
|
|
|
|
class InvalidCloseRequestError(CloseOutboxError):
|
|
"""A close intent is ambiguous, unbounded, or unsupported."""
|
|
|
|
|
|
class OutboxConflictError(CloseOutboxError):
|
|
"""One run/transaction identity was reused for a different close intent."""
|
|
|
|
|
|
class OutboxCorruptError(CloseOutboxError):
|
|
"""A durable record cannot be safely decoded or validated."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CloseRequest:
|
|
"""One bounded Activity Core completion or failure intent."""
|
|
|
|
run_id: str
|
|
transaction_id: str
|
|
action: str
|
|
result: dict[str, Any]
|
|
error: str = ""
|
|
reopen: bool = False
|
|
version: str = OUTBOX_VERSION
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.version != OUTBOX_VERSION:
|
|
raise InvalidCloseRequestError(
|
|
f"unsupported close request version {self.version!r}"
|
|
)
|
|
_validate_identifier(self.run_id, "run_id")
|
|
_validate_identifier(self.transaction_id, "transaction_id")
|
|
if self.action not in {"complete", "fail"}:
|
|
raise InvalidCloseRequestError("action must be 'complete' or 'fail'")
|
|
if not isinstance(self.result, dict):
|
|
raise InvalidCloseRequestError("result must be an object")
|
|
normalized = _normalize_json(self.result)
|
|
object.__setattr__(self, "result", normalized)
|
|
if not isinstance(self.error, str):
|
|
raise InvalidCloseRequestError("error must be a string")
|
|
error = _bounded_text(self.error, 2000)
|
|
object.__setattr__(self, "error", error)
|
|
if not isinstance(self.reopen, bool):
|
|
raise InvalidCloseRequestError("reopen must be a boolean")
|
|
if self.action == "complete" and (error or self.reopen):
|
|
raise InvalidCloseRequestError(
|
|
"complete requests cannot carry error or reopen"
|
|
)
|
|
if self.action == "fail" and not error:
|
|
raise InvalidCloseRequestError("fail requests require a bounded error")
|
|
if len(_json_bytes(self.intent())) > _MAX_PAYLOAD_BYTES:
|
|
raise InvalidCloseRequestError(
|
|
f"close request exceeds {_MAX_PAYLOAD_BYTES} encoded bytes"
|
|
)
|
|
|
|
@property
|
|
def entry_id(self) -> str:
|
|
identity = f"{self.run_id}\0{self.transaction_id}"
|
|
return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:32]
|
|
|
|
@property
|
|
def intent_digest(self) -> str:
|
|
return hashlib.sha256(_json_bytes(self.intent())).hexdigest()
|
|
|
|
def intent(self) -> dict[str, Any]:
|
|
return {
|
|
"version": self.version,
|
|
"entry_id": self.entry_id,
|
|
"run_id": self.run_id,
|
|
"transaction_id": self.transaction_id,
|
|
"action": self.action,
|
|
"result": self.result,
|
|
"error": self.error,
|
|
"reopen": self.reopen,
|
|
}
|
|
|
|
def detached_copy(self) -> "CloseRequest":
|
|
return CloseRequest(
|
|
run_id=self.run_id,
|
|
transaction_id=self.transaction_id,
|
|
action=self.action,
|
|
result=json.loads(json.dumps(self.result)),
|
|
error=self.error,
|
|
reopen=self.reopen,
|
|
version=self.version,
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EnqueueReceipt:
|
|
entry_id: str
|
|
created: bool
|
|
state: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReplayReport:
|
|
attempted: int = 0
|
|
delivered: int = 0
|
|
failed: int = 0
|
|
quarantined: int = 0
|
|
remaining: int = 0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _Record:
|
|
request: CloseRequest
|
|
state: str
|
|
created_at: str
|
|
attempts: int = 0
|
|
last_attempt_at: str | None = None
|
|
last_error: str | None = None
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.state not in {"pending", "delivered"}:
|
|
raise OutboxCorruptError("invalid outbox state")
|
|
if isinstance(self.attempts, bool) or not isinstance(self.attempts, int):
|
|
raise OutboxCorruptError("attempts must be a non-negative integer")
|
|
if self.attempts < 0:
|
|
raise OutboxCorruptError("attempts must be a non-negative integer")
|
|
_validate_timestamp(self.created_at, "created_at")
|
|
if self.last_attempt_at is not None:
|
|
_validate_timestamp(self.last_attempt_at, "last_attempt_at")
|
|
if self.last_error is not None and not isinstance(self.last_error, str):
|
|
raise OutboxCorruptError("last_error must be a string or null")
|
|
if self.last_error is not None and len(self.last_error) > 1000:
|
|
raise OutboxCorruptError("last_error exceeds the durable size bound")
|
|
|
|
def payload(self) -> dict[str, Any]:
|
|
return {
|
|
**self.request.intent(),
|
|
"state": self.state,
|
|
"created_at": self.created_at,
|
|
"attempts": self.attempts,
|
|
"last_attempt_at": self.last_attempt_at,
|
|
"last_error": self.last_error,
|
|
}
|
|
|
|
@classmethod
|
|
def from_payload(cls, value: Any) -> "_Record":
|
|
if not isinstance(value, dict):
|
|
raise OutboxCorruptError("outbox record must be an object")
|
|
if any(not isinstance(key, str) for key in value):
|
|
raise OutboxCorruptError("outbox record field names must be strings")
|
|
actual = set(value)
|
|
if actual != _RECORD_KEYS:
|
|
missing = sorted(_RECORD_KEYS - actual)
|
|
unknown = sorted(actual - _RECORD_KEYS)
|
|
raise OutboxCorruptError(
|
|
f"outbox record fields invalid: missing={missing} unknown={unknown}"
|
|
)
|
|
try:
|
|
request = CloseRequest(
|
|
run_id=value["run_id"],
|
|
transaction_id=value["transaction_id"],
|
|
action=value["action"],
|
|
result=value["result"],
|
|
error=value["error"],
|
|
reopen=value["reopen"],
|
|
version=value["version"],
|
|
)
|
|
except InvalidCloseRequestError as exc:
|
|
raise OutboxCorruptError(str(exc)) from exc
|
|
if value["entry_id"] != request.entry_id:
|
|
raise OutboxCorruptError("entry_id does not match run/transaction identity")
|
|
return cls(
|
|
request=request,
|
|
state=value["state"],
|
|
created_at=value["created_at"],
|
|
attempts=value["attempts"],
|
|
last_attempt_at=value["last_attempt_at"],
|
|
last_error=value["last_error"],
|
|
)
|
|
|
|
|
|
class CloseOutbox:
|
|
"""Process-safe durable store for close intents and replay outcomes."""
|
|
|
|
def __init__(self, *, state_dir: Path | None = None) -> None:
|
|
self.state_dir = (
|
|
Path(state_dir).expanduser().resolve() if state_dir else _state_dir()
|
|
)
|
|
self.root = self.state_dir / "close-outbox"
|
|
self.pending_dir = self.root / "pending"
|
|
self.delivered_dir = self.root / "delivered"
|
|
self.quarantine_dir = self.root / "quarantine"
|
|
self.lock_path = self.root / ".lock"
|
|
for directory in (
|
|
self.root,
|
|
self.pending_dir,
|
|
self.delivered_dir,
|
|
self.quarantine_dir,
|
|
):
|
|
_ensure_private_dir(directory)
|
|
|
|
def enqueue(self, request: CloseRequest) -> EnqueueReceipt:
|
|
"""Persist one intent or return its existing identical durable state."""
|
|
if not isinstance(request, CloseRequest):
|
|
raise InvalidCloseRequestError("request must be a CloseRequest")
|
|
with self._locked():
|
|
for state, directory in (
|
|
("delivered", self.delivered_dir),
|
|
("pending", self.pending_dir),
|
|
):
|
|
path = directory / f"{request.entry_id}.json"
|
|
if path.exists():
|
|
record = self._load(path)
|
|
if record.state != state:
|
|
raise OutboxCorruptError(
|
|
f"outbox record directory/state mismatch: expected {state}"
|
|
)
|
|
if record.request.intent_digest != request.intent_digest:
|
|
raise OutboxConflictError(
|
|
"run/transaction identity already has a different close intent"
|
|
)
|
|
return EnqueueReceipt(request.entry_id, False, state)
|
|
if tuple(self.quarantine_dir.glob(f"{request.entry_id}.*.json")):
|
|
raise OutboxCorruptError(
|
|
"run/transaction identity has quarantined outbox material"
|
|
)
|
|
record = _Record(
|
|
request=request,
|
|
state="pending",
|
|
created_at=_now(),
|
|
)
|
|
_atomic_write_json(
|
|
self.pending_dir / f"{request.entry_id}.json",
|
|
record.payload(),
|
|
)
|
|
return EnqueueReceipt(request.entry_id, True, "pending")
|
|
|
|
def replay(
|
|
self,
|
|
deliver: Callable[[CloseRequest], Any],
|
|
*,
|
|
limit: int = 100,
|
|
) -> ReplayReport:
|
|
"""Attempt pending deliveries once each without executing workload code."""
|
|
if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
|
|
raise ValueError("limit must be an integer between 1 and 1000")
|
|
attempted = delivered = failed = quarantined = 0
|
|
with self._locked():
|
|
paths = sorted(self.pending_dir.glob("*.json"))[:limit]
|
|
for path in paths:
|
|
try:
|
|
record = self._load(path)
|
|
except OutboxCorruptError as exc:
|
|
self._quarantine(path, str(exc))
|
|
quarantined += 1
|
|
continue
|
|
if record.state != "pending":
|
|
self._quarantine(
|
|
path,
|
|
f"pending directory contains state {record.state!r}",
|
|
)
|
|
quarantined += 1
|
|
continue
|
|
|
|
destination = self.delivered_dir / path.name
|
|
if destination.exists():
|
|
self._quarantine(
|
|
path,
|
|
"pending entry conflicts with existing delivered material",
|
|
)
|
|
quarantined += 1
|
|
continue
|
|
|
|
attempted += 1
|
|
attempted_at = _now()
|
|
trying = _Record(
|
|
request=record.request,
|
|
state="pending",
|
|
created_at=record.created_at,
|
|
attempts=record.attempts + 1,
|
|
last_attempt_at=attempted_at,
|
|
last_error=None,
|
|
)
|
|
_atomic_write_json(path, trying.payload())
|
|
try:
|
|
deliver(record.request.detached_copy())
|
|
except Exception as exc:
|
|
retained = _Record(
|
|
request=record.request,
|
|
state="pending",
|
|
created_at=record.created_at,
|
|
attempts=trying.attempts,
|
|
last_attempt_at=attempted_at,
|
|
last_error=(
|
|
f"{type(exc).__name__}: close delivery failed"
|
|
)[:1000],
|
|
)
|
|
_atomic_write_json(path, retained.payload())
|
|
failed += 1
|
|
continue
|
|
|
|
completed = _Record(
|
|
request=record.request,
|
|
state="delivered",
|
|
created_at=record.created_at,
|
|
attempts=trying.attempts,
|
|
last_attempt_at=attempted_at,
|
|
last_error=None,
|
|
)
|
|
_atomic_write_json(path, completed.payload())
|
|
os.replace(path, destination)
|
|
_fsync_dir(self.pending_dir)
|
|
_fsync_dir(self.delivered_dir)
|
|
delivered += 1
|
|
|
|
remaining = len(tuple(self.pending_dir.glob("*.json")))
|
|
return ReplayReport(
|
|
attempted=attempted,
|
|
delivered=delivered,
|
|
failed=failed,
|
|
quarantined=quarantined,
|
|
remaining=remaining,
|
|
)
|
|
|
|
def pending_count(self) -> int:
|
|
with self._locked():
|
|
return len(tuple(self.pending_dir.glob("*.json")))
|
|
|
|
def _load(self, path: Path) -> _Record:
|
|
try:
|
|
if path.stat().st_size > _MAX_PAYLOAD_BYTES + 8192:
|
|
raise OutboxCorruptError("outbox record exceeds the durable size bound")
|
|
raw = path.read_text(encoding="utf-8")
|
|
value = json.loads(raw)
|
|
return _Record.from_payload(value)
|
|
except OutboxCorruptError:
|
|
raise
|
|
except (OSError, UnicodeError, ValueError, TypeError) as exc:
|
|
raise OutboxCorruptError(
|
|
f"cannot decode outbox record {path.name}: {type(exc).__name__}"
|
|
) from exc
|
|
|
|
def _quarantine(self, path: Path, reason: str) -> None:
|
|
suffix = uuid.uuid4().hex[:12]
|
|
target = self.quarantine_dir / f"{path.stem}.{suffix}.json"
|
|
os.replace(path, target)
|
|
_fsync_dir(self.pending_dir)
|
|
_fsync_dir(self.quarantine_dir)
|
|
_atomic_write_json(
|
|
self.quarantine_dir / f"{path.stem}.{suffix}.error.json",
|
|
{
|
|
"entry_id": path.stem,
|
|
"quarantined_at": _now(),
|
|
"reason": _bounded_text(reason, 1000),
|
|
},
|
|
)
|
|
|
|
@contextmanager
|
|
def _locked(self) -> Iterator[None]:
|
|
lock_file = _open_private_file(self.lock_path)
|
|
try:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
|
|
yield
|
|
finally:
|
|
try:
|
|
fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
|
|
finally:
|
|
lock_file.close()
|
|
|
|
|
|
def _state_dir() -> Path:
|
|
explicit = os.environ.get("REIN_AHARNESS_STATE_DIR", "").strip()
|
|
if explicit:
|
|
return Path(explicit).expanduser().resolve()
|
|
xdg = os.environ.get("XDG_STATE_HOME", "").strip()
|
|
if xdg:
|
|
return (Path(xdg).expanduser() / "rein-aharness").resolve()
|
|
return (Path.home() / ".local" / "state" / "rein-aharness").resolve()
|
|
|
|
|
|
def _validate_identifier(value: Any, name: str) -> None:
|
|
if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None:
|
|
raise InvalidCloseRequestError(
|
|
f"{name} must be a bounded, path-safe identifier"
|
|
)
|
|
|
|
|
|
def _normalize_json(value: Any, *, depth: int = 0) -> Any:
|
|
if depth > _MAX_JSON_DEPTH:
|
|
raise InvalidCloseRequestError("result exceeds the JSON depth bound")
|
|
if value is None or isinstance(value, bool):
|
|
return value
|
|
if isinstance(value, str):
|
|
if len(value) > _MAX_STRING_LENGTH:
|
|
raise InvalidCloseRequestError(
|
|
f"result string exceeds {_MAX_STRING_LENGTH} characters"
|
|
)
|
|
return value
|
|
if isinstance(value, int):
|
|
if abs(value) > 2**63 - 1:
|
|
raise InvalidCloseRequestError("result integer exceeds signed 64-bit range")
|
|
return value
|
|
if isinstance(value, float):
|
|
if not math.isfinite(value):
|
|
raise InvalidCloseRequestError("result floats must be finite")
|
|
return value
|
|
if isinstance(value, list):
|
|
if len(value) > _MAX_COLLECTION_ITEMS:
|
|
raise InvalidCloseRequestError("result array exceeds the item bound")
|
|
return [_normalize_json(item, depth=depth + 1) for item in value]
|
|
if isinstance(value, dict):
|
|
if len(value) > _MAX_COLLECTION_ITEMS:
|
|
raise InvalidCloseRequestError("result object exceeds the field bound")
|
|
normalized: dict[str, Any] = {}
|
|
for key, item in value.items():
|
|
if not isinstance(key, str) or not key or len(key) > 200:
|
|
raise InvalidCloseRequestError(
|
|
"result field names must be non-empty strings of at most 200 characters"
|
|
)
|
|
normalized[key] = _normalize_json(item, depth=depth + 1)
|
|
return normalized
|
|
raise InvalidCloseRequestError(
|
|
f"result contains unsupported JSON type: {type(value).__name__}"
|
|
)
|
|
|
|
|
|
def _validate_timestamp(value: Any, name: str) -> None:
|
|
if not isinstance(value, str):
|
|
raise OutboxCorruptError(f"{name} must be a timestamp string")
|
|
try:
|
|
parsed = datetime.fromisoformat(value)
|
|
except ValueError as exc:
|
|
raise OutboxCorruptError(f"{name} is not an ISO timestamp") from exc
|
|
if parsed.tzinfo is None:
|
|
raise OutboxCorruptError(f"{name} must include a timezone")
|
|
|
|
|
|
def _ensure_private_dir(path: Path) -> None:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
os.chmod(path, 0o700)
|
|
|
|
|
|
def _open_private_file(path: Path) -> TextIO:
|
|
fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600)
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
return os.fdopen(fd, "r+", encoding="utf-8")
|
|
except BaseException:
|
|
os.close(fd)
|
|
raise
|
|
|
|
|
|
def _atomic_write_json(path: Path, value: dict[str, Any]) -> None:
|
|
data = _json_bytes(value) + b"\n"
|
|
temp = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp"
|
|
fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
try:
|
|
os.fchmod(fd, 0o600)
|
|
with os.fdopen(fd, "wb") as handle:
|
|
fd = -1
|
|
handle.write(data)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
os.replace(temp, path)
|
|
os.chmod(path, 0o600)
|
|
_fsync_dir(path.parent)
|
|
except BaseException:
|
|
if fd >= 0:
|
|
os.close(fd)
|
|
try:
|
|
temp.unlink()
|
|
except FileNotFoundError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def _fsync_dir(path: Path) -> None:
|
|
fd = os.open(path, os.O_RDONLY)
|
|
try:
|
|
os.fsync(fd)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def _json_bytes(value: Any) -> bytes:
|
|
return json.dumps(
|
|
value,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=False,
|
|
allow_nan=False,
|
|
).encode("utf-8")
|
|
|
|
|
|
def _bounded_text(value: str, limit: int) -> str:
|
|
return " ".join(value.split())[:limit]
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|