"""Durable, idempotent close-evidence outbox core. The outbox is intentionally not wired into the live claim loop yet. It stores bounded Activity Core completion/failure intents outside target checkouts so a future reconciler can retry close delivery without re-running repository work. """ from __future__ import annotations import fcntl import hashlib import json import math import os import re import uuid from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable, Iterator, TextIO OUTBOX_VERSION = "1" _MAX_PAYLOAD_BYTES = 65536 _MAX_STRING_LENGTH = 4000 _MAX_COLLECTION_ITEMS = 200 _MAX_JSON_DEPTH = 8 _IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@+-]{0,199}") _RECORD_KEYS = frozenset( { "version", "entry_id", "run_id", "transaction_id", "action", "result", "error", "reopen", "state", "created_at", "attempts", "last_attempt_at", "last_error", } ) class CloseOutboxError(RuntimeError): """Base error for invalid records or durable outbox operations.""" class InvalidCloseRequestError(CloseOutboxError): """A close intent is ambiguous, unbounded, or unsupported.""" class OutboxConflictError(CloseOutboxError): """One run/transaction identity was reused for a different close intent.""" class OutboxCorruptError(CloseOutboxError): """A durable record cannot be safely decoded or validated.""" @dataclass(frozen=True) class CloseRequest: """One bounded Activity Core completion or failure intent.""" run_id: str transaction_id: str action: str result: dict[str, Any] error: str = "" reopen: bool = False version: str = OUTBOX_VERSION def __post_init__(self) -> None: if self.version != OUTBOX_VERSION: raise InvalidCloseRequestError( f"unsupported close request version {self.version!r}" ) _validate_identifier(self.run_id, "run_id") _validate_identifier(self.transaction_id, "transaction_id") if self.action not in {"complete", "fail"}: raise InvalidCloseRequestError("action must be 'complete' or 'fail'") if not isinstance(self.result, dict): raise InvalidCloseRequestError("result must be an object") normalized = _normalize_json(self.result) object.__setattr__(self, "result", normalized) if not isinstance(self.error, str): raise InvalidCloseRequestError("error must be a string") error = _bounded_text(self.error, 2000) object.__setattr__(self, "error", error) if not isinstance(self.reopen, bool): raise InvalidCloseRequestError("reopen must be a boolean") if self.action == "complete" and (error or self.reopen): raise InvalidCloseRequestError( "complete requests cannot carry error or reopen" ) if self.action == "fail" and not error: raise InvalidCloseRequestError("fail requests require a bounded error") if len(_json_bytes(self.intent())) > _MAX_PAYLOAD_BYTES: raise InvalidCloseRequestError( f"close request exceeds {_MAX_PAYLOAD_BYTES} encoded bytes" ) @property def entry_id(self) -> str: identity = f"{self.run_id}\0{self.transaction_id}" return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:32] @property def intent_digest(self) -> str: return hashlib.sha256(_json_bytes(self.intent())).hexdigest() def intent(self) -> dict[str, Any]: return { "version": self.version, "entry_id": self.entry_id, "run_id": self.run_id, "transaction_id": self.transaction_id, "action": self.action, "result": self.result, "error": self.error, "reopen": self.reopen, } def detached_copy(self) -> "CloseRequest": return CloseRequest( run_id=self.run_id, transaction_id=self.transaction_id, action=self.action, result=json.loads(json.dumps(self.result)), error=self.error, reopen=self.reopen, version=self.version, ) @dataclass(frozen=True) class EnqueueReceipt: entry_id: str created: bool state: str @dataclass(frozen=True) class ReplayReport: attempted: int = 0 delivered: int = 0 failed: int = 0 quarantined: int = 0 remaining: int = 0 @dataclass(frozen=True) class _Record: request: CloseRequest state: str created_at: str attempts: int = 0 last_attempt_at: str | None = None last_error: str | None = None def __post_init__(self) -> None: if self.state not in {"pending", "delivered"}: raise OutboxCorruptError("invalid outbox state") if isinstance(self.attempts, bool) or not isinstance(self.attempts, int): raise OutboxCorruptError("attempts must be a non-negative integer") if self.attempts < 0: raise OutboxCorruptError("attempts must be a non-negative integer") _validate_timestamp(self.created_at, "created_at") if self.last_attempt_at is not None: _validate_timestamp(self.last_attempt_at, "last_attempt_at") if self.last_error is not None and not isinstance(self.last_error, str): raise OutboxCorruptError("last_error must be a string or null") if self.last_error is not None and len(self.last_error) > 1000: raise OutboxCorruptError("last_error exceeds the durable size bound") def payload(self) -> dict[str, Any]: return { **self.request.intent(), "state": self.state, "created_at": self.created_at, "attempts": self.attempts, "last_attempt_at": self.last_attempt_at, "last_error": self.last_error, } @classmethod def from_payload(cls, value: Any) -> "_Record": if not isinstance(value, dict): raise OutboxCorruptError("outbox record must be an object") if any(not isinstance(key, str) for key in value): raise OutboxCorruptError("outbox record field names must be strings") actual = set(value) if actual != _RECORD_KEYS: missing = sorted(_RECORD_KEYS - actual) unknown = sorted(actual - _RECORD_KEYS) raise OutboxCorruptError( f"outbox record fields invalid: missing={missing} unknown={unknown}" ) try: request = CloseRequest( run_id=value["run_id"], transaction_id=value["transaction_id"], action=value["action"], result=value["result"], error=value["error"], reopen=value["reopen"], version=value["version"], ) except InvalidCloseRequestError as exc: raise OutboxCorruptError(str(exc)) from exc if value["entry_id"] != request.entry_id: raise OutboxCorruptError("entry_id does not match run/transaction identity") return cls( request=request, state=value["state"], created_at=value["created_at"], attempts=value["attempts"], last_attempt_at=value["last_attempt_at"], last_error=value["last_error"], ) class CloseOutbox: """Process-safe durable store for close intents and replay outcomes.""" def __init__(self, *, state_dir: Path | None = None) -> None: self.state_dir = ( Path(state_dir).expanduser().resolve() if state_dir else _state_dir() ) self.root = self.state_dir / "close-outbox" self.pending_dir = self.root / "pending" self.delivered_dir = self.root / "delivered" self.quarantine_dir = self.root / "quarantine" self.lock_path = self.root / ".lock" for directory in ( self.root, self.pending_dir, self.delivered_dir, self.quarantine_dir, ): _ensure_private_dir(directory) def enqueue(self, request: CloseRequest) -> EnqueueReceipt: """Persist one intent or return its existing identical durable state.""" if not isinstance(request, CloseRequest): raise InvalidCloseRequestError("request must be a CloseRequest") with self._locked(): for state, directory in ( ("delivered", self.delivered_dir), ("pending", self.pending_dir), ): path = directory / f"{request.entry_id}.json" if path.exists(): record = self._load(path) if record.state != state: raise OutboxCorruptError( f"outbox record directory/state mismatch: expected {state}" ) if record.request.intent_digest != request.intent_digest: raise OutboxConflictError( "run/transaction identity already has a different close intent" ) return EnqueueReceipt(request.entry_id, False, state) if tuple(self.quarantine_dir.glob(f"{request.entry_id}.*.json")): raise OutboxCorruptError( "run/transaction identity has quarantined outbox material" ) record = _Record( request=request, state="pending", created_at=_now(), ) _atomic_write_json( self.pending_dir / f"{request.entry_id}.json", record.payload(), ) return EnqueueReceipt(request.entry_id, True, "pending") def replay( self, deliver: Callable[[CloseRequest], Any], *, limit: int = 100, ) -> ReplayReport: """Attempt pending deliveries once each without executing workload code.""" if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000: raise ValueError("limit must be an integer between 1 and 1000") attempted = delivered = failed = quarantined = 0 with self._locked(): paths = sorted(self.pending_dir.glob("*.json"))[:limit] for path in paths: try: record = self._load(path) except OutboxCorruptError as exc: self._quarantine(path, str(exc)) quarantined += 1 continue if record.state != "pending": self._quarantine( path, f"pending directory contains state {record.state!r}", ) quarantined += 1 continue destination = self.delivered_dir / path.name if destination.exists(): self._quarantine( path, "pending entry conflicts with existing delivered material", ) quarantined += 1 continue attempted += 1 attempted_at = _now() trying = _Record( request=record.request, state="pending", created_at=record.created_at, attempts=record.attempts + 1, last_attempt_at=attempted_at, last_error=None, ) _atomic_write_json(path, trying.payload()) try: deliver(record.request.detached_copy()) except Exception as exc: retained = _Record( request=record.request, state="pending", created_at=record.created_at, attempts=trying.attempts, last_attempt_at=attempted_at, last_error=( f"{type(exc).__name__}: close delivery failed" )[:1000], ) _atomic_write_json(path, retained.payload()) failed += 1 continue completed = _Record( request=record.request, state="delivered", created_at=record.created_at, attempts=trying.attempts, last_attempt_at=attempted_at, last_error=None, ) _atomic_write_json(path, completed.payload()) os.replace(path, destination) _fsync_dir(self.pending_dir) _fsync_dir(self.delivered_dir) delivered += 1 remaining = len(tuple(self.pending_dir.glob("*.json"))) return ReplayReport( attempted=attempted, delivered=delivered, failed=failed, quarantined=quarantined, remaining=remaining, ) def pending_count(self) -> int: with self._locked(): return len(tuple(self.pending_dir.glob("*.json"))) def _load(self, path: Path) -> _Record: try: if path.stat().st_size > _MAX_PAYLOAD_BYTES + 8192: raise OutboxCorruptError("outbox record exceeds the durable size bound") raw = path.read_text(encoding="utf-8") value = json.loads(raw) return _Record.from_payload(value) except OutboxCorruptError: raise except (OSError, UnicodeError, ValueError, TypeError) as exc: raise OutboxCorruptError( f"cannot decode outbox record {path.name}: {type(exc).__name__}" ) from exc def _quarantine(self, path: Path, reason: str) -> None: suffix = uuid.uuid4().hex[:12] target = self.quarantine_dir / f"{path.stem}.{suffix}.json" os.replace(path, target) _fsync_dir(self.pending_dir) _fsync_dir(self.quarantine_dir) _atomic_write_json( self.quarantine_dir / f"{path.stem}.{suffix}.error.json", { "entry_id": path.stem, "quarantined_at": _now(), "reason": _bounded_text(reason, 1000), }, ) @contextmanager def _locked(self) -> Iterator[None]: lock_file = _open_private_file(self.lock_path) try: fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) yield finally: try: fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) finally: lock_file.close() def _state_dir() -> Path: explicit = os.environ.get("REIN_AHARNESS_STATE_DIR", "").strip() if explicit: return Path(explicit).expanduser().resolve() xdg = os.environ.get("XDG_STATE_HOME", "").strip() if xdg: return (Path(xdg).expanduser() / "rein-aharness").resolve() return (Path.home() / ".local" / "state" / "rein-aharness").resolve() def _validate_identifier(value: Any, name: str) -> None: if not isinstance(value, str) or _IDENTIFIER.fullmatch(value) is None: raise InvalidCloseRequestError( f"{name} must be a bounded, path-safe identifier" ) def _normalize_json(value: Any, *, depth: int = 0) -> Any: if depth > _MAX_JSON_DEPTH: raise InvalidCloseRequestError("result exceeds the JSON depth bound") if value is None or isinstance(value, bool): return value if isinstance(value, str): if len(value) > _MAX_STRING_LENGTH: raise InvalidCloseRequestError( f"result string exceeds {_MAX_STRING_LENGTH} characters" ) return value if isinstance(value, int): if abs(value) > 2**63 - 1: raise InvalidCloseRequestError("result integer exceeds signed 64-bit range") return value if isinstance(value, float): if not math.isfinite(value): raise InvalidCloseRequestError("result floats must be finite") return value if isinstance(value, list): if len(value) > _MAX_COLLECTION_ITEMS: raise InvalidCloseRequestError("result array exceeds the item bound") return [_normalize_json(item, depth=depth + 1) for item in value] if isinstance(value, dict): if len(value) > _MAX_COLLECTION_ITEMS: raise InvalidCloseRequestError("result object exceeds the field bound") normalized: dict[str, Any] = {} for key, item in value.items(): if not isinstance(key, str) or not key or len(key) > 200: raise InvalidCloseRequestError( "result field names must be non-empty strings of at most 200 characters" ) normalized[key] = _normalize_json(item, depth=depth + 1) return normalized raise InvalidCloseRequestError( f"result contains unsupported JSON type: {type(value).__name__}" ) def _validate_timestamp(value: Any, name: str) -> None: if not isinstance(value, str): raise OutboxCorruptError(f"{name} must be a timestamp string") try: parsed = datetime.fromisoformat(value) except ValueError as exc: raise OutboxCorruptError(f"{name} is not an ISO timestamp") from exc if parsed.tzinfo is None: raise OutboxCorruptError(f"{name} must include a timezone") def _ensure_private_dir(path: Path) -> None: path.mkdir(parents=True, exist_ok=True) os.chmod(path, 0o700) def _open_private_file(path: Path) -> TextIO: fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) try: os.fchmod(fd, 0o600) return os.fdopen(fd, "r+", encoding="utf-8") except BaseException: os.close(fd) raise def _atomic_write_json(path: Path, value: dict[str, Any]) -> None: data = _json_bytes(value) + b"\n" temp = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" fd = os.open(temp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) try: os.fchmod(fd, 0o600) with os.fdopen(fd, "wb") as handle: fd = -1 handle.write(data) handle.flush() os.fsync(handle.fileno()) os.replace(temp, path) os.chmod(path, 0o600) _fsync_dir(path.parent) except BaseException: if fd >= 0: os.close(fd) try: temp.unlink() except FileNotFoundError: pass raise def _fsync_dir(path: Path) -> None: fd = os.open(path, os.O_RDONLY) try: os.fsync(fd) finally: os.close(fd) def _json_bytes(value: Any) -> bytes: return json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False, ).encode("utf-8") def _bounded_text(value: str, limit: int) -> str: return " ".join(value.split())[:limit] def _now() -> str: return datetime.now(timezone.utc).isoformat()