Implement P05 checked services and safe selected delivery recovery
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s
Build and Publish Container Image / build-and-push (push) Successful in 19s
Account journey acceptance / journeys (push) Successful in 5s

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a092fe-13b1-7f12-ac74-7d258af4d79c
This commit is contained in:
tegwick 2026-09-13 22:11:49 +02:00
parent 99618488e0
commit ac0eb14b75
15 changed files with 735 additions and 88 deletions

View file

@ -3,7 +3,20 @@
from __future__ import annotations
import json
from urllib.request import Request, urlopen
from urllib.request import Request, urlopen, build_opener, HTTPRedirectHandler
from urllib.error import HTTPError
class DeliveryError(RuntimeError):
def __init__(self, code):
self.code = code
super().__init__(code)
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
from user_engine.domain import OutboxEvent
@ -39,6 +52,8 @@ class HTTPOutboxDeliveryAdapter:
"occurred_at": event.occurred_at.isoformat(),
"data": dict(event.payload),
}
if event.event_type in _MAIL_EVENTS and not self.mail_url:
raise DeliveryError("mail_unconfigured")
if self.mail_url and event.event_type in _MAIL_EVENTS:
if not self.mail_bearer_token:
raise RuntimeError("mail delivery token is required")
@ -51,21 +66,78 @@ class HTTPOutboxDeliveryAdapter:
event_envelope["data"] = event_data
self._post(self.event_url, event_envelope, self.event_bearer_token)
def _post(
self, url: str, envelope: dict[str, object], bearer_token: str
) -> None:
with urlopen(
Request(
url,
data=json.dumps(envelope).encode(),
headers={
"Authorization": f"Bearer {bearer_token}",
"Content-Type": "application/json",
"Idempotency-Key": str(envelope["id"]),
},
method="POST",
),
timeout=self.timeout_seconds,
def _post(self, url: str, envelope: dict[str, object], bearer_token: str) -> None:
request = Request(
url,
data=json.dumps(envelope).encode(),
headers={
"Authorization": f"Bearer {bearer_token}",
"Content-Type": "application/json",
"Idempotency-Key": str(envelope["id"]),
},
method="POST",
)
try:
response = build_opener(NoRedirect()).open(
request, timeout=self.timeout_seconds
)
except HTTPError as exc:
code = "provider_unavailable"
try:
body = json.loads(exc.read(8192))
candidate = body.get("error")
if candidate in {
"delivery_outcome_unknown",
"temporary_deferral",
"permanent_rejection",
"recipient_suppressed",
}:
code = candidate
except Exception:
pass
raise DeliveryError(code) from None
except Exception:
raise DeliveryError("provider_unavailable") from None
with response:
if not 200 <= response.status < 300:
raise DeliveryError("provider_unavailable")
def mail_status(self):
if not self.mail_url or not self.mail_bearer_token:
return {"checks": {"smtp_authentication": "unconfigured"}}
from urllib.parse import urlsplit, urlunsplit
parsed = urlsplit(self.mail_url)
url = urlunsplit((parsed.scheme, parsed.netloc, "/v1/status", "", ""))
req = Request(
url,
data=b"{}",
headers={
"Authorization": "Bearer " + self.mail_bearer_token,
"Content-Type": "application/json",
},
method="POST",
)
with build_opener(NoRedirect()).open(req, timeout=7) as response:
return json.loads(response.read(8192))
def mail_delivery_status(self, event_id):
if not self.mail_url or not self.mail_bearer_token:
return {"state": "unconfigured"}
from urllib.parse import urlsplit, urlunsplit
parsed = urlsplit(self.mail_url)
url = urlunsplit((parsed.scheme, parsed.netloc, "/v1/delivery-status", "", ""))
request = Request(
url,
data=json.dumps({"event_id": event_id}).encode(),
headers={
"Authorization": "Bearer " + self.mail_bearer_token,
"Content-Type": "application/json",
},
method="POST",
)
with build_opener(NoRedirect()).open(
request, timeout=self.timeout_seconds
) as response:
if response.status < 200 or response.status >= 300:
raise RuntimeError(f"delivery rejected with HTTP {response.status}")
return json.loads(response.read(8192))

View file

@ -87,6 +87,12 @@ class InMemoryUserEngineStore:
with self._lifecycle_lock:
yield
@contextmanager
def outbox_delivery_guard(self, event_id: str):
# Separate namespace; reuse the existing cross-connection lock primitive.
with self.tenant_lifecycle_guard("outbox-event:" + event_id):
yield
def outbox_history(self) -> tuple[OutboxEvent, ...]:
return tuple(self.outbox_events)

View file

@ -103,6 +103,12 @@ class PostgresUserEngineStore:
with self._cursor() as cursor:
cursor.execute("SELECT pg_advisory_unlock(hashtextextended(%s, 0))", (key,))
@contextmanager
def outbox_delivery_guard(self, event_id: str):
# Separate namespace; reuse the existing cross-connection lock primitive.
with self.tenant_lifecycle_guard("outbox-event:" + event_id):
yield
def outbox_history(self) -> tuple[OutboxEvent, ...]:
with self._cursor() as cursor:
cursor.execute("SELECT payload FROM user_engine_outbox_events ORDER BY occurred_at, event_id")

View file

@ -0,0 +1,143 @@
"""Bounded, non-sending diagnostics rendered only after platform authorization."""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from html import escape
import json
from urllib.request import Request, build_opener, HTTPRedirectHandler
from urllib.error import HTTPError
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
def sign_in_status(base):
try:
response = build_opener(NoRedirect()).open(
base.rstrip("/") + "/readyz", timeout=5
)
except HTTPError as exc:
response = exc
with response:
raw = response.read(16385)
if len(raw) > 16384:
raise ValueError("oversized status")
return json.loads(raw)
def check_services(oidc, delivery):
def safe(call):
try:
return call()
except Exception:
return {}
with ThreadPoolExecutor(max_workers=2) as pool:
sign = pool.submit(
safe, lambda: sign_in_status(oidc.backend_url) if oidc else {}
)
mail = pool.submit(
safe,
lambda: delivery.mail_status() if hasattr(delivery, "mail_status") else {},
)
sign, mail = sign.result(), mail.result()
values = {}
sign_checks = sign.get("checks", []) if isinstance(sign, dict) else []
for row in sign_checks if isinstance(sign_checks, list) else []:
if isinstance(row, dict):
values[row.get("name")] = row.get("status")
checks = [
(
"Directory sign-in",
values.get("lldap"),
"Sign-in and identity lookup",
"Retry the check; if it still fails, ask the identity service owner to restore directory connectivity.",
),
(
"Primary authentication",
values.get("authelia"),
"Fresh sign-in",
"Retry after the authentication service recovers. Existing sessions do not prove new sign-in works.",
),
(
"Authenticator provider",
values.get("privacyidea"),
"MFA verification and enrollment",
"Retry after the MFA service recovers. Do not disable MFA to work around an outage.",
),
(
"Factor-read credential",
values.get("factor_reader"),
"Enrollment decisions and MFA verification",
"Renewal is automatic. A failure can mean expiry, lost permissions or projection failure; ask the credential owner to run the scoped renewal check.",
),
]
statuses = mail.get("checks", {}) if isinstance(mail, dict) else {}
if not isinstance(statuses, dict):
statuses = {}
checks += [
(
"Mail record store",
statuses.get("database"),
"Durable delivery tracking",
"Restore the mail store before resuming delivery.",
),
(
"SMTP authentication",
statuses.get("smtp_authentication"),
"Invitation and verification mail",
"Check the configured mail lane. Use assisted password setup when the person cannot receive email.",
),
]
rows = ""
for name, status, impact, recovery in checks:
label = {
"ok": "Check passed",
"failed": "Unavailable",
"unconfigured": "Not configured",
}.get(status, "Unknown — check unavailable")
rows += (
"<tr><td>"
+ escape(name)
+ "</td><td>"
+ escape(label)
+ "</td><td>"
+ escape(impact)
+ "</td><td>"
+ escape(recovery)
+ "</td></tr>"
)
return (
"<section><h2>Checked service state</h2><p>Checked at "
+ escape(datetime.now(timezone.utc).isoformat(timespec="seconds"))
+ ". SMTP authentication may be cached for 30 seconds. These checks send no email and do not prove inbox receipt.</p><table><thead><tr><th>Component</th><th>State</th><th>Affected journeys</th><th>Recovery</th></tr></thead><tbody>"
+ rows
+ '</tbody></table><p><a href="/platform/operations">Check services again</a></p></section>'
)
def selected_mail_status(delivery, event):
if event.event_type not in {"family_member.invited", "family_invitation.resent"}:
return ""
try:
receipt = delivery.mail_delivery_status(event.event_id)
state = receipt.get("state") if isinstance(receipt, dict) else None
except Exception:
state = None
detail = {
"provider_accepted": "The mail provider accepted this event. This does not prove inbox receipt. Retrying this same record reconciles the remaining event delivery without submitting another invitation.",
"unknown": "The SMTP outcome is uncertain. Automatic resending is blocked. Ask the mail service owner to reconcile the provider record using this delivery ID; use assisted password setup if access is urgent.",
"failed": "The previous mail attempt definitely failed. Review a delivery attempt to retry this same record.",
"not_found": "No mail submission is recorded. Review a delivery attempt to submit this selected record.",
"unconfigured": "The mail lane is not configured. Use assisted password setup or restore the mail lane.",
}.get(
state,
"Mail evidence is unavailable. Check the mail service and retry this status check; do not infer that a message was sent or received.",
)
return (
"<section><h2>Selected mail evidence</h2><p>"
+ escape(detail)
+ "</p></section>"
)

View file

@ -437,6 +437,9 @@ class UserEngineStore(Protocol):
def pending_outbox(self) -> tuple[OutboxEvent, ...]:
"""Return pending outbox events in write order."""
def outbox_delivery_guard(self, event_id: str) -> ContextManager[None]:
"""Serialize delivery/replay for one event across processes."""
def save_outbox(self, event: OutboxEvent) -> None:
"""Persist outbox delivery state."""

View file

@ -63,7 +63,7 @@ def create_application() -> PortalApplication:
),
timeout_seconds=float(os.environ.get("USER_ENGINE_DELIVERY_TIMEOUT", "5")),
)
return PortalApplication(
app = PortalApplication(
service,
trusted_proxy_secret=_required("USER_ENGINE_PROXY_SECRET"),
login_url=_required("USER_ENGINE_LOGIN_URL"),
@ -120,6 +120,9 @@ def create_application() -> PortalApplication:
),
)
from user_engine.operations_status import check_services
app.operations_probe = lambda: check_services(app.oidc_client, app.outbox_delivery)
return app
def main() -> None:
host = os.environ.get("USER_ENGINE_HOST", "0.0.0.0")

View file

@ -2867,71 +2867,65 @@ class UserEngineService:
return self.store.pending_outbox()
def deliver_outbox(
self,
actor: Actor,
deliver: Callable[[OutboxEvent], None],
*,
worker_id: str,
max_attempts: int = 3,
correlation_id: str | None = None,
self, actor: Actor, deliver: Callable[[OutboxEvent], None], *,
worker_id: str, max_attempts: int = 3, correlation_id: str | None = None,
event_id: str | None = None, expected_attempts: int | None = None,
) -> tuple[OutboxEvent, ...]:
"""Claim and deliver pending events, retaining bounded failure details."""
if not worker_id.strip():
raise ValidationError("outbox worker_id is required")
if not 1 <= max_attempts <= 20:
raise ValidationError("outbox max_attempts must be between 1 and 20")
"""Serialize each external attempt and retain only classified failures."""
if not worker_id.strip() or not 1 <= max_attempts <= 20:
raise ValidationError("invalid outbox worker or attempt limit")
correlation_id = correlation_id or new_id("corr")
self._authorize(
actor, action="outbox.deliver", resource_type="user-engine:outbox",
resource_id="pending", tenant=actor.tenant,
correlation_id=correlation_id,
)
results: list[OutboxEvent] = []
for event in self.store.pending_outbox():
claimed = replace(
event, claimed_by=worker_id, claimed_at=utc_now(),
delivery_attempts=event.delivery_attempts + 1,
failed_at=None, failure_reason=None,
)
with self.store.transaction():
self.store.save_outbox(claimed)
try:
deliver(claimed)
except Exception as exc:
reason = str(exc).strip()[:200] or type(exc).__name__
failed = replace(
claimed, failed_at=utc_now(), failure_reason=reason,
dead_lettered_at=(utc_now() if claimed.delivery_attempts >= max_attempts else None),
)
with self.store.transaction():
self.store.save_outbox(failed)
results.append(failed)
else:
delivered = replace(claimed, delivered_at=utc_now())
with self.store.transaction():
self.store.save_outbox(delivered)
results.append(delivered)
self._authorize(actor, action="outbox.deliver", resource_type="user-engine:outbox",
resource_id="pending", tenant=actor.tenant, correlation_id=correlation_id)
selected = self.store.outbox_event(event_id) if event_id else None
if event_id and selected is None:
raise NotFoundError("delivery record not found")
events = (selected,) if selected else self.store.pending_outbox()
results = []
for candidate in events:
with self.store.outbox_delivery_guard(candidate.event_id):
event = self.store.outbox_event(candidate.event_id)
if event is None or event.delivered_at is not None:
continue
if expected_attempts is not None and event.delivery_attempts != expected_attempts:
raise ConflictError("Delivery changed. Refresh and review the attempt again.")
# Owning this cross-process guard proves any previous claim is orphaned.
# Receiver idempotency reconciles a lost receipt without blind resending.
if event.dead_lettered_at and not event_id:
continue
claimed = replace(event, claimed_by=worker_id, claimed_at=utc_now(),
delivery_attempts=event.delivery_attempts+1, failed_at=None, failure_reason=None)
with self.store.transaction(): self.store.save_outbox(claimed)
try:
deliver(claimed)
except Exception as exc:
code = getattr(exc, "code", "delivery_unavailable")
if code not in {"mail_unconfigured", "delivery_outcome_unknown", "temporary_deferral", "permanent_rejection", "recipient_suppressed", "provider_timeout", "provider_unavailable"}:
code = "delivery_unavailable"
outcome = replace(claimed, claimed_by=None, claimed_at=None,
failed_at=utc_now(), failure_reason=code,
dead_lettered_at=utc_now() if claimed.delivery_attempts >= max_attempts else None)
else:
outcome = replace(claimed, claimed_by=None, claimed_at=None,
delivered_at=utc_now(), dead_lettered_at=None)
with self.store.transaction(): self.store.save_outbox(outcome)
results.append(outcome)
return tuple(results)
def replay_outbox(
self, actor: Actor, event_id: str, *, correlation_id: str | None = None
) -> OutboxEvent:
def replay_outbox(self, actor: Actor, event_id: str, *, correlation_id: str | None = None) -> OutboxEvent:
correlation_id = correlation_id or new_id("corr")
event = self.store.outbox_event(event_id)
if event is None:
raise NotFoundError("outbox event not found")
self.resolve_tenant_context(actor, event.tenant)
self._authorize(
actor, action="outbox.replay", resource_type="user-engine:outbox-event",
resource_id=event_id, tenant=event.tenant, correlation_id=correlation_id,
)
replayed = replace(
event, claimed_by=None, claimed_at=None, failed_at=None,
failure_reason=None, dead_lettered_at=None,
)
with self.store.transaction():
self.store.save_outbox(replayed)
return replayed
with self.store.outbox_delivery_guard(event_id):
event = self.store.outbox_event(event_id)
if event is None: raise NotFoundError("outbox event not found")
self.resolve_tenant_context(actor, event.tenant)
self._authorize(actor, action="outbox.replay", resource_type="user-engine:outbox-event",
resource_id=event_id, tenant=event.tenant, correlation_id=correlation_id)
if event.delivered_at or (event.claimed_by and not event.failed_at):
raise ConflictError("Delivery is completed or its previous attempt is unresolved.")
replayed = replace(event, claimed_by=None, claimed_at=None, failed_at=None,
failure_reason=None, dead_lettered_at=None)
with self.store.transaction(): self.store.save_outbox(replayed)
return replayed
def outbox_diagnostics(self) -> OutboxDiagnostics:
event_types: dict[str, int] = {}

View file

@ -109,6 +109,7 @@ class PortalApplication:
self.provisioning = provisioning
self.tenant_management = tenant_management
self.outbox_delivery = outbox_delivery
self.operations_probe = None
self.registration_verification = registration_verification
self.registration_clients = frozenset(registration_clients)
self.registration_tenants = frozenset(registration_tenants)
@ -791,6 +792,27 @@ class PortalApplication:
query = parse_qs(str(environ.get("QUERY_STRING", "")))
return self._html(start_response, self._platform_activity(
query.get("reference", [""])[0], query.get("tenant", [""])[0]), correlation_id)
if path == "/platform/operations/deliver" and method == "POST":
self.service.resolve_tenant_context(actor, PLATFORM_TENANT)
body = self._form_body(environ)
self._require_csrf(environ, body.get("csrf_token", ""))
event_id = body.get("event_id", "")
with self.service.store.outbox_delivery_guard(event_id):
event = self.service.store.outbox_event(event_id)
if event is None: raise NotFoundError("delivery record not found")
if event.delivered_at:
raise ConflictError("Delivery is already completed.")
if self.outbox_delivery is None: raise ValidationError("Delivery is unavailable")
confirmation = self._confirm_change(environ, start_response, body,
json.dumps(_jsonable(event),sort_keys=True), "Attempt this delivery",
"This submits only delivery " + event.event_id + " for " + event.tenant +
". Invitation messages will be sent to the recorded recipient. Provider acceptance does not prove the person received it. Keep support reference " + event.correlation_id + ".",
correlation_id)
if confirmation is not None: return confirmation
self.service.deliver_outbox(actor, self.outbox_delivery,
worker_id="platform-operator",event_id=event.event_id,
expected_attempts=event.delivery_attempts,correlation_id=correlation_id)
return self._redirect(start_response,"/platform/operations?"+urlencode({"event_id":event_id}),correlation_id)
if path in {"/platform/operations", "/platform/operations/replay"}:
self.service.resolve_tenant_context(actor, PLATFORM_TENANT)
if method == "POST" and path.endswith("/replay"):
@ -799,7 +821,7 @@ class PortalApplication:
event = self.service.store.outbox_event(str(body.get("event_id", "")))
if event is None:
raise NotFoundError("delivery record not found")
if event.delivered_at is not None or event.claimed_by:
if event.delivered_at is not None or (event.claimed_by and not event.failed_at):
raise ConflictError("Delivery is already completed or being processed. Refresh its status.")
self.service.replay_outbox(actor, event.event_id, correlation_id=correlation_id)
return self._redirect(start_response, "/platform/operations?"+urlencode({"event_id":event.event_id}), correlation_id)
@ -1191,6 +1213,8 @@ class PortalApplication:
@staticmethod
def _delivery_status(event: Any) -> str:
if event.delivered_at: return "Accepted by delivery adapter; receipt by the person is unverified"
if event.failure_reason == "delivery_outcome_unknown": return "Mail outcome unknown — check provider evidence before sending another message"
if event.failure_reason == "mail_unconfigured": return "Mail is not configured — use assisted setup or restore the mail lane"
if event.dead_lettered_at: return "Delivery stopped after repeated failures"
if event.failed_at: return "Delivery failed; retry pending"
if event.claimed_by: return "Being processed"
@ -1242,15 +1266,19 @@ class PortalApplication:
rows = ""
for event in events:
action = ""
if event.delivered_at is None and not event.claimed_by and (event.failed_at or event.dead_lettered_at):
action = f'<form method="post" action="/platform/operations/replay"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Queue a retry</button></form>'
if event.delivered_at is None:
action = f'<form method="post" action="/platform/operations/deliver"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Review delivery attempt</button></form>'
if event.delivered_at is None and (not event.claimed_by or event.failed_at) and (event.failed_at or event.dead_lettered_at):
action += f'<form method="post" action="/platform/operations/replay"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Queue a retry</button></form>'
rows += f'<tr><td>{escape(event.event_id)}</td><td>{escape(event.tenant)}</td><td>{escape(event.event_type)}</td><td>{escape(self._delivery_status(event))}</td><td>{escape(event.correlation_id)}</td><td>{action}</td></tr>'
configuration = self._operation_capabilities()
return self._page_html("Service recovery", '<h1>Service recovery</h1>' + configuration + '<p><a href="/platform/activity">Investigate a support reference</a></p><p>This view shows local delivery records. Live sign-in, email receipt and authenticator health are not verified here.</p>'
from user_engine.operations_status import check_services, selected_mail_status
evidence = selected_mail_status(self.outbox_delivery, events[0]) if event_id else ""
configuration = (self.operations_probe() if self.operations_probe else check_services(None,None)) + self._operation_capabilities()
return self._page_html("Service recovery", '<h1>Service recovery</h1>' + configuration + evidence + '<p><a href="/platform/activity">Investigate a support reference</a></p><p>Delivery records below describe recorded attempts. Component checks do not prove a complete sign-in or inbox receipt.</p>'
'<form method="get" action="/platform/operations"><label>Delivery record ID <input name="event_id"></label><button type="submit">Find delivery</button></form>'
'<table><thead><tr><th>Delivery</th><th>Tenant</th><th>Kind</th><th>Status</th><th>Support reference</th><th>Recovery</th></tr></thead><tbody>'
+ (rows or '<tr><td colspan="6">No delivery records. This does not prove mail was received.</td></tr>')
+ '</tbody></table><p>Queued retries are processed by the delivery worker. Check the record again for the result.</p><p><a href="/platform">Return to platform administration</a></p>')
+ '</tbody></table><p>Queueing a retry does not send it. Use Review delivery attempt to submit one selected record, then check its result. No background worker is enabled here.</p><p><a href="/platform">Return to platform administration</a></p>')
def _operation_capabilities(self) -> str:
capabilities = (