Implement P05 checked services and safe selected delivery recovery
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a092fe-13b1-7f12-ac74-7d258af4d79c
This commit is contained in:
parent
99618488e0
commit
ac0eb14b75
15 changed files with 735 additions and 88 deletions
|
|
@ -3,7 +3,20 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.request import Request, urlopen, build_opener, HTTPRedirectHandler
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
class DeliveryError(RuntimeError):
|
||||
def __init__(self, code):
|
||||
self.code = code
|
||||
super().__init__(code)
|
||||
|
||||
|
||||
class NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
from user_engine.domain import OutboxEvent
|
||||
|
||||
|
|
@ -39,6 +52,8 @@ class HTTPOutboxDeliveryAdapter:
|
|||
"occurred_at": event.occurred_at.isoformat(),
|
||||
"data": dict(event.payload),
|
||||
}
|
||||
if event.event_type in _MAIL_EVENTS and not self.mail_url:
|
||||
raise DeliveryError("mail_unconfigured")
|
||||
if self.mail_url and event.event_type in _MAIL_EVENTS:
|
||||
if not self.mail_bearer_token:
|
||||
raise RuntimeError("mail delivery token is required")
|
||||
|
|
@ -51,21 +66,78 @@ class HTTPOutboxDeliveryAdapter:
|
|||
event_envelope["data"] = event_data
|
||||
self._post(self.event_url, event_envelope, self.event_bearer_token)
|
||||
|
||||
def _post(
|
||||
self, url: str, envelope: dict[str, object], bearer_token: str
|
||||
) -> None:
|
||||
with urlopen(
|
||||
Request(
|
||||
url,
|
||||
data=json.dumps(envelope).encode(),
|
||||
headers={
|
||||
"Authorization": f"Bearer {bearer_token}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": str(envelope["id"]),
|
||||
},
|
||||
method="POST",
|
||||
),
|
||||
timeout=self.timeout_seconds,
|
||||
def _post(self, url: str, envelope: dict[str, object], bearer_token: str) -> None:
|
||||
request = Request(
|
||||
url,
|
||||
data=json.dumps(envelope).encode(),
|
||||
headers={
|
||||
"Authorization": f"Bearer {bearer_token}",
|
||||
"Content-Type": "application/json",
|
||||
"Idempotency-Key": str(envelope["id"]),
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
response = build_opener(NoRedirect()).open(
|
||||
request, timeout=self.timeout_seconds
|
||||
)
|
||||
except HTTPError as exc:
|
||||
code = "provider_unavailable"
|
||||
try:
|
||||
body = json.loads(exc.read(8192))
|
||||
candidate = body.get("error")
|
||||
if candidate in {
|
||||
"delivery_outcome_unknown",
|
||||
"temporary_deferral",
|
||||
"permanent_rejection",
|
||||
"recipient_suppressed",
|
||||
}:
|
||||
code = candidate
|
||||
except Exception:
|
||||
pass
|
||||
raise DeliveryError(code) from None
|
||||
except Exception:
|
||||
raise DeliveryError("provider_unavailable") from None
|
||||
with response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise DeliveryError("provider_unavailable")
|
||||
|
||||
def mail_status(self):
|
||||
if not self.mail_url or not self.mail_bearer_token:
|
||||
return {"checks": {"smtp_authentication": "unconfigured"}}
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
parsed = urlsplit(self.mail_url)
|
||||
url = urlunsplit((parsed.scheme, parsed.netloc, "/v1/status", "", ""))
|
||||
req = Request(
|
||||
url,
|
||||
data=b"{}",
|
||||
headers={
|
||||
"Authorization": "Bearer " + self.mail_bearer_token,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with build_opener(NoRedirect()).open(req, timeout=7) as response:
|
||||
return json.loads(response.read(8192))
|
||||
|
||||
def mail_delivery_status(self, event_id):
|
||||
if not self.mail_url or not self.mail_bearer_token:
|
||||
return {"state": "unconfigured"}
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
parsed = urlsplit(self.mail_url)
|
||||
url = urlunsplit((parsed.scheme, parsed.netloc, "/v1/delivery-status", "", ""))
|
||||
request = Request(
|
||||
url,
|
||||
data=json.dumps({"event_id": event_id}).encode(),
|
||||
headers={
|
||||
"Authorization": "Bearer " + self.mail_bearer_token,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with build_opener(NoRedirect()).open(
|
||||
request, timeout=self.timeout_seconds
|
||||
) as response:
|
||||
if response.status < 200 or response.status >= 300:
|
||||
raise RuntimeError(f"delivery rejected with HTTP {response.status}")
|
||||
return json.loads(response.read(8192))
|
||||
|
|
|
|||
|
|
@ -87,6 +87,12 @@ class InMemoryUserEngineStore:
|
|||
with self._lifecycle_lock:
|
||||
yield
|
||||
|
||||
@contextmanager
|
||||
def outbox_delivery_guard(self, event_id: str):
|
||||
# Separate namespace; reuse the existing cross-connection lock primitive.
|
||||
with self.tenant_lifecycle_guard("outbox-event:" + event_id):
|
||||
yield
|
||||
|
||||
def outbox_history(self) -> tuple[OutboxEvent, ...]:
|
||||
return tuple(self.outbox_events)
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,12 @@ class PostgresUserEngineStore:
|
|||
with self._cursor() as cursor:
|
||||
cursor.execute("SELECT pg_advisory_unlock(hashtextextended(%s, 0))", (key,))
|
||||
|
||||
@contextmanager
|
||||
def outbox_delivery_guard(self, event_id: str):
|
||||
# Separate namespace; reuse the existing cross-connection lock primitive.
|
||||
with self.tenant_lifecycle_guard("outbox-event:" + event_id):
|
||||
yield
|
||||
|
||||
def outbox_history(self) -> tuple[OutboxEvent, ...]:
|
||||
with self._cursor() as cursor:
|
||||
cursor.execute("SELECT payload FROM user_engine_outbox_events ORDER BY occurred_at, event_id")
|
||||
|
|
|
|||
143
src/user_engine/operations_status.py
Normal file
143
src/user_engine/operations_status.py
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
"""Bounded, non-sending diagnostics rendered only after platform authorization."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from html import escape
|
||||
import json
|
||||
from urllib.request import Request, build_opener, HTTPRedirectHandler
|
||||
from urllib.error import HTTPError
|
||||
|
||||
|
||||
class NoRedirect(HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
def sign_in_status(base):
|
||||
try:
|
||||
response = build_opener(NoRedirect()).open(
|
||||
base.rstrip("/") + "/readyz", timeout=5
|
||||
)
|
||||
except HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
raw = response.read(16385)
|
||||
if len(raw) > 16384:
|
||||
raise ValueError("oversized status")
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def check_services(oidc, delivery):
|
||||
def safe(call):
|
||||
try:
|
||||
return call()
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
sign = pool.submit(
|
||||
safe, lambda: sign_in_status(oidc.backend_url) if oidc else {}
|
||||
)
|
||||
mail = pool.submit(
|
||||
safe,
|
||||
lambda: delivery.mail_status() if hasattr(delivery, "mail_status") else {},
|
||||
)
|
||||
sign, mail = sign.result(), mail.result()
|
||||
values = {}
|
||||
sign_checks = sign.get("checks", []) if isinstance(sign, dict) else []
|
||||
for row in sign_checks if isinstance(sign_checks, list) else []:
|
||||
if isinstance(row, dict):
|
||||
values[row.get("name")] = row.get("status")
|
||||
checks = [
|
||||
(
|
||||
"Directory sign-in",
|
||||
values.get("lldap"),
|
||||
"Sign-in and identity lookup",
|
||||
"Retry the check; if it still fails, ask the identity service owner to restore directory connectivity.",
|
||||
),
|
||||
(
|
||||
"Primary authentication",
|
||||
values.get("authelia"),
|
||||
"Fresh sign-in",
|
||||
"Retry after the authentication service recovers. Existing sessions do not prove new sign-in works.",
|
||||
),
|
||||
(
|
||||
"Authenticator provider",
|
||||
values.get("privacyidea"),
|
||||
"MFA verification and enrollment",
|
||||
"Retry after the MFA service recovers. Do not disable MFA to work around an outage.",
|
||||
),
|
||||
(
|
||||
"Factor-read credential",
|
||||
values.get("factor_reader"),
|
||||
"Enrollment decisions and MFA verification",
|
||||
"Renewal is automatic. A failure can mean expiry, lost permissions or projection failure; ask the credential owner to run the scoped renewal check.",
|
||||
),
|
||||
]
|
||||
statuses = mail.get("checks", {}) if isinstance(mail, dict) else {}
|
||||
if not isinstance(statuses, dict):
|
||||
statuses = {}
|
||||
checks += [
|
||||
(
|
||||
"Mail record store",
|
||||
statuses.get("database"),
|
||||
"Durable delivery tracking",
|
||||
"Restore the mail store before resuming delivery.",
|
||||
),
|
||||
(
|
||||
"SMTP authentication",
|
||||
statuses.get("smtp_authentication"),
|
||||
"Invitation and verification mail",
|
||||
"Check the configured mail lane. Use assisted password setup when the person cannot receive email.",
|
||||
),
|
||||
]
|
||||
rows = ""
|
||||
for name, status, impact, recovery in checks:
|
||||
label = {
|
||||
"ok": "Check passed",
|
||||
"failed": "Unavailable",
|
||||
"unconfigured": "Not configured",
|
||||
}.get(status, "Unknown — check unavailable")
|
||||
rows += (
|
||||
"<tr><td>"
|
||||
+ escape(name)
|
||||
+ "</td><td>"
|
||||
+ escape(label)
|
||||
+ "</td><td>"
|
||||
+ escape(impact)
|
||||
+ "</td><td>"
|
||||
+ escape(recovery)
|
||||
+ "</td></tr>"
|
||||
)
|
||||
return (
|
||||
"<section><h2>Checked service state</h2><p>Checked at "
|
||||
+ escape(datetime.now(timezone.utc).isoformat(timespec="seconds"))
|
||||
+ ". SMTP authentication may be cached for 30 seconds. These checks send no email and do not prove inbox receipt.</p><table><thead><tr><th>Component</th><th>State</th><th>Affected journeys</th><th>Recovery</th></tr></thead><tbody>"
|
||||
+ rows
|
||||
+ '</tbody></table><p><a href="/platform/operations">Check services again</a></p></section>'
|
||||
)
|
||||
|
||||
|
||||
def selected_mail_status(delivery, event):
|
||||
if event.event_type not in {"family_member.invited", "family_invitation.resent"}:
|
||||
return ""
|
||||
try:
|
||||
receipt = delivery.mail_delivery_status(event.event_id)
|
||||
state = receipt.get("state") if isinstance(receipt, dict) else None
|
||||
except Exception:
|
||||
state = None
|
||||
detail = {
|
||||
"provider_accepted": "The mail provider accepted this event. This does not prove inbox receipt. Retrying this same record reconciles the remaining event delivery without submitting another invitation.",
|
||||
"unknown": "The SMTP outcome is uncertain. Automatic resending is blocked. Ask the mail service owner to reconcile the provider record using this delivery ID; use assisted password setup if access is urgent.",
|
||||
"failed": "The previous mail attempt definitely failed. Review a delivery attempt to retry this same record.",
|
||||
"not_found": "No mail submission is recorded. Review a delivery attempt to submit this selected record.",
|
||||
"unconfigured": "The mail lane is not configured. Use assisted password setup or restore the mail lane.",
|
||||
}.get(
|
||||
state,
|
||||
"Mail evidence is unavailable. Check the mail service and retry this status check; do not infer that a message was sent or received.",
|
||||
)
|
||||
return (
|
||||
"<section><h2>Selected mail evidence</h2><p>"
|
||||
+ escape(detail)
|
||||
+ "</p></section>"
|
||||
)
|
||||
|
|
@ -437,6 +437,9 @@ class UserEngineStore(Protocol):
|
|||
def pending_outbox(self) -> tuple[OutboxEvent, ...]:
|
||||
"""Return pending outbox events in write order."""
|
||||
|
||||
def outbox_delivery_guard(self, event_id: str) -> ContextManager[None]:
|
||||
"""Serialize delivery/replay for one event across processes."""
|
||||
|
||||
def save_outbox(self, event: OutboxEvent) -> None:
|
||||
"""Persist outbox delivery state."""
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ def create_application() -> PortalApplication:
|
|||
),
|
||||
timeout_seconds=float(os.environ.get("USER_ENGINE_DELIVERY_TIMEOUT", "5")),
|
||||
)
|
||||
return PortalApplication(
|
||||
app = PortalApplication(
|
||||
service,
|
||||
trusted_proxy_secret=_required("USER_ENGINE_PROXY_SECRET"),
|
||||
login_url=_required("USER_ENGINE_LOGIN_URL"),
|
||||
|
|
@ -120,6 +120,9 @@ def create_application() -> PortalApplication:
|
|||
),
|
||||
)
|
||||
|
||||
from user_engine.operations_status import check_services
|
||||
app.operations_probe = lambda: check_services(app.oidc_client, app.outbox_delivery)
|
||||
return app
|
||||
|
||||
def main() -> None:
|
||||
host = os.environ.get("USER_ENGINE_HOST", "0.0.0.0")
|
||||
|
|
|
|||
|
|
@ -2867,71 +2867,65 @@ class UserEngineService:
|
|||
return self.store.pending_outbox()
|
||||
|
||||
def deliver_outbox(
|
||||
self,
|
||||
actor: Actor,
|
||||
deliver: Callable[[OutboxEvent], None],
|
||||
*,
|
||||
worker_id: str,
|
||||
max_attempts: int = 3,
|
||||
correlation_id: str | None = None,
|
||||
self, actor: Actor, deliver: Callable[[OutboxEvent], None], *,
|
||||
worker_id: str, max_attempts: int = 3, correlation_id: str | None = None,
|
||||
event_id: str | None = None, expected_attempts: int | None = None,
|
||||
) -> tuple[OutboxEvent, ...]:
|
||||
"""Claim and deliver pending events, retaining bounded failure details."""
|
||||
if not worker_id.strip():
|
||||
raise ValidationError("outbox worker_id is required")
|
||||
if not 1 <= max_attempts <= 20:
|
||||
raise ValidationError("outbox max_attempts must be between 1 and 20")
|
||||
"""Serialize each external attempt and retain only classified failures."""
|
||||
if not worker_id.strip() or not 1 <= max_attempts <= 20:
|
||||
raise ValidationError("invalid outbox worker or attempt limit")
|
||||
correlation_id = correlation_id or new_id("corr")
|
||||
self._authorize(
|
||||
actor, action="outbox.deliver", resource_type="user-engine:outbox",
|
||||
resource_id="pending", tenant=actor.tenant,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
results: list[OutboxEvent] = []
|
||||
for event in self.store.pending_outbox():
|
||||
claimed = replace(
|
||||
event, claimed_by=worker_id, claimed_at=utc_now(),
|
||||
delivery_attempts=event.delivery_attempts + 1,
|
||||
failed_at=None, failure_reason=None,
|
||||
)
|
||||
with self.store.transaction():
|
||||
self.store.save_outbox(claimed)
|
||||
try:
|
||||
deliver(claimed)
|
||||
except Exception as exc:
|
||||
reason = str(exc).strip()[:200] or type(exc).__name__
|
||||
failed = replace(
|
||||
claimed, failed_at=utc_now(), failure_reason=reason,
|
||||
dead_lettered_at=(utc_now() if claimed.delivery_attempts >= max_attempts else None),
|
||||
)
|
||||
with self.store.transaction():
|
||||
self.store.save_outbox(failed)
|
||||
results.append(failed)
|
||||
else:
|
||||
delivered = replace(claimed, delivered_at=utc_now())
|
||||
with self.store.transaction():
|
||||
self.store.save_outbox(delivered)
|
||||
results.append(delivered)
|
||||
self._authorize(actor, action="outbox.deliver", resource_type="user-engine:outbox",
|
||||
resource_id="pending", tenant=actor.tenant, correlation_id=correlation_id)
|
||||
selected = self.store.outbox_event(event_id) if event_id else None
|
||||
if event_id and selected is None:
|
||||
raise NotFoundError("delivery record not found")
|
||||
events = (selected,) if selected else self.store.pending_outbox()
|
||||
results = []
|
||||
for candidate in events:
|
||||
with self.store.outbox_delivery_guard(candidate.event_id):
|
||||
event = self.store.outbox_event(candidate.event_id)
|
||||
if event is None or event.delivered_at is not None:
|
||||
continue
|
||||
if expected_attempts is not None and event.delivery_attempts != expected_attempts:
|
||||
raise ConflictError("Delivery changed. Refresh and review the attempt again.")
|
||||
# Owning this cross-process guard proves any previous claim is orphaned.
|
||||
# Receiver idempotency reconciles a lost receipt without blind resending.
|
||||
if event.dead_lettered_at and not event_id:
|
||||
continue
|
||||
claimed = replace(event, claimed_by=worker_id, claimed_at=utc_now(),
|
||||
delivery_attempts=event.delivery_attempts+1, failed_at=None, failure_reason=None)
|
||||
with self.store.transaction(): self.store.save_outbox(claimed)
|
||||
try:
|
||||
deliver(claimed)
|
||||
except Exception as exc:
|
||||
code = getattr(exc, "code", "delivery_unavailable")
|
||||
if code not in {"mail_unconfigured", "delivery_outcome_unknown", "temporary_deferral", "permanent_rejection", "recipient_suppressed", "provider_timeout", "provider_unavailable"}:
|
||||
code = "delivery_unavailable"
|
||||
outcome = replace(claimed, claimed_by=None, claimed_at=None,
|
||||
failed_at=utc_now(), failure_reason=code,
|
||||
dead_lettered_at=utc_now() if claimed.delivery_attempts >= max_attempts else None)
|
||||
else:
|
||||
outcome = replace(claimed, claimed_by=None, claimed_at=None,
|
||||
delivered_at=utc_now(), dead_lettered_at=None)
|
||||
with self.store.transaction(): self.store.save_outbox(outcome)
|
||||
results.append(outcome)
|
||||
return tuple(results)
|
||||
|
||||
def replay_outbox(
|
||||
self, actor: Actor, event_id: str, *, correlation_id: str | None = None
|
||||
) -> OutboxEvent:
|
||||
def replay_outbox(self, actor: Actor, event_id: str, *, correlation_id: str | None = None) -> OutboxEvent:
|
||||
correlation_id = correlation_id or new_id("corr")
|
||||
event = self.store.outbox_event(event_id)
|
||||
if event is None:
|
||||
raise NotFoundError("outbox event not found")
|
||||
self.resolve_tenant_context(actor, event.tenant)
|
||||
self._authorize(
|
||||
actor, action="outbox.replay", resource_type="user-engine:outbox-event",
|
||||
resource_id=event_id, tenant=event.tenant, correlation_id=correlation_id,
|
||||
)
|
||||
replayed = replace(
|
||||
event, claimed_by=None, claimed_at=None, failed_at=None,
|
||||
failure_reason=None, dead_lettered_at=None,
|
||||
)
|
||||
with self.store.transaction():
|
||||
self.store.save_outbox(replayed)
|
||||
return replayed
|
||||
with self.store.outbox_delivery_guard(event_id):
|
||||
event = self.store.outbox_event(event_id)
|
||||
if event is None: raise NotFoundError("outbox event not found")
|
||||
self.resolve_tenant_context(actor, event.tenant)
|
||||
self._authorize(actor, action="outbox.replay", resource_type="user-engine:outbox-event",
|
||||
resource_id=event_id, tenant=event.tenant, correlation_id=correlation_id)
|
||||
if event.delivered_at or (event.claimed_by and not event.failed_at):
|
||||
raise ConflictError("Delivery is completed or its previous attempt is unresolved.")
|
||||
replayed = replace(event, claimed_by=None, claimed_at=None, failed_at=None,
|
||||
failure_reason=None, dead_lettered_at=None)
|
||||
with self.store.transaction(): self.store.save_outbox(replayed)
|
||||
return replayed
|
||||
|
||||
def outbox_diagnostics(self) -> OutboxDiagnostics:
|
||||
event_types: dict[str, int] = {}
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ class PortalApplication:
|
|||
self.provisioning = provisioning
|
||||
self.tenant_management = tenant_management
|
||||
self.outbox_delivery = outbox_delivery
|
||||
self.operations_probe = None
|
||||
self.registration_verification = registration_verification
|
||||
self.registration_clients = frozenset(registration_clients)
|
||||
self.registration_tenants = frozenset(registration_tenants)
|
||||
|
|
@ -791,6 +792,27 @@ class PortalApplication:
|
|||
query = parse_qs(str(environ.get("QUERY_STRING", "")))
|
||||
return self._html(start_response, self._platform_activity(
|
||||
query.get("reference", [""])[0], query.get("tenant", [""])[0]), correlation_id)
|
||||
if path == "/platform/operations/deliver" and method == "POST":
|
||||
self.service.resolve_tenant_context(actor, PLATFORM_TENANT)
|
||||
body = self._form_body(environ)
|
||||
self._require_csrf(environ, body.get("csrf_token", ""))
|
||||
event_id = body.get("event_id", "")
|
||||
with self.service.store.outbox_delivery_guard(event_id):
|
||||
event = self.service.store.outbox_event(event_id)
|
||||
if event is None: raise NotFoundError("delivery record not found")
|
||||
if event.delivered_at:
|
||||
raise ConflictError("Delivery is already completed.")
|
||||
if self.outbox_delivery is None: raise ValidationError("Delivery is unavailable")
|
||||
confirmation = self._confirm_change(environ, start_response, body,
|
||||
json.dumps(_jsonable(event),sort_keys=True), "Attempt this delivery",
|
||||
"This submits only delivery " + event.event_id + " for " + event.tenant +
|
||||
". Invitation messages will be sent to the recorded recipient. Provider acceptance does not prove the person received it. Keep support reference " + event.correlation_id + ".",
|
||||
correlation_id)
|
||||
if confirmation is not None: return confirmation
|
||||
self.service.deliver_outbox(actor, self.outbox_delivery,
|
||||
worker_id="platform-operator",event_id=event.event_id,
|
||||
expected_attempts=event.delivery_attempts,correlation_id=correlation_id)
|
||||
return self._redirect(start_response,"/platform/operations?"+urlencode({"event_id":event_id}),correlation_id)
|
||||
if path in {"/platform/operations", "/platform/operations/replay"}:
|
||||
self.service.resolve_tenant_context(actor, PLATFORM_TENANT)
|
||||
if method == "POST" and path.endswith("/replay"):
|
||||
|
|
@ -799,7 +821,7 @@ class PortalApplication:
|
|||
event = self.service.store.outbox_event(str(body.get("event_id", "")))
|
||||
if event is None:
|
||||
raise NotFoundError("delivery record not found")
|
||||
if event.delivered_at is not None or event.claimed_by:
|
||||
if event.delivered_at is not None or (event.claimed_by and not event.failed_at):
|
||||
raise ConflictError("Delivery is already completed or being processed. Refresh its status.")
|
||||
self.service.replay_outbox(actor, event.event_id, correlation_id=correlation_id)
|
||||
return self._redirect(start_response, "/platform/operations?"+urlencode({"event_id":event.event_id}), correlation_id)
|
||||
|
|
@ -1191,6 +1213,8 @@ class PortalApplication:
|
|||
@staticmethod
|
||||
def _delivery_status(event: Any) -> str:
|
||||
if event.delivered_at: return "Accepted by delivery adapter; receipt by the person is unverified"
|
||||
if event.failure_reason == "delivery_outcome_unknown": return "Mail outcome unknown — check provider evidence before sending another message"
|
||||
if event.failure_reason == "mail_unconfigured": return "Mail is not configured — use assisted setup or restore the mail lane"
|
||||
if event.dead_lettered_at: return "Delivery stopped after repeated failures"
|
||||
if event.failed_at: return "Delivery failed; retry pending"
|
||||
if event.claimed_by: return "Being processed"
|
||||
|
|
@ -1242,15 +1266,19 @@ class PortalApplication:
|
|||
rows = ""
|
||||
for event in events:
|
||||
action = ""
|
||||
if event.delivered_at is None and not event.claimed_by and (event.failed_at or event.dead_lettered_at):
|
||||
action = f'<form method="post" action="/platform/operations/replay"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Queue a retry</button></form>'
|
||||
if event.delivered_at is None:
|
||||
action = f'<form method="post" action="/platform/operations/deliver"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Review delivery attempt</button></form>'
|
||||
if event.delivered_at is None and (not event.claimed_by or event.failed_at) and (event.failed_at or event.dead_lettered_at):
|
||||
action += f'<form method="post" action="/platform/operations/replay"><input type="hidden" name="csrf_token" value="{escape(csrf)}"><input type="hidden" name="event_id" value="{escape(event.event_id)}"><button type="submit">Queue a retry</button></form>'
|
||||
rows += f'<tr><td>{escape(event.event_id)}</td><td>{escape(event.tenant)}</td><td>{escape(event.event_type)}</td><td>{escape(self._delivery_status(event))}</td><td>{escape(event.correlation_id)}</td><td>{action}</td></tr>'
|
||||
configuration = self._operation_capabilities()
|
||||
return self._page_html("Service recovery", '<h1>Service recovery</h1>' + configuration + '<p><a href="/platform/activity">Investigate a support reference</a></p><p>This view shows local delivery records. Live sign-in, email receipt and authenticator health are not verified here.</p>'
|
||||
from user_engine.operations_status import check_services, selected_mail_status
|
||||
evidence = selected_mail_status(self.outbox_delivery, events[0]) if event_id else ""
|
||||
configuration = (self.operations_probe() if self.operations_probe else check_services(None,None)) + self._operation_capabilities()
|
||||
return self._page_html("Service recovery", '<h1>Service recovery</h1>' + configuration + evidence + '<p><a href="/platform/activity">Investigate a support reference</a></p><p>Delivery records below describe recorded attempts. Component checks do not prove a complete sign-in or inbox receipt.</p>'
|
||||
'<form method="get" action="/platform/operations"><label>Delivery record ID <input name="event_id"></label><button type="submit">Find delivery</button></form>'
|
||||
'<table><thead><tr><th>Delivery</th><th>Tenant</th><th>Kind</th><th>Status</th><th>Support reference</th><th>Recovery</th></tr></thead><tbody>'
|
||||
+ (rows or '<tr><td colspan="6">No delivery records. This does not prove mail was received.</td></tr>')
|
||||
+ '</tbody></table><p>Queued retries are processed by the delivery worker. Check the record again for the result.</p><p><a href="/platform">Return to platform administration</a></p>')
|
||||
+ '</tbody></table><p>Queueing a retry does not send it. Use Review delivery attempt to submit one selected record, then check its result. No background worker is enabled here.</p><p><a href="/platform">Return to platform administration</a></p>')
|
||||
|
||||
def _operation_capabilities(self) -> str:
|
||||
capabilities = (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue