fix(edge): retry+degrade side-effect POSTs; finish ACTIVITY-WP-0027

Consistency sweep and recently-on-scope retry transient 502/503/504 then
return a degraded context snapshot instead of thrashing Temporal.
Document edge-relay resilience. Retire Binky dual-clock host timers after
claim-loop smoke.

src/activity_core/context_resolvers/state_hub.py

@@ -26,13 +26,21 @@ When STATE_HUB_URL points at the state-hub edge relay, allowlisted GET reads may
 be served from a stale local cache during upstream outages (`X-StateHub-Edge-Cache:
 stale`). activity-core treats those as ordinary successful reads so workflows can
 continue with last-known hub state.
+
+Side-effect POSTs (`consistency_sweep_remote_all`, `recently_on_scope_hourly`)
+retry transient 502/503/504 and network errors, then optionally **degrade**
+instead of hard-failing the workflow (ACTIVITY-WP-0027-T06). See
+`docs/edge-relay-resilience.md`.
+
 Config: STATE_HUB_URL env var (default: http://127.0.0.1:8000).
 """
 
 from __future__ import annotations
 
 import json
+import logging
 import os
+import time
 from datetime import datetime, timezone
 from typing import Any
 
@@ -40,6 +48,8 @@ import httpx
 
 from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY, ContextResolver
 
+logger = logging.getLogger(__name__)
+
 _DEFAULT_STATE_HUB_URL = "http://127.0.0.1:8000"
 _TIMEOUT_SECONDS = 10.0
 _SWEEP_TIMEOUT_SECONDS = 330.0
@@ -50,6 +60,9 @@ _OPEN_TASK_STATUSES = {"wait", "todo", "progress"}
 # forcing the rule expression to special-case None.
 _NEVER_SCANNED_AGE_DAYS = 99999
 
+# Transient HTTP statuses worth retrying against the edge relay / hub.
+_TRANSIENT_HTTP = frozenset({502, 503, 504})
+
 
 def _base_url() -> str:
     return os.environ.get("STATE_HUB_URL", _DEFAULT_STATE_HUB_URL).rstrip("/")
@@ -65,11 +78,92 @@ def _fetch_json(path: str, params: dict[str, Any] | None = None) -> Any:
         return {}
 
 
-def _post_json(path: str, payload: dict[str, Any], *, timeout: float = _TIMEOUT_SECONDS) -> Any:
+def _post_retry_attempts() -> int:
+    try:
+        return max(1, int(os.environ.get("STATE_HUB_POST_RETRIES", "3")))
+    except ValueError:
+        return 3
+
+
+def _post_retry_backoff_seconds() -> float:
+    try:
+        return max(0.0, float(os.environ.get("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "2")))
+    except ValueError:
+        return 2.0
+
+
+def _side_effect_degrade_default() -> bool:
+    """Default degrade-on-unavailable for side-effect POSTs after retries."""
+    raw = (os.environ.get("STATE_HUB_SIDE_EFFECT_DEGRADE") or "true").strip().lower()
+    return raw in {"1", "true", "yes", "on", ""}
+
+
+def _post_json(
+    path: str,
+    payload: dict[str, Any],
+    *,
+    timeout: float = _TIMEOUT_SECONDS,
+    retries: int | None = None,
+) -> Any:
+    """POST JSON with retries on transient edge/hub failures (ACTIVITY-WP-0027)."""
     url = f"{_base_url()}{path}"
-    resp = httpx.post(url, json=payload, timeout=timeout)
-    resp.raise_for_status()
-    return resp.json()
+    attempts = _post_retry_attempts() if retries is None else max(1, retries)
+    backoff = _post_retry_backoff_seconds()
+    last_exc: Exception | None = None
+
+    for attempt in range(attempts):
+        try:
+            resp = httpx.post(url, json=payload, timeout=timeout)
+            status = getattr(resp, "status_code", None)
+            if status in _TRANSIENT_HTTP and attempt + 1 < attempts:
+                logger.warning(
+                    "state-hub POST %s returned %s (attempt %s/%s); retrying",
+                    path,
+                    status,
+                    attempt + 1,
+                    attempts,
+                )
+                if backoff:
+                    time.sleep(backoff * (attempt + 1))
+                continue
+            resp.raise_for_status()
+            return resp.json()
+        except (
+            httpx.TimeoutException,
+            httpx.NetworkError,
+            httpx.RemoteProtocolError,
+        ) as exc:
+            last_exc = exc
+            if attempt + 1 >= attempts:
+                raise
+            logger.warning(
+                "state-hub POST %s network error %s (attempt %s/%s); retrying",
+                path,
+                exc,
+                attempt + 1,
+                attempts,
+            )
+            if backoff:
+                time.sleep(backoff * (attempt + 1))
+        except httpx.HTTPStatusError as exc:
+            # Non-transient HTTP errors fail immediately.
+            code = exc.response.status_code if exc.response is not None else None
+            if code in _TRANSIENT_HTTP and attempt + 1 < attempts:
+                last_exc = exc
+                if backoff:
+                    time.sleep(backoff * (attempt + 1))
+                continue
+            raise
+
+    if last_exc is not None:
+        raise last_exc
+    raise RuntimeError(f"state-hub POST {path} failed without exception")
+
+
+def _want_degrade(params: dict[str, Any]) -> bool:
+    if "degrade_on_unavailable" in params:
+        return bool(params.get("degrade_on_unavailable"))
+    return _side_effect_degrade_default()
 
 
 def _validate_consistency_sweep_remote_all(result: Any) -> dict[str, Any]:
@@ -138,21 +232,58 @@ class StateHubContextResolver(ContextResolver):
             payload = {
                 key: value
                 for key, value in params.items()
-                if key not in {"required"}
+                if key not in {"required", "degrade_on_unavailable"}
             }
-            result = _post_json("/recently-on-scope/hourly", payload)
+            try:
+                result = _post_json("/recently-on-scope/hourly", payload)
+            except (httpx.HTTPError, ValueError) as exc:
+                if not _want_degrade(params):
+                    raise
+                logger.warning(
+                    "recently_on_scope_hourly degraded after retries: %s", exc
+                )
+                return {
+                    "generated": [],
+                    "skipped": [],
+                    "failed": [
+                        {
+                            "reason": "edge_unavailable",
+                            "detail": str(exc)[:300],
+                        }
+                    ],
+                    "degraded": True,
+                    "degraded_reason": str(exc)[:300],
+                }
             return _validate_recently_on_scope_hourly(result)
         if query == "consistency_sweep_remote_all":
             payload = {
                 key: value
                 for key, value in params.items()
-                if key not in {"required"}
+                if key not in {"required", "degrade_on_unavailable"}
             }
-            result = _post_json(
-                "/consistency/sweep/remote-all",
-                payload,
-                timeout=_SWEEP_TIMEOUT_SECONDS,
-            )
+            try:
+                result = _post_json(
+                    "/consistency/sweep/remote-all",
+                    payload,
+                    timeout=_SWEEP_TIMEOUT_SECONDS,
+                )
+            except (httpx.HTTPError, ValueError) as exc:
+                if not _want_degrade(params):
+                    raise
+                logger.warning(
+                    "consistency_sweep_remote_all degraded after retries: %s", exc
+                )
+                # Shape matches validator; exit_code 75 = temporary failure.
+                return {
+                    "exit_code": 75,
+                    "lock_skipped": True,
+                    "repos_processed": [],
+                    "skipped_clean": [],
+                    "skipped_missing": [],
+                    "skipped_budget": [],
+                    "degraded": True,
+                    "degraded_reason": str(exc)[:300],
+                }
             return _validate_consistency_sweep_remote_all(result)
         if query == "phase5_stabilization_check":
             return _phase5_stabilization_check(params)