fix(edge): retry+degrade side-effect POSTs; finish ACTIVITY-WP-0027
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 1s
Build and Publish Container Image / build-and-push (push) Successful in 33s

Consistency sweep and recently-on-scope retry transient 502/503/504 then
return a degraded context snapshot instead of thrashing Temporal.
Document edge-relay resilience. Retire Binky dual-clock host timers after
claim-loop smoke.
This commit is contained in:
tegwick 2026-08-05 17:53:00 +02:00
parent 2b3fc6a556
commit 4e605a6839
6 changed files with 334 additions and 20 deletions

View file

@ -9,13 +9,25 @@ from activity_core.context_resolvers.state_hub import StateHubContextResolver
class DummyResponse:
def __init__(self, payload: Any, status_error: Exception | None = None) -> None:
def __init__(
self,
payload: Any,
status_error: Exception | None = None,
status_code: int = 200,
) -> None:
self.payload = payload
self.status_error = status_error
self.status_code = status_code
def raise_for_status(self) -> None:
if self.status_error is not None:
raise self.status_error
if self.status_code >= 400:
req = httpx.Request("POST", "http://dummy")
resp = httpx.Response(self.status_code, request=req)
raise httpx.HTTPStatusError(
f"{self.status_code}", request=req, response=resp
)
def json(self) -> Any:
return self.payload
@ -498,9 +510,30 @@ def test_recently_on_scope_hourly_failure_bubbles(monkeypatch) -> None:
raise httpx.ConnectError("offline")
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "1")
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
with pytest.raises(httpx.ConnectError):
StateHubContextResolver().resolve("recently_on_scope_hourly", None, {"range": "1h"})
StateHubContextResolver().resolve(
"recently_on_scope_hourly",
None,
{"range": "1h", "degrade_on_unavailable": False},
)
def test_recently_on_scope_hourly_degrades(monkeypatch) -> None:
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
raise httpx.ConnectError("offline")
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "2")
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
result = StateHubContextResolver().resolve(
"recently_on_scope_hourly", None, {"range": "1h"}
)
assert result["degraded"] is True
assert result["failed"][0]["reason"] == "edge_unavailable"
def test_consistency_sweep_remote_all_posts_batch(monkeypatch) -> None:
@ -544,15 +577,69 @@ def test_consistency_sweep_remote_all_failure_bubbles(monkeypatch) -> None:
raise httpx.ConnectError("offline")
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "1")
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
with pytest.raises(httpx.ConnectError):
StateHubContextResolver().resolve(
"consistency_sweep_remote_all",
None,
{"max_seconds": 300},
{"max_seconds": 300, "degrade_on_unavailable": False},
)
def test_consistency_sweep_degrades_after_retries(monkeypatch) -> None:
calls: list[int] = []
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
calls.append(1)
raise httpx.ConnectError("offline")
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "3")
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
monkeypatch.setenv("STATE_HUB_SIDE_EFFECT_DEGRADE", "true")
result = StateHubContextResolver().resolve(
"consistency_sweep_remote_all",
None,
{"max_seconds": 300},
)
assert result["degraded"] is True
assert result["exit_code"] == 75
assert result["lock_skipped"] is True
assert len(calls) == 3
def test_consistency_sweep_retries_then_succeeds(monkeypatch) -> None:
calls: list[int] = []
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
calls.append(1)
if len(calls) < 2:
return DummyResponse({}, status_code=503)
return DummyResponse(
{
"exit_code": 0,
"lock_skipped": False,
"repos_processed": [],
},
status_code=200,
)
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "3")
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
result = StateHubContextResolver().resolve(
"consistency_sweep_remote_all",
None,
{"max_seconds": 300},
)
assert result["exit_code"] == 0
assert len(calls) == 2
def test_consistency_sweep_remote_all_rejects_empty_response(monkeypatch) -> None:
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
return DummyResponse({})