fix(edge): retry+degrade side-effect POSTs; finish ACTIVITY-WP-0027
Consistency sweep and recently-on-scope retry transient 502/503/504 then return a degraded context snapshot instead of thrashing Temporal. Document edge-relay resilience. Retire Binky dual-clock host timers after claim-loop smoke.
This commit is contained in:
parent
2b3fc6a556
commit
4e605a6839
6 changed files with 334 additions and 20 deletions
|
|
@ -9,13 +9,25 @@ from activity_core.context_resolvers.state_hub import StateHubContextResolver
|
|||
|
||||
|
||||
class DummyResponse:
|
||||
def __init__(self, payload: Any, status_error: Exception | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
payload: Any,
|
||||
status_error: Exception | None = None,
|
||||
status_code: int = 200,
|
||||
) -> None:
|
||||
self.payload = payload
|
||||
self.status_error = status_error
|
||||
self.status_code = status_code
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
if self.status_error is not None:
|
||||
raise self.status_error
|
||||
if self.status_code >= 400:
|
||||
req = httpx.Request("POST", "http://dummy")
|
||||
resp = httpx.Response(self.status_code, request=req)
|
||||
raise httpx.HTTPStatusError(
|
||||
f"{self.status_code}", request=req, response=resp
|
||||
)
|
||||
|
||||
def json(self) -> Any:
|
||||
return self.payload
|
||||
|
|
@ -498,9 +510,30 @@ def test_recently_on_scope_hourly_failure_bubbles(monkeypatch) -> None:
|
|||
raise httpx.ConnectError("offline")
|
||||
|
||||
monkeypatch.setattr(httpx, "post", fake_post)
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "1")
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
|
||||
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
StateHubContextResolver().resolve("recently_on_scope_hourly", None, {"range": "1h"})
|
||||
StateHubContextResolver().resolve(
|
||||
"recently_on_scope_hourly",
|
||||
None,
|
||||
{"range": "1h", "degrade_on_unavailable": False},
|
||||
)
|
||||
|
||||
|
||||
def test_recently_on_scope_hourly_degrades(monkeypatch) -> None:
|
||||
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
|
||||
raise httpx.ConnectError("offline")
|
||||
|
||||
monkeypatch.setattr(httpx, "post", fake_post)
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "2")
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
|
||||
|
||||
result = StateHubContextResolver().resolve(
|
||||
"recently_on_scope_hourly", None, {"range": "1h"}
|
||||
)
|
||||
assert result["degraded"] is True
|
||||
assert result["failed"][0]["reason"] == "edge_unavailable"
|
||||
|
||||
|
||||
def test_consistency_sweep_remote_all_posts_batch(monkeypatch) -> None:
|
||||
|
|
@ -544,15 +577,69 @@ def test_consistency_sweep_remote_all_failure_bubbles(monkeypatch) -> None:
|
|||
raise httpx.ConnectError("offline")
|
||||
|
||||
monkeypatch.setattr(httpx, "post", fake_post)
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "1")
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
|
||||
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
StateHubContextResolver().resolve(
|
||||
"consistency_sweep_remote_all",
|
||||
None,
|
||||
{"max_seconds": 300},
|
||||
{"max_seconds": 300, "degrade_on_unavailable": False},
|
||||
)
|
||||
|
||||
|
||||
def test_consistency_sweep_degrades_after_retries(monkeypatch) -> None:
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
|
||||
calls.append(1)
|
||||
raise httpx.ConnectError("offline")
|
||||
|
||||
monkeypatch.setattr(httpx, "post", fake_post)
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "3")
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
|
||||
monkeypatch.setenv("STATE_HUB_SIDE_EFFECT_DEGRADE", "true")
|
||||
|
||||
result = StateHubContextResolver().resolve(
|
||||
"consistency_sweep_remote_all",
|
||||
None,
|
||||
{"max_seconds": 300},
|
||||
)
|
||||
assert result["degraded"] is True
|
||||
assert result["exit_code"] == 75
|
||||
assert result["lock_skipped"] is True
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
def test_consistency_sweep_retries_then_succeeds(monkeypatch) -> None:
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
|
||||
calls.append(1)
|
||||
if len(calls) < 2:
|
||||
return DummyResponse({}, status_code=503)
|
||||
return DummyResponse(
|
||||
{
|
||||
"exit_code": 0,
|
||||
"lock_skipped": False,
|
||||
"repos_processed": [],
|
||||
},
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(httpx, "post", fake_post)
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRIES", "3")
|
||||
monkeypatch.setenv("STATE_HUB_POST_RETRY_BACKOFF_SECONDS", "0")
|
||||
|
||||
result = StateHubContextResolver().resolve(
|
||||
"consistency_sweep_remote_all",
|
||||
None,
|
||||
{"max_seconds": 300},
|
||||
)
|
||||
assert result["exit_code"] == 0
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
def test_consistency_sweep_remote_all_rejects_empty_response(monkeypatch) -> None:
|
||||
def fake_post(url: str, **kwargs: Any) -> DummyResponse:
|
||||
return DummyResponse({})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue