feat: complete reliable coordination adapter
Some checks failed
tamq-ci / test (push) Failing after 5s
Some checks failed
tamq-ci / test (push) Failing after 5s
Assistant: codex Assistant-Model: gpt-5.6-sol Assistant-Session: 01a03397-4d51-7fd1-8ff2-946eb22ea2bc
This commit is contained in:
parent
25113f463e
commit
6d2ccc7760
30 changed files with 2553 additions and 144 deletions
|
|
@ -1,3 +1,5 @@
|
|||
import json
|
||||
|
||||
from tamq.control import PaneDisplay
|
||||
from tamq.service import Service
|
||||
from tamq.store import Store
|
||||
|
|
@ -96,6 +98,13 @@ def test_service_trigger_mode_submits_exactly_once(tmp_path, monkeypatch):
|
|||
service._deliver_once()
|
||||
assert control.submitted == [("tamq:repo-b", "From:repo-a/o: go")]
|
||||
assert control.placed == []
|
||||
lifecycle = store.protocol_events(message_id=store.list()[0]["message_id"])
|
||||
assert [event["event_type"] for event in lifecycle] == [
|
||||
"message.accepted",
|
||||
"delivery.attempted",
|
||||
"delivery.injected",
|
||||
]
|
||||
assert lifecycle[-1]["delivery_mode"] == "trigger"
|
||||
|
||||
|
||||
def test_failed_pushy_submission_remains_pending(tmp_path, monkeypatch):
|
||||
|
|
@ -111,6 +120,12 @@ def test_failed_pushy_submission_remains_pending(tmp_path, monkeypatch):
|
|||
Service(store=store, input_grace=0)._deliver_once()
|
||||
|
||||
assert store.list()[0]["state"] == "pending"
|
||||
failure = store.protocol_events(event_type="delivery.failed")[0]
|
||||
assert failure["delivery_mode"] == "pushy"
|
||||
detail = json.loads(failure["detail"])
|
||||
assert detail["reason"] == "RuntimeError"
|
||||
assert detail["attempt"] == 1
|
||||
assert detail["attempt_limit"] == 4
|
||||
|
||||
|
||||
def test_input_delivery_waits_for_new_endpoint_grace(tmp_path, monkeypatch):
|
||||
|
|
@ -126,7 +141,7 @@ def test_input_delivery_waits_for_new_endpoint_grace(tmp_path, monkeypatch):
|
|||
assert store.list()[0]["state"] == "pending"
|
||||
|
||||
|
||||
def test_service_writes_output_once_and_retains_pending_ack(tmp_path, monkeypatch):
|
||||
def test_default_injected_policy_completes_output_delivery_once(tmp_path, monkeypatch):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
store.register_endpoint("tmux-amq-9-boot", 9, "tamq", ["repo-b"], "output")
|
||||
message_id = store.add("repo-a", "repo-b", "continue")
|
||||
|
|
@ -148,7 +163,7 @@ def test_service_writes_output_once_and_retains_pending_ack(tmp_path, monkeypatc
|
|||
)
|
||||
]
|
||||
row = store.list()[0]
|
||||
assert row["state"] == "pending"
|
||||
assert row["state"] == "injected"
|
||||
assert row["displayed_at"] is not None
|
||||
assert store.db.execute("SELECT COUNT(*) FROM leases").fetchone()[0] == 0
|
||||
|
||||
|
|
@ -166,18 +181,125 @@ def test_failed_terminal_output_remains_undisplayed_and_retryable(tmp_path, monk
|
|||
raise OSError("temporary failure")
|
||||
|
||||
monkeypatch.setattr("tamq.service.write_terminal_output", fail_once)
|
||||
service = Service(store=store)
|
||||
service = Service(store=store, retry_backoff=(0,))
|
||||
|
||||
service._deliver_once()
|
||||
assert store.list()[0]["displayed_at"] is None
|
||||
store.db.execute("UPDATE leases SET expires_at=0")
|
||||
store.db.commit()
|
||||
service._deliver_once()
|
||||
|
||||
assert len(attempts) == 2
|
||||
assert store.list()[0]["displayed_at"] is not None
|
||||
|
||||
|
||||
def test_acknowledged_policy_waits_then_redelivers_same_message(tmp_path, monkeypatch):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
store.register_endpoint(
|
||||
"ep", 9, "tamq", ["repo-b"], "trigger",
|
||||
delivery_ack_mode="acknowledged",
|
||||
delivery_max_attempts=3,
|
||||
ack_timeout_seconds=10,
|
||||
)
|
||||
message_id = store.add("repo-a", "repo-b", "confirm")
|
||||
monkeypatch.setattr("tamq.service.ControlModeClient", FakeControl)
|
||||
service = Service(store=store, input_grace=0, retry_backoff=(0,))
|
||||
|
||||
service._deliver_once()
|
||||
first_control = FakeControl.instances[-1]
|
||||
assert store.message(message_id)["state"] == "awaiting_ack"
|
||||
assert store.message(message_id)["attempt_count"] == 1
|
||||
assert len(first_control.submitted) == 1
|
||||
service._deliver_once()
|
||||
assert FakeControl.instances[-1].submitted == []
|
||||
|
||||
store.db.execute(
|
||||
"UPDATE messages SET next_attempt_at=0 WHERE message_id=?", (message_id,)
|
||||
)
|
||||
store.db.commit()
|
||||
service._deliver_once()
|
||||
second_control = FakeControl.instances[-1]
|
||||
assert store.message(message_id)["attempt_count"] == 2
|
||||
assert len(second_control.submitted) == 1
|
||||
assert store.acknowledge(message_id) is True
|
||||
service._deliver_once()
|
||||
assert FakeControl.instances[-1].submitted == []
|
||||
|
||||
|
||||
def test_delivery_attempt_cap_is_terminal_and_operator_retry_resets(tmp_path, monkeypatch):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
store.register_endpoint(
|
||||
"ep", 9, "tamq", ["repo-b"], "pushy", delivery_max_attempts=2
|
||||
)
|
||||
message_id = store.add("repo-a", "repo-b", "fail")
|
||||
|
||||
class FailingControl(FakeControl):
|
||||
def place(self, window, text):
|
||||
raise OSError("no pane")
|
||||
|
||||
monkeypatch.setattr("tamq.service.ControlModeClient", FailingControl)
|
||||
service = Service(store=store, input_grace=0, retry_backoff=(0,))
|
||||
service._deliver_once()
|
||||
service._deliver_once()
|
||||
service._deliver_once()
|
||||
|
||||
row = store.message(message_id)
|
||||
assert row["state"] == "failed"
|
||||
assert row["attempt_count"] == 2
|
||||
assert row["last_failure_reason"] == "OSError"
|
||||
assert store.retry_message(message_id) is True
|
||||
assert store.message(message_id)["state"] == "pending"
|
||||
assert store.message(message_id)["attempt_count"] == 0
|
||||
|
||||
|
||||
def test_ack_timeout_exhaustion_is_terminal_but_late_ack_wins(tmp_path, monkeypatch):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
store.register_endpoint(
|
||||
"ep", 9, "tamq", ["repo-b"], "trigger",
|
||||
delivery_ack_mode="acknowledged",
|
||||
delivery_max_attempts=2,
|
||||
ack_timeout_seconds=10,
|
||||
)
|
||||
message_id = store.add("repo-a", "repo-b", "confirm")
|
||||
monkeypatch.setattr("tamq.service.ControlModeClient", FakeControl)
|
||||
service = Service(store=store, input_grace=0, retry_backoff=(0,))
|
||||
|
||||
service._deliver_once()
|
||||
for _ in range(2):
|
||||
store.db.execute(
|
||||
"UPDATE messages SET next_attempt_at=0 WHERE message_id=?", (message_id,)
|
||||
)
|
||||
store.db.commit()
|
||||
service._deliver_once()
|
||||
|
||||
row = store.message(message_id)
|
||||
assert row["state"] == "failed"
|
||||
assert row["attempt_count"] == 2
|
||||
assert row["last_failure_reason"] == "ack_timeout"
|
||||
assert store.acknowledge(message_id) is True
|
||||
assert store.message(message_id)["state"] == "acknowledged"
|
||||
assert store.protocol_events(message_id=message_id)[-1]["outcome"] == "late_acknowledged"
|
||||
|
||||
|
||||
def test_expired_delivery_lease_consumes_attempt_and_schedules_retry(tmp_path):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
message_id = store.add("repo-a", "repo-b", "lease")
|
||||
claim = store.claim_delivery(
|
||||
message_id,
|
||||
"ep",
|
||||
attempt_limit=2,
|
||||
retry_delay=7,
|
||||
delivery_mode="output",
|
||||
ttl=1,
|
||||
now=10,
|
||||
)
|
||||
assert claim is not None
|
||||
assert store.expire_delivery_leases(now=12) == 1
|
||||
row = store.message(message_id)
|
||||
assert row["state"] == "pending"
|
||||
assert row["attempt_count"] == 1
|
||||
assert row["last_failure_reason"] == "lease_expired"
|
||||
assert row["next_attempt_at"] == 19
|
||||
|
||||
|
||||
def test_service_disconnects_disappeared_tmux_endpoint(tmp_path, monkeypatch):
|
||||
store = Store(tmp_path / "queue.sqlite3")
|
||||
store.register_endpoint("tmux-amq-9-boot", 9, "tamq", ["repo-b"])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue