Implement worker coordination runtime and finish WP-0003

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a07b5b-ea58-7ad2-bdbb-0b1c995cfc35

tests/test_runtime.py

@@ -0,0 +1,351 @@
+import copy
+import json
+import sqlite3
+
+import pytest
+
+from coordination_engine.adapters import AdapterError, SafetyError
+from coordination_engine.config import Config
+from coordination_engine.runtime import Runtime
+from coordination_engine.store import Store
+
+
+class FakeHub:
+    def __init__(self):
+        self.data = {
+            "repos": {"r": "demo"},
+            "plans": [{"id": "p", "repo_id": "r", "status": "active"}],
+            "tasks": [
+                {
+                    "id": "t",
+                    "workplan_id": "p",
+                    "status": "todo",
+                    "title": "Fix parser",
+                    "updated_at": "1",
+                }
+            ],
+            "dependencies": [],
+            "messages": [],
+        }
+        self.receipts = []
+        self.offline = False
+        self.projection_offline = False
+
+    def snapshot(self):
+        if self.offline:
+            raise OSError("offline")
+        return copy.deepcopy(self.data)
+
+    def project(self, receipt):
+        if self.projection_offline:
+            raise OSError("offline")
+        self.receipts.append(receipt)
+
+
+class FakeTamq:
+    def __init__(self):
+        self.calls = []
+        self.messages = {}
+        self.error = None
+        self.state = "injected"
+
+    def wake(self, lease, prompt):
+        self.calls.append((lease, prompt))
+        if self.error:
+            raise self.error
+        self.messages.setdefault(lease["lease_id"], str(len(self.messages)))
+        return {
+            "endpoint_id": "e",
+            "message_id": self.messages[lease["lease_id"]],
+            "transport_state": self.state,
+        }
+
+    def receipt(self, message_id):
+        return {"state": self.state}
+
+
+@pytest.fixture
+def env(tmp_path):
+    config = Config(state_dir=tmp_path / "state", repos=["demo"]).validate()
+    store = Store(config)
+    hub, tamq = FakeHub(), FakeTamq()
+    clock = [100.0]
+    runtime = Runtime(config, store, hub, tamq, lambda: clock[0])
+    yield config, store, hub, tamq, clock, runtime
+    store.close()
+
+
+def test_dedupe_and_delivery_is_not_completion(env):
+    _, store, hub, tamq, clock, runtime = env
+    runtime.tick()
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    assert store.leases()[0]["state"] == "offered"
+    assert store.leases()[0]["transport_state"] == "injected"
+    assert all("prompt" not in r and "checkpoint" not in r for r in hub.receipts)
+
+
+@pytest.mark.parametrize(
+    "field,value",
+    [
+        ("needs_human", True),
+        ("intervention_note", "private review note"),
+        ("title", "Get a secret"),
+        ("action_classes", ["external_publish"]),
+        ("blocking_reason", "operator decision"),
+    ],
+)
+def test_safety_stops_never_retry(env, field, value):
+    _, store, hub, tamq, clock, runtime = env
+    hub.data["tasks"][0][field] = value
+    for i in range(8):
+        clock[0] += 300
+        runtime.tick()
+    assert not tamq.calls
+    assert store.leases()[0]["state"] == "stopped"
+    assert value not in json.dumps(hub.receipts) if isinstance(value, str) else True
+
+
+def test_dependencies_change_without_task_revision(env):
+    _, store, hub, tamq, clock, runtime = env
+    hub.data["plans"].append({"id": "upstream", "repo_id": "other", "status": "active"})
+    hub.data["dependencies"] = [
+        {
+            "id": "d",
+            "from_workplan_id": "p",
+            "to_workplan_id": "upstream",
+            "relationship_type": "blocks",
+        }
+    ]
+    runtime.tick()
+    assert not tamq.calls
+    assert store.leases()[0]["state"] == "waiting"
+    hub.data["plans"][1]["status"] = "finished"
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 1
+
+
+def test_unresolved_dependency_and_wait_task_do_not_dispatch(env):
+    _, _, hub, tamq, _, runtime = env
+    hub.data["tasks"][0]["status"] = "wait"
+    runtime.tick()
+    assert not tamq.calls
+
+
+def test_incoming_dependency_does_not_block(env):
+    _, _, hub, tamq, _, runtime = env
+    hub.data["dependencies"] = [
+        {
+            "id": "d",
+            "from_workplan_id": "other",
+            "to_workplan_id": "p",
+            "relationship_type": "blocks",
+        }
+    ]
+    runtime.tick()
+    assert len(tamq.calls) == 1
+
+
+def test_retry_identity_and_exhaustion(env):
+    config, store, _, tamq, clock, runtime = env
+    tamq.error = AdapterError("offline")
+    for i in range(7):
+        clock[0] += 400
+        runtime.tick()
+    assert len(tamq.calls) == config.max_attempts
+    assert len({row["lease_id"] for row, _ in tamq.calls}) == 1
+    assert len({prompt for _, prompt in tamq.calls}) == 1
+    assert store.leases()[0]["state"] == "stopped"
+
+
+def test_crash_after_send_recovers_same_message(env):
+    config, store, hub, tamq, clock, runtime = env
+    runtime.tick()
+    recovered = Runtime(config, store, hub, tamq, lambda: clock[0])
+    clock[0] += 31
+    recovered.tick()
+    clock[0] += 15
+    recovered.tick()
+    assert len(tamq.calls) == 2
+    assert len(tamq.messages) == 1
+
+
+def test_worker_lifecycle_and_checkpoint_sanitization(env):
+    _, store, hub, _, clock, runtime = env
+    runtime.tick()
+    row = store.leases()[0]
+    with pytest.raises(ValueError):
+        runtime.worker_update("renew", row["lease_id"], "demo")
+    with pytest.raises(ValueError):
+        runtime.worker_update("ack", row["lease_id"], "wrong")
+    runtime.worker_update("ack", row["lease_id"], "demo")
+    runtime.worker_update("renew", row["lease_id"], "demo")
+    runtime.worker_update(
+        "complete", row["lease_id"], "demo", {"summary": "private diagnostic detail"}
+    )
+    runtime.project()
+    assert store.leases()[0]["state"] == "completed"
+    assert "private diagnostic detail" not in json.dumps(hub.receipts)
+
+
+def test_expired_worker_cannot_renew(env):
+    _, store, _, _, clock, runtime = env
+    runtime.tick()
+    clock[0] += 31
+    with pytest.raises(ValueError):
+        runtime.worker_update("ack", store.leases()[0]["lease_id"], "demo")
+
+
+def test_hub_outage_and_projection_recovery(env):
+    _, store, hub, tamq, clock, runtime = env
+    hub.projection_offline = True
+    runtime.tick()
+    assert store.pending_receipts()
+    hub.offline = True
+    clock[0] += 40
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    hub.offline = hub.projection_offline = False
+    clock[0] += 300
+    runtime.tick()
+    assert not store.pending_receipts()
+
+
+def test_one_active_worker_per_repo(env):
+    _, store, hub, tamq, _, runtime = env
+    hub.data["tasks"].append(dict(hub.data["tasks"][0], id="t2"))
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    assert [r["state"] for r in store.leases()] == ["offered", "pending"]
+
+
+def test_explicit_action_message_only(env):
+    _, _, hub, tamq, clock, runtime = env
+    hub.data["tasks"] = []
+    hub.data["messages"] = [
+        {"id": "m", "to_agent": "demo", "subject": "FYI", "body": "ordinary text"}
+    ]
+    runtime.tick()
+    assert not tamq.calls
+    hub.data["messages"][0]["subject"] = "[action] Inspect work"
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    assert "ordinary text" not in tamq.calls[0][1]
+
+
+def test_adapter_safety_error_is_terminal(env):
+    _, store, _, tamq, _, runtime = env
+    tamq.error = SafetyError("ambiguous_endpoint")
+    runtime.tick()
+    assert store.leases()[0]["state"] == "stopped"
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {"max_attempts": 10},
+        {"max_attempts": True},
+        {"timeout": float("nan")},
+        {"retry_backoff": []},
+        {"policy_profile": "unsafe"},
+        {"api_base": "http://user:secret@localhost"},
+        {"allow": ["secrets"]},
+    ],
+)
+def test_invalid_config(kwargs):
+    with pytest.raises(ValueError):
+        Config(**kwargs).validate()
+
+
+def test_backup_schema_and_exclusive_writer(env):
+    config, store, _, _, _, _ = env
+    backup = store.backup()
+    assert backup.stat().st_mode & 0o777 == 0o600
+    with sqlite3.connect(backup) as db:
+        assert db.execute("PRAGMA user_version").fetchone()[0] == 1
+    other = Store(config)
+    try:
+        with store.writer_lock():
+            with pytest.raises(BlockingIOError):
+                with other.writer_lock():
+                    pass
+    finally:
+        other.close()
+    store.db.execute("PRAGMA user_version=99")
+    with pytest.raises(ValueError, match="newer"):
+        Store(config)
+
+
+def test_checkpoint_continuation_gets_new_transport_identity(env):
+    _, store, _, tamq, clock, runtime = env
+    runtime.tick()
+    row = store.leases()[0]
+    runtime.worker_update("ack", row["lease_id"], "demo")
+    runtime.worker_update(
+        "checkpoint",
+        row["lease_id"],
+        "demo",
+        {"summary": "Parser fixed", "next_action": "Run local checks"},
+    )
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 2
+    assert len(tamq.messages) == 2
+    assert store.leases()[0]["state"] == "checkpointed"
+    child = store.leases()[1]
+    assert "using coordination-engine history" in tamq.calls[1][1]
+    assert "State Hub checkpoint" not in tamq.calls[1][1]
+    assert child["source"] == "checkpoint"
+    assert child["source_id"] == row["trigger_id"]
+    runtime.worker_update("ack", child["lease_id"], "demo")
+    runtime.worker_update("complete", child["lease_id"], "demo")
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 2
+
+
+def test_revision_change_does_not_duplicate_active_worker(env):
+    _, store, hub, tamq, clock, runtime = env
+    runtime.tick()
+    hub.data["tasks"][0]["status"] = "progress"
+    hub.data["tasks"][0]["updated_at"] = "2"
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    assert store.leases()[0]["state"] == "offered"
+
+
+def test_new_safety_gate_stops_old_revision_immediately(env):
+    _, store, hub, tamq, clock, runtime = env
+    runtime.tick()
+    hub.data["tasks"][0]["needs_human"] = True
+    clock[0] += 15
+    runtime.tick()
+    assert len(tamq.calls) == 1
+    assert all(r["state"] == "stopped" for r in store.leases())
+
+
+def test_failed_transport_stops_coordination(env):
+    _, store, _, tamq, clock, runtime = env
+    runtime.tick()
+    tamq.state = "failed"
+    clock[0] += 15
+    runtime.tick()
+    assert store.leases()[0]["state"] == "stopped"
+
+
+@pytest.mark.parametrize(
+    "checkpoint",
+    [[], {"summary": {}}, {"files_changed": "file"}, {"blocked_reason": []}],
+)
+def test_malformed_checkpoint_does_not_release_lease(env, checkpoint):
+    _, store, _, _, _, runtime = env
+    runtime.tick()
+    lease = store.leases()[0]["lease_id"]
+    runtime.worker_update("ack", lease, "demo")
+    with pytest.raises(ValueError):
+        runtime.worker_update("checkpoint", lease, "demo", checkpoint)
+    assert store.leases()[0]["state"] == "acknowledged"