Harden SBOM retries and align hub evidence

Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a028de-e2c8-7732-8521-46a7fc5db82f

tests/test_ops_evidence_sinks.py

@@ -367,6 +367,90 @@ def test_core_hub_interaction_event_sink_posts_and_verifies_compact_event(monkey
     assert "token=secret" not in serialized
 
 
+def test_hub_core_interaction_port_posts_and_verifies_sanitized_event(monkeypatch) -> None:
+    posts: list[dict[str, Any]] = []
+
+    def fake_post(url: str, **kwargs: Any) -> DummyResponse:
+        assert url == "http://hub-core.test/ports/events/interaction"
+        assert "Authorization" not in kwargs["headers"]
+        posts.append({"url": url, **kwargs})
+        return DummyResponse(
+            {
+                "id": "event-port-1",
+                "status": "accepted",
+                "correlation_id": _run_id(),
+            },
+            status_code=202,
+        )
+
+    def fake_get(url: str, **kwargs: Any) -> DummyResponse:
+        assert url == "http://hub-core.test/ports/projections/interaction_events"
+        return DummyResponse(
+            {
+                "id": "interaction_events",
+                "data": {"items": [{"id": "event-port-1"}]},
+                "provenance": {},
+            }
+        )
+
+    monkeypatch.setattr(httpx, "post", fake_post)
+    monkeypatch.setattr(httpx, "get", fake_get)
+
+    result = persist_ops_inventory_evidence(
+        _payload([
+            {
+                "type": "hub-core-interaction-event",
+                "hub_core_url": "http://hub-core.test",
+                "event_type": "ops-endpoint-verified",
+            }
+        ])
+    )
+
+    assert result == [
+        {
+            "type": "hub-core-interaction-event",
+            "status": "posted",
+            "event_type": "hub.interaction.recorded",
+            "reported_event_type": "ops-endpoint-verified",
+            "event_id": "event-port-1",
+            "correlation_id": _run_id(),
+            "verified": True,
+            "context_key": "ops_probe",
+        }
+    ]
+    body = posts[0]["json"]
+    assert body["schema_version"] == "0.1.0"
+    assert body["correlation_id"] == _run_id()
+    assert body["event_type"] == "hub.interaction.recorded"
+    assert body["payload"]["reported_event_type"] == "ops-endpoint-verified"
+    assert body["payload"]["endpoint"]["url"] == "http://state-hub.test/health"
+    assert body["subject_refs"]["endpoint"] == "state-hub-health"
+
+    serialized = json.dumps(body, sort_keys=True)
+    assert "secret response body" not in serialized
+    assert "Authorization" not in serialized
+    assert "user:pass" not in serialized
+    assert "token=secret" not in serialized
+
+
+def test_hub_core_interaction_port_skips_when_base_url_missing(monkeypatch) -> None:
+    monkeypatch.delenv("HUB_CORE_BASE_URL", raising=False)
+
+    result = persist_ops_inventory_evidence(
+        _payload([{"type": "hub-core-interaction-event"}])
+    )
+
+    assert result == [
+        {
+            "type": "hub-core-interaction-event",
+            "status": "skipped",
+            "reason": "missing_hub_core_config",
+            "missing": ["HUB_CORE_BASE_URL"],
+            "context_key": "ops_probe",
+        }
+    ]
+
+
 def test_core_hub_sink_skips_cleanly_when_config_missing(monkeypatch) -> None:
     monkeypatch.delenv("CORE_HUB_BASE_URL", raising=False)
     monkeypatch.delenv("CORE_HUB_RUNTIME_TOKEN", raising=False)

tests/test_railiance_ops_inventory_wiring.py

@@ -41,6 +41,9 @@ def test_runtime_config_has_ops_inventory_placeholders() -> None:
     assert config["data"]["OPS_INVENTORY_PATH"] == (
         "/etc/activity-core/ops/service-inventory.yml"
     )
+    assert config["data"]["HUB_CORE_BASE_URL"] == (
+        "http://core-hub-api.core-hub.svc.cluster.local:8010"
+    )
     assert config["data"]["INTER_HUB_URL"] == ""
     assert config["data"]["OPS_HUB_WIDGET_MAPPING"] == ""
 
@@ -68,9 +71,8 @@ def test_external_configmap_projects_disabled_ops_probe_definition(tmp_path) ->
                 "allow_network": True,
                 "evidence_sinks": [
                     {
-                        "type": "state-hub-progress",
+                        "type": "hub-core-interaction-event",
                         "event_type": "ops_inventory_probe",
-                        "author": "activity-core",
                     }
                 ],
             },
@@ -266,12 +268,23 @@ def test_disabled_ops_probe_definition_can_emit_fixture_evidence(
     posts: list[dict[str, Any]] = []
 
     def fake_progress_get(url: str, **kwargs: Any) -> _JsonResponse:
-        return _JsonResponse([])
+        assert url.endswith("/ports/projections/interaction_events")
+        return _JsonResponse({"data": {"items": [{"id": "event-1"}]}})
 
     def fake_progress_post(url: str, **kwargs: Any) -> _JsonResponse:
+        assert url.endswith("/ports/events/interaction")
         posts.append({"url": url, **kwargs})
-        return _JsonResponse({"id": "progress-1"})
+        return _JsonResponse(
+            {
+                "id": "event-1",
+                "status": "accepted",
+                "correlation_id": "12345678-aaaa-bbbb-cccc-123456789abc",
+            },
+            status_code=202,
+        )
 
+    runtime_config = _by_kind_name("ConfigMap", "actcore-runtime-config")
+    monkeypatch.setenv("HUB_CORE_BASE_URL", runtime_config["data"]["HUB_CORE_BASE_URL"])
     monkeypatch.setattr(httpx, "get", fake_progress_get)
     monkeypatch.setattr(httpx, "post", fake_progress_post)
 
@@ -288,8 +301,9 @@ def test_disabled_ops_probe_definition_can_emit_fixture_evidence(
 
     assert definition.enabled is False
     assert result[0]["status"] == "posted"
-    assert posts[0]["json"]["event_type"] == "ops_inventory_probe"
-    assert posts[0]["json"]["detail"]["probe"]["summary"]["ok"] == 4
+    assert posts[0]["json"]["event_type"] == "hub.interaction.recorded"
+    assert posts[0]["json"]["payload"]["reported_event_type"] == "ops_inventory_probe"
+    assert posts[0]["json"]["payload"]["probe"]["summary"]["ok"] == 4
 
 
 class _HttpResponse:

tests/test_sbom_nexus_context_resolver.py

@@ -8,7 +8,10 @@ import httpx
 import pytest
 
 from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY
-from activity_core.context_resolvers.sbom_nexus import SbomNexusContextResolver
+from activity_core.context_resolvers.sbom_nexus import (
+    SbomNexusContextResolver,
+    apply_bounded_ingest,
+)
 
 
 class DummyResponse:
@@ -36,7 +39,7 @@ class DummyClient:
         self._payload = payload
         self._status_code = status_code
         self.calls: list[tuple[str, dict[str, Any] | None]] = []
-        self.posts: list[tuple[str, dict[str, Any] | None]] = []
+        self.posts: list[tuple[str, dict[str, Any] | None, dict[str, str]]] = []
         self.post_results: dict[str, list[DummyResponse]] = {}
 
     def __enter__(self) -> "DummyClient":
@@ -49,8 +52,13 @@ class DummyClient:
         self.calls.append((url, params))
         return DummyResponse(self._payload, self._status_code)
 
-    def post(self, url: str, json: dict[str, Any] | None = None) -> DummyResponse:
-        self.posts.append((url, json))
+    def post(
+        self,
+        url: str,
+        json: dict[str, Any] | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> DummyResponse:
+        self.posts.append((url, json, headers or {}))
         for suffix, results in self.post_results.items():
             if url.endswith(suffix) and results:
                 return results.pop(0)
@@ -160,7 +168,7 @@ def test_catch_up_truncates_an_over_long_response(monkeypatch) -> None:
     assert result["selected_count"] == 2
 
 
-def test_apply_processes_at_most_limit_and_records_terminal_outcomes(monkeypatch) -> None:
+def test_declared_apply_remains_read_only_during_selection(monkeypatch) -> None:
     repos = [
         {
             "repo_slug": "no-checkout",
@@ -182,6 +190,31 @@ def test_apply_processes_at_most_limit_and_records_terminal_outcomes(monkeypatch
         },
     ]
     client = _install(monkeypatch, _payload(repos))
+    result = SbomNexusContextResolver().resolve(
+        "catch_up", None, {"limit": 2, "apply": True}
+    )
+
+    assert result["selected_count"] == 2
+    assert "attempted_count" not in result
+    assert client.posts == []
+
+
+def test_apply_processes_only_fixed_targets_and_records_terminal_outcomes(monkeypatch) -> None:
+    repos = [
+        {
+            "repo_slug": "no-checkout",
+            "last_sbom_at": None,
+            "has_sbom": False,
+            "checkout_available": False,
+        },
+        {
+            "repo_slug": "scan-me",
+            "last_sbom_at": None,
+            "has_sbom": False,
+            "checkout_available": True,
+        },
+    ]
+    client = _install(monkeypatch, _payload([]))
     client.post_results = {
         "/sbom/no-checkout/skip": [
             DummyResponse(
@@ -205,8 +238,9 @@ def test_apply_processes_at_most_limit_and_records_terminal_outcomes(monkeypatch
         ],
     }
 
-    result = SbomNexusContextResolver().resolve(
-        "catch_up", None, {"limit": 2, "apply": True}
+    result = apply_bounded_ingest(
+        repos,
+        operation_id="run-1",
     )
 
     assert result["attempted_count"] == 2
@@ -227,48 +261,141 @@ def test_apply_processes_at_most_limit_and_records_terminal_outcomes(monkeypatch
         }
     ]
     assert len(client.posts) == 2
-    assert all("not-selected" not in url for url, _body in client.posts)
+    first_key = client.posts[0][2]["Idempotency-Key"]
+    assert first_key == client.posts[0][2]["X-Activity-Core-Operation-ID"]
+    assert first_key != client.posts[1][2]["Idempotency-Key"]
 
 
-def test_apply_records_ingest_error_when_ingest_fails(monkeypatch) -> None:
+def test_ambiguous_ingest_failure_does_not_create_synthetic_skip(monkeypatch) -> None:
     client = _install(
         monkeypatch,
-        _payload(
-            [
-                {
-                    "repo_slug": "broken",
-                    "last_sbom_at": None,
-                    "has_sbom": False,
-                    "checkout_available": True,
-                }
-            ]
-        ),
+        _payload([]),
     )
     client.post_results = {
         "/sbom/broken/ingest": [DummyResponse({}, status_code=503)],
-        "/sbom/broken/skip": [
+    }
+
+    with pytest.raises(httpx.HTTPStatusError):
+        apply_bounded_ingest(
+            [{"repo_slug": "broken", "checkout_available": True}],
+            operation_id="run-ambiguous",
+        )
+
+    assert len(client.posts) == 1
+    assert client.posts[0][0].endswith("/sbom/broken/ingest")
+
+
+def test_retry_resumes_heartbeat_outcomes(monkeypatch) -> None:
+    repos = [
+        {"repo_slug": "first", "checkout_available": True},
+        {"repo_slug": "second", "checkout_available": True},
+    ]
+    first_client = _install(monkeypatch, _payload([]))
+    first_client.post_results = {
+        "/sbom/first/ingest": [
             DummyResponse(
                 {
-                    "repo_slug": "broken",
-                    "status": "skipped",
-                    "reason": "ingest-error",
-                    "snapshot_id": "skip-error-1",
+                    "repo_slug": "first",
+                    "status": "ingested",
+                    "snapshot_id": "snapshot-first",
+                }
+            )
+        ],
+        "/sbom/second/ingest": [DummyResponse({}, status_code=503)],
+    }
+    heartbeats: list[list[dict[str, Any]]] = []
+
+    with pytest.raises(httpx.HTTPStatusError):
+        apply_bounded_ingest(
+            repos,
+            operation_id="run-retry",
+            on_progress=lambda outcomes: heartbeats.append(outcomes),
+        )
+
+    completed = heartbeats[-1]
+    retry_client = _install(monkeypatch, _payload([]))
+    retry_client.post_results = {
+        "/sbom/second/ingest": [
+            DummyResponse(
+                {
+                    "repo_slug": "second",
+                    "status": "ingested",
+                    "snapshot_id": "snapshot-second",
+                }
+            )
+        ]
+    }
+
+    result = apply_bounded_ingest(
+        repos,
+        operation_id="run-retry",
+        completed=completed,
+    )
+
+    assert [outcome["repo_slug"] for outcome in result["updated"]] == [
+        "first",
+        "second",
+    ]
+    assert len(retry_client.posts) == 1
+    assert retry_client.posts[0][0].endswith("/sbom/second/ingest")
+
+
+def test_retry_reuses_stable_identity_for_same_run_and_repo(monkeypatch) -> None:
+    repo = {"repo_slug": "ambiguous", "checkout_available": True}
+    first_client = _install(monkeypatch, _payload([]))
+    first_client.post_results = {
+        "/sbom/ambiguous/ingest": [DummyResponse({}, status_code=503)],
+    }
+
+    with pytest.raises(httpx.HTTPStatusError):
+        apply_bounded_ingest([repo], operation_id="run-stable")
+
+    retry_client = _install(monkeypatch, _payload([]))
+    retry_client.post_results = {
+        "/sbom/ambiguous/ingest": [
+            DummyResponse(
+                {
+                    "repo_slug": "ambiguous",
+                    "status": "ingested",
+                    "snapshot_id": "snapshot-ambiguous",
                 }
             )
         ],
     }
+    apply_bounded_ingest([repo], operation_id="run-stable")
 
-    result = SbomNexusContextResolver().resolve(
-        "catch_up", None, {"limit": 3, "apply": True}
+    first_headers = first_client.posts[0][2]
+    retry_headers = retry_client.posts[0][2]
+    assert retry_headers["Idempotency-Key"] == first_headers["Idempotency-Key"]
+    assert retry_headers["X-Activity-Core-Operation-ID"] == first_headers[
+        "X-Activity-Core-Operation-ID"
+    ]
+
+
+def test_apply_collapses_duplicate_targets(monkeypatch) -> None:
+    client = _install(monkeypatch, _payload([]))
+    client.post_results = {
+        "/sbom/duplicate/skip": [
+            DummyResponse(
+                {
+                    "repo_slug": "duplicate",
+                    "status": "skipped",
+                    "reason": "no-checkout",
+                }
+            )
+        ]
+    }
+
+    result = apply_bounded_ingest(
+        [
+            {"repo_slug": "duplicate", "checkout_available": False},
+            {"repo_slug": "duplicate", "checkout_available": False},
+        ],
+        operation_id="run-duplicates",
     )
 
     assert result["attempted_count"] == 1
-    assert result["updated"] == []
-    assert result["skipped"][0]["reason"] == "ingest-error"
-    assert client.posts[1][1] == {
-        "reason": "ingest-error",
-        "detail": "HTTPStatusError",
-    }
+    assert len(client.posts) == 1
 
 
 def test_read_only_default_never_posts(monkeypatch) -> None:
@@ -281,6 +408,106 @@ def test_read_only_default_never_posts(monkeypatch) -> None:
     assert client.posts == []
 
 
+@pytest.mark.asyncio
+async def test_dedicated_activity_applies_truncated_workflow_selection(monkeypatch) -> None:
+    from activity_core import activities
+
+    captured: dict[str, Any] = {}
+
+    def fake_apply(repos, *, operation_id, completed, on_progress):
+        captured.update(
+            repos=repos,
+            operation_id=operation_id,
+            completed=completed,
+        )
+        return {"attempted_count": len(repos), "updated": [], "skipped": []}
+
+    monkeypatch.setattr(
+        "activity_core.context_resolvers.sbom_nexus.apply_bounded_ingest",
+        fake_apply,
+    )
+    patch = await activities.apply_sbom_catchup(
+        {
+            "run_id": "run-fixed",
+            "context_sources": [
+                {
+                    "type": "sbom-nexus",
+                    "query": "catch_up",
+                    "bind_to": "context.catchup",
+                    "params": {"limit": 2, "apply": True},
+                }
+            ],
+            "context": {
+                "catchup": {
+                    "limit": 2,
+                    "repos": [
+                        {"repo_slug": "a"},
+                        {"repo_slug": "b"},
+                        {"repo_slug": "must-not-run"},
+                    ],
+                }
+            },
+        }
+    )
+
+    assert [repo["repo_slug"] for repo in captured["repos"]] == ["a", "b"]
+    assert captured["operation_id"] == "run-fixed"
+    assert patch["catchup"]["attempted_count"] == 2
+
+
+@pytest.mark.asyncio
+async def test_dedicated_activity_forwards_heartbeat_outcomes(monkeypatch) -> None:
+    from activity_core import activities
+
+    completed = [
+        {
+            "repo_slug": "already-done",
+            "status": "skipped",
+            "reason": "no-checkout",
+        }
+    ]
+    captured: dict[str, Any] = {}
+
+    monkeypatch.setattr(
+        activities,
+        "_sbom_heartbeat_state",
+        lambda run_id: {
+            "run_id": run_id,
+            "outcomes_by_bind": {"catchup": completed},
+        },
+    )
+
+    def fake_apply(repos, *, operation_id, completed, on_progress):
+        captured["completed"] = completed
+        return {"attempted_count": 1, "updated": [], "skipped": completed}
+
+    monkeypatch.setattr(
+        "activity_core.context_resolvers.sbom_nexus.apply_bounded_ingest",
+        fake_apply,
+    )
+    await activities.apply_sbom_catchup(
+        {
+            "run_id": "run-resumed",
+            "context_sources": [
+                {
+                    "type": "sbom-nexus",
+                    "query": "catch_up",
+                    "bind_to": "context.catchup",
+                    "params": {"limit": 1, "apply": True},
+                }
+            ],
+            "context": {
+                "catchup": {
+                    "limit": 1,
+                    "repos": [{"repo_slug": "already-done"}],
+                }
+            },
+        }
+    )
+
+    assert captured["completed"] == completed
+
+
 def test_catch_up_normalises_partial_entries(monkeypatch) -> None:
     _install(
         monkeypatch,