diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 89f71f2..532a4ff 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -19,7 +19,7 @@ | workplan | REUSE-WP-0009 | finished | — | workplans/REUSE-WP-0009-cli-hardening.md | | workplan | REUSE-WP-0010 | finished | — | workplans/REUSE-WP-0010-network-federation.md | | workplan | REUSE-WP-0016 | finished | — | workplans/REUSE-WP-0016-interactive-registry-maintain.md | -| workplan | REUSE-WP-0020 | active | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | +| workplan | REUSE-WP-0020 | finished | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0001-T01 | done | — | workplans/REUSE-WP-0001-statehub-bootstrap.md | | task | REUSE-WP-0001-T02 | done | — | workplans/REUSE-WP-0001-statehub-bootstrap.md | | task | REUSE-WP-0001-T03 | done | — | workplans/REUSE-WP-0001-statehub-bootstrap.md | @@ -72,9 +72,10 @@ | task | REUSE-WP-0016-T09 | done | — | workplans/REUSE-WP-0016-interactive-registry-maintain.md | | task | REUSE-WP-0020-T01 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0020-T02 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | -| task | REUSE-WP-0020-T03 | wait | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | -| task | REUSE-WP-0020-T04 | wait | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | +| task | REUSE-WP-0020-T03 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | +| task | REUSE-WP-0020-T04 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0020-T05 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0020-T06 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0020-T07 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | | task | REUSE-WP-0020-T08 | done | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | +| task | REUSE-WP-0020-T09 | todo | — | workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md | diff --git a/reuse_surface/hub/app.py b/reuse_surface/hub/app.py index 5cd00a4..b784f6d 100644 --- a/reuse_surface/hub/app.py +++ b/reuse_surface/hub/app.py @@ -20,6 +20,11 @@ from reuse_surface.hub.webhooks import ( HUB_VERSION = "0.1.0" +# Registration fields that change what the federated index composes to. A +# PATCH touching any of them invalidates the composed index; a PATCH that only +# edits, say, the description does not. +COMPOSITION_FIELDS = frozenset({"enabled", "url", "index", "required", "domain"}) + def _db_path() -> Path: return Path(os.environ.get("REUSE_SURFACE_DB", "/data/reuse.db")) @@ -82,11 +87,13 @@ def create_app() -> FastAPI: async def register_repo(request: Request) -> dict[str, Any]: payload = await request.json() try: - return store.create_repo(payload) + registration = store.create_repo(payload) except FileExistsError as exc: raise _http_error(409, "conflict", str(exc)) from exc except ValueError as exc: raise _http_error(400, "validation_error", str(exc)) from exc + store.mark_stale() + return registration @app.get("/v1/repos/{repo}") def get_repo(repo: str) -> dict[str, Any]: @@ -99,16 +106,20 @@ def create_app() -> FastAPI: async def update_repo(repo: str, request: Request) -> dict[str, Any]: payload = await request.json() try: - return store.update_repo(repo, payload) + registration = store.update_repo(repo, payload) except KeyError as exc: raise _http_error(404, "not_found", str(exc)) from exc except ValueError as exc: raise _http_error(400, "validation_error", str(exc)) from exc + if COMPOSITION_FIELDS & set(payload): + store.mark_stale() + return registration @app.delete("/v1/repos/{repo}", status_code=204, dependencies=[Depends(_require_auth)]) def delete_repo(repo: str) -> Response: if not store.delete_repo(repo): raise _http_error(404, "not_found", f"repo not found: {repo}") + store.mark_stale() return Response(status_code=204) async def _federated_response( @@ -116,19 +127,26 @@ def create_app() -> FastAPI: accept: str | None, format_param: str, ) -> Response: - # composed_at/stale track *forced* recomposes (refresh=True: manual - # POST, webhook, scheduled fallback), not every plain GET -- a plain - # GET still serves current best-effort data (compose_from_store's own - # per-source cache_ttl_seconds still applies) but must not silently - # clear a staleness signal nothing actually refreshed. + # composed_at/stale track recomposes that actually refetched, not + # every plain GET -- a plain GET otherwise serves current best-effort + # data (compose_from_store's own per-source cache_ttl_seconds still + # applies) and must not clear a staleness signal nothing acted on. + # + # A GET *does* refresh when the index is marked stale. Registration + # writes set that flag, and without this a newly registered or + # re-enabled repo stays invisible for as long as its cached index + # survives -- silently, since the response would keep reporting + # stale: false. Clearing the flag here is honest because this pass + # really did refetch. async with compose_lock: + effective_refresh = refresh or store.get_compose_state()["stale"] try: federated, warnings = compose_from_store( - store, refresh=refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN + store, refresh=effective_refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN ) except FileNotFoundError as exc: raise _http_error(502, "compose_error", str(exc)) from exc - if refresh: + if effective_refresh: store.record_compose() compose_state = store.get_compose_state() diff --git a/specs/FederationHubAPI.md b/specs/FederationHubAPI.md index 2b46f33..acc8a86 100644 --- a/specs/FederationHubAPI.md +++ b/specs/FederationHubAPI.md @@ -199,14 +199,28 @@ composed_at: "2026-07-07T16:22:09+00:00" # REUSE-WP-0019-T02 stale: false # REUSE-WP-0019-T02 ``` -`composed_at`/`stale` (added REUSE-WP-0019-T02) track *forced* recomposes -only (`refresh=true`, the webhook, or the scheduled fallback) — a plain -`GET` still serves current best-effort data (per-source `cache_ttl_seconds` -still applies) but never silently clears a staleness signal nothing -actually refreshed. `composed_at` is `null` until the first forced -recompose since the hub process's SQLite DB was created. `stale: true` -means a `registry/indexes/` change was pushed (via webhook) since the last -forced recompose completed. +`composed_at`/`stale` (added REUSE-WP-0019-T02) track recomposes that +actually refetched — `refresh=true`, the webhook, the scheduled fallback, or +a plain `GET` that found the index marked stale. A plain `GET` on a +non-stale index still serves current best-effort data (per-source +`cache_ttl_seconds` still applies) and never clears a staleness signal +nothing acted on. `composed_at` is `null` until the first such recompose +since the hub process's SQLite DB was created. + +`stale: true` means the composed index no longer reflects its inputs, set by +either of two triggers: + +- a `registry/indexes/` change pushed by a member repo (via webhook), or +- a **registration change through this API** (REUSE-WP-0020-T09) — `POST + /v1/repos`, `DELETE /v1/repos/{repo}`, or a `PATCH /v1/repos/{repo}` that + touches `enabled`, `url`, `index`, `required`, or `domain`. + +The second trigger closes a silent failure: before it, enabling a source left +the composed index untouched and still reporting `stale: false`, so a repo +could be correctly registered and yet invisible in `/v1/federated` for as long +as its cached index survived. The next `GET` now recomposes and clears the +flag — which is not a silent clear, because that pass really did refetch. A +recompose that *fails* leaves `stale: true` set. Query parameters: diff --git a/tests/test_hub.py b/tests/test_hub.py index 2cc411d..767aa5d 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -208,8 +208,12 @@ def test_get_federated_reports_stale_after_mark(hub_client, monkeypatch): db_path = os.environ["REUSE_SURFACE_DB"] _HubStore(Path(db_path)).mark_stale() - response = hub_client.get("/v1/federated") - assert response.json()["stale"] is True + # A stale index recomposes on the next plain GET and the flag clears -- + # not a silent clear, since that pass refetched (REUSE-WP-0020-T09). + with patch("urllib.request.urlopen", return_value=FakeResponse()): + response = hub_client.get("/v1/federated") + assert response.json()["stale"] is False + assert _HubStore(Path(db_path)).get_compose_state()["stale"] is False # --- T02: Forgejo webhook receiver --- @@ -393,4 +397,154 @@ def test_store_record_reuse_event_and_list(tmp_path): def test_store_record_reuse_event_rejects_invalid(tmp_path): store = HubStore(tmp_path / "hub.db") with pytest.raises(ValueError): - store.record_reuse_event({**VALID_REUSE_EVENT, "verdict": "nope"}) \ No newline at end of file + store.record_reuse_event({**VALID_REUSE_EVENT, "verdict": "nope"}) + +# --- T09: a registration change invalidates the composed index --- + +SECOND_INDEX_BEFORE = """ +version: 1 +domain: helix_forge +updated: "2026-08-20" +capabilities: [] +""" + +SECOND_INDEX_AFTER = """ +version: 1 +domain: helix_forge +updated: "2026-08-21" +capabilities: + - id: capability.remote.second + name: Second Remote + domain: helix_forge + vector: D3/A2/C3/R3 + owner: example + path: registry/capabilities/capability.remote.second.md + summary: Capability a member repo published after its index was repaired + tags: [sample] + consumption_modes: [planning] +""" + +SECOND_URL = "https://example.com/second.yaml" + + +class _Forge: + """Serves a per-URL index body that the test can change mid-flight.""" + + def __init__(self): + self.bodies = { + "https://example.com/capabilities.yaml": REMOTE_INDEX, + SECOND_URL: SECOND_INDEX_BEFORE, + } + + def publish(self, url, body): + self.bodies[url] = body + + def open(self, request, *args, **kwargs): + url = request.full_url if hasattr(request, "full_url") else str(request) + body = self.bodies[url].encode("utf-8") + + class FakeResponse: + def __enter__(self_inner): + return self_inner + + def __exit__(self_inner, *args): + return False + + def read(self_inner): + return body + + return FakeResponse() + + +def _register(hub_client, repo, url): + return hub_client.post( + "/v1/repos", + json={"repo": repo, "url": url, "domain": "helix_forge"}, + headers={"Authorization": "Bearer test-token"}, + ) + + +def _set_enabled(hub_client, repo, value): + return hub_client.patch( + f"/v1/repos/{repo}", + json={"enabled": value}, + headers={"Authorization": "Bearer test-token"}, + ) + + +def test_re_enabled_source_refetches_a_stale_cache(hub_client): + """The evidence-binder incident, reproduced. + + A member's cached index is the broken version. The member repairs and + republishes, and the source is re-enabled -- but the per-source cache is + still inside its 24h TTL, so a plain GET would keep serving the old empty + copy and report stale: false while doing it. The registration change must + force a refetch. + """ + forge = _Forge() + _register(hub_client, "remote-repo", "https://example.com/capabilities.yaml") + _register(hub_client, "second-repo", SECOND_URL) + + with patch("urllib.request.urlopen", side_effect=forge.open): + # cache both sources while second-repo still publishes nothing usable + hub_client.get("/v1/federated?refresh=true") + _set_enabled(hub_client, "second-repo", False) + hub_client.get("/v1/federated?refresh=true") + + # the member repairs its index, then asks to be re-enabled + forge.publish(SECOND_URL, SECOND_INDEX_AFTER) + _set_enabled(hub_client, "second-repo", True) + + response = hub_client.get("/v1/federated") + + ids = {c["id"] for c in response.json()["capabilities"]} + assert "capability.remote.second" in ids, ( + "re-enabled source served from a stale cache -- registered but invisible" + ) + assert response.json()["stale"] is False + + +def test_registration_write_marks_the_index_stale(hub_client): + """A registration write must flag the composed index as no longer current. + + Asserted on the flag rather than on composed_at, which has second + resolution and ties when two composes land in the same second. + """ + from reuse_surface.hub.store import HubStore as _HubStore + + store = _HubStore(Path(os.environ["REUSE_SURFACE_DB"])) + forge = _Forge() + _register(hub_client, "remote-repo", "https://example.com/capabilities.yaml") + + with patch("urllib.request.urlopen", side_effect=forge.open): + hub_client.get("/v1/federated?refresh=true") + assert store.get_compose_state()["stale"] is False + + _register(hub_client, "second-repo", SECOND_URL) + assert store.get_compose_state()["stale"] is True, ( + "registering a repo left the composed index looking current" + ) + + hub_client.get("/v1/federated") + + assert store.get_compose_state()["stale"] is False + + +def test_cosmetic_patch_does_not_force_a_recompose(hub_client): + """A description edit changes nothing about composition; leave the cache alone.""" + forge = _Forge() + _register(hub_client, "remote-repo", "https://example.com/capabilities.yaml") + + with patch("urllib.request.urlopen", side_effect=forge.open): + first = hub_client.get("/v1/federated?refresh=true") + composed_at = first.json()["composed_at"] + + hub_client.patch( + "/v1/repos/remote-repo", + json={"description": "just a label change"}, + headers={"Authorization": "Bearer test-token"}, + ) + after = hub_client.get("/v1/federated") + + assert after.json()["composed_at"] == composed_at + assert after.json()["stale"] is False diff --git a/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md b/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md index 88395a0..97e31db 100644 --- a/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md +++ b/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md @@ -371,7 +371,7 @@ derived from `STALE_DAYS`. Full suite: 179 passed. ```task id: REUSE-WP-0020-T09 -status: todo +status: done priority: medium ``` @@ -393,8 +393,30 @@ Fix: have the write paths on `/v1/repos` (create, and update where `enabled`, `GET /v1/federated` recomposes — or at minimum report `stale: true` so a caller can tell the served view no longer matches the registrations behind it. -Add a hub test: register or enable a source, then `GET /v1/federated` without -`refresh`, and assert the new source's capabilities are present. +**Done 2026-08-21.** `POST /v1/repos`, `DELETE /v1/repos/{repo}`, and a +`PATCH` touching `COMPOSITION_FIELDS` (`enabled`, `url`, `index`, `required`, +`domain`) now call `mark_stale()`; a plain `GET /v1/federated` recomposes when +the flag is set and clears it. A cosmetic `PATCH` (description only) leaves the +cache alone. `specs/FederationHubAPI.md` updated — this changes a documented +contract, so the spec's staleness section was rewritten rather than left to +drift. + +Three tests added. **The first two I wrote were worthless** and it is worth +recording why: they passed with the fix removed. A *newly* registered repo has +no cache entry, so a plain `GET` fetches it regardless — that was never the +bug. The real failure needs a **populated cache holding the old content**: +evidence-binder's cached index was the broken pre-repair copy, still inside the +24h `cache_ttl_seconds`, so the plain `GET` kept serving zero rows from it. + +`test_re_enabled_source_refetches_a_stale_cache` reproduces exactly that — +cache the broken index, disable, republish repaired content, re-enable, plain +`GET` — and fails with an assertion error on pre-T09 code. + +Verified by mutation: reverting the three `mark_stale()` calls makes both +behavioural tests fail. 184 tests pass on the fix. + +**Not yet deployed.** Production still has the bug; it needs an image build of +this commit and a `make reuse-deploy`. ---