From 0c6b1e2538de54713309e7310734d8aa590e95b1 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 21 Aug 2026 00:04:39 +0200 Subject: [PATCH] Harden federated compose against malformed member indexes (REUSE-WP-0020) Repointing the production hub's 50 Gitea-hosted federation sources to Forgejo ahead of the 2026-08-31 CoulombCore retirement took /v1/federated to HTTP 500. One member index (evidence-binder) has capability rows with no `id`, and compose_federated_index dereferenced item["id"] unguarded. Its Gitea copy was a stale snapshot returning a non-mapping, so those rows had never been parsed. A single malformed member index must not take down the whole endpoint. Extract _read_index_entries(): unparseable YAML, a non-mapping body, an empty file, and a non-list `capabilities` each degrade to a warning and an empty row list, and rows without an `id` are skipped individually. A failed source stays listed with count 0 so it remains visible to operators rather than silently disappearing. Also fix wall-clock rot in tests/test_plan_check.py, which was already failing at clean HEAD: three tests pinned the compose date to a literal that has now aged past STALE_DAYS. Add workplan REUSE-WP-0020 covering the full cutover. Co-Authored-By: Claude Opus 5 --- registry/indexes/federated.yaml | 44 +++- reuse_surface/federation.py | 35 ++- tests/test_federation.py | 79 +++++- tests/test_plan_check.py | 20 +- ...-WP-0020-coulombcore-retirement-cutover.md | 241 ++++++++++++++++++ 5 files changed, 403 insertions(+), 16 deletions(-) create mode 100644 workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md diff --git a/registry/indexes/federated.yaml b/registry/indexes/federated.yaml index 0bb55d8..542c451 100644 --- a/registry/indexes/federated.yaml +++ b/registry/indexes/federated.yaml @@ -1,7 +1,7 @@ # Composed federated capability index. Regenerate with: # reuse-surface federation compose version: 1 -updated: '2026-07-07' +updated: '2026-08-21' domain: helix_forge collision_policy: warn sources: @@ -66,7 +66,7 @@ sources: url: https://forgejo.coulomb.social/coulomb/evidence-binder/raw/main/registry/indexes/capabilities.yaml cache: registry/federation/cache/evidence-binder.yaml - repo: evidence-source - count: 0 + count: 1 url: https://forgejo.coulomb.social/coulomb/evidence-source/raw/main/registry/indexes/capabilities.yaml cache: registry/federation/cache/evidence-source.yaml - repo: feature-control @@ -309,9 +309,9 @@ capabilities: source_index: registry/federation/cache/the-custodian.yaml - id: capability.agents.kaizen-framework name: Kaizen Agentic Framework - summary: AI agency framework providing 18 specialized deployable agent instruction - sets plus persistent, project-scoped memory and cross-agent coordination via a - Coach meta-agent. + summary: AI agency framework providing 20 deployable agent instruction sets, project + memory, metrics, role and engagement contracts, and scheduled preparation for + governed execution. vector: D3 / A2 / C1 / R0 domain: agents status: draft @@ -321,9 +321,13 @@ capabilities: - agents - memory - coordination + - metrics + - scheduling + - engagements consumption_modes: - cli - library import + - file contracts source_repo: kaizen-agentic source_url: https://forgejo.coulomb.social/coulomb/kaizen-agentic/raw/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/kaizen-agentic.yaml @@ -372,9 +376,11 @@ capabilities: name: Audit Event Retention summary: Collect, normalize, retain, and search audit events with integrity evidence across tenants. - vector: D4 / A2 / C2 / R1 - domain: helix_forge - status: draft + joins: operations.audit + provision: data/capability/audit-core-operational.json + vector: D4 provision / A4 / C3 / R2 + domain: infotech + status: production owner: audit-core path: registry/capabilities/capability.audit.event-retain.md tags: @@ -382,6 +388,7 @@ capabilities: - retention - compliance consumption_modes: + - http ingest - source module source_repo: audit-core source_url: https://forgejo.coulomb.social/coulomb/audit-core/raw/main/registry/indexes/capabilities.yaml @@ -950,6 +957,27 @@ capabilities: source_repo: open-reuse source_url: https://forgejo.coulomb.social/coulomb/open-reuse/raw/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/open-reuse.yaml +- id: capability.infotech.pdf-evidence-ingest + name: Headless PDF Evidence Ingest + summary: "Turns raw PDF bytes into an engine-shaped Document + DocumentRepresentation\ + \ \u2014 SHA-256 fingerprint, canonical text, page map, and gap-free offset map\ + \ \u2014 as a runtime-agnostic library with no viewer, persistence, or React coupling." + vector: D2 / A1 / C1 / R1 + domain: infotech + status: draft + owner: evidence-source + path: registry/capabilities/capability.infotech.pdf-evidence-ingest.md + tags: + - pdf + - ingest + - fingerprint + - canonical-text + - evidence + consumption_modes: + - library import + source_repo: evidence-source + source_url: https://forgejo.coulomb.social/coulomb/evidence-source/raw/main/registry/indexes/capabilities.yaml + source_index: registry/federation/cache/evidence-source.yaml - id: capability.infotech.repo-template name: Coulomb Repository Template summary: Bootstrap new git repositories with agent instructions, registry scaffold, diff --git a/reuse_surface/federation.py b/reuse_surface/federation.py index 8e58dff..3c8c03e 100644 --- a/reuse_surface/federation.py +++ b/reuse_surface/federation.py @@ -183,6 +183,27 @@ def resolve_source_index_path( return _write_remote_cache(source["repo"], url, content, cache_dir), warnings +def _read_index_entries( + index_path: Path, repo: str +) -> tuple[list[Any], list[str]]: + """Read the capability rows of one member index. + + A malformed member index must not abort the whole compose, so every + failure here degrades to a warning and an empty row list. + """ + try: + with index_path.open(encoding="utf-8") as handle: + index_data = yaml.safe_load(handle) + except yaml.YAMLError as exc: + return [], [f"{repo}: unparseable index, skipped ({exc.__class__.__name__})"] + if not isinstance(index_data, dict): + return [], [f"{repo}: index is not a mapping, skipped"] + entries = index_data.get("capabilities") or [] + if not isinstance(entries, list): + return [], [f"{repo}: capabilities is not a list, skipped"] + return entries, [] + + def compose_federated_index( manifest: dict[str, Any] | None = None, *, @@ -204,11 +225,17 @@ def compose_federated_index( warnings.extend(source_warnings) if index_path is None: continue - with index_path.open(encoding="utf-8") as handle: - index_data = yaml.safe_load(handle) + entries, read_warnings = _read_index_entries(index_path, source["repo"]) + warnings.extend(read_warnings) count = 0 - for item in index_data.get("capabilities", []): - cap_id = item["id"] + for position, item in enumerate(entries): + if not isinstance(item, dict): + warnings.append(f"{source['repo']}: capability #{position} is not a mapping, skipped") + continue + cap_id = item.get("id") + if not cap_id: + warnings.append(f"{source['repo']}: capability #{position} has no id, skipped") + continue if cap_id in seen_ids: warnings.append( f"duplicate id {cap_id}: {seen_ids[cap_id]} and {source['repo']}" diff --git a/tests/test_federation.py b/tests/test_federation.py index eafd490..1119100 100644 --- a/tests/test_federation.py +++ b/tests/test_federation.py @@ -205,4 +205,81 @@ def test_compose_merges_remote_capabilities(tmp_path, monkeypatch): def test_load_production_manifest_still_validates(): manifest = load_federation_manifest() assert manifest["domain"] == "helix_forge" - assert any(source["repo"] == "reuse-surface" for source in manifest["sources"]) \ No newline at end of file + assert any(source["repo"] == "reuse-surface" for source in manifest["sources"]) + +def _compose_with_remote_body(body: str, tmp_path, monkeypatch): + """Compose with the remote member index serving `body` verbatim.""" + monkeypatch.setattr("reuse_surface.federation.CACHE_DIR", tmp_path / "cache") + payload = body.encode("utf-8") + + class FakeResponse: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return payload + + with patch("urllib.request.urlopen", return_value=FakeResponse()): + return compose_federated_index(_remote_manifest()) + + +def _local_ids_survived(federated) -> bool: + return any(item["source_repo"] == "local" for item in federated["capabilities"]) + + +def test_compose_skips_capability_without_id(tmp_path, monkeypatch): + body = """ +version: 1 +domain: helix_forge +capabilities: + - type: library + title: No id at all + - id: capability.remote.sample + name: Remote Sample +""" + federated, warnings = _compose_with_remote_body(body, tmp_path, monkeypatch) + ids = {item["id"] for item in federated["capabilities"]} + assert "capability.remote.sample" in ids + assert _local_ids_survived(federated) + assert any("remote-repo" in w and "no id" in w for w in warnings) + + +def test_compose_skips_non_mapping_index(tmp_path, monkeypatch): + federated, warnings = _compose_with_remote_body( + "just a string, not a mapping\n", tmp_path, monkeypatch + ) + assert _local_ids_survived(federated) + assert any("not a mapping" in w for w in warnings) + + +def test_compose_skips_empty_index(tmp_path, monkeypatch): + federated, warnings = _compose_with_remote_body("", tmp_path, monkeypatch) + assert _local_ids_survived(federated) + assert any("remote-repo" in w for w in warnings) + + +def test_compose_skips_unparseable_index(tmp_path, monkeypatch): + federated, warnings = _compose_with_remote_body( + "capabilities: [unclosed\n", tmp_path, monkeypatch + ) + assert _local_ids_survived(federated) + assert any("unparseable" in w for w in warnings) + + +def test_compose_skips_capabilities_not_a_list(tmp_path, monkeypatch): + federated, warnings = _compose_with_remote_body( + "version: 1\ncapabilities: not-a-list\n", tmp_path, monkeypatch + ) + assert _local_ids_survived(federated) + assert any("not a list" in w for w in warnings) + + +def test_malformed_source_still_listed_with_zero_count(tmp_path, monkeypatch): + federated, _ = _compose_with_remote_body( + "just a string, not a mapping\n", tmp_path, monkeypatch + ) + remote = next(s for s in federated["sources"] if s["repo"] == "remote-repo") + assert remote["count"] == 0 diff --git a/tests/test_plan_check.py b/tests/test_plan_check.py index 5731353..fd9fccb 100644 --- a/tests/test_plan_check.py +++ b/tests/test_plan_check.py @@ -1,6 +1,20 @@ from __future__ import annotations import json +from datetime import datetime, timedelta, timezone + +from reuse_surface.plan_check import STALE_DAYS + + +def _recent_date() -> str: + """A compose date that is always inside the staleness window. + + A hardcoded literal here made the test rot: it passed when written and + began failing once wall-clock time drifted past STALE_DAYS. + """ + fresh = datetime.now(timezone.utc) - timedelta(days=max(STALE_DAYS - 1, 0)) + return fresh.strftime("%Y-%m-%d") + from reuse_surface.plan_check import ( Match, @@ -93,7 +107,7 @@ Build a single interface for issue tracking across Gitea, GitHub, and GitLab. def test_run_plan_check_reuse_verdict(monkeypatch): monkeypatch.setattr( "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-06"), + lambda: (SAMPLE_CAPABILITIES, _recent_date()), ) query = MatchQuery( source="intent", @@ -236,7 +250,7 @@ def test_cmd_plan_check_file_request_flag(monkeypatch): monkeypatch.setattr( "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-06"), + lambda: (SAMPLE_CAPABILITIES, _recent_date()), ) filed_calls = [] monkeypatch.setattr( @@ -255,7 +269,7 @@ def test_cmd_plan_check_intent_json(monkeypatch): monkeypatch.setattr( "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-06"), + lambda: (SAMPLE_CAPABILITIES, _recent_date()), ) exit_code = main( ["plan-check", "--intent", "issue tracking gitea github gitlab", "--format", "json"] diff --git a/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md b/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md new file mode 100644 index 0000000..e45067a --- /dev/null +++ b/workplans/REUSE-WP-0020-coulombcore-retirement-cutover.md @@ -0,0 +1,241 @@ +--- +id: REUSE-WP-0020 +type: workplan +title: "CoulombCore retirement cutover: federation sources, image, and compose resilience" +domain: infotech +repo: reuse-surface +status: active +owner: claude +topic_slug: helix-forge +created: "2026-08-20" +updated: "2026-08-20" +--- + +# CoulombCore retirement cutover + +CoulombCore is switched off **2026-08-31** (operator decision, 2026-08-20). +`gitea.coulomb.social` runs on it. Raised by `prj-state-hub-retirement` +(SHR-WP-0002-T07) as a stale-image report; investigation found a second, +more urgent problem underneath it. + +**Two independent failures**, not one: + +1. **Federation source data** — the production hub's *registrations* pointed + 50 of 61 sources at `gitea.coulomb.social`. This breaks on 08-31 with + nobody restarting anything. It is database state, not code: + `registry/federation/sources.yaml` at HEAD has been 61/61 Forgejo since + `d1c1313` (RAIL-HO-WP-0006). +2. **Deployed image** — `railiance-apps/helm/reuse-surface-values.yaml` pins + `image.tag: "e3ae22e"`. The chart's `repository` was already migrated to + Forgejo in `railiance-apps@04be416`, but `e3ae22e` is commit-dated + 2026-07-07 18:25 and CI only began publishing to Forgejo at 21:25 the same + day — so a Forgejo `:e3ae22e` tag most likely never existed. Any restart, + reschedule, or node reboot risks `ImagePullBackOff` **now**, not on 08-31. + +Item 1 was resolved on 2026-08-20 (see T01). Doing so exposed a third issue: +a single malformed member index takes the whole federated endpoint down. + +**Baseline vector:** `D5 / A4 / C5 / R3` — unchanged by this workplan; +this is operational continuity, not capability growth. + +## Scope boundary + +The Deployment manifest is **not in this repo**. It lives in `railiance-apps` +(`charts/reuse-surface/`, `helm/reuse-surface-values.yaml`, RAILIANCE-WP-0007). +This repo owns the image contents, the deploy guide, and the federation data; +applying the manifest change requires a working `KUBECONFIG=~/.kube/config-hosteurope`, +which is not reachable from the workstation without an ops-bridge tunnel. + +--- + +## Repoint Production Federation Sources To Forgejo + +```task +id: REUSE-WP-0020-T01 +status: done +priority: high +``` + +**Completed 2026-08-20.** Updated 50 enabled hub registrations from +`gitea.coulomb.social//raw/main/...` to +`forgejo.coulomb.social//raw/branch/main/...` via +`reuse-surface hub update --repo --url `. + +Notes for the record: + +- All 50 new URLs were pre-verified `200` before any write. +- The canonical Forgejo raw form is `/raw/branch/main/`. `/raw/main/` answers + `303` and works only because `urllib.request.urlopen` follows redirects — + `registry/federation/sources.yaml` still uses the redirecting form. Harmless, + but prefer the canonical form in new writes. +- One registration (`inter-hub`, already `disabled`) still carries a Gitea URL. + Its Forgejo equivalent answers `307`, so it was left alone rather than + repointed to something unverified. It contributes nothing while disabled. + +Verified: `GET /v1/federated` → 60 sources, all Forgejo, 62 capabilities. + +## Harden Compose Against A Malformed Member Index + +```task +id: REUSE-WP-0020-T02 +status: done +priority: high +``` + +T01 briefly took `/v1/federated` to HTTP 500. Root cause: `evidence-binder`'s +live index has two capability entries with **no `id` field**, and +`compose_federated_index` does `cap_id = item["id"]` unguarded. Its Gitea copy +was a stale snapshot returning a non-mapping, so the entries had never been +parsed before. + +**One bad member index must not take down the federated endpoint.** In +`reuse_surface/federation.py::compose_federated_index`, guard the per-source +body so a failure degrades to a warning and the remaining sources still +compose: + +| Failure | Current | Wanted | +|---|---|---| +| `item["id"]` missing | `KeyError`, 500 | skip entry, warn, keep source | +| `yaml.safe_load` → non-mapping | `AttributeError`, 500 | skip source, warn | +| `yaml.safe_load` → `None` (empty file) | `AttributeError`, 500 | skip source, warn | +| YAML parse error | `YAMLError`, 500 | skip source, warn | +| `sorted(key=item["id"])` | `KeyError` | unreachable once entries are filtered | + +Confirmed present at HEAD, not only in the deployed image — deploying HEAD +would **not** have fixed this. Land T02 before T04. + +Pytest in `tests/test_federation.py`: member index with a missing `id`, an +empty file, a non-mapping body, and unparseable YAML — each composes the +other sources successfully and emits a warning naming the source. + +**Done 2026-08-20.** Extracted `_read_index_entries()`; every read failure now +returns an empty row list plus a warning, and rows without an `id` are skipped +individually. A failed source stays in `sources` with `count: 0` rather than +vanishing, so operators can still see it. Six tests added. Verified against the +real `evidence-binder` index: composes clean with two skip warnings instead of +raising. + +## Fix The evidence-binder Capability Index + +```task +id: REUSE-WP-0020-T03 +status: todo +priority: medium +``` + +Cross-repo, in `~/evidence-binder`. Both entries in +`registry/indexes/capabilities.yaml` ("Evidence-to-target binding", +"Visual-guide rect registry") lack the required `id`. Add ids consistent with +the registry naming convention, confirm against `schemas/capability.schema.yaml`, +and check whether `registry/capabilities/*.md` entries exist behind them or +whether the index rows are orphans. + +Route rather than edit directly if that repo has an owning agent. + +## Re-Enable The evidence-binder Source + +```task +id: REUSE-WP-0020-T04 +status: wait +priority: medium +``` + +Blocked on T02 and T03. `evidence-binder` was set `--no-enabled` on the +production hub to restore service. Once its index is valid and compose is +hardened, re-enable and confirm `/v1/federated` returns 61 sources. + +## Repoint The Production Deployment To A Forgejo Image + +```task +id: REUSE-WP-0020-T05 +status: todo +priority: high +``` + +Deadline-bound: must land before **2026-08-31**. + +1. Verify which tags actually exist in + `forgejo.coulomb.social/coulomb/reuse-surface` (registry API needs auth; + anonymous `GET /v2/.../tags/list` returns `401`) and that + `.forgejo/workflows/image.yaml` is green on `main`. +2. Bump `image.tag` in `railiance-apps/helm/reuse-surface-values.yaml` from + `e3ae22e` to a verified Forgejo tag at current HEAD. +3. Correct `docs/deploy/reuse-kubernetes.md`, which still describes Gitea as + the live registry and its manual `docker build`/`push` commands still + target `gitea.coulomb.social`. +4. Apply from a host with cluster access; verify rollout, `/v1/federated`, + and that the PVC at `/data` survived. + +Ships REUSE-WP-0019 **T04/T05/T06** as a side effect — telemetry store, +R-axis aggregation, and hub freshness monitoring were closed as finished but +landed after the deployed commit and have never run in production. +`GET /v1/reuse-events` returning `404` on production confirms this. + +Review the six weeks of change between `e3ae22e` and HEAD before applying. + +## Correct Hub Freshness And Health Routing + +```task +id: REUSE-WP-0020-T06 +status: todo +priority: low +``` + +Two smaller production inconsistencies found while verifying T01: + +- `GET /health` returns nginx `404` through the ingress, though + `docs/deploy/reuse-kubernetes.md` documents it as the liveness path and the + landing page links it. Only `/v1/*` routes. Pod probes hit the container + directly so the service is unaffected, but the documented URL is wrong — + fix the ingress route or the docs, whichever matches intent. +- `composed_at` stayed at `2026-08-20T20:44:12` with `stale: false` across a + recompose that demonstrably changed output (61 → 62 capabilities). The + freshness timestamp is not tracking recomposes. Re-check after T05, since + REUSE-WP-0019-T06 touches exactly this and is not deployed. + +## Refresh SCOPE.md Standard Sections + +```task +id: REUSE-WP-0020-T07 +status: todo +priority: low +``` + +Unrelated to the retirement, recorded so it is not lost. The hub's repo scope +check reports `C5b`/`C5c` warnings: `SCOPE.md` is missing the standard H2 +sections *Relevant When*, *Not Relevant When*, *How It Fits*, *Terminology*, +*Related / Overlapping*, *Provided Capabilities*, and carries no fenced +capability block. + +## Fix Wall-Clock Rot In The Plan-Check Test Suite + +```task +id: REUSE-WP-0020-T08 +status: done +priority: medium +``` + +Found while verifying T02: `tests/test_plan_check.py` was already failing at +clean HEAD, independent of any change here. Three tests pinned the federated +index compose date to the literal `"2026-07-06"`, and +`test_run_plan_check_reuse_verdict` asserts no staleness warning — so the test +passed when written and started failing once wall-clock time drifted past +`STALE_DAYS`. **CI is red today for this reason alone**, which matters because +T05 calls for verifying the image workflow is green. + +**Done 2026-08-20.** Replaced the literals with a `_recent_date()` helper +derived from `STALE_DAYS`. Full suite: 179 passed. + +--- + +## Verification + +```bash +python3 -m pytest tests/test_federation.py -q +python3 -m reuse_surface.cli validate +python3 -m reuse_surface.cli federation compose --refresh +REUSE_SURFACE_URL=https://reuse.coulomb.social python3 -m reuse_surface.cli hub list +curl -s https://reuse.coulomb.social/v1/federated | python3 -m json.tool | head -20 +``` + +No source may reference `gitea.coulomb.social` after this workplan closes.