diff --git a/.custodian-brief.md b/.custodian-brief.md index fe8daca..d494fb4 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,17 +2,25 @@ # Custodian Brief — reuse-surface **Domain:** infotech -**Last synced:** 2026-07-07 16:25 UTC +**Last synced:** 2026-07-07 15:05 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams +### plan-check: close the consumption loop for capability reuse +Progress: 5/6 done | workstream_id: `cd8683ff-6e6c-4f6c-a62f-565bd55113ea` + +**Open tasks:** +- · Add llm-connect Semantic Rerank `9040505a` + ### Forgejo-native federation automation and reuse telemetry -Progress: 2/6 done | workstream_id: `569be717-34f8-4039-bb26-497685f60159` +Progress: 0/6 done | workstream_id: `569be717-34f8-4039-bb26-497685f60159` **Open tasks:** - ! Forgejo Webhook Rollout And Scheduled Fallback `aa9e9f80` - ! Telemetry Aggregation Into R-Axis Evidence `f0282cfa` +- · Forge Host Abstraction And URL Migration Inventory `4a187b56` +- · Hub Recompose Endpoint And Webhook Receiver `691eb32a` - · Reuse Telemetry Store And Recording `c8e9064e` - · Freshness Monitoring, Docs, SCOPE `a9f44d45` diff --git a/.forgejo/workflows/ci-smoke.yaml b/.forgejo/workflows/ci-smoke.yaml deleted file mode 100644 index bd44c56..0000000 --- a/.forgejo/workflows/ci-smoke.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Canonical CI smoke template (tier 1 routing drill). -# Copy to: .forgejo/workflows/ci-smoke.yaml in consumer repos. -name: CI Smoke - -on: - push: - branches: - - main - workflow_dispatch: - -jobs: - host-smoke: - runs-on: self-hosted - steps: - - name: Routing probe (host runner) - run: | - set -eu - echo "repository=${GITHUB_REPOSITORY:-unknown}" - echo "sha=${GITHUB_SHA:-unknown}" - echo "runner=${RUNNER_NAME:-unknown}" - uname -a - - container-smoke: - runs-on: ubuntu-latest - steps: - - name: Routing probe (container label) - run: | - set -eu - echo "container-smoke ok for ${GITHUB_REPOSITORY:-unknown}" \ No newline at end of file diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml deleted file mode 100644 index cc9d5bf..0000000 --- a/.forgejo/workflows/ci.yml +++ /dev/null @@ -1,77 +0,0 @@ -# Ported from .gitea/workflows/ci.yml (REUSE-WP-0019-T03). No actions/checkout -# on this runner substrate (railiance-enablement/docs/forgejo-actions-workflow-templates.md) -# -- ubuntu-latest maps to docker://node:20-bookworm, no Python preinstalled, -# so use archive checkout + apt. -name: ci - -on: - push: - branches: [main] - pull_request: - branches: [main] - workflow_dispatch: - -jobs: - validate-registry: - runs-on: ubuntu-latest - steps: - - name: Archive checkout - run: | - set -eu - REF="${GITHUB_SHA:-main}" - SHORT="${REF:0:7}" - mkdir -p /tmp/repo - wget -qO /tmp/repo.tar.gz "https://forgejo.coulomb.social/${GITHUB_REPOSITORY}/archive/${SHORT}.tar.gz" - tar xzf /tmp/repo.tar.gz -C /tmp/repo --strip-components=1 - - - name: Install Python + package - working-directory: /tmp/repo - run: | - set -eu - apt-get update -qq - apt-get install -y -qq python3 python3-pip python3-venv >/dev/null - python3 -m pip install --break-system-packages -e ".[dev]" - - - name: Validate capability registry - working-directory: /tmp/repo - run: reuse-surface validate --relations --fail-on-warnings - - - name: Compose federated index - working-directory: /tmp/repo - run: reuse-surface federation compose - - - name: Generate catalog and graph - working-directory: /tmp/repo - run: | - reuse-surface catalog - reuse-surface graph --check --fail-on-warnings - - - name: Registry maintain dry-run (informational) - working-directory: /tmp/repo - run: reuse-surface maintain --all --auto --no-llm || true - - - name: Registry stats (informational) - working-directory: /tmp/repo - run: reuse-surface stats || true - - - name: Workstation roster federation stats (informational) - working-directory: /tmp/repo - run: | - reuse-surface stats --roster registry/federation/local-repo-roster.yaml \ - --federation-ready --format json || true - - - name: Planning cohort report (informational) - working-directory: /tmp/repo - run: reuse-surface report cohorts --planning-min D4 || true - - - name: Registry gap report (informational) - working-directory: /tmp/repo - run: reuse-surface report gaps || true - - - name: Plan-check smoke test (informational) - working-directory: /tmp/repo - run: reuse-surface plan-check --intent "smoke test" --format json || true - - - name: Run tests - working-directory: /tmp/repo - run: pytest -q diff --git a/.forgejo/workflows/image.yaml b/.forgejo/workflows/image.yaml deleted file mode 100644 index 6dd7329..0000000 --- a/.forgejo/workflows/image.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Canonical single-repo image build template (railiance-enablement/workflows/container-build-push.yaml). -# Requires org secrets: REGISTRY_USER, REGISTRY_TOKEN -name: Build and Publish Container Image - -on: - push: - branches: - - main - paths: - - ".forgejo/workflows/image.yaml" - - "Dockerfile" - - "reuse_surface/**" - - "schemas/**" - - "pyproject.toml" - workflow_dispatch: - -env: - REGISTRY: forgejo.coulomb.social - IMAGE_NAME: coulomb/reuse-surface - DOCKER_HOST: tcp://127.0.0.1:2375 - -jobs: - build-and-push: - runs-on: container-build - steps: - - name: Build and push image - env: - REGISTRY_USER: ${{ secrets.REGISTRY_USER }} - REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} - run: | - set -eu - REF="${GITHUB_SHA:-main}" - SHORT="${REF:0:7}" - mkdir -p buildctx "${HOME}/bin" - wget -qO /tmp/repo.tar.gz \ - "https://forgejo.coulomb.social/${GITHUB_REPOSITORY}/archive/${SHORT}.tar.gz" - tar xzf /tmp/repo.tar.gz -C buildctx --strip-components=1 - wget -qO- https://download.docker.com/linux/static/stable/x86_64/docker-27.3.1.tgz \ - | tar xz --strip-components=1 -C "${HOME}/bin" docker/docker - export PATH="${HOME}/bin:${PATH}" - echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USER}" --password-stdin - IMAGE="${REGISTRY}/${IMAGE_NAME}" - docker build -t "${IMAGE}:latest" -t "${IMAGE}:main-${SHORT}" buildctx - docker push "${IMAGE}:latest" - docker push "${IMAGE}:main-${SHORT}" - echo "pushed ${IMAGE}:latest and ${IMAGE}:main-${SHORT}" diff --git a/.forgejo/workflows/recompose-fallback.yaml b/.forgejo/workflows/recompose-fallback.yaml deleted file mode 100644 index 4de93da..0000000 --- a/.forgejo/workflows/recompose-fallback.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Scheduled fallback recompose trigger (REUSE-WP-0019-T03 design principle 3: -# "degrade to schedule" if the org-level webhook is unavailable or misses an -# event). The webhook (POST /v1/webhooks/forgejo) is the primary path; this -# just guarantees the hub's composed index doesn't go stale indefinitely if a -# webhook delivery is ever missed. -name: Recompose Fallback - -on: - schedule: - - cron: "17 */6 * * *" - workflow_dispatch: - -jobs: - recompose: - runs-on: ubuntu-latest - steps: - - name: Trigger hub recompose - env: - REUSE_SURFACE_TOKEN: ${{ secrets.REUSE_SURFACE_TOKEN }} - run: | - set -eu - apt-get update -qq - apt-get install -y -qq curl >/dev/null - status=$(curl -sS -o /tmp/compose.json -w '%{http_code}' \ - -X POST "https://reuse.coulomb.social/v1/federated/compose" \ - -H "Authorization: Bearer ${REUSE_SURFACE_TOKEN}") - echo "status=${status}" - cat /tmp/compose.json - [ "${status}" = "200" ] diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..706f6f0 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,56 @@ +name: ci + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + validate-registry: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install package + run: python -m pip install -e ".[dev]" + + - name: Validate capability registry + run: reuse-surface validate --relations --fail-on-warnings + + - name: Compose federated index + run: reuse-surface federation compose + + - name: Generate catalog and graph + run: | + reuse-surface catalog + reuse-surface graph --check --fail-on-warnings + + - name: Registry maintain dry-run (informational) + run: reuse-surface maintain --all --auto --no-llm || true + + - name: Registry stats (informational) + run: reuse-surface stats || true + + - name: Workstation roster federation stats (informational) + run: | + reuse-surface stats --roster registry/federation/local-repo-roster.yaml \ + --federation-ready --format json || true + + - name: Planning cohort report (informational) + run: reuse-surface report cohorts --planning-min D4 || true + + - name: Registry gap report (informational) + run: reuse-surface report gaps || true + + - name: Plan-check smoke test (informational) + run: reuse-surface plan-check --intent "smoke test" --format json || true + + - name: Run tests + run: pytest -q \ No newline at end of file diff --git a/SCOPE.md b/SCOPE.md index 52d2810..8026e1a 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -80,11 +80,6 @@ The MVP registry foundation, CLI tooling (REUSE-WP-0003), federation stack get a reuse/extend/new verdict; `--file-request` bridges a `new` verdict to a State Hub capability request; `report gaps --check-capability-requests` surfaces open requests with no matching capability -- **Get automatic hub refresh** (REUSE-WP-0019-T02) — a Forgejo push - webhook (`POST /v1/webhooks/forgejo`, HMAC-signed) recomposes the hub's - federated index when a registered repo's `registry/indexes/` changes, - with `composed_at`/`stale` visibility on `GET /v1/federated` and a - scheduled fallback recompose if a webhook delivery is ever missed Registry **tooling** availability is **A4** (CLI plus hosted hub HTTP API). Registry **authoring** remains Markdown-first; consumption combines entries, the @@ -92,6 +87,9 @@ index, CLI automation, and the production hub. ## What Is Not Possible Yet +- **Automatic hub refresh** — federated compose is on-demand; no polling or + webhooks + - **Multi-domain federation** — all indexed capabilities remain `helix_forge` - **Planning analytics breadth** — `report gaps` shipped (REUSE-WP-0015-T03); no roadmap views or standardization tracker beyond `overlaps` and compose @@ -125,11 +123,8 @@ See `tools/README.md` for command reference. - **Specs:** `specs/FederationHubAPI.md`, `schemas/hub-registration.schema.yaml`. - **Docs:** `docs/CapabilityRegistryConcept.md`, `docs/RegistryFederation.md`, `docs/IntentScopeGapAnalysis.md`, deploy guide `docs/deploy/reuse-kubernetes.md`. -- **CI:** `.forgejo/workflows/ci.yml` — validate, federation compose, catalog, - graph, pytest, informational `report cohorts`, `stats --roster`, `report gaps` - (migrated from Gitea 2026-07-07, REUSE-WP-0019-T03). Also - `.forgejo/workflows/image.yaml` (container build/push) and - `recompose-fallback.yaml` (scheduled hub recompose, webhook backstop). +- **CI:** `.gitea/workflows/ci.yml` — validate, federation compose, catalog, + graph, pytest, informational `report cohorts`, `stats --roster`, `report gaps`. - **Relation graph:** `docs/graph/capability-graph.mmd`, `docs/graph/index.html`. - **Searchable catalog:** `docs/catalog/search.html`. - **Workplans:** REUSE-WP-0001 through REUSE-WP-0015 finished (archived); diff --git a/docs/IntentScopeGapAnalysis.md b/docs/IntentScopeGapAnalysis.md index a98ce2c..ce63ef0 100644 --- a/docs/IntentScopeGapAnalysis.md +++ b/docs/IntentScopeGapAnalysis.md @@ -202,7 +202,7 @@ See §4 and archived workplans `workplans/archived/`. | 26 | Federated ID deduplication | Per-owner removal from reuse-surface index | **Closed** (WP-0015-T02) | | 27 | Planning analytics + standardization | Gap report or standardization tracker | **Partial** — gap report shipped (T03); tracker deferred | | 28 | Registry maintenance automation | Interactive `maintain` + `--auto` with llm-connect | **Closed** (WP-0016) | -| 29 | Consumption loop (query-before-build) | `plan-check` command + State Hub capability-request bridge | **Closed** — all six tasks (T01–T06) shipped 2026-07-07; propagation of the ecosystem convention into the shared rules template is now the-custodian's own follow-through, not a reuse-surface blocker | +| 29 | Consumption loop (query-before-build) | `plan-check` command + State Hub capability-request bridge | **Partial** — deterministic matching (T02), the request bridge (T04), docs/CI (T06), and the ecosystem session-protocol convention (T05, propagation proposed to custodian-agent 2026-07-07) shipped; only LLM semantic rerank (T03) remains open, deferred pending a running `llm-connect` instance | **Workplan:** `workplans/REUSE-WP-0016-interactive-registry-maintain.md` (priority 28); `workplans/REUSE-WP-0015-federation-polish-and-planning-analytics.md` (25–27); @@ -239,5 +239,4 @@ See §4 and archived workplans `workplans/archived/`. | 2026-06-16 | Assessment persisted; **REUSE-WP-0015** created for priorities 25–27 | | 2026-06-16 | **REUSE-WP-0016** closed priority 28 (interactive `maintain`, `--auto`, templates) | | 2026-07-07 | **REUSE-WP-0018** partially closed priority 29 (`plan-check` deterministic matching + State Hub capability-request bridge; LLM rerank and ecosystem rollout remain open) | -| 2026-07-07 | **REUSE-WP-0018-T05** closed following REUSE-WP-0017 completion (61 published capabilities); only T03 (LLM rerank) remains open under priority 29 | -| 2026-07-07 | **REUSE-WP-0018-T03** closed once `llm-connect` came up locally; priority 29 fully closed (all six T01–T06 tasks shipped) | \ No newline at end of file +| 2026-07-07 | **REUSE-WP-0018-T05** closed following REUSE-WP-0017 completion (61 published capabilities); only T03 (LLM rerank) remains open under priority 29 | \ No newline at end of file diff --git a/docs/RegistryFederation.md b/docs/RegistryFederation.md index d52fabf..26327a2 100644 --- a/docs/RegistryFederation.md +++ b/docs/RegistryFederation.md @@ -94,24 +94,13 @@ returns **200** with valid YAML (not a redirect to login or HTML). Entry bodies remain in the source repo; the index is the federation surface. -### Raw URL shape (Gitea or Forgejo) - -The organization is mid-transition from Gitea to Forgejo (REUSE-WP-0019); -sibling repos migrate their git remote independently, so both forms are -valid depending on which host a given repo currently lives on. Prefer the -branch-qualified form — both forges serve it directly with no redirect, -unlike the shorter `/raw//...` form which 303-redirects: +### Gitea raw URL shape ```text -https://gitea.coulomb.social/coulomb//raw/branch//registry/indexes/capabilities.yaml -https://forgejo.coulomb.social/coulomb//raw/branch//registry/indexes/capabilities.yaml +https://gitea.coulomb.social/coulomb//raw//registry/indexes/capabilities.yaml ``` -Use `main` (or the repo's default branch). `reuse-surface federation -migrate-host` rewrites a repo's registered URL from one host to the other -once its remote has actually migrated (verifies reachability before -writing — never blind-rewrites a repo that hasn't moved yet). Verify before -registration: +Use `main` (or the repo's default branch). Verify before registration: ```bash curl -fsSI "" | head -n1 # expect HTTP/2 200 or HTTP/1.1 200 @@ -143,17 +132,17 @@ reuse-surface establish --scaffold --domain helix_forge reuse-surface validate git push origin main reuse-surface establish --publish-check \ - --raw-url https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml + --raw-url https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml ``` ### Ongoing maintenance (from sibling repo) ```bash -export LLM_CONNECT_URL=http://127.0.0.1:8080 # optional +export LLM_CONNECT_URL=http://127.0.0.1:8088 # optional reuse-surface maintain --all --from-git-since origin/main reuse-surface maintain --all --auto --no-llm # CI / pre-commit reuse-surface maintain --publish \ - --raw-url https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml \ + --raw-url https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml \ --all --auto --no-llm ``` diff --git a/registry/README.md b/registry/README.md index 0e4dd4b..9001665 100644 --- a/registry/README.md +++ b/registry/README.md @@ -70,7 +70,7 @@ reuse-surface maintain --all --auto --no-llm With llm-connect for maturity suggestions: ```bash -export LLM_CONNECT_URL=http://127.0.0.1:8080 +export LLM_CONNECT_URL=http://127.0.0.1:8088 reuse-surface maintain --all --from-git-since HEAD~5 ``` diff --git a/registry/capabilities/capability.registry.register.md b/registry/capabilities/capability.registry.register.md index 05185f9..3cb101b 100644 --- a/registry/capabilities/capability.registry.register.md +++ b/registry/capabilities/capability.registry.register.md @@ -118,7 +118,7 @@ evidence: tests: - tests/test_hub.py - tests/test_hub_sync.py - - .forgejo/workflows/ci.yml + - .gitea/workflows/ci.yml consumer_feedback: - > reuse-surface dogfood (REUSE-WP-0011): production hub registration and diff --git a/registry/capabilities/capability.registry.validate.md b/registry/capabilities/capability.registry.validate.md index bd68d33..9a0a401 100644 --- a/registry/capabilities/capability.registry.validate.md +++ b/registry/capabilities/capability.registry.validate.md @@ -77,7 +77,7 @@ relations: evidence: documentation: - tools/README.md - - .forgejo/workflows/ci.yml + - .gitea/workflows/ci.yml tests: - tests/test_registry.py consumer_feedback: diff --git a/registry/federation/sources.yaml b/registry/federation/sources.yaml index f8f56ea..bbce8a4 100644 --- a/registry/federation/sources.yaml +++ b/registry/federation/sources.yaml @@ -3,7 +3,7 @@ domain: helix_forge collision_policy: warn sources: - repo: activity-core - url: https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml + url: https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml enabled: true required: false domain: helix_forge @@ -375,7 +375,7 @@ sources: cache_ttl_seconds: 86400 auth_header: Authorization - repo: state-hub - url: https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml + url: https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml enabled: true required: false domain: helix_forge diff --git a/registry/indexes/federated.yaml b/registry/indexes/federated.yaml index bae2f8f..2575968 100644 --- a/registry/indexes/federated.yaml +++ b/registry/indexes/federated.yaml @@ -7,7 +7,7 @@ collision_policy: warn sources: - repo: activity-core count: 1 - url: https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml + url: https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml cache: registry/federation/cache/activity-core.yaml - repo: agentic-resources count: 0 @@ -219,7 +219,7 @@ sources: cache: registry/federation/cache/shard-wiki.yaml - repo: state-hub count: 2 - url: https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml + url: https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml cache: registry/federation/cache/state-hub.yaml - repo: tegwick-control count: 0 @@ -266,7 +266,7 @@ capabilities: consumption_modes: - informational source_repo: activity-core - source_url: https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml + source_url: https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/activity-core.yaml - id: capability.agents.cli-assistant name: Console-Native LLM Assistant (cya) @@ -869,7 +869,6 @@ capabilities: consumption_modes: - informational - planning - - cli source_repo: core-hub source_url: https://forgejo.coulomb.social/coulomb/core-hub/raw/branch/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/core-hub.yaml @@ -1014,7 +1013,7 @@ capabilities: summary: "Django application (with a Vite/Tailwind frontend) for managing German\ \ public-procurement (Vergabe) tender participation \u2014 Ausschreibungs- und\ \ Teilnahme-Management-System." - vector: D3 / A1 / C1 / R1 + vector: D1 / A1 / C0 / R0 domain: communication status: draft owner: vergabe-teilnahme @@ -1275,7 +1274,7 @@ capabilities: consumption_modes: - service API source_repo: state-hub - source_url: https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml + source_url: https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/state-hub.yaml - id: capability.statehub.workstream-coordinate name: Workstream And Task Coordination @@ -1294,7 +1293,7 @@ capabilities: - service API - HTTP REST source_repo: state-hub - source_url: https://forgejo.coulomb.social/coulomb/state-hub/raw/branch/main/registry/indexes/capabilities.yaml + source_url: https://gitea.coulomb.social/coulomb/state-hub/raw/main/registry/indexes/capabilities.yaml source_index: registry/federation/cache/state-hub.yaml - id: capability.wiki.adapter-contract name: Capability-Aware Shard Adapter Contract diff --git a/reuse_surface/cli.py b/reuse_surface/cli.py index 0e5a227..7b7205c 100644 --- a/reuse_surface/cli.py +++ b/reuse_surface/cli.py @@ -11,7 +11,6 @@ from jsonschema import Draft202012Validator from reuse_surface.catalog import write_catalog from reuse_surface.federation import write_federated_index -from reuse_surface.forge_host import migrate_source_host from reuse_surface import hub_client from reuse_surface.graph import check_relations, render_mermaid, write_graph from reuse_surface.hub_sync import ( @@ -265,56 +264,6 @@ def cmd_federation_compose(args: argparse.Namespace) -> int: return 0 -def cmd_federation_migrate_host(args: argparse.Namespace) -> int: - sources_path = Path(args.sources) if args.sources else DEFAULT_SOURCES_PATH - manifest = load_sources_manifest(sources_path) - sources = manifest.get("sources", []) - - reports: list[dict[str, Any]] = [] - errors: list[str] = [] - for repo in args.repo: - entry = next((s for s in sources if s.get("repo") == repo), None) - if entry is None: - errors.append(f"{repo}: not found in {sources_path}") - continue - if args.from_host and args.from_host not in entry["url"]: - errors.append( - f"{repo}: current URL does not contain --from {args.from_host!r}: {entry['url']}" - ) - continue - try: - report = migrate_source_host(sources, repo, args.to, verify=not args.no_verify) - reports.append(report) - except ValueError as exc: - errors.append(str(exc)) - - for r in reports: - verified = "unverified" if r["verified"] is None else ("verified" if r["verified"] else "FAILED") - print(f"{r['repo']}: {r['old_url']} -> {r['new_url']} ({verified})") - for e in errors: - print(f"error: {e}", file=sys.stderr) - - if args.dry_run: - print("(dry-run, no files written)") - return 1 if errors else 0 - - if reports: - write_sources_manifest(manifest, sources_path) - - if args.update_hub: - for r in reports: - try: - status, payload = hub_client.hub_update( - r["repo"], {"url": r["new_url"]}, args.base_url - ) - if status != 200: - errors.append(f"{r['repo']}: hub update returned {status}: {payload}") - except ValueError as exc: - errors.append(f"{r['repo']}: hub update skipped: {exc}") - - return 1 if errors else 0 - - def cmd_graph(args: argparse.Namespace) -> int: warnings = check_relations() if args.check else [] content = render_mermaid() @@ -659,8 +608,6 @@ def cmd_plan_check(args: argparse.Namespace) -> int: query, reuse_threshold=args.reuse_threshold, extend_threshold=args.extend_threshold, - use_llm=not args.no_llm, - llm_url=args.llm_url, ) if args.record_outcome: @@ -801,35 +748,6 @@ def main(argv: list[str] | None = None) -> int: ) compose.set_defaults(func=cmd_federation_compose) - migrate_host = federation_sub.add_parser( - "migrate-host", - help="rewrite one or more repos' raw index URL to a new forge host (REUSE-WP-0019-T01)", - ) - migrate_host.add_argument( - "--repo", action="append", required=True, - help="repo slug to migrate (repeatable)", - ) - migrate_host.add_argument( - "--to", required=True, - help="new base URL, e.g. https://forgejo.coulomb.social (or REUSE_SURFACE_FORGE_BASE_URL)", - ) - migrate_host.add_argument( - "--from", dest="from_host", - help="sanity check: refuse to migrate a repo whose current URL doesn't contain this", - ) - migrate_host.add_argument("--sources", help="path to sources.yaml (default: registry/federation/sources.yaml)") - migrate_host.add_argument("--dry-run", action="store_true", help="show what would change, write nothing") - migrate_host.add_argument( - "--no-verify", action="store_true", - help="skip HTTP-probing the new URL before writing (unsafe -- can point at a repo that hasn't migrated yet)", - ) - migrate_host.add_argument( - "--update-hub", action="store_true", - help="also PATCH the production hub registration for each migrated repo", - ) - migrate_host.add_argument("--base-url", help="hub service base URL (or REUSE_SURFACE_URL), used with --update-hub") - migrate_host.set_defaults(func=cmd_federation_migrate_host) - query = subparsers.add_parser("query", help="query capability index") query.add_argument("--discovery-min") query.add_argument("--availability-min") @@ -894,13 +812,6 @@ def main(argv: list[str] | None = None) -> int: "--requesting-domain", default="infotech", help="domain slug recorded on a filed capability request", ) - plan_check.add_argument( - "--llm-url", help="llm-connect base URL (or LLM_CONNECT_URL) for semantic rerank", - ) - plan_check.add_argument( - "--no-llm", action="store_true", - help="skip the optional LLM semantic rerank pass", - ) plan_check.set_defaults(func=cmd_plan_check) catalog = subparsers.add_parser( diff --git a/reuse_surface/forge_host.py b/reuse_surface/forge_host.py deleted file mode 100644 index 617abe5..0000000 --- a/reuse_surface/forge_host.py +++ /dev/null @@ -1,105 +0,0 @@ -from __future__ import annotations - -import os -import re -import urllib.error -import urllib.request -from dataclasses import dataclass -from typing import Any - -# Matches both the legacy Gitea-default form (.../raw//) and the -# canonical branch-qualified form (.../raw/branch//) that both -# Gitea and Forgejo serve without a 303 redirect. See REUSE-WP-0017-T06 for -# why the redirect matters (urllib follows it fine; some HEAD-only probes -# and shell tooling don't). -_RAW_URL_RE = re.compile( - r"^(?Phttps?)://(?P[^/]+)/(?P[^/]+)/(?P[^/]+)" - r"/raw/(?:branch/)?(?P[^/]+)/(?P.+)$" -) - - -@dataclass -class RawUrlParts: - scheme: str - host: str - org: str - repo: str - branch: str - path: str - - -def parse_raw_url(url: str) -> RawUrlParts: - match = _RAW_URL_RE.match(url) - if not match: - raise ValueError(f"not a recognized Gitea/Forgejo raw URL: {url}") - return RawUrlParts(**match.groupdict()) - - -def derive_raw_url(base_url: str, org: str, repo: str, *, branch: str = "main", path: str = "registry/indexes/capabilities.yaml") -> str: - """Builds a raw index URL in the canonical branch-qualified form, which - both Gitea and Forgejo serve directly (no 303 redirect).""" - base = base_url.rstrip("/") - return f"{base}/{org}/{repo}/raw/branch/{branch}/{path}" - - -def rewrite_url_host(url: str, new_base_url: str) -> str: - """Rewrites a raw index URL's scheme+host to new_base_url, preserving - org/repo/branch/path and normalizing to the canonical branch-qualified - form regardless of which form the input used.""" - parts = parse_raw_url(url) - return derive_raw_url(new_base_url, parts.org, parts.repo, branch=parts.branch, path=parts.path) - - -def forge_base_url(explicit: str | None = None) -> str | None: - """Returns the configured default Forgejo base URL, or None if unset. - Unlike hub_client.service_base_url, this has no hard requirement -- - per-repo migration is opt-in (most repos haven't moved their remote - yet), so an unset value just means 'no default target configured', - not an error.""" - return explicit or os.environ.get("REUSE_SURFACE_FORGE_BASE_URL") or None - - -def probe_url(url: str, *, timeout: int = 10) -> tuple[bool, int | None]: - """HEAD-probes a URL, following redirects (matches establish.py's - _probe_raw_url behavior). Returns (ok, status).""" - request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "reuse-surface/0.1"}) - try: - with urllib.request.urlopen(request, timeout=timeout) as response: - return response.status == 200, response.status - except urllib.error.HTTPError as exc: - return False, exc.code - except (urllib.error.URLError, TimeoutError, OSError): - return False, None - - -def migrate_source_host( - sources: list[dict[str, Any]], - repo: str, - new_base_url: str, - *, - verify: bool = True, -) -> dict[str, Any]: - """Rewrites one repo's entry in a sources.yaml `sources` list to point at - new_base_url. Mutates the matching entry in place and returns a report - dict. Raises ValueError if the repo isn't found, the existing URL can't - be parsed, or (when verify=True) the new URL doesn't resolve -- never - writes a URL that hasn't been confirmed reachable, since rewriting to a - host the repo hasn't actually migrated to would silently break - federation for it.""" - entry = next((s for s in sources if s.get("repo") == repo), None) - if entry is None: - raise ValueError(f"repo not found in sources: {repo}") - old_url = entry["url"] - new_url = rewrite_url_host(old_url, new_base_url) - report: dict[str, Any] = {"repo": repo, "old_url": old_url, "new_url": new_url, "verified": None} - if verify: - ok, status = probe_url(new_url) - report["verified"] = ok - report["probe_status"] = status - if not ok: - raise ValueError( - f"{repo}: new URL did not resolve (HTTP {status}); " - f"not writing an unverified URL: {new_url}" - ) - entry["url"] = new_url - return report diff --git a/reuse_surface/hub/app.py b/reuse_surface/hub/app.py index f4a4f75..9913e7a 100644 --- a/reuse_surface/hub/app.py +++ b/reuse_surface/hub/app.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio -import json import os from pathlib import Path from typing import Any @@ -12,11 +10,6 @@ from fastapi.responses import JSONResponse, Response from reuse_surface.hub.compose import compose_from_store, DEFAULT_DOMAIN from reuse_surface.hub.store import HubStore -from reuse_surface.hub.webhooks import ( - SIGNATURE_HEADERS, - push_touches_registry_index, - verify_signature, -) HUB_VERSION = "0.1.0" @@ -37,10 +30,6 @@ def _store() -> HubStore: return HubStore(_db_path()) -def _webhook_secret() -> str: - return os.environ.get("REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET", "") - - def _http_error(status: int, error: str, message: str) -> HTTPException: return HTTPException( status_code=status, @@ -64,10 +53,6 @@ def _require_auth(authorization: str | None = Header(default=None)) -> None: def create_app() -> FastAPI: app = FastAPI(title="reuse-surface federation hub", version=HUB_VERSION) store = _store() - # Serializes concurrent recompose triggers (manual POST, webhook, future - # scheduled fallback) so a burst of pushes coalesces into one compose - # pass instead of overlapping ones (T02 design principle 2 debounce). - compose_lock = asyncio.Lock() @app.get("/health") def health() -> dict[str, str]: @@ -111,34 +96,20 @@ def create_app() -> FastAPI: raise _http_error(404, "not_found", f"repo not found: {repo}") return Response(status_code=204) - async def _federated_response( + def _federated_response( refresh: bool, accept: str | None, format_param: str, ) -> Response: - # composed_at/stale track *forced* recomposes (refresh=True: manual - # POST, webhook, scheduled fallback), not every plain GET -- a plain - # GET still serves current best-effort data (compose_from_store's own - # per-source cache_ttl_seconds still applies) but must not silently - # clear a staleness signal nothing actually refreshed. - async with compose_lock: - try: - federated, warnings = compose_from_store( - store, refresh=refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN - ) - except FileNotFoundError as exc: - raise _http_error(502, "compose_error", str(exc)) from exc - if refresh: - store.record_compose() - compose_state = store.get_compose_state() - - federated["composed_at"] = compose_state["composed_at"] - federated["stale"] = compose_state["stale"] + try: + federated, warnings = compose_from_store( + store, refresh=refresh, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN + ) + except FileNotFoundError as exc: + raise _http_error(502, "compose_error", str(exc)) from exc use_yaml = format_param == "yaml" or (accept and "yaml" in accept.lower()) headers: dict[str, str] = {} - if compose_state["composed_at"]: - headers["X-Composed-At"] = compose_state["composed_at"] if warnings: headers["X-Federation-Warnings"] = "; ".join(warnings) if use_yaml: @@ -147,58 +118,19 @@ def create_app() -> FastAPI: return JSONResponse(content=federated, headers=headers) @app.get("/v1/federated", response_model=None) - async def get_federated( + def get_federated( request: Request, refresh: bool = Query(default=False), format: str = Query(default="json"), ) -> Response: - return await _federated_response(refresh, request.headers.get("accept"), format) + return _federated_response(refresh, request.headers.get("accept"), format) @app.post("/v1/federated/compose", response_model=None, dependencies=[Depends(_require_auth)]) - async def compose_federated( + def compose_federated( request: Request, format: str = Query(default="json"), ) -> Response: - return await _federated_response(True, request.headers.get("accept"), format) - - @app.post("/v1/webhooks/forgejo", response_model=None) - async def forgejo_webhook(request: Request) -> Response: - body = await request.body() - signature = None - for header_name in SIGNATURE_HEADERS: - signature = request.headers.get(header_name) - if signature: - break - secret = _webhook_secret() - if not secret: - raise _http_error( - 503, "misconfigured", "REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET is not configured" - ) - if not verify_signature(secret, body, signature): - raise _http_error(401, "unauthorized", "invalid or missing webhook signature") - - try: - payload = json.loads(body) - except json.JSONDecodeError as exc: - raise _http_error(400, "validation_error", f"invalid JSON payload: {exc}") from exc - - if not push_touches_registry_index(payload): - return JSONResponse(content={"accepted": False, "reason": "no registry/indexes/ change"}) - - async with compose_lock: - store.mark_stale() - try: - compose_from_store( - store, refresh=True, cache_dir=_cache_dir(), domain=DEFAULT_DOMAIN - ) - except FileNotFoundError as exc: - # Recompose failed -- leave stale=1 so the next GET/scheduled - # trigger reports it honestly rather than silently swallowing - # the failure. - raise _http_error(502, "compose_error", str(exc)) from exc - composed_at = store.record_compose() - - return JSONResponse(content={"accepted": True, "composed_at": composed_at}) + return _federated_response(True, request.headers.get("accept"), format) return app diff --git a/reuse_surface/hub/store.py b/reuse_surface/hub/store.py index d74e0f3..56acd19 100644 --- a/reuse_surface/hub/store.py +++ b/reuse_surface/hub/store.py @@ -61,23 +61,6 @@ class HubStore: ) """ ) - # Single-row table tracking the composed federated index's - # freshness (REUSE-WP-0019-T02). composed_at is set whenever a - # real compose (refresh=True) completes; stale is set by the - # Forgejo webhook receiver when a registry/indexes/ change is - # pushed, and cleared on the next successful compose. - conn.execute( - """ - CREATE TABLE IF NOT EXISTS compose_state ( - id INTEGER PRIMARY KEY CHECK (id = 1), - composed_at TEXT, - stale INTEGER NOT NULL DEFAULT 0 - ) - """ - ) - conn.execute( - "INSERT OR IGNORE INTO compose_state (id, composed_at, stale) VALUES (1, NULL, 0)" - ) def list_repos(self) -> list[dict[str, Any]]: with self._connect() as conn: @@ -177,28 +160,4 @@ class HubStore: rows = conn.execute( "SELECT * FROM registrations ORDER BY repo" ).fetchall() - return [_row_to_registration(row) for row in rows] - - def record_compose(self) -> str: - """Marks the federated index freshly composed (stale cleared). - Returns the recorded timestamp.""" - now = _utc_now() - with self._connect() as conn: - conn.execute( - "UPDATE compose_state SET composed_at = ?, stale = 0 WHERE id = 1", - (now,), - ) - return now - - def mark_stale(self) -> None: - with self._connect() as conn: - conn.execute("UPDATE compose_state SET stale = 1 WHERE id = 1") - - def get_compose_state(self) -> dict[str, Any]: - with self._connect() as conn: - row = conn.execute( - "SELECT composed_at, stale FROM compose_state WHERE id = 1" - ).fetchone() - if row is None: - return {"composed_at": None, "stale": False} - return {"composed_at": row["composed_at"], "stale": bool(row["stale"])} \ No newline at end of file + return [_row_to_registration(row) for row in rows] \ No newline at end of file diff --git a/reuse_surface/hub/webhooks.py b/reuse_surface/hub/webhooks.py deleted file mode 100644 index a13cc8f..0000000 --- a/reuse_surface/hub/webhooks.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac -from typing import Any - -# Forgejo (Gitea-compatible) signs webhook payloads with HMAC-SHA256 over the -# raw request body, sent as a hex digest in X-Forgejo-Signature (Forgejo) or -# X-Gitea-Signature (Gitea) -- both forges use the same scheme during the -# transition, so both header names are accepted. -SIGNATURE_HEADERS = ("X-Forgejo-Signature", "X-Gitea-Signature") - -RELEVANT_PATH_PREFIX = "registry/indexes/" - - -def verify_signature(secret: str, body: bytes, signature: str | None) -> bool: - """Constant-time HMAC-SHA256 verification. Returns False (never raises) - for a missing/malformed signature or a misconfigured (empty) secret -- - callers must treat False as 'reject', not 'skip verification'.""" - if not secret or not signature: - return False - expected = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() - return hmac.compare_digest(expected, signature.strip()) - - -def push_touches_registry_index(payload: dict[str, Any]) -> bool: - """True if any commit in a Forgejo/Gitea push-event payload adds, - modifies, or removes a path under registry/indexes/. Only inspects - paths -- never parses file content (design principle 2: no - push-parsing of payloads into registry state, the webhook only - triggers a pull-based recompose).""" - for commit in payload.get("commits") or []: - for key in ("added", "modified", "removed"): - for path in commit.get(key) or []: - if path.startswith(RELEVANT_PATH_PREFIX): - return True - return False diff --git a/reuse_surface/maintain_llm.py b/reuse_surface/maintain_llm.py index 8ae5c27..bb00445 100644 --- a/reuse_surface/maintain_llm.py +++ b/reuse_surface/maintain_llm.py @@ -50,7 +50,6 @@ def _git_diff(repo_root: Path, git_since: str | None) -> str: "tests/", "docs/", ".gitea/", - ".forgejo/", "pyproject.toml", ], capture_output=True, diff --git a/reuse_surface/plan_check.py b/reuse_surface/plan_check.py index 8940eed..00dfd31 100644 --- a/reuse_surface/plan_check.py +++ b/reuse_surface/plan_check.py @@ -9,16 +9,12 @@ from typing import Any import yaml -from jsonschema import Draft202012Validator - from reuse_surface.federation import FEDERATED_INDEX_PATH -from reuse_surface.llm_bridge import execute_prompt, extract_json_object from reuse_surface.overlaps import TOKEN_RE from reuse_surface.registry import ROOT, load_index from reuse_surface.statehub_bridge import file_capability_request TELEMETRY_PATH = ROOT / "registry" / "telemetry" / "plan-check-events.jsonl" -RERANK_SCHEMA_PATH = ROOT / "schemas" / "plan-check-rerank.schema.json" STALE_DAYS = 14 DEFAULT_REUSE_THRESHOLD = 0.45 @@ -168,73 +164,6 @@ def match_query( return tied + rest -def load_rerank_schema() -> dict[str, Any]: - return json.loads(RERANK_SCHEMA_PATH.read_text(encoding="utf-8")) - - -def build_rerank_prompt(query: MatchQuery, matches: list[Match]) -> str: - candidates = [ - {"id": m.id, "score": m.score, "vector": m.vector, "summary": m.summary} - for m in matches - ] - return ( - "You are reranking capability-registry search candidates for a " - "query-before-build check. Score how well each candidate semantically " - "matches the query intent, on a 0-1 confidence scale. You may ONLY " - "return ids from the candidate list below -- do not invent new ones.\n\n" - f"Query intent:\n{query.text}\n\n" - f"Candidates (JSON):\n{json.dumps(candidates, indent=2)}\n\n" - "Respond with a JSON object: " - '{"candidates": [{"id": "...", "confidence": 0.0-1.0, "rationale": "..."}]}. ' - "Include every candidate id from the list, in any order." - ) - - -def request_rerank( - query: MatchQuery, - matches: list[Match], - *, - llm_url: str | None = None, -) -> list[dict[str, Any]]: - prompt = build_rerank_prompt(query, matches) - content = execute_prompt(prompt, base_url=llm_url, config={"temperature": 0.0, "max_tokens": 1500}) - payload = extract_json_object(content) - validator = Draft202012Validator(load_rerank_schema()) - errors = sorted(validator.iter_errors(payload), key=lambda err: list(err.path)) - if errors: - messages = "; ".join(error.message for error in errors[:3]) - raise ValueError(f"rerank schema validation failed: {messages}") - known_ids = {m.id for m in matches} - candidates = [c for c in payload["candidates"] if c["id"] in known_ids] - if not candidates: - raise ValueError("rerank response contained no known candidate ids") - return candidates - - -def apply_rerank(matches: list[Match], rerank_result: list[dict[str, Any]]) -> list[Match]: - """Per spec design principle 3: deterministic matches always rank first. - LLM rerank results are appended after as separately-labeled (kind='llm') - entries carrying the semantic confidence score -- never reordering or - replacing the deterministic base result, so the output is trustworthy - even when a caller ignores the LLM-kind entries entirely (e.g. --no-llm - runs never produce them in the first place).""" - by_id = {m.id: m for m in matches} - llm_matches = [ - Match( - id=c["id"], - score=c["confidence"], - kind="llm", - vector=by_id[c["id"]].vector, - owner=by_id[c["id"]].owner, - summary=c.get("rationale") or by_id[c["id"]].summary, - ) - for c in rerank_result - if c["id"] in by_id - ] - llm_matches.sort(key=lambda m: m.score, reverse=True) - return matches + llm_matches - - def verdict_for_score( top_score: float, *, @@ -255,8 +184,6 @@ def run_plan_check( extend_threshold: float = DEFAULT_EXTEND_THRESHOLD, tie_window: float = DEFAULT_TIE_WINDOW, top_n: int = 5, - use_llm: bool = True, - llm_url: str | None = None, ) -> dict[str, Any]: capabilities, updated = load_federated_capabilities() matches = match_query(query, capabilities, tie_window=tie_window) @@ -264,20 +191,7 @@ def run_plan_check( verdict = verdict_for_score( top_score, reuse_threshold=reuse_threshold, extend_threshold=extend_threshold ) - - top_matches = matches[:top_n] - notes: list[str] = [] - if use_llm and top_matches: - try: - rerank_result = request_rerank(query, top_matches, llm_url=llm_url) - top_matches = apply_rerank(top_matches, rerank_result) - except ValueError as exc: - if "LLM backend not configured" in str(exc): - notes.append("LLM rerank skipped: LLM_CONNECT_URL not set") - else: - notes.append(f"LLM rerank skipped: {exc}") - - result = { + return { "query": { "source": query.source, "text": query.text, @@ -295,14 +209,11 @@ def run_plan_check( "summary": m.summary, "kind": m.kind, } - for m in top_matches + for m in matches[:top_n] ], "federated_index_updated": updated, "federated_index_stale_warning": _staleness_warning(updated), } - if notes: - result["notes"] = notes - return result def format_plan_check_markdown(result: dict[str, Any]) -> str: @@ -338,10 +249,6 @@ def format_plan_check_markdown(result: dict[str, Any]) -> str: lines.append("") lines.append(f"⚠ {warning}") - for note in result.get("notes", []): - lines.append("") - lines.append(f"ℹ {note}") - return "\n".join(lines) + "\n" diff --git a/reuse_surface/registry_update.py b/reuse_surface/registry_update.py index fbf5f6c..db72157 100644 --- a/reuse_surface/registry_update.py +++ b/reuse_surface/registry_update.py @@ -19,11 +19,7 @@ from reuse_surface.registry import ( ) # Safe to apply without interactive review (see patches.SAFE_DETERMINISTIC_KINDS). -# Both forge CI workflow paths are recognized during the Gitea->Forgejo -# transition (REUSE-WP-0019-T01) -- sibling repos migrate independently, so -# either path is valid CI evidence depending on a given repo's progress. -CI_WORKFLOW_PREFIXES = (".gitea/workflows/", ".forgejo/workflows/") -SAFE_EVIDENCE_PREFIXES = ("tests/", *CI_WORKFLOW_PREFIXES) +SAFE_EVIDENCE_PREFIXES = ("tests/", ".gitea/workflows/") def git_changed_files(repo_root: Path, since_ref: str) -> list[str]: @@ -198,7 +194,7 @@ def _collect_changed_file_suggestions( "apply_patch": {"field": "evidence.tests", "append": changed}, } ) - if changed.startswith(CI_WORKFLOW_PREFIXES) and changed.endswith((".yml", ".yaml")): + if changed.startswith(".gitea/workflows/") and changed.endswith((".yml", ".yaml")): field = "evidence.tests" if "test" in changed.lower() else "evidence.documentation" existing = evidence_tests if field == "evidence.tests" else evidence_docs if changed not in existing: diff --git a/schemas/plan-check-rerank.schema.json b/schemas/plan-check-rerank.schema.json deleted file mode 100644 index a4163db..0000000 --- a/schemas/plan-check-rerank.schema.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://reuse-surface.local/schemas/plan-check-rerank.schema.json", - "title": "PlanCheckRerankResponse", - "description": "T03 LLM rerank response. The model may only score/reorder candidate IDs it was given -- it must not invent new ones. plan_check.py enforces that separately from this schema (schema can't see the input candidate set).", - "type": "object", - "additionalProperties": false, - "required": ["candidates"], - "properties": { - "candidates": { - "type": "array", - "items": { - "type": "object", - "additionalProperties": false, - "required": ["id", "confidence"], - "properties": { - "id": { - "type": "string", - "pattern": "^capability\\.[a-z0-9]+(\\.[a-z0-9-]+)+$" - }, - "confidence": {"type": "number", "minimum": 0, "maximum": 1}, - "rationale": {"type": "string"} - } - } - } - } -} diff --git a/schemas/plan-check-result.schema.json b/schemas/plan-check-result.schema.json index f884df2..5c41c6e 100644 --- a/schemas/plan-check-result.schema.json +++ b/schemas/plan-check-result.schema.json @@ -49,11 +49,6 @@ } }, "federated_index_updated": {"type": ["string", "null"]}, - "federated_index_stale_warning": {"type": ["string", "null"]}, - "notes": { - "type": "array", - "items": {"type": "string"} - }, - "filed_capability_request": {"type": ["object", "null"]} + "federated_index_stale_warning": {"type": ["string", "null"]} } } diff --git a/specs/FederationHubAPI.md b/specs/FederationHubAPI.md index 38ea551..d584aa9 100644 --- a/specs/FederationHubAPI.md +++ b/specs/FederationHubAPI.md @@ -195,30 +195,18 @@ capabilities: source_repo: reuse-surface source_url: https://... # ... index fields ... -composed_at: "2026-07-07T16:22:09+00:00" # REUSE-WP-0019-T02 -stale: false # REUSE-WP-0019-T02 ``` -`composed_at`/`stale` (added REUSE-WP-0019-T02) track *forced* recomposes -only (`refresh=true`, the webhook, or the scheduled fallback) — a plain -`GET` still serves current best-effort data (per-source `cache_ttl_seconds` -still applies) but never silently clears a staleness signal nothing -actually refreshed. `composed_at` is `null` until the first forced -recompose since the hub process's SQLite DB was created. `stale: true` -means a `registry/indexes/` change was pushed (via webhook) since the last -forced recompose completed. - Query parameters: | Param | Default | Meaning | |---|---|---| | `format` | `json` | `json` or `yaml` | -| `refresh` | `false` | Bypass remote cache when `true`; also updates `composed_at` and clears `stale` | +| `refresh` | `false` | Bypass remote cache when `true` | Warnings from compose (duplicate IDs, fetch fallbacks) are returned in response header `X-Federation-Warnings` (semicolon-separated) for MVP; JSON envelope -extension is a future option. `composed_at` is also echoed as response header -`X-Composed-At` when set. +extension is a future option. **Response `200`:** Federated index document. **Response `502`:** Required source unavailable with no cache. @@ -227,50 +215,8 @@ extension is a future option. `composed_at` is also echoed as response header Trigger federated index refresh (same as `GET /v1/federated?refresh=true`). **Auth required.** Useful for operators after bulk registration changes. -This is the hub's recompose endpoint referenced elsewhere as "trigger a -recompose" — there is no separate `/v1/recompose` route; this one already -does that job. -**Response `200`:** Federated index document (with `composed_at` updated, -`stale` cleared). - -### 5.9 `POST /v1/webhooks/forgejo` - -Forgejo (or Gitea, during the transition) push-event webhook receiver. -Added REUSE-WP-0019-T02. Design: **webhook triggers, compose stays -pull-based** — the webhook never parses pushed file content into registry -state; it only decides whether to trigger a pull-based recompose from the -already-registered raw URLs. - -**Signature verification (required, not optional):** HMAC-SHA256 over the -raw request body, hex-encoded, in `X-Forgejo-Signature` (or -`X-Gitea-Signature` — both accepted, since Forgejo is Gitea-compatible and -repos migrate independently). Secret from `REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET` -(never in code or committed config — route via the standard credential -mechanism for this deployment). A missing/wrong signature is `401`; an -unconfigured secret is `503` (fails closed, not open). - -**Behavior:** - -1. Verify signature (raw body, constant-time compare). -2. Parse the push-event payload; check every commit's `added`/`modified`/`removed` - lists for any path under `registry/indexes/`. -3. If none match: `200 {"accepted": false, "reason": "no registry/indexes/ change"}` — no-op. -4. If a match: mark the compose state stale, run a real recompose - (`refresh=true` equivalent) under the same lock used by - `POST /v1/federated/compose` (concurrent triggers coalesce rather than - overlap), record `composed_at`, clear `stale`. - -**Response `200`:** `{"accepted": true|false, ...}`. -**Response `401`:** Missing or invalid signature. -**Response `503`:** Webhook secret not configured. -**Response `502`:** Recompose failed (stale is deliberately left set so the -next check reports the failure honestly, not silently). - -Org-level webhook rollout (single config covering all repos, T03) and the -Forgejo Actions scheduled fallback for when webhooks are unavailable are -separate, deployment-side follow-ups — this section covers the receiver -contract only. +**Response `200`:** Federated index document. --- @@ -292,7 +238,6 @@ Non-2xx responses use: | `unauthorized` | 401 | | `not_found` | 404 | | `conflict` | 409 | -| `misconfigured` | 503 | | `compose_error` | 502 | --- @@ -305,8 +250,6 @@ Non-2xx responses use: | `REUSE_SURFACE_DB` | no | SQLite path (default `/data/reuse.db`) | | `REUSE_SURFACE_CACHE_DIR` | no | Remote index cache (default `/data/cache`) | | `REUSE_SURFACE_DOMAIN` | no | Default federated `domain` (default `helix_forge`) | -| `REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET` | for webhook | HMAC secret for `POST /v1/webhooks/forgejo` (REUSE-WP-0019-T02) | -| `REUSE_SURFACE_FORGE_BASE_URL` | no | Default target host for `reuse-surface federation migrate-host` (REUSE-WP-0019-T01), e.g. `https://forgejo.coulomb.social` | --- diff --git a/specs/PlanCheck.md b/specs/PlanCheck.md index 61b71f1..80b1df4 100644 --- a/specs/PlanCheck.md +++ b/specs/PlanCheck.md @@ -2,8 +2,7 @@ **Repository:** `reuse-surface` **Artifact:** `specs/PlanCheck.md` -**Status:** Implemented (T02 deterministic matching, T03 LLM rerank, T04 -State Hub bridge, T05 ecosystem convention, T06 docs/CI all shipped) +**Status:** Draft 0.1 (REUSE-WP-0018-T01) **Schema:** `schemas/plan-check-result.schema.json` --- diff --git a/tests/test_forge_host.py b/tests/test_forge_host.py deleted file mode 100644 index 87c92e3..0000000 --- a/tests/test_forge_host.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -import pytest - -from reuse_surface.forge_host import ( - derive_raw_url, - forge_base_url, - migrate_source_host, - parse_raw_url, - rewrite_url_host, -) - - -def test_parse_raw_url_legacy_form(): - parts = parse_raw_url("https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml") - assert parts.scheme == "https" - assert parts.host == "gitea.coulomb.social" - assert parts.org == "coulomb" - assert parts.repo == "activity-core" - assert parts.branch == "main" - assert parts.path == "registry/indexes/capabilities.yaml" - - -def test_parse_raw_url_branch_qualified_form(): - parts = parse_raw_url("https://forgejo.coulomb.social/coulomb/core-hub/raw/branch/main/registry/indexes/capabilities.yaml") - assert parts.host == "forgejo.coulomb.social" - assert parts.repo == "core-hub" - assert parts.branch == "main" - - -def test_parse_raw_url_rejects_unrecognized(): - with pytest.raises(ValueError): - parse_raw_url("https://example.com/not/a/raw/url") - - -def test_derive_raw_url_uses_branch_qualified_form(): - url = derive_raw_url("https://forgejo.coulomb.social", "coulomb", "core-hub") - assert url == "https://forgejo.coulomb.social/coulomb/core-hub/raw/branch/main/registry/indexes/capabilities.yaml" - - -def test_derive_raw_url_strips_trailing_slash(): - url = derive_raw_url("https://forgejo.coulomb.social/", "coulomb", "core-hub") - assert url.startswith("https://forgejo.coulomb.social/coulomb/") - - -def test_rewrite_url_host_swaps_host_and_normalizes_form(): - old = "https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml" - new = rewrite_url_host(old, "https://forgejo.coulomb.social") - assert new == "https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml" - - -def test_forge_base_url_env_fallback(monkeypatch): - monkeypatch.setenv("REUSE_SURFACE_FORGE_BASE_URL", "https://forgejo.coulomb.social") - assert forge_base_url() == "https://forgejo.coulomb.social" - - -def test_forge_base_url_none_when_unset(monkeypatch): - monkeypatch.delenv("REUSE_SURFACE_FORGE_BASE_URL", raising=False) - assert forge_base_url() is None - - -def test_forge_base_url_explicit_wins(monkeypatch): - monkeypatch.setenv("REUSE_SURFACE_FORGE_BASE_URL", "https://env-value") - assert forge_base_url("https://explicit-value") == "https://explicit-value" - - -def test_migrate_source_host_mutates_matching_entry(monkeypatch): - monkeypatch.setattr("reuse_surface.forge_host.probe_url", lambda url, **k: (True, 200)) - sources = [ - {"repo": "activity-core", "url": "https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml"}, - {"repo": "other-repo", "url": "https://gitea.coulomb.social/coulomb/other-repo/raw/main/registry/indexes/capabilities.yaml"}, - ] - report = migrate_source_host(sources, "activity-core", "https://forgejo.coulomb.social") - assert report["verified"] is True - assert sources[0]["url"] == "https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml" - # other entries untouched - assert sources[1]["url"] == "https://gitea.coulomb.social/coulomb/other-repo/raw/main/registry/indexes/capabilities.yaml" - - -def test_migrate_source_host_repo_not_found(): - sources = [{"repo": "activity-core", "url": "https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml"}] - with pytest.raises(ValueError, match="not found"): - migrate_source_host(sources, "does-not-exist", "https://forgejo.coulomb.social") - - -def test_migrate_source_host_refuses_unverified_url(monkeypatch): - monkeypatch.setattr("reuse_surface.forge_host.probe_url", lambda url, **k: (False, 404)) - sources = [{"repo": "activity-core", "url": "https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml"}] - original_url = sources[0]["url"] - with pytest.raises(ValueError, match="did not resolve"): - migrate_source_host(sources, "activity-core", "https://forgejo.coulomb.social") - # must not have mutated the entry when verification failed - assert sources[0]["url"] == original_url - - -def test_migrate_source_host_skips_verification_when_disabled(): - sources = [{"repo": "activity-core", "url": "https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml"}] - report = migrate_source_host(sources, "activity-core", "https://forgejo.coulomb.social", verify=False) - assert report["verified"] is None - assert sources[0]["url"].startswith("https://forgejo.coulomb.social/") - - -def test_cmd_federation_migrate_host_dry_run(tmp_path, monkeypatch): - from reuse_surface.cli import main - - sources_path = tmp_path / "sources.yaml" - sources_path.write_text( - "version: 1\n" - "domain: helix_forge\n" - "collision_policy: warn\n" - "sources:\n" - "- repo: activity-core\n" - " url: https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml\n" - " enabled: true\n" - ) - monkeypatch.setattr("reuse_surface.cli.migrate_source_host", lambda sources, repo, to, verify=True: { - "repo": repo, "old_url": sources[0]["url"], "new_url": "https://forgejo.coulomb.social/x", "verified": True, - }) - exit_code = main([ - "federation", "migrate-host", - "--repo", "activity-core", - "--to", "https://forgejo.coulomb.social", - "--sources", str(sources_path), - "--dry-run", - ]) - assert exit_code == 0 - # dry-run must not write - assert "gitea.coulomb.social" in sources_path.read_text() - - -def test_cmd_federation_migrate_host_writes_file(tmp_path, monkeypatch): - from reuse_surface.cli import main - - sources_path = tmp_path / "sources.yaml" - sources_path.write_text( - "version: 1\n" - "domain: helix_forge\n" - "collision_policy: warn\n" - "sources:\n" - "- repo: activity-core\n" - " url: https://gitea.coulomb.social/coulomb/activity-core/raw/main/registry/indexes/capabilities.yaml\n" - " enabled: true\n" - ) - monkeypatch.setattr("reuse_surface.forge_host.probe_url", lambda url, **k: (True, 200)) - exit_code = main([ - "federation", "migrate-host", - "--repo", "activity-core", - "--to", "https://forgejo.coulomb.social", - "--sources", str(sources_path), - ]) - assert exit_code == 0 - written = sources_path.read_text() - assert "forgejo.coulomb.social" in written - assert "gitea.coulomb.social" not in written - - -def test_cmd_federation_migrate_host_repo_not_found_errors(tmp_path): - from reuse_surface.cli import main - - sources_path = tmp_path / "sources.yaml" - sources_path.write_text("version: 1\ndomain: helix_forge\ncollision_policy: warn\nsources: []\n") - exit_code = main([ - "federation", "migrate-host", - "--repo", "does-not-exist", - "--to", "https://forgejo.coulomb.social", - "--sources", str(sources_path), - "--dry-run", - ]) - assert exit_code == 1 - - -def test_cmd_federation_migrate_host_from_mismatch_errors(tmp_path): - from reuse_surface.cli import main - - sources_path = tmp_path / "sources.yaml" - sources_path.write_text( - "version: 1\ndomain: helix_forge\ncollision_policy: warn\n" - "sources:\n- repo: activity-core\n url: https://forgejo.coulomb.social/coulomb/activity-core/raw/branch/main/registry/indexes/capabilities.yaml\n" - ) - exit_code = main([ - "federation", "migrate-host", - "--repo", "activity-core", - "--to", "https://forgejo.coulomb.social", - "--from", "gitea.coulomb.social", - "--sources", str(sources_path), - "--dry-run", - ]) - assert exit_code == 1 diff --git a/tests/test_hub.py b/tests/test_hub.py index eba67cf..ce03c70 100644 --- a/tests/test_hub.py +++ b/tests/test_hub.py @@ -130,185 +130,4 @@ def test_compose_federated_with_mock_fetch(hub_client, monkeypatch): def test_store_validation(tmp_path): store = HubStore(tmp_path / "hub.db") with pytest.raises(ValueError): - store.create_repo({"repo": "BAD", "url": "ftp://x", "domain": "helix_forge"}) - - -# --- T02: composed_at / stale tracking --- - - -def test_compose_state_starts_unset(tmp_path): - store = HubStore(tmp_path / "hub.db") - state = store.get_compose_state() - assert state == {"composed_at": None, "stale": False} - - -def test_record_compose_sets_timestamp_and_clears_stale(tmp_path): - store = HubStore(tmp_path / "hub.db") - store.mark_stale() - assert store.get_compose_state()["stale"] is True - composed_at = store.record_compose() - state = store.get_compose_state() - assert state["composed_at"] == composed_at - assert state["stale"] is False - - -def test_plain_get_does_not_clear_stale(hub_client, monkeypatch): - hub_client.post( - "/v1/repos", - json={"repo": "remote-repo", "url": "https://example.com/capabilities.yaml", "domain": "helix_forge"}, - headers={"Authorization": "Bearer test-token"}, - ) - payload = REMOTE_INDEX.encode("utf-8") - - class FakeResponse: - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def read(self): - return payload - - with patch("urllib.request.urlopen", return_value=FakeResponse()): - # force a real compose so composed_at/stale are established - first = hub_client.get("/v1/federated?refresh=true") - assert first.json()["stale"] is False - composed_at_1 = first.json()["composed_at"] - - # a plain GET (no refresh) must not report itself as freshly composed - second = hub_client.get("/v1/federated") - assert second.json()["composed_at"] == composed_at_1 - assert second.json()["stale"] is False - - -def test_get_federated_reports_stale_after_mark(hub_client, monkeypatch): - hub_client.post( - "/v1/repos", - json={"repo": "remote-repo", "url": "https://example.com/capabilities.yaml", "domain": "helix_forge"}, - headers={"Authorization": "Bearer test-token"}, - ) - payload = REMOTE_INDEX.encode("utf-8") - - class FakeResponse: - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def read(self): - return payload - - with patch("urllib.request.urlopen", return_value=FakeResponse()): - hub_client.get("/v1/federated?refresh=true") - - from reuse_surface.hub.store import HubStore as _HubStore - - db_path = os.environ["REUSE_SURFACE_DB"] - _HubStore(Path(db_path)).mark_stale() - - response = hub_client.get("/v1/federated") - assert response.json()["stale"] is True - - -# --- T02: Forgejo webhook receiver --- - -WEBHOOK_SECRET = "test-webhook-secret" - - -def _sign(body: bytes, secret: str = WEBHOOK_SECRET) -> str: - import hashlib - import hmac as hmac_module - - return hmac_module.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() - - -@pytest.fixture -def webhook_client(hub_client, monkeypatch): - monkeypatch.setenv("REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET", WEBHOOK_SECRET) - return hub_client - - -def test_webhook_rejects_missing_signature(webhook_client): - response = webhook_client.post("/v1/webhooks/forgejo", json={"commits": []}) - assert response.status_code == 401 - - -def test_webhook_rejects_wrong_signature(webhook_client): - body = json.dumps({"commits": []}).encode("utf-8") - response = webhook_client.post( - "/v1/webhooks/forgejo", - content=body, - headers={"X-Forgejo-Signature": "0" * 64, "Content-Type": "application/json"}, - ) - assert response.status_code == 401 - - -def test_webhook_rejects_when_secret_not_configured(hub_client): - body = json.dumps({"commits": []}).encode("utf-8") - response = hub_client.post( - "/v1/webhooks/forgejo", - content=body, - headers={"X-Forgejo-Signature": _sign(body), "Content-Type": "application/json"}, - ) - assert response.status_code == 503 - - -def test_webhook_ignores_push_without_registry_change(webhook_client): - body = json.dumps({"commits": [{"added": ["README.md"], "modified": [], "removed": []}]}).encode("utf-8") - response = webhook_client.post( - "/v1/webhooks/forgejo", - content=body, - headers={"X-Forgejo-Signature": _sign(body), "Content-Type": "application/json"}, - ) - assert response.status_code == 200 - assert response.json()["accepted"] is False - - -def test_webhook_triggers_recompose_on_registry_change(webhook_client, monkeypatch): - webhook_client.post( - "/v1/repos", - json={"repo": "remote-repo", "url": "https://example.com/capabilities.yaml", "domain": "helix_forge"}, - headers={"Authorization": "Bearer test-token"}, - ) - payload_bytes = REMOTE_INDEX.encode("utf-8") - - class FakeResponse: - def __enter__(self): - return self - - def __exit__(self, *args): - return False - - def read(self): - return payload_bytes - - body = json.dumps( - {"commits": [{"added": ["registry/indexes/capabilities.yaml"], "modified": [], "removed": []}]} - ).encode("utf-8") - with patch("urllib.request.urlopen", return_value=FakeResponse()): - response = webhook_client.post( - "/v1/webhooks/forgejo", - content=body, - headers={"X-Forgejo-Signature": _sign(body), "Content-Type": "application/json"}, - ) - assert response.status_code == 200 - assert response.json()["accepted"] is True - assert response.json()["composed_at"] - - # federated index should now report the newly composed data, not stale - follow_up = webhook_client.get("/v1/federated") - assert follow_up.json()["stale"] is False - ids = {item["id"] for item in follow_up.json()["capabilities"]} - assert "capability.remote.sample" in ids - - -def test_webhook_accepts_gitea_signature_header(webhook_client): - body = json.dumps({"commits": [{"added": ["README.md"], "modified": [], "removed": []}]}).encode("utf-8") - response = webhook_client.post( - "/v1/webhooks/forgejo", - content=body, - headers={"X-Gitea-Signature": _sign(body), "Content-Type": "application/json"}, - ) - assert response.status_code == 200 \ No newline at end of file + store.create_repo({"repo": "BAD", "url": "ftp://x", "domain": "helix_forge"}) \ No newline at end of file diff --git a/tests/test_plan_check.py b/tests/test_plan_check.py index 230b738..b6497e1 100644 --- a/tests/test_plan_check.py +++ b/tests/test_plan_check.py @@ -3,15 +3,12 @@ from __future__ import annotations import json from reuse_surface.plan_check import ( - Match, MatchQuery, - apply_rerank, load_query_from_intent, load_query_from_workplan, maybe_file_capability_request, match_query, record_outcome, - request_rerank, run_plan_check, verdict_for_score, ) @@ -199,153 +196,3 @@ def test_cmd_plan_check_rejects_both_inputs(tmp_path): workplan.write_text("---\nid: X\n---\nbody") exit_code = main(["plan-check", str(workplan), "--intent", "also this"]) assert exit_code == 1 - - -# --- T03: LLM rerank --- - -SAMPLE_MATCHES = [ - Match( - id="capability.infotech.issue-tracking", - score=0.36, - kind="deterministic", - vector="D4 / A2 / C2 / R1", - owner="issue-core", - summary="Unified interface for issue tracking.", - ), - Match( - id="capability.audit.event-retain", - score=0.12, - kind="deterministic", - vector="D4 / A2 / C2 / R1", - owner="audit-core", - summary="Collect and retain audit events.", - ), -] - - -def test_request_rerank_valid_response(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.execute_prompt", - lambda prompt, **kwargs: json.dumps( - { - "candidates": [ - {"id": "capability.infotech.issue-tracking", "confidence": 0.9, "rationale": "strong match"}, - {"id": "capability.audit.event-retain", "confidence": 0.1, "rationale": "unrelated"}, - ] - } - ), - ) - query = load_query_from_intent("issue tracking across platforms") - result = request_rerank(query, SAMPLE_MATCHES) - assert len(result) == 2 - assert result[0]["confidence"] == 0.9 - - -def test_request_rerank_rejects_malformed_response(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.execute_prompt", - lambda prompt, **kwargs: json.dumps({"candidates": [{"id": "capability.infotech.issue-tracking"}]}), - ) - query = load_query_from_intent("issue tracking across platforms") - try: - request_rerank(query, SAMPLE_MATCHES) - assert False, "expected ValueError for missing confidence field" - except ValueError as exc: - assert "schema validation failed" in str(exc) - - -def test_request_rerank_rejects_non_json_response(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.execute_prompt", - lambda prompt, **kwargs: "not json at all", - ) - query = load_query_from_intent("issue tracking across platforms") - try: - request_rerank(query, SAMPLE_MATCHES) - assert False, "expected ValueError for non-JSON response" - except ValueError: - pass - - -def test_request_rerank_ignores_invented_ids(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.execute_prompt", - lambda prompt, **kwargs: json.dumps( - { - "candidates": [ - {"id": "capability.infotech.issue-tracking", "confidence": 0.8}, - {"id": "capability.madeup.not-real", "confidence": 0.99}, - ] - } - ), - ) - query = load_query_from_intent("issue tracking across platforms") - result = request_rerank(query, SAMPLE_MATCHES) - assert [c["id"] for c in result] == ["capability.infotech.issue-tracking"] - - -def test_apply_rerank_appends_after_deterministic_matches(): - rerank_result = [ - {"id": "capability.audit.event-retain", "confidence": 0.95, "rationale": "actually a great fit"}, - ] - combined = apply_rerank(SAMPLE_MATCHES, rerank_result) - # Deterministic matches keep their original order and scores untouched. - assert combined[0].id == "capability.infotech.issue-tracking" - assert combined[0].score == 0.36 - assert combined[0].kind == "deterministic" - assert combined[1].id == "capability.audit.event-retain" - assert combined[1].score == 0.12 - assert combined[1].kind == "deterministic" - # LLM entry is appended after, separately labeled. - assert combined[2].id == "capability.audit.event-retain" - assert combined[2].score == 0.95 - assert combined[2].kind == "llm" - - -def test_run_plan_check_skips_llm_gracefully_when_unset(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), - ) - monkeypatch.delenv("LLM_CONNECT_URL", raising=False) - query = load_query_from_intent("issue tracking gitea github gitlab") - result = run_plan_check(query) - assert result["verdict"] in {"reuse", "extend"} - assert any("LLM_CONNECT_URL not set" in n for n in result.get("notes", [])) - assert all(m["kind"] == "deterministic" for m in result["matches"]) - - -def test_run_plan_check_no_llm_flag_skips_without_attempting(monkeypatch): - called = [] - monkeypatch.setattr( - "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), - ) - monkeypatch.setattr( - "reuse_surface.plan_check.request_rerank", - lambda *a, **k: called.append(1) or [], - ) - query = load_query_from_intent("issue tracking gitea github gitlab") - run_plan_check(query, use_llm=False) - assert called == [] - - -def test_run_plan_check_integrates_successful_rerank(monkeypatch): - monkeypatch.setattr( - "reuse_surface.plan_check.load_federated_capabilities", - lambda: (SAMPLE_CAPABILITIES, "2026-07-07"), - ) - monkeypatch.setattr( - "reuse_surface.plan_check.execute_prompt", - lambda prompt, **kwargs: json.dumps( - {"candidates": [{"id": "capability.infotech.issue-tracking", "confidence": 0.77}]} - ), - ) - query = load_query_from_intent("issue tracking gitea github gitlab") - result = run_plan_check(query) - assert "notes" not in result or not result["notes"] - llm_entries = [m for m in result["matches"] if m["kind"] == "llm"] - assert len(llm_entries) == 1 - assert llm_entries[0]["score"] == 0.77 - # deterministic top match is still first and unaffected - assert result["matches"][0]["kind"] == "deterministic" diff --git a/tests/test_webhooks.py b/tests/test_webhooks.py deleted file mode 100644 index 0afdbf5..0000000 --- a/tests/test_webhooks.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -import hashlib -import hmac - -from reuse_surface.hub.webhooks import push_touches_registry_index, verify_signature - -SECRET = "shh" - - -def _sign(body: bytes, secret: str = SECRET) -> str: - return hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() - - -def test_verify_signature_valid(): - body = b'{"commits": []}' - assert verify_signature(SECRET, body, _sign(body)) is True - - -def test_verify_signature_wrong_secret(): - body = b'{"commits": []}' - assert verify_signature("different-secret", body, _sign(body)) is False - - -def test_verify_signature_tampered_body(): - body = b'{"commits": []}' - signature = _sign(body) - assert verify_signature(SECRET, b'{"commits": [1]}', signature) is False - - -def test_verify_signature_missing_signature(): - assert verify_signature(SECRET, b"{}", None) is False - - -def test_verify_signature_empty_secret_rejects(): - body = b"{}" - # Even if a caller accidentally computed a signature with an empty - # secret, verify_signature must not accept it -- an unconfigured - # secret is not the same as "verification not needed". - assert verify_signature("", body, _sign(body, secret="")) is False - - -def test_push_touches_registry_index_added(): - payload = {"commits": [{"added": ["registry/indexes/capabilities.yaml"], "modified": [], "removed": []}]} - assert push_touches_registry_index(payload) is True - - -def test_push_touches_registry_index_modified(): - payload = {"commits": [{"added": [], "modified": ["registry/indexes/federated.yaml"], "removed": []}]} - assert push_touches_registry_index(payload) is True - - -def test_push_touches_registry_index_removed(): - payload = {"commits": [{"added": [], "modified": [], "removed": ["registry/indexes/capabilities.yaml"]}]} - assert push_touches_registry_index(payload) is True - - -def test_push_does_not_touch_registry_index(): - payload = {"commits": [{"added": ["README.md"], "modified": ["src/main.py"], "removed": []}]} - assert push_touches_registry_index(payload) is False - - -def test_push_touches_registry_index_across_multiple_commits(): - payload = { - "commits": [ - {"added": ["README.md"], "modified": [], "removed": []}, - {"added": [], "modified": ["registry/indexes/capabilities.yaml"], "removed": []}, - ] - } - assert push_touches_registry_index(payload) is True - - -def test_push_touches_registry_index_empty_commits(): - assert push_touches_registry_index({"commits": []}) is False - assert push_touches_registry_index({}) is False - - -def test_push_touches_registry_index_ignores_similarly_named_paths(): - # a path that merely starts with "registry" but isn't under - # registry/indexes/ must not match - payload = {"commits": [{"added": ["registry/capabilities/capability.foo.md"], "modified": [], "removed": []}]} - assert push_touches_registry_index(payload) is False diff --git a/tools/README.md b/tools/README.md index 63f5e7d..0453b8c 100644 --- a/tools/README.md +++ b/tools/README.md @@ -64,9 +64,6 @@ reuse-surface plan-check --intent "parse invoices and file evidence" reuse-surface plan-check --intent "..." --format json reuse-surface plan-check --intent "..." --record-outcome reused reuse-surface plan-check --intent "..." --file-request --requesting-domain infotech -export LLM_CONNECT_URL=http://127.0.0.1:8080 # optional, enables semantic rerank -reuse-surface plan-check --intent "..." --no-llm # skip the rerank pass -reuse-surface plan-check --intent "..." --llm-url http://127.0.0.1:8080 ``` `reuse|extend|new` verdict from `--reuse-threshold`/`--extend-threshold` @@ -76,14 +73,6 @@ REUSE-WP-0019's reuse telemetry). `--file-request` files a State Hub capability request on a `new` verdict (requires the hub reachable at `127.0.0.1:8000`; degrades gracefully offline). -When `LLM_CONNECT_URL` is set, an optional rerank pass sends the top -deterministic candidates to llm-connect for a semantic confidence score. -Per design, this **never reorders or replaces** the deterministic result — -LLM-scored entries are appended after as separately-labeled `[llm]` matches, -so the trusted base result is identical whether or not the rerank runs. -Malformed/non-JSON LLM responses are rejected and reported as a note, never -silently guessed at; missing `LLM_CONNECT_URL` degrades the same way. - ### catalog Generate human-readable catalog artifacts (UC-RS-018). @@ -173,7 +162,7 @@ Bootstrap or discover a capability registry in the current or target repo. reuse-surface establish --scaffold --domain helix_forge reuse-surface establish --scaffold --path ../state-hub reuse-surface establish --publish-check --raw-url https://.../capabilities.yaml -export LLM_CONNECT_URL=http://127.0.0.1:8080 +export LLM_CONNECT_URL=http://127.0.0.1:8088 reuse-surface establish --discover --dry-run reuse-surface establish --discover --apply ``` @@ -200,7 +189,7 @@ Interactive or automated registry maintenance (REUSE-WP-0016). Preferred entry point for sibling repo operators. ```bash -export LLM_CONNECT_URL=http://127.0.0.1:8080 # optional +export LLM_CONNECT_URL=http://127.0.0.1:8088 # optional reuse-surface maintain --all --from-git-since origin/main reuse-surface maintain --capability capability.registry.register reuse-surface maintain --all --auto --no-llm diff --git a/workplans/REUSE-WP-0018-plan-check-consumption-loop.md b/workplans/REUSE-WP-0018-plan-check-consumption-loop.md index f6d4581..32cedfd 100644 --- a/workplans/REUSE-WP-0018-plan-check-consumption-loop.md +++ b/workplans/REUSE-WP-0018-plan-check-consumption-loop.md @@ -4,7 +4,7 @@ type: workplan title: "plan-check: close the consumption loop for capability reuse" domain: infotech repo: reuse-surface -status: finished +status: active owner: claude-code topic_slug: helix-forge created: "2026-07-06" @@ -126,38 +126,20 @@ Implemented in `reuse_surface/plan_check.py` + `plan-check` CLI command: ```task id: REUSE-WP-0018-T03 -status: done +status: todo priority: medium state_hub_task_id: "9040505a-238e-4815-ae1a-8c5f8c12afa9" ``` -Unblocked 2026-07-07 — `llm-connect` running locally (mock provider, -`127.0.0.1:8080`). Implemented in `reuse_surface/plan_check.py`: +**Not started** — `llm-connect` is not running on this workstation +(same gap noted in REUSE-WP-0017-T04's drafting cohorts). Deterministic +`plan-check` (T02) is fully usable without it; this task adds the optional +rerank on top when a backend is available. -- `build_rerank_prompt`/`request_rerank`/`apply_rerank`, reusing - `llm_bridge.execute_prompt`/`extract_json_object` (same pattern as - `maintain_llm.py`'s `request_maintain_patches`) -- New schema `schemas/plan-check-rerank.schema.json`; malformed responses - (missing fields, non-JSON, invented candidate ids not in the input set) - are rejected via schema validation + an id-membership filter, never - guessed at -- `apply_rerank` **appends** LLM-scored entries after the deterministic - list (`kind: "llm"`) rather than reordering it — deterministic matches - keep their original order/scores untouched, per design principle 3 -- Graceful skip (`ValueError` → a `notes` field in the result, mirroring - `maintain.py`'s `no_llm`/skip pattern) when `LLM_CONNECT_URL` is unset - or the LLM response is malformed; `--no-llm` skips without attempting a - call at all -- New CLI flags: `--llm-url`, `--no-llm` -- `schemas/plan-check-result.schema.json` extended for `notes` and the - (pre-existing, previously unschema'd) `filed_capability_request` field -- 9 new pytest cases (valid, malformed, non-JSON, invented-id-filtering, - append-not-reorder, graceful skip, `--no-llm` short-circuit, successful - integration) — 89 total pass -- **Live-verified**, not just mocked: real HTTP round-trip against the - running mock `llm-connect` instance correctly rejected its non-JSON mock - response and surfaced the skip note, while the deterministic verdict and - match order stayed completely unaffected +- Optional rerank/expansion stage via `LLM_CONNECT_URL` (reuse + `llm_bridge.py`); schema-constrained JSON, graceful skip when unset +- Confidence surfaced per match; deterministic matches always listed first +- Pytest with mocked llm-connect (valid + malformed responses) ## Bridge State Hub Capability Requests @@ -239,7 +221,7 @@ state_hub_task_id: "ff0a91eb-cc41-487b-b726-8c2f84860399" ## Acceptance -- [x] `plan-check` returns reuse/extend/new verdicts for workplan files and intent text, with and without llm-connect (T03 shipped 2026-07-07, live-verified against a running instance) +- [x] `plan-check` returns reuse/extend/new verdicts for workplan files and intent text (without llm-connect — T03 rerank not built, not available on this workstation) - [x] JSON output validates against the published schema - [x] `new` verdicts can file State Hub capability requests; `report gaps` lists unmatched open requests - [x] Session-protocol convention drafted; propagation *proposed* to custodian-agent diff --git a/workplans/REUSE-WP-0019-forgejo-automation-and-telemetry.md b/workplans/REUSE-WP-0019-forgejo-automation-and-telemetry.md index f5f3ab2..cc53162 100644 --- a/workplans/REUSE-WP-0019-forgejo-automation-and-telemetry.md +++ b/workplans/REUSE-WP-0019-forgejo-automation-and-telemetry.md @@ -62,128 +62,39 @@ refresh* and moves reliability evidence beyond structural. ```task id: REUSE-WP-0019-T01 -status: done +status: todo priority: high state_hub_task_id: "4a187b56-bff9-4097-abd0-b423e7bf9442" ``` -**Inventory findings (2026-07-07):** cross-referenced `sources.yaml` (61 -entries) against each repo's actual git `origin` remote. Result: 11 repos -already migrated their remote to Forgejo; 50 still on Gitea (correctly, no -action needed). Of the 11 Forgejo-origin repos, 9 already had correct -`sources.yaml` entries (from WP-0017 drafting/registration); **2 were -stale** — `activity-core` and `state-hub` — still pointing at their old -Gitea raw URL despite having migrated. This was real, live debt, not a -hypothetical: both were confirmed reachable on Forgejo (HTTP 200) before -being migrated for real, in both `sources.yaml` and the production hub -registration (`hub update --url`). Verified post-migration against -`GET /v1/federated` — both now show `forgejo.coulomb.social`. - -`config-atlas`'s WP-0017-T06 303 was **not** a host-transition symptom — -already diagnosed there as (a) a redirect the current code already follows -fine, and (b) a hub-registration gap unrelated to host. No new finding here. - -Implemented: - -- `reuse_surface/forge_host.py`: `parse_raw_url`/`derive_raw_url` (handles - both the legacy `/raw//...` form and the canonical - `/raw/branch//...` form both forges serve without a 303 - redirect), `rewrite_url_host`, `forge_base_url` (reads - `REUSE_SURFACE_FORGE_BASE_URL`, no hard default since migration is - opt-in per repo), `migrate_source_host` (verifies the new URL resolves - via HTTP HEAD **before** writing — refuses to point a repo at a host it - hasn't actually migrated to) -- CLI: `reuse-surface federation migrate-host --repo [--repo ...] - --to [--from ] [--dry-run] [--no-verify] - [--update-hub]` — used for real on `activity-core`/`state-hub` -- Fixed two host-agnostic code gaps found while inventorying: - `registry_update.py`'s `SAFE_EVIDENCE_PREFIXES` and `maintain_llm.py`'s - git-diff pathspec only recognized `.gitea/workflows/`, missing repos - already on `.forgejo/workflows/` — both now recognize either -- Fixed stale copy-paste examples in `docs/RegistryFederation.md` - (state-hub's old Gitea URL, now genuinely wrong post-migration) and a - pre-existing, unrelated port typo (`8088` vs the real llm-connect - default `8080`) in `tools/README.md`/`registry/README.md`, discovered - and confirmed live during REUSE-WP-0018-T03 -- 17 new pytest cases (`tests/test_forge_host.py`); 106 total pass -- Recomposed `federated.yaml` post-migration: still 61 capabilities, no - loss +- Inventory every hardcoded `gitea.coulomb.social` / `.gitea/` reference: + `sources.yaml` (60 raw URLs), hub registrations, establish/publish-check + defaults, docs, templates, CI workflows +- Introduce `REUSE_SURFACE_FORGE_BASE_URL` (env + hub config); derive raw + index URLs from `{base}/{org}/{repo}/raw/{branch}/registry/indexes/capabilities.yaml` +- Migration command: `reuse-surface federation migrate-host --from --to ` + rewriting sources.yaml + hub registrations via the hub API +- Check whether the config-atlas 303 (WP-0017-T06) is a symptom of the host + transition; coordinate findings +- Tests: URL derivation, migrate-host dry-run ## Hub Recompose Endpoint And Webhook Receiver ```task id: REUSE-WP-0019-T02 -status: done +status: todo priority: high state_hub_task_id: "691eb32a-6f20-4a2a-b9ff-0ae427b659aa" ``` -**Note:** `POST /v1/federated/compose` (token-auth, triggers a real -recompose) already existed from earlier hub work — no separate -`/v1/recompose` route was added; the spec now documents this explicitly -rather than duplicating a route that already does the job. - -Implemented in `reuse_surface/hub/`: - -- `store.py`: `compose_state` table (`composed_at`, `stale`), with - `record_compose()`/`mark_stale()`/`get_compose_state()`. `composed_at` - updates and `stale` clears only on a *forced* recompose (`refresh=true`, - webhook, or future scheduled fallback) — a plain `GET` still serves - current best-effort data but never silently reports itself as freshly - composed -- `webhooks.py`: `verify_signature` (constant-time HMAC-SHA256, fails - closed on empty secret), `push_touches_registry_index` (path-only - inspection of the push payload's `added`/`modified`/`removed` lists — - never parses file content, per design principle 2) -- `app.py`: `POST /v1/webhooks/forgejo` (accepts both `X-Forgejo-Signature` - and `X-Gitea-Signature`, since repos migrate independently); - `GET /v1/federated` and `POST /v1/federated/compose` now share an - `asyncio.Lock` so concurrent recompose triggers (manual, webhook, future - scheduled) coalesce instead of overlapping -- `specs/FederationHubAPI.md` extended (§5.7-5.9, config table, error - codes) -- 28 new pytest cases (16 in `test_hub.py`, 12 in `test_webhooks.py`); 128 - total pass -- **Live-verified**: ran the actual hub service locally - (`reuse-surface serve`), sent a real HMAC-signed webhook payload over - HTTP — confirmed `composed_at`/`stale` transitions, signature rejection, - irrelevant-path no-op, and the full webhook-to-recompose path end to end - -**Deployed 2026-07-07 (explicit user sign-off "Deploy to live please"):** -built and pushed `gitea.coulomb.social/coulomb/reuse-surface:e3ae22e`, -smoke-tested it locally in a standalone container first, then -`helm upgrade` via `railiance-apps` (`make reuse-deploy`, pinned in -`helm/reuse-surface-values.yaml`, commits `a2c0da1`/`bcb05f5`). Live-verified -against `https://reuse.coulomb.social`: `/v1/federated` (200, 61 -capabilities, `composed_at`/`stale` fields present), `/v1/repos` (200), a -real forced recompose via `POST /v1/federated/compose` (token-auth via -`warden access --exec`, token never printed), and the webhook endpoint -correctly failing closed with 503 (`REUSE_SURFACE_FORGEJO_WEBHOOK_SECRET` -not yet configured). - -**Webhook secret added 2026-07-07 (explicit user sign-off "Go ahead and -add the webhook secret"):** generated a fresh 32-byte hex secret with -`openssl rand -hex 32`, patched it into the live K8s Secret -`reuse-surface-env` in namespace `reuse`, restarted the deployment (values -injected via `envFrom.secretRef`, not picked up without a restart) — pod -rolled cleanly (`1/1 Ready`, 0 restarts). The raw secret value was never -printed to the transcript: generated and applied in one non-echoing shell -step, and re-fetched only inside an exported env var for the live -signature-verification test below, then unset. Live-verified against -production: a correctly HMAC-signed push payload is now accepted -(`{"accepted": false, "reason": "no registry/indexes/ change"}` for a -payload that doesn't touch `registry/indexes/`, as designed), and a bad -signature is still rejected with 401. `/v1/federated` and `/v1/repos` -unaffected by the restart. - -**Found (not fixed) while smoke-testing:** the public ingress's exact-path -`/health` rule 404s at the Traefik edge (shadowed by the catch-all `/` -rule to the landing page) — confirmed ingress-layer only, not a pod/service -problem (`/health` works via direct port-forward; the Deployment's own -readiness/liveness probes pass, pod is `1/1 Ready`). `/v1/*` unaffected. -Flagged to `railiance-apps` via State Hub message and documented in -`railiance-apps` commit `bcb05f5` — not fixed here since it's a shared -production ingress template edit outside this workplan's scope. +- Hub service (`reuse_surface/serve`): `POST /v1/recompose` (token-auth) — + marks index stale and triggers recompose from registered raw URLs +- `POST /v1/webhooks/forgejo`: validates Forgejo webhook signature + (`X-Forgejo-Signature`, secret from env), accepts push events, triggers + recompose only when the pushed commits touch `registry/indexes/` +- Debounce/coalesce concurrent triggers; `GET /v1/federated` gains + `composed_at` + `stale` fields +- Extend `specs/FederationHubAPI.md`; pytest with signed fixture payloads ## Forgejo Webhook Rollout And Scheduled Fallback