From ea1fd234812ff6f19de2583bd86aed2af0a8b9dd Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 2 Jul 2026 01:47:47 +0200 Subject: [PATCH 01/10] Workplan terminology: templates, updater guard, add_progress_event alias - project_rules templates: rename workstream->workplan in prose; registration guidance is now file-first + fix-consistency C-06 (manual create_workplan/ create_workstream calls create duplicates); progress examples use workplan_id; legacy field names (state_hub_workstream_id) annotated - update_agent_instruction_files: never overwrite filled-in stack-and-commands/repo-boundary/architecture rules (TODO-marker guard) - mcp_server: add_progress_event accepts workplan_id (preferred) with workstream_id kept as legacy alias, mirroring create_task Co-Authored-By: Claude Fable 5 --- mcp_server/server.py | 6 +++-- scripts/project_rules/agents-codex.template | 12 ++++----- .../project_rules/credential-routing.template | 2 +- scripts/project_rules/first-session.template | 26 +++++++++++-------- .../project_rules/session-protocol.template | 20 +++++++++----- .../workplan-convention.template | 13 +++++++--- scripts/update_agent_instruction_files.py | 12 ++++++++- 7 files changed, 60 insertions(+), 31 deletions(-) diff --git a/mcp_server/server.py b/mcp_server/server.py index c571fe6..4287d37 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -989,6 +989,7 @@ def add_progress_event( summary: str, event_type: str = "note", topic_id: str | None = None, + workplan_id: str | None = None, workstream_id: str | None = None, task_id: str | None = None, detail: dict | str | None = None, @@ -999,7 +1000,8 @@ def add_progress_event( summary: human-readable summary of what happened event_type: free-form label, e.g. note | milestone | blocker | insight topic_id: optional topic UUID - workstream_id: optional workstream UUID + workplan_id: optional workplan UUID (preferred) + workstream_id: legacy alias for workplan_id task_id: optional task UUID detail: optional structured data (JSONB); accepts a dict or a JSON string """ @@ -1010,7 +1012,7 @@ def add_progress_event( detail = {"raw": detail} event = _post("/progress", { "topic_id": topic_id, - "workstream_id": workstream_id, + "workplan_id": workplan_id or workstream_id, "task_id": task_id, "event_type": event_type, "summary": summary, diff --git a/scripts/project_rules/agents-codex.template b/scripts/project_rules/agents-codex.template index f48ed78..7e379dc 100644 --- a/scripts/project_rules/agents-codex.template +++ b/scripts/project_rules/agents-codex.template @@ -33,8 +33,8 @@ statehub outbox status/replay after connectivity returns. # Offline brief — works without hub connection cat .custodian-brief.md -# Active workstreams for this domain -curl -s "http://127.0.0.1:8000/workstreams/?topic_id={TOPIC_ID}&status=active" \ +# Active workplans for this domain +curl -s "http://127.0.0.1:8000/workplans/?topic_id={TOPIC_ID}&status=active" \ | python3 -m json.tool # Check inbox @@ -57,12 +57,12 @@ curl -s -X POST http://127.0.0.1:8000/progress/ \ "summary": "what was done", "event_type": "note", "author": "codex", - "workstream_id": "", + "workplan_id": "", "task_id": "" }' ``` -Omit `workstream_id` / `task_id` when not applicable. +Omit `workplan_id` / `task_id` when not applicable. ### Update task status @@ -86,7 +86,7 @@ curl -s -X PATCH "http://127.0.0.1:8000/tasks/" \ ## Session Protocol **Start:** -1. `cat .custodian-brief.md` — domain goal and open workstreams (offline-safe) +1. `cat .custodian-brief.md` — domain goal and open workplans (offline-safe) 2. Check inbox: `GET /messages/?to_agent={REPO_SLUG}&unread_only=true`; mark read 3. Scan workplans: `ls workplans/` — note `status: ready`, `active`, or `blocked` files and open tasks 4. Check human-needed tasks: `GET /tasks/?needs_human=true` @@ -145,7 +145,7 @@ owner: codex topic_slug: ... created: "YYYY-MM-DD" updated: "YYYY-MM-DD" -state_hub_workstream_id: "" # written by fix-consistency — do not edit +state_hub_workstream_id: "" # written by fix-consistency — do not edit (legacy name; holds the workplan id) --- ``` diff --git a/scripts/project_rules/credential-routing.template b/scripts/project_rules/credential-routing.template index 68ec02c..b1e18fd 100644 --- a/scripts/project_rules/credential-routing.template +++ b/scripts/project_rules/credential-routing.template @@ -20,7 +20,7 @@ Requires the `warden` CLI from `~/ops-warden` (`uv tool install .` or `uv run wa | Agent runtime | How to orient | | --- | --- | | **Codex / Grok** (shell, HTTP State Hub) | `warden route` commands above; inbox `to_agent={REPO_SLUG}` is for coordination, not secret vending | -| **Claude Code** (MCP when available) | `get_domain_summary("custodian")` for workstreams; **still** use `warden route` for credential ownership | +| **Claude Code** (MCP when available) | `get_domain_summary("custodian")` for workplans; **still** use `warden route` for credential ownership | | **llm-connect** (inference service) | Never put secret retrieval in prompts; route custody to OpenBao/operator paths surfaced by `warden route` | ### Quick routing table diff --git a/scripts/project_rules/first-session.template b/scripts/project_rules/first-session.template index 2f2da88..9aabb6d 100644 --- a/scripts/project_rules/first-session.template +++ b/scripts/project_rules/first-session.template @@ -1,6 +1,6 @@ ## First Session Protocol -Triggered when `get_domain_summary("{DOMAIN}")` shows **no workstreams**. +Triggered when `get_domain_summary("{DOMAIN}")` shows **no workplans**. The project is registered but work has not yet been structured. **Step 1 — Read, don't write** @@ -11,27 +11,31 @@ The project is registered but work has not yet been structured. **Step 2 — Survey in-progress work** Look for TODOs, open branches, half-finished files. Note done vs. started but incomplete. -**Step 3 — Propose workstreams to Bernd** -Propose 1–3 workstreams — each a coherent strand, weeks to months, anchored to a +**Step 3 — Propose workplans to Bernd** +Propose 1–3 workplans — each a coherent strand, weeks to months, anchored to a roadmap phase. **Wait for approval before creating.** -**Step 4 — Create workplan file first, then DB record (ADR-001)** +**Step 4 — Write the workplan file; fix-consistency registers it (ADR-001)** ``` -workplans/{WP_PREFIX}-NNNN-.md ← write this first +workplans/{WP_PREFIX}-NNNN-.md ← write this, commit it ``` -Then register in the hub: -``` -create_workstream(topic_id="{TOPIC_ID}", title="...", owner="...", description="...") -create_task(workstream_id="", title="...", priority="high|medium|low") +Then register by running the consistency check — do **not** call +`create_workplan`/`create_task` (or legacy `create_workstream`) yourself; +manual registration duplicates what C-06 creates from the file: +```bash +statehub fix-consistency --repo {REPO_SLUG} ``` +C-06 creates the hub workplan + tasks and writes `state_hub_workstream_id` / +`state_hub_task_id` back into the file (legacy field names, kept for +compatibility — they hold workplan/task IDs). **Step 5 — Record the setup** ``` add_progress_event( - summary="First session: structured {DOMAIN} into N workstreams, M tasks", + summary="First session: structured {DOMAIN} into N workplans, M tasks", event_type="milestone", topic_id="{TOPIC_ID}", - detail={"workstreams": [...], "tasks_created": M} + detail={"workplans": [...], "tasks_created": M} ) ``` diff --git a/scripts/project_rules/session-protocol.template b/scripts/project_rules/session-protocol.template index bad3f75..bc5d40d 100644 --- a/scripts/project_rules/session-protocol.template +++ b/scripts/project_rules/session-protocol.template @@ -44,7 +44,7 @@ For each file with `status: ready`, `active`, or `blocked`, note pending **Step 4 — Present brief** -1. **Active workstreams** for `{DOMAIN}` — title, task counts, blocking decisions +1. **Active workplans** for `{DOMAIN}` — title, task counts, blocking decisions 2. **Pending tasks** from `workplans/` + any `[repo:{REPO_SLUG}]` hub tasks 3. **Goal guidance** — if `goal_guidance` in summary: - `needs_workplan`: surface as top action — *"Repo goal '{title}' has no workplan yet"* @@ -52,23 +52,31 @@ For each file with `status: ready`, `active`, or `blocked`, note pending 4. **Suggested next action** — highest-priority open item 5. **SBOM status** — flag if `last_sbom_at` is unset for this repo -If no workstreams: follow First Session Protocol (`first-session.md`). +If no workplans: follow First Session Protocol (`first-session.md`). **During work:** `record_decision()` · `add_progress_event()` · `resolve_decision()` -> State Hub is a *read model*. Bootstrap tools (`create_workstream`, `create_task`) -> are First Session Protocol only. Work structure belongs in repo files (ADR-001). +> State Hub is a *read model*. **Never register workplans or tasks by hand** +> (`create_workplan`, `create_task`, or the legacy `create_workstream`) — write +> the workplan file in `workplans/` and run `fix-consistency`; its C-06 check +> registers the workplan and its tasks in the hub and writes the IDs back into +> the file. Manual registration creates duplicates the moment fix-consistency +> runs. Work structure belongs in repo files (ADR-001). +> +> Terminology: "workstream" is the legacy name for workplan. Some API/frontmatter +> field names keep it for compatibility (`state_hub_workstream_id`, +> `workstream_id` params) — treat them as workplan IDs. **Session close:** With MCP tools: ``` -add_progress_event(summary="...", topic_id="{TOPIC_ID}", workstream_id="") +add_progress_event(summary="...", topic_id="{TOPIC_ID}", workplan_id="") ``` Without MCP tools: ```bash curl -s -X POST http://127.0.0.1:8000/progress/ \ -H "Content-Type: application/json" \ - -d '{"topic_id":"{TOPIC_ID}","workstream_id":"","event_type":"note","summary":"what changed","author":"codex"}' + -d '{"topic_id":"{TOPIC_ID}","workplan_id":"","event_type":"note","summary":"what changed","author":"codex"}' ``` If workplan files were modified, ensure the local copy is up to date first, then sync from the repo checkout: diff --git a/scripts/project_rules/workplan-convention.template b/scripts/project_rules/workplan-convention.template index c0e27df..ff22391 100644 --- a/scripts/project_rules/workplan-convention.template +++ b/scripts/project_rules/workplan-convention.template @@ -5,7 +5,7 @@ ID prefix: `{WP_PREFIX}-` Work items originate as files in this repo **before** being registered in the hub. -Canonical workplan/workstream frontmatter statuses are: +Canonical workplan frontmatter statuses are: `proposed`, `ready`, `active`, `blocked`, `backlog`, `finished`, `archived`. Use `proposed` for a newly drafted plan, `ready` after review against current repo state, and `finished` when implementation is complete. `stalled` and @@ -16,14 +16,15 @@ prefix: `YYMMDD-{WP_PREFIX}-NNNN-.md`. The frontmatter id remains unchanged; the prefix is only for quick visual reference. Small opportunistic tasks discovered during another session use **Ad Hoc Tasks**: -`workplans/ADHOC-YYYY-MM-DD.md`, workstream slug `adhoc-YYYY-MM-DD`, and task ids +`workplans/ADHOC-YYYY-MM-DD.md`, workplan slug `adhoc-YYYY-MM-DD`, and task ids `ADHOC-YYYY-MM-DD-T01`, `T02`, etc. Use adhocs only for low-risk work completed directly. Promote anything requiring analysis, design, approval, dependencies, or multiple planned phases into a normal workplan. Ecosystem todos from other agents arrive as `[repo:{REPO_SLUG}]` hub tasks — -visible at session start. Pick one up by creating the workplan file, then registering -the workstream. +visible at session start. Pick one up by creating the workplan file, committing, +and running `statehub fix-consistency` — C-06 registers the workplan in the hub. +Never register by hand with `create_workplan`/`create_workstream`. Task blocks use this shape: @@ -37,4 +38,8 @@ state_hub_task_id: "" # written by fix-consistency — do not edit Status progression is `todo` → `progress` → `done`; use `wait` for waiting or blocked work and `cancel` for stopped work. +Workplan frontmatter carries `state_hub_workstream_id` — a legacy field name +kept for compatibility ("workstream" is the old term for workplan); it holds +the hub workplan id and is written by fix-consistency. Do not edit or rename it. + diff --git a/scripts/update_agent_instruction_files.py b/scripts/update_agent_instruction_files.py index e2522f2..cf1ec57 100644 --- a/scripts/update_agent_instruction_files.py +++ b/scripts/update_agent_instruction_files.py @@ -27,6 +27,10 @@ def fetch(path: str): EXTENSION_MARKER = "" +# Rule files that repos fill in with local content; only (re)write them while +# they still contain the template's TODO markers. +PRESERVE_IF_CUSTOMIZED = {"stack-and-commands", "repo-boundary", "architecture"} + def render(template: str, values: dict[str, str]) -> str: for key, value in values.items(): @@ -188,7 +192,13 @@ def update_repo( rules_dir = path / ".claude" / "rules" rules_dir.mkdir(parents=True, exist_ok=True) for name, template in rule_templates.items(): - (rules_dir / f"{name}.md").write_text(render(template, values), encoding="utf-8") + target = rules_dir / f"{name}.md" + if name in PRESERVE_IF_CUSTOMIZED and target.exists(): + # These files start as TODO templates and get filled per repo; + # never overwrite a filled-in version with the blank template. + if "TODO" not in target.read_text(encoding="utf-8"): + continue + target.write_text(render(template, values), encoding="utf-8") return f"{repo_slug}\t{path}\t{prefix}" From 8bd4a67639744ebe3a9d5b2b10706fd42c4b4615 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 11:20:16 +0200 Subject: [PATCH 02/10] CUST-WP-0011-T07 done: cluster State Hub is primary (exact-count restore, tunnel rewire) Co-Authored-By: Claude Fable 5 --- ...T-WP-0011-state-hub-threephoenix-migration.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md b/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md index 76096a2..158a9a5 100644 --- a/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md +++ b/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md @@ -343,11 +343,9 @@ port-forwards were removed after verification. ```task id: CUST-WP-0011-T07 -status: todo +status: done priority: medium state_hub_task_id: "ff1de25e-c301-4b86-9420-84dfe72e565e" -needs_human: true -intervention_note: "Requires explicit approval to freeze WSL2 writes and make the cluster State Hub the primary endpoint." ``` With human approval, freeze WSL2 writes, take a final dump, restore it to the @@ -363,6 +361,18 @@ Accepted approaches: **Done when:** `get_state_summary()` and dashboard live data are served by the cluster State Hub, and WSL2 is no longer receiving normal writes. +Result: completed 2026-07-03 with explicit operator approval ("go forward with +1 and 2 and 3"). Sequence: cluster image refreshed to `ea1fd23` (adds the +add_progress_event workplan_id alias; schema head `e9f0a1b2c3d4` unchanged); +WSL2 uvicorn stopped (freeze); final `pg_dump` from `infra-postgres-1` +restored into CNPG `state-hub-db`/`state_hub` with `SET ROLE state_hub` +ownership; row counts matched exactly (633 workplans, 3964 tasks, 8192 +progress events, 14 topics, 1933 token events); private access rewired via +ops-bridge `state-hub-primary` forward tunnel so `127.0.0.1:8000` serves the +cluster hub. The railiance01 automation chain (`:18000`) verified intact. +First primary-served write: progress event `56aab39b`. WSL2 fallback restart: +`bridge down state-hub-primary && cd ~/state-hub && make api`. + --- ### T08 — Stabilise with WSL2 retained as fallback From 2b6a3ef5213c53ad743870889e25df9a891cbb95 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 11:22:22 +0200 Subject: [PATCH 03/10] chore(consistency): sync task status from DB [auto] Updated by fix-consistency on 2026-07-03: - update .custodian-brief.md for state-hub --- .custodian-brief.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.custodian-brief.md b/.custodian-brief.md index 7dfc5ff..9a5110c 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,7 +2,7 @@ # Custodian Brief — state-hub **Domain:** infotech -**Last synced:** 2026-07-01 21:50 UTC +**Last synced:** 2026-07-03 09:22 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams @@ -21,10 +21,9 @@ Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` - … and 1 more open tasks ### Pragmatic State Hub Migration to railiance01 -Progress: 6/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` +Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` **Open tasks:** -- · T07 — Cutover: redirect MCP config to cluster `ff1de25e` - · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` - · T09 — Retire WSL2 instance `d75a2d49` From a0167ff3869dd6e5e28ed6bf81a1745c4035a4ad Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 18:06:57 +0200 Subject: [PATCH 04/10] chore(consistency): sync task status from DB [auto] Updated by fix-consistency on 2026-07-03: - update .custodian-brief.md for state-hub --- .custodian-brief.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.custodian-brief.md b/.custodian-brief.md index 9a5110c..971d021 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,11 +2,18 @@ # Custodian Brief — state-hub **Domain:** infotech -**Last synced:** 2026-07-03 09:22 UTC +**Last synced:** 2026-07-03 16:06 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams +### Pragmatic State Hub Migration to railiance01 +Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` + +**Open tasks:** +- · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` +- · T09 — Retire WSL2 instance `d75a2d49` + ### State Hub Full ThreePhoenix HA Migration Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` @@ -20,13 +27,6 @@ Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` - · T07 — Update agent access and runbooks for HA endpoint `959062d8` - … and 1 more open tasks -### Pragmatic State Hub Migration to railiance01 -Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` - -**Open tasks:** -- · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` -- · T09 — Retire WSL2 instance `d75a2d49` - --- ## MCP Orientation (when available) From 5388aad77a8ed4cfd61ad58886592f67b1089bab Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 19:27:09 +0200 Subject: [PATCH 05/10] chore(consistency): sync task status from DB [auto] Updated by fix-consistency on 2026-07-03: - update .custodian-brief.md for state-hub --- .custodian-brief.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.custodian-brief.md b/.custodian-brief.md index 971d021..d731de3 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,18 +2,11 @@ # Custodian Brief — state-hub **Domain:** infotech -**Last synced:** 2026-07-03 16:06 UTC +**Last synced:** 2026-07-03 17:27 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams -### Pragmatic State Hub Migration to railiance01 -Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` - -**Open tasks:** -- · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` -- · T09 — Retire WSL2 instance `d75a2d49` - ### State Hub Full ThreePhoenix HA Migration Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` @@ -27,6 +20,13 @@ Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` - · T07 — Update agent access and runbooks for HA endpoint `959062d8` - … and 1 more open tasks +### Pragmatic State Hub Migration to railiance01 +Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` + +**Open tasks:** +- · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` +- · T09 — Retire WSL2 instance `d75a2d49` + --- ## MCP Orientation (when available) From dbe917ceae121ee8f630975b84f764cc9c719922 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 4 Jul 2026 00:42:56 +0200 Subject: [PATCH 06/10] Workplan consistency optimization --- api/models/workplan.py | 6 +++ api/routers/workstreams.py | 52 ++++++++++++++++++ api/schemas/workplan.py | 17 ++++++ .../f1a2b3c4d5e6_workplan_file_backing.py | 31 +++++++++++ scripts/consistency_check.py | 51 ++++++++++++++++++ tests/test_routers_core.py | 53 +++++++++++++++++++ 6 files changed, 210 insertions(+) create mode 100644 migrations/versions/f1a2b3c4d5e6_workplan_file_backing.py diff --git a/api/models/workplan.py b/api/models/workplan.py index 7accd52..7fcf51f 100644 --- a/api/models/workplan.py +++ b/api/models/workplan.py @@ -52,6 +52,12 @@ class Workplan(Base, TimestampMixin): nullable=True, index=True, ) + backing_filename: Mapped[str | None] = mapped_column(String(255), nullable=True) + backing_relative_path: Mapped[str | None] = mapped_column(Text, nullable=True) + backing_archived: Mapped[bool | None] = mapped_column(nullable=True) + backing_synced_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) topic: Mapped["Topic | None"] = relationship("Topic", back_populates="workplans") # noqa: F821 repo: Mapped["ManagedRepo"] = relationship("ManagedRepo", lazy="selectin") # noqa: F821 diff --git a/api/routers/workstreams.py b/api/routers/workstreams.py index 05e6a3e..8737648 100644 --- a/api/routers/workstreams.py +++ b/api/routers/workstreams.py @@ -17,6 +17,7 @@ from api.events import EventEnvelope, publish_event from api.models.managed_repo import ManagedRepo from api.models.workplan import Workplan from api.schemas.workplan import ( + WorkplanBindingsSync, WorkplanCreate, WorkplanRead, WorkplanUpdate, @@ -212,9 +213,38 @@ async def _build_workplan_index(session: AsyncSession) -> dict[str, Any]: "needs_review": bool(review and review.needs_review), "health_labels": ["needs_review"] if review and review.needs_review else [], } + await _merge_db_backing_index(session, index) return {"workplans": index, "workstreams": index} +async def _merge_db_backing_index(session: AsyncSession, index: dict[str, Any]) -> None: + """Fill index gaps from DB-backed file bindings synced by fix-consistency.""" + result = await session.execute( + select(Workplan, ManagedRepo.slug) + .join(ManagedRepo, Workplan.repo_id == ManagedRepo.id) + .where(Workplan.backing_filename.isnot(None)) + ) + for wp, repo_slug in result.all(): + key = str(wp.id) + if key in index: + continue + index[key] = { + "filename": wp.backing_filename, + "relative_path": wp.backing_relative_path, + "repo_slug": repo_slug, + "archived": bool(wp.backing_archived), + "status": normalize_workplan_status(wp.status) if wp.status else None, + "needs_review": False, + "health_labels": [], + } + + +def _invalidate_workplan_index_cache() -> None: + global _INDEX_CACHE, _INDEX_CACHE_AT + _INDEX_CACHE = None + _INDEX_CACHE_AT = 0.0 + + def _index_with_meta(*, stale: bool, refresh_in_progress: bool) -> dict[str, Any]: age = time.monotonic() - _INDEX_CACHE_AT if _INDEX_CACHE_AT else None return { @@ -459,6 +489,28 @@ async def workplan_index_preferred( return await _workplan_index(refresh=refresh, session=session) +@workplan_router.put("/index/bindings") +async def sync_workplan_bindings( + body: WorkplanBindingsSync, + session: AsyncSession = Depends(get_session), +) -> dict[str, int]: + """Upsert workstation workplan file bindings for remote API index fallback.""" + synced_at = datetime.now(timezone.utc) + updated = 0 + for entry in body.bindings: + wp = await session.get(Workplan, entry.workplan_id) + if wp is None: + continue + wp.backing_filename = entry.filename + wp.backing_relative_path = entry.relative_path + wp.backing_archived = entry.archived + wp.backing_synced_at = synced_at + updated += 1 + await session.commit() + _invalidate_workplan_index_cache() + return {"updated": updated, "received": len(body.bindings)} + + @router.post("/", response_model=WorkplanRead, status_code=status.HTTP_201_CREATED) async def create_workstream( request: Request, diff --git a/api/schemas/workplan.py b/api/schemas/workplan.py index 4d52256..90be8b8 100644 --- a/api/schemas/workplan.py +++ b/api/schemas/workplan.py @@ -67,6 +67,19 @@ class WorkplanUpdate(WorkplanStatusMixin): repo_goal_id: uuid.UUID | None = None +class WorkplanFileBinding(BaseModel): + workplan_id: uuid.UUID + filename: str + relative_path: str + repo_slug: str + archived: bool = False + status: WorkplanStatus | None = None + + +class WorkplanBindingsSync(BaseModel): + bindings: list[WorkplanFileBinding] + + class WorkplanRead(WorkplanStatusMixin): model_config = ConfigDict(from_attributes=True) id: uuid.UUID @@ -87,6 +100,10 @@ class WorkplanRead(WorkplanStatusMixin): queue_rank: int | None = None execution_group: str | None = None scheduled_for: datetime | None = None + backing_filename: str | None = None + backing_relative_path: str | None = None + backing_archived: bool | None = None + backing_synced_at: datetime | None = None created_at: datetime updated_at: datetime diff --git a/migrations/versions/f1a2b3c4d5e6_workplan_file_backing.py b/migrations/versions/f1a2b3c4d5e6_workplan_file_backing.py new file mode 100644 index 0000000..f120bcd --- /dev/null +++ b/migrations/versions/f1a2b3c4d5e6_workplan_file_backing.py @@ -0,0 +1,31 @@ +"""add workplan file backing metadata for remote API index + +Revision ID: f1a2b3c4d5e6 +Revises: e9f0a1b2c3d4 +Create Date: 2026-07-03 + +""" +from alembic import op +import sqlalchemy as sa + +revision = "f1a2b3c4d5e6" +down_revision = "e9f0a1b2c3d4" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("workplans", sa.Column("backing_filename", sa.String(255), nullable=True)) + op.add_column("workplans", sa.Column("backing_relative_path", sa.Text(), nullable=True)) + op.add_column("workplans", sa.Column("backing_archived", sa.Boolean(), nullable=True)) + op.add_column( + "workplans", + sa.Column("backing_synced_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("workplans", "backing_synced_at") + op.drop_column("workplans", "backing_archived") + op.drop_column("workplans", "backing_relative_path") + op.drop_column("workplans", "backing_filename") \ No newline at end of file diff --git a/scripts/consistency_check.py b/scripts/consistency_check.py index a4a01de..4388e19 100644 --- a/scripts/consistency_check.py +++ b/scripts/consistency_check.py @@ -570,6 +570,20 @@ def _api_patch(api_base: str, path: str, body: dict) -> Any: return {"_error": str(exc)} +def _api_put(api_base: str, path: str, body: dict) -> Any: + if not _HAS_HTTPX: + return {"_error": "httpx is not installed"} + if not path.endswith("/"): + path += "/" + try: + with _httpx.Client(base_url=api_base, timeout=30.0, follow_redirects=True) as c: + r = c.put(path, json=body) + r.raise_for_status() + return r.json() + except Exception as exc: + return {"_error": str(exc)} + + def _api_post(api_base: str, path: str, body: dict) -> Any: if not _HAS_HTTPX: return {"_error": "httpx is not installed"} @@ -1263,9 +1277,46 @@ def check_repo(api_base: str, repo_slug: str, repo_path_override: str | None = N # workstream from the file, leaving the first as an invisible orphan. _check_ghost_duplicates(api_base, workplan_infos, file_ws_ids, report) + _sync_workplan_bindings(api_base, repo_slug, workplan_infos, repo_dir, report) + return report +def _sync_workplan_bindings( + api_base: str, + repo_slug: str, + workplan_infos: list[tuple[Path, dict, str]], + repo_dir: Path, + report: ConsistencyReport, +) -> None: + bindings: list[dict[str, Any]] = [] + for wp_file, meta, _ in workplan_infos: + ws_id = str(meta.get("state_hub_workstream_id", "")).strip().strip('"') + if not ws_id: + continue + archived = wp_file.parent.name == "archived" + file_status = normalise_workstream_status(str(meta.get("status", "")).strip()) + bindings.append( + { + "workplan_id": ws_id, + "filename": wp_file.name, + "relative_path": workplan_display_path(repo_dir, wp_file), + "repo_slug": repo_slug, + "archived": archived, + "status": file_status or None, + } + ) + if not bindings: + return + result = _api_put(api_base, "/workplans/index/bindings", {"bindings": bindings}) + if isinstance(result, dict) and "_error" in result: + report.fixes_applied.append(f"bindings WARN: {result['_error']}") + elif isinstance(result, dict): + report.fixes_applied.append( + f"bindings: synced {result.get('updated', 0)}/{result.get('received', len(bindings))}" + ) + + def _check_orphan_db( api_base: str, repo_id: str, diff --git a/tests/test_routers_core.py b/tests/test_routers_core.py index 961fbd2..da75d63 100644 --- a/tests/test_routers_core.py +++ b/tests/test_routers_core.py @@ -192,6 +192,59 @@ class TestWorkstreams: assert r.status_code == 200 assert "workstreams" in r.json() + async def test_workplan_bindings_sync_populates_index(self, client, tmp_path): + await _create_domain(client) + topic = await _create_topic(client) + repo = await _create_repo(client, slug="binding-repo", local_path=str(tmp_path)) + ws = await _create_workplan( + client, + repo["id"], + topic_id=topic["id"], + slug="binding-wp", + title="Binding WP", + ) + + workplans_dir = tmp_path / "workplans" + workplans_dir.mkdir() + wp_file = workplans_dir / "BIND-WP-0001-demo.md" + wp_file.write_text( + "---\n" + f"id: BIND-WP-0001\n" + "type: workplan\n" + "title: Binding WP\n" + "status: active\n" + f'state_hub_workstream_id: "{ws["id"]}"\n' + "---\n", + encoding="utf-8", + ) + + sync = await client.put( + "/workplans/index/bindings", + json={ + "bindings": [ + { + "workplan_id": ws["id"], + "filename": wp_file.name, + "relative_path": "workplans/BIND-WP-0001-demo.md", + "repo_slug": "binding-repo", + "archived": False, + "status": "active", + } + ] + }, + ) + assert sync.status_code == 200 + assert sync.json()["updated"] == 1 + + hide = await client.patch("/repos/binding-repo", json={"local_path": "/nonexistent/path"}) + assert hide.status_code == 200 + + r = await client.get("/workplans/index?refresh=true") + assert r.status_code == 200 + entry = r.json()["workplans"][ws["id"]] + assert entry["filename"] == wp_file.name + assert entry["repo_slug"] == "binding-repo" + # --------------------------------------------------------------------------- # Task tests From cac9a6b1e04710e55903e35145a1bc60f30f4957 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 6 Jul 2026 10:36:30 +0200 Subject: [PATCH 07/10] chore(consistency): sync task status from DB [auto] Updated by fix-consistency on 2026-07-06: - update .custodian-brief.md for state-hub --- .custodian-brief.md | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.custodian-brief.md b/.custodian-brief.md index d731de3..a52af6d 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,7 +2,7 @@ # Custodian Brief — state-hub **Domain:** infotech -**Last synced:** 2026-07-03 17:27 UTC +**Last synced:** 2026-07-06 08:36 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams @@ -20,13 +20,6 @@ Progress: 0/8 done | workstream_id: `8d0c1b5d-44da-4b91-8357-e6526d3e0a85` - · T07 — Update agent access and runbooks for HA endpoint `959062d8` - … and 1 more open tasks -### Pragmatic State Hub Migration to railiance01 -Progress: 7/9 done | workstream_id: `967baafb-d92d-405a-ba0b-0d00d37c4940` - -**Open tasks:** -- · T08 — Stabilisation period (2 weeks minimum) `e06a59a0` -- · T09 — Retire WSL2 instance `d75a2d49` - --- ## MCP Orientation (when available) From f2e042a27863fdd57d57d4e0e256efe3ad6e3c89 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 6 Jul 2026 10:52:49 +0200 Subject: [PATCH 08/10] Finish CUST-WP-0011 and implement STATE-WP-0061 suggestion backlog. Add persisted Suggestion entities with WSJF ranking, relevance bumps, REST/MCP write surfaces, /suggestions dashboard page, and daily triage digest integration. Document the cluster operating model, archive the completed migration workplan, and seed WARDEN-WP-0012 routing scenarios. --- AGENTS.md | 7 +- INTENT.md | 5 + README.md | 5 + SCOPE.md | 3 +- api/main.py | 3 +- api/models/__init__.py | 7 + api/models/suggestion.py | 120 ++++++++ api/routers/capability_requests.py | 20 +- api/routers/state.py | 71 ++++- api/routers/suggestions.py | 259 ++++++++++++++++++ api/schemas/state.py | 2 + api/schemas/suggestion.py | 96 +++++++ api/services/suggestion_relevance.py | 118 ++++++++ api/services/suggestion_wsjf.py | 17 ++ api/services/write_idempotency.py | 5 + dashboard/observablehq.config.js | 2 + dashboard/src/docs/capabilities.md | 5 +- dashboard/src/docs/suggestions.md | 49 ++++ dashboard/src/suggestions.md | 79 ++++++ dashboard/src/wsjf-triage.md | 2 +- docs/activity-core-delegation.md | 2 +- docs/cluster-operating-model.md | 200 ++++++++++++++ docs/onboarding.md | 11 +- infra/README.md | 11 +- mcp_server/TOOLS.md | 7 +- mcp_server/server.py | 95 ++++++- .../versions/f0a1b2c3d4e5_suggestions.py | 82 ++++++ scripts/seed_wp0012_suggestions.py | 55 ++++ tests/test_suggestions.py | 158 +++++++++++ ...-0011-state-hub-threephoenix-migration.md} | 34 ++- ...061-demand-weighted-suggestion-backlog.md} | 73 +++-- 31 files changed, 1537 insertions(+), 66 deletions(-) create mode 100644 api/models/suggestion.py create mode 100644 api/routers/suggestions.py create mode 100644 api/schemas/suggestion.py create mode 100644 api/services/suggestion_relevance.py create mode 100644 api/services/suggestion_wsjf.py create mode 100644 dashboard/src/docs/suggestions.md create mode 100644 dashboard/src/suggestions.md create mode 100644 docs/cluster-operating-model.md create mode 100644 migrations/versions/f0a1b2c3d4e5_suggestions.py create mode 100644 scripts/seed_wp0012_suggestions.py create mode 100644 tests/test_suggestions.py rename workplans/{CUST-WP-0011-state-hub-threephoenix-migration.md => archived/260706-CUST-WP-0011-state-hub-threephoenix-migration.md} (92%) rename workplans/{STATE-WP-0061-demand-weighted-suggestion-backlog.md => archived/260706-STATE-WP-0061-demand-weighted-suggestion-backlog.md} (68%) diff --git a/AGENTS.md b/AGENTS.md index 55fbaf1..5adfcdd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,11 @@ there is no MCP server for Codex agents. | Context | URL | |---------|-----| -| Local workstation | `http://127.0.0.1:8000` | -| Remote via tunnel | `http://127.0.0.1:18000` | +| Primary (cluster via tunnel) | `http://127.0.0.1:8000` | +| Remote machine mesh | `http://127.0.0.1:18000` | + +Cluster operating model (access, rollback, backups): +[`docs/cluster-operating-model.md`](docs/cluster-operating-model.md) ### Orient at session start diff --git a/INTENT.md b/INTENT.md index 483d1ac..099f48b 100644 --- a/INTENT.md +++ b/INTENT.md @@ -42,6 +42,11 @@ general application platform. When State Hub detects facts, it records and exposes them; when new work must be spawned from events, that responsibility belongs to activity-core or a human-approved workflow. +**Sanctioned writes** (STATE-WP-0061): besides `resolve_decision` and progress +events, the hub may persist gated needs as `Suggestion` records, bump demand +`relevance`, vet/decline/promote them into real tasks, and project WSJF ranking +as a read model for daily triage. + --- ## What it is diff --git a/README.md b/README.md index 1bf290f..f73cf83 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ then run consistency sync. All services bind to `127.0.0.1` only — nothing exposed to the network. +**Production:** the primary State Hub API runs on coulombcore-k3s. Workstation +port `8000` reaches it through the ops-bridge `state-hub-primary` tunnel. See +[`docs/cluster-operating-model.md`](docs/cluster-operating-model.md) for access, +rollback, backups, and pragmatic limitations. + --- ## Setup diff --git a/SCOPE.md b/SCOPE.md index 6737a7e..d0975ac 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -15,7 +15,8 @@ tooling, and dashboard telemetry. - repo registration (classification-driven) and consistency synchronization - repo classification spine (14 market domains, `.repo-classification.yaml`) - task-flow engine and flow definitions -- SBOM, contribution, capability, TPSC, DoI, token, and interface-change tracking +- SBOM, contribution, capability, demand-weighted suggestion backlog, TPSC, DoI, + token, and interface-change tracking - State Hub tests, operational docs, policies, prompts, and local infra ## Out Of Scope diff --git a/api/main.py b/api/main.py index bc14b74..db63efc 100644 --- a/api/main.py +++ b/api/main.py @@ -12,7 +12,7 @@ from starlette.responses import Response as StarletteResponse from api.database import engine from api.events import shutdown_publisher from api.services.write_idempotency import WriteIdempotencyMiddleware -from api.routers import decisions, extension_points, progress, state, tasks, technical_debt, topics, workstreams, workstream_dependencies +from api.routers import decisions, extension_points, progress, state, suggestions, tasks, technical_debt, topics, workstreams, workstream_dependencies from api.routers import domains, repos, contributions, sbom, policy, domain_goals, repo_goals, messages, capability_requests, tpsc, services from api.routers import token_events from api.routers import interface_changes @@ -123,6 +123,7 @@ app.include_router(contributions.router) app.include_router(sbom.router) app.include_router(messages.router) app.include_router(capability_requests.router) +app.include_router(suggestions.router) app.include_router(tpsc.router) app.include_router(services.router) app.include_router(token_events.router) diff --git a/api/models/__init__.py b/api/models/__init__.py index f40b01b..522c3b0 100644 --- a/api/models/__init__.py +++ b/api/models/__init__.py @@ -34,6 +34,12 @@ from api.models.workplan_launch_request import WorkplanLaunchRequest from api.models.fabric_graph import FabricGraphImport, FabricGraphNode, FabricGraphEdge from api.models.legacy_meter import LegacyInterface, LegacyInterfaceUsageBucket from api.models.write_idempotency_key import WriteIdempotencyKey +from api.models.suggestion import ( + Suggestion, + SuggestionNote, + SuggestionRelevanceBump, + SuggestionStage, +) __all__ = [ "Base", @@ -67,4 +73,5 @@ __all__ = [ "FabricGraphImport", "FabricGraphNode", "FabricGraphEdge", "LegacyInterface", "LegacyInterfaceUsageBucket", "WriteIdempotencyKey", + "Suggestion", "SuggestionNote", "SuggestionRelevanceBump", "SuggestionStage", ] \ No newline at end of file diff --git a/api/models/suggestion.py b/api/models/suggestion.py new file mode 100644 index 0000000..df5a791 --- /dev/null +++ b/api/models/suggestion.py @@ -0,0 +1,120 @@ +import enum +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql import func + +from api.models.base import Base, TimestampMixin, new_uuid + + +class SuggestionStage(str, enum.Enum): + suggestion = "suggestion" + requirement = "requirement" + promoted = "promoted" + declined = "declined" + + +OPEN_SUGGESTION_STAGES = (SuggestionStage.suggestion, SuggestionStage.requirement) + + +class Suggestion(Base, TimestampMixin): + __tablename__ = "suggestions" + + id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), primary_key=True, default=new_uuid + ) + domain_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("domains.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + topic_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("topics.id", ondelete="SET NULL"), nullable=True + ) + workplan_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("workplans.id", ondelete="SET NULL"), nullable=True + ) + title: Mapped[str] = mapped_column(String(500), nullable=False) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + origin: Mapped[str | None] = mapped_column(String(200), nullable=True) + origin_ref: Mapped[str | None] = mapped_column(String(200), nullable=True, index=True) + stage: Mapped[SuggestionStage] = mapped_column( + Enum(SuggestionStage, name="suggestionstage"), + nullable=False, + default=SuggestionStage.suggestion, + index=True, + ) + relevance: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + relevance_events: Mapped[int] = mapped_column( + Integer, nullable=False, default=0, server_default="0" + ) + last_requested_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + base_value: Mapped[float] = mapped_column(Float, nullable=False, default=3.0, server_default="3") + job_size: Mapped[float] = mapped_column(Float, nullable=False, default=3.0, server_default="3") + relevance_weight: Mapped[float] = mapped_column( + Float, nullable=False, default=1.0, server_default="1" + ) + promoted_task_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("tasks.id", ondelete="SET NULL"), nullable=True + ) + + domain: Mapped["Domain"] = relationship("Domain", lazy="selectin") # noqa: F821 + topic: Mapped["Topic | None"] = relationship("Topic", lazy="selectin") # noqa: F821 + workplan: Mapped["Workplan | None"] = relationship("Workplan", lazy="selectin") # noqa: F821 + promoted_task: Mapped["Task | None"] = relationship("Task", lazy="selectin") # noqa: F821 + notes: Mapped[list["SuggestionNote"]] = relationship( + "SuggestionNote", + back_populates="suggestion", + lazy="selectin", + order_by="SuggestionNote.created_at", + ) + + @property + def domain_slug(self) -> str: + return self.domain.slug if self.domain is not None else "" + + +class SuggestionNote(Base): + __tablename__ = "suggestion_notes" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=new_uuid) + suggestion_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("suggestions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + stage: Mapped[str] = mapped_column(String(30), nullable=False) + author: Mapped[str | None] = mapped_column(String(100), nullable=True) + content: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) + + suggestion: Mapped["Suggestion"] = relationship("Suggestion", back_populates="notes") + + +class SuggestionRelevanceBump(Base): + """Audit trail for relevance bumps; supports debounce lookups.""" + + __tablename__ = "suggestion_relevance_bumps" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=new_uuid) + suggestion_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("suggestions.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + source: Mapped[str] = mapped_column(String(50), nullable=False) + source_key: Mapped[str] = mapped_column(String(200), nullable=False) + reason: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), nullable=False + ) \ No newline at end of file diff --git a/api/routers/capability_requests.py b/api/routers/capability_requests.py index db3df74..3192004 100644 --- a/api/routers/capability_requests.py +++ b/api/routers/capability_requests.py @@ -13,6 +13,7 @@ from api.models.capability_request import CapabilityRequest from api.models.domain import Domain from api.models.managed_repo import ManagedRepo from api.models.task import Task +from api.services.suggestion_relevance import bump_matching_for_capability_request from api.schemas.capability_request import ( CapabilityRequestAccept, CapabilityRequestCreate, @@ -68,6 +69,23 @@ def _build_capability_request( ) +async def _on_capability_request_persisted( + session: AsyncSession, + req: CapabilityRequest, + body: CapabilityRequestCreate, +) -> None: + await session.flush() + await bump_matching_for_capability_request( + session, + request_id=req.id, + title=body.title, + description=body.description or "", + capability_type=body.capability_type, + catalog_entry_id=req.catalog_entry_id, + ) + await _notify_on_create(session, req, body) + + async def _notify_on_create( session: AsyncSession, req: CapabilityRequest, @@ -405,7 +423,7 @@ router.include_router( request_read_schema=CapabilityRequestRead, route_request=_route_capability, build_request=_build_capability_request, - on_request_persisted=_notify_on_create, + on_request_persisted=_on_capability_request_persisted, check_transition=_check_transition, apply_accept_fields=_apply_accept_fields, after_accept=_notify_on_accept, diff --git a/api/routers/state.py b/api/routers/state.py index cd7fd89..0f0f218 100644 --- a/api/routers/state.py +++ b/api/routers/state.py @@ -21,8 +21,12 @@ from api.models.sbom_snapshot import SBOMSnapshot from api.models.task import Task, TaskPriority, TaskStatus from api.models.technical_debt import TechnicalDebt from api.models.topic import Topic, TopicStatus +from api.models.suggestion import OPEN_SUGGESTION_STAGES, Suggestion from api.models.workplan import Workplan from api.models.workplan_dependency import WorkplanDependency +from api.schemas.suggestion import RankedSuggestionDigest +from api.services.suggestion_relevance import bump_suggestions_for_next_steps +from api.services.suggestion_wsjf import compute_wsjf, suggestion_sort_key from api.schemas.decision import DecisionRead from api.schemas.domain import DomainSummary from api.schemas.progress_event import ProgressEventRead @@ -329,7 +333,7 @@ async def build_state_summary(session: AsyncSession) -> StateSummary: ), ) - next_steps = await _derive_next_steps(session) + next_steps, _ = await _derive_next_steps(session) # Domain summary stats domain_summaries = await _build_domain_summaries(session) @@ -368,6 +372,8 @@ async def build_state_summary(session: AsyncSession) -> StateSummary: ) )).scalar() or 0 + ranked_suggestions = await _ranked_suggestion_digest(session, limit=10) + result = StateSummary( generated_at=datetime.now(tz=timezone.utc), totals=totals, @@ -387,6 +393,7 @@ async def build_state_summary(session: AsyncSession) -> StateSummary: contribution_counts=contribution_counts, licence_risk_count=licence_risk_count, open_capability_requests=open_cap_req_count, + ranked_suggestions=ranked_suggestions, open_workstreams=[ WorkstreamWithDeps( **{ @@ -700,7 +707,7 @@ async def _build_dashboard_overview(session: AsyncSession) -> DashboardOverview: waiting_tasks=[TaskRead.model_validate(t) for t in waiting], blocked_tasks=[TaskRead.model_validate(t) for t in waiting], recent_progress=[ProgressEventRead.model_validate(e) for e in recent], - next_steps=await _derive_next_steps(session), + next_steps=(await _derive_next_steps(session))[0], contribution_counts=contribution_counts, licence_risk_count=licence_risk_count, open_capability_requests=open_cap_req_count, @@ -852,7 +859,33 @@ _PRIORITY_RANK = { } -async def _derive_next_steps(session: AsyncSession) -> list[NextStep]: +async def _ranked_suggestion_digest( + session: AsyncSession, + *, + limit: int = 10, +) -> list[RankedSuggestionDigest]: + rows = ( + await session.execute( + select(Suggestion).where(Suggestion.stage.in_(OPEN_SUGGESTION_STAGES)) + ) + ).scalars().all() + ranked = sorted(rows, key=suggestion_sort_key)[:limit] + return [ + RankedSuggestionDigest( + id=s.id, + title=s.title, + stage=s.stage, + domain_slug=s.domain_slug, + origin_ref=s.origin_ref, + relevance=s.relevance, + wsjf=compute_wsjf(s), + last_requested_at=s.last_requested_at, + ) + for s in ranked + ] + + +async def _derive_next_steps(session: AsyncSession) -> tuple[list[NextStep], list[Suggestion]]: """Derive contextual next-action suggestions from current hub state. Two signal sources: @@ -991,7 +1024,31 @@ async def _derive_next_steps(session: AsyncSession) -> list[NextStep]: )) seen_task_ids.add(task.id) - return steps + # Signal 3: open demand-weighted suggestions (needed-but-unmet backlog) + open_suggestions = sorted( + ( + await session.execute( + select(Suggestion).where(Suggestion.stage.in_(OPEN_SUGGESTION_STAGES)) + ) + ).scalars().all(), + key=suggestion_sort_key, + )[:5] + for suggestion in open_suggestions: + steps.append(NextStep( + type="open_suggestion", + domain=suggestion.domain_slug, + workstream_id=suggestion.workplan_id, + workstream_title=None, + workstream_slug=suggestion.origin_ref, + task_id=None, + task_title=None, + message=( + f"Gated need '{suggestion.title}' (stage={suggestion.stage.value}, " + f"relevance={suggestion.relevance}) — vet or promote when unblocked" + ), + )) + + return steps, open_suggestions async def _get_domain_slug_for_workstream(ws: Workplan | None, session: AsyncSession) -> str | None: @@ -1019,8 +1076,12 @@ async def get_next_steps(session: AsyncSession = Depends(get_session)) -> list[N Returns suggestions based on: - Recently resolved decisions → first open task in the same workstream - Workstreams whose every dependency workstream is now finished -> first todo task + - Open demand-weighted suggestions (gated needs accruing relevance) """ - return await _derive_next_steps(session) + steps, surfaced = await _derive_next_steps(session) + await bump_suggestions_for_next_steps(session, surfaced) + await session.commit() + return steps @router.get("/health") diff --git a/api/routers/suggestions.py b/api/routers/suggestions.py new file mode 100644 index 0000000..b04c044 --- /dev/null +++ b/api/routers/suggestions.py @@ -0,0 +1,259 @@ +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.database import get_session +from api.models.domain import Domain +from api.models.suggestion import ( + OPEN_SUGGESTION_STAGES, + Suggestion, + SuggestionNote, + SuggestionStage, +) +from api.models.task import Task, TaskPriority, TaskStatus +from api.schemas.suggestion import ( + SuggestionBumpRelevance, + SuggestionCreate, + SuggestionDecline, + SuggestionNoteRead, + SuggestionPromote, + SuggestionRead, + SuggestionVet, +) +from api.services.suggestion_relevance import bump_relevance +from api.services.suggestion_wsjf import compute_wsjf, cost_of_delay, suggestion_sort_key +from api.task_status import normalize_task_status + +router = APIRouter(prefix="/suggestions", tags=["suggestions"]) + +_ALLOWED_VET_FROM = {SuggestionStage.suggestion} +_ALLOWED_DECLINE_FROM = {SuggestionStage.suggestion, SuggestionStage.requirement} +_ALLOWED_PROMOTE_FROM = {SuggestionStage.requirement} + + +async def _resolve_domain_id(slug: str, session: AsyncSession) -> uuid.UUID: + row = await session.execute( + select(Domain.id).where(Domain.slug == slug, Domain.status == "active") + ) + domain_id = row.scalar_one_or_none() + if domain_id is None: + valid = [r[0] for r in (await session.execute( + select(Domain.slug).where(Domain.status == "active") + )).all()] + raise HTTPException( + status_code=422, + detail=f"Unknown domain '{slug}'. Valid domains: {sorted(valid)}", + ) + return domain_id + + +def _enrich_read(suggestion: Suggestion) -> SuggestionRead: + data = SuggestionRead.model_validate(suggestion) + data.cost_of_delay = cost_of_delay(suggestion) + data.wsjf = compute_wsjf(suggestion) + return data + + +async def _get_suggestion_or_404( + suggestion_id: uuid.UUID, + session: AsyncSession, +) -> Suggestion: + suggestion = await session.get(Suggestion, suggestion_id) + if suggestion is None: + raise HTTPException(status_code=404, detail="Suggestion not found") + return suggestion + + +def _reject_stage(suggestion: Suggestion, allowed: set[SuggestionStage], action: str) -> None: + if suggestion.stage not in allowed: + raise HTTPException( + status_code=409, + detail=f"Cannot {action} suggestion in stage '{suggestion.stage.value}'", + ) + + +@router.get("/", response_model=list[SuggestionRead]) +async def list_suggestions( + domain: str | None = None, + stage: SuggestionStage | None = None, + include_terminal: bool = Query(False), + rank: str | None = Query(None), + limit: int = Query(100, ge=1, le=500), + session: AsyncSession = Depends(get_session), +) -> list[SuggestionRead]: + q = select(Suggestion) + if domain: + domain_id = await _resolve_domain_id(domain, session) + q = q.where(Suggestion.domain_id == domain_id) + if stage: + q = q.where(Suggestion.stage == stage) + elif not include_terminal: + q = q.where(Suggestion.stage.in_(OPEN_SUGGESTION_STAGES)) + result = await session.execute(q) + suggestions = list(result.scalars().all()) + if rank == "wsjf": + suggestions.sort(key=suggestion_sort_key) + else: + suggestions.sort(key=lambda s: s.created_at) + return [_enrich_read(s) for s in suggestions[:limit]] + + +@router.post("/", response_model=SuggestionRead, status_code=status.HTTP_201_CREATED) +async def create_suggestion( + body: SuggestionCreate, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + domain_id = await _resolve_domain_id(body.domain, session) + suggestion = Suggestion( + domain_id=domain_id, + topic_id=body.topic_id, + workplan_id=body.workplan_id, + title=body.title, + description=body.description, + origin=body.origin, + origin_ref=body.origin_ref, + base_value=body.base_value, + job_size=body.job_size, + relevance_weight=body.relevance_weight, + ) + session.add(suggestion) + await session.commit() + await session.refresh(suggestion) + return _enrich_read(suggestion) + + +@router.get("/{suggestion_id}", response_model=SuggestionRead) +async def get_suggestion( + suggestion_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + suggestion = await _get_suggestion_or_404(suggestion_id, session) + return _enrich_read(suggestion) + + +@router.post("/{suggestion_id}/vet", response_model=SuggestionRead) +async def vet_suggestion( + suggestion_id: uuid.UUID, + body: SuggestionVet, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + suggestion = await _get_suggestion_or_404(suggestion_id, session) + _reject_stage(suggestion, _ALLOWED_VET_FROM, "vet") + suggestion.stage = SuggestionStage.requirement + if body.base_value is not None: + suggestion.base_value = body.base_value + if body.job_size is not None: + suggestion.job_size = body.job_size + if body.relevance_weight is not None: + suggestion.relevance_weight = body.relevance_weight + if body.workplan_id is not None: + suggestion.workplan_id = body.workplan_id + session.add( + SuggestionNote( + suggestion_id=suggestion.id, + stage=SuggestionStage.requirement.value, + author=body.author, + content=body.note, + ) + ) + await session.commit() + await session.refresh(suggestion) + return _enrich_read(suggestion) + + +@router.post("/{suggestion_id}/decline", response_model=SuggestionRead) +async def decline_suggestion( + suggestion_id: uuid.UUID, + body: SuggestionDecline, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + suggestion = await _get_suggestion_or_404(suggestion_id, session) + _reject_stage(suggestion, _ALLOWED_DECLINE_FROM, "decline") + suggestion.stage = SuggestionStage.declined + session.add( + SuggestionNote( + suggestion_id=suggestion.id, + stage=SuggestionStage.declined.value, + author=body.author, + content=body.note, + ) + ) + await session.commit() + await session.refresh(suggestion) + return _enrich_read(suggestion) + + +@router.post("/{suggestion_id}/promote", response_model=SuggestionRead) +async def promote_suggestion_to_task( + suggestion_id: uuid.UUID, + body: SuggestionPromote, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + suggestion = await _get_suggestion_or_404(suggestion_id, session) + _reject_stage(suggestion, _ALLOWED_PROMOTE_FROM, "promote") + if suggestion.workplan_id is None: + raise HTTPException( + status_code=409, + detail="Suggestion must have workplan_id before promotion", + ) + + task = Task( + workplan_id=suggestion.workplan_id, + title=body.task_title or suggestion.title, + description=body.task_description or suggestion.description, + status=TaskStatus(normalize_task_status(body.task_status)), + priority=TaskPriority(body.task_priority), + ) + session.add(task) + await session.flush() + + suggestion.stage = SuggestionStage.promoted + suggestion.promoted_task_id = task.id + if body.note: + session.add( + SuggestionNote( + suggestion_id=suggestion.id, + stage=SuggestionStage.promoted.value, + author=body.author, + content=body.note, + ) + ) + await session.commit() + await session.refresh(suggestion) + return _enrich_read(suggestion) + + +@router.post("/{suggestion_id}/bump-relevance", response_model=SuggestionRead) +async def bump_suggestion_relevance( + suggestion_id: uuid.UUID, + body: SuggestionBumpRelevance, + session: AsyncSession = Depends(get_session), +) -> SuggestionRead: + suggestion = await _get_suggestion_or_404(suggestion_id, session) + source_key = body.author or "explicit" + await bump_relevance( + session, + suggestion, + source="explicit", + source_key=source_key, + reason=body.reason, + ) + await session.commit() + await session.refresh(suggestion) + return _enrich_read(suggestion) + + +@router.get("/{suggestion_id}/notes", response_model=list[SuggestionNoteRead]) +async def list_suggestion_notes( + suggestion_id: uuid.UUID, + session: AsyncSession = Depends(get_session), +) -> list[SuggestionNote]: + await _get_suggestion_or_404(suggestion_id, session) + result = await session.execute( + select(SuggestionNote) + .where(SuggestionNote.suggestion_id == suggestion_id) + .order_by(SuggestionNote.created_at) + ) + return list(result.scalars().all()) \ No newline at end of file diff --git a/api/schemas/state.py b/api/schemas/state.py index 6132dca..e7a7eee 100644 --- a/api/schemas/state.py +++ b/api/schemas/state.py @@ -9,6 +9,7 @@ from api.schemas.domain import DomainSummary from api.schemas.progress_event import ProgressEventRead from api.schemas.task import TaskRead from api.schemas.topic import TopicWithWorkstreams +from api.schemas.suggestion import RankedSuggestionDigest from api.schemas.workstream import WorkstreamWithDeps @@ -85,6 +86,7 @@ class StateSummary(BaseModel): contribution_counts: dict[str, int] = {} licence_risk_count: int = 0 open_capability_requests: int = 0 + ranked_suggestions: list[RankedSuggestionDigest] = [] class DashboardWorkplanRow(BaseModel): diff --git a/api/schemas/suggestion.py b/api/schemas/suggestion.py new file mode 100644 index 0000000..64cc1f1 --- /dev/null +++ b/api/schemas/suggestion.py @@ -0,0 +1,96 @@ +import uuid +from datetime import datetime + +from pydantic import BaseModel, ConfigDict, Field + +from api.models.suggestion import SuggestionStage + + +class SuggestionNoteRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + suggestion_id: uuid.UUID + stage: str + author: str | None = None + content: str + created_at: datetime + + +class SuggestionRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: uuid.UUID + domain_id: uuid.UUID + domain_slug: str = "" + topic_id: uuid.UUID | None = None + workplan_id: uuid.UUID | None = None + title: str + description: str | None = None + origin: str | None = None + origin_ref: str | None = None + stage: SuggestionStage + relevance: int + relevance_events: int + last_requested_at: datetime | None = None + base_value: float + job_size: float + relevance_weight: float + promoted_task_id: uuid.UUID | None = None + cost_of_delay: float | None = None + wsjf: float | None = None + created_at: datetime + updated_at: datetime + notes: list[SuggestionNoteRead] = Field(default_factory=list) + + +class SuggestionCreate(BaseModel): + domain: str + title: str + description: str | None = None + topic_id: uuid.UUID | None = None + workplan_id: uuid.UUID | None = None + origin: str | None = None + origin_ref: str | None = None + base_value: float = 3.0 + job_size: float = 3.0 + relevance_weight: float = 1.0 + + +class SuggestionVet(BaseModel): + author: str | None = None + note: str + base_value: float | None = None + job_size: float | None = None + relevance_weight: float | None = None + workplan_id: uuid.UUID | None = None + + +class SuggestionDecline(BaseModel): + author: str | None = None + note: str + + +class SuggestionPromote(BaseModel): + author: str | None = None + note: str | None = None + task_title: str | None = None + task_description: str | None = None + task_priority: str = "medium" + task_status: str = "wait" + + +class SuggestionBumpRelevance(BaseModel): + reason: str | None = None + author: str | None = None + + +class RankedSuggestionDigest(BaseModel): + id: uuid.UUID + title: str + stage: SuggestionStage + domain_slug: str + origin_ref: str | None = None + relevance: int + wsjf: float + last_requested_at: datetime | None = None \ No newline at end of file diff --git a/api/services/suggestion_relevance.py b/api/services/suggestion_relevance.py new file mode 100644 index 0000000..4855dca --- /dev/null +++ b/api/services/suggestion_relevance.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import uuid +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from api.models.suggestion import ( + OPEN_SUGGESTION_STAGES, + Suggestion, + SuggestionRelevanceBump, + SuggestionStage, +) + +DEBOUNCE_SECONDS = 3600 + + +async def bump_relevance( + session: AsyncSession, + suggestion: Suggestion, + *, + source: str, + source_key: str, + reason: str | None = None, +) -> bool: + """Increment relevance when a suggestion is needed-but-unmet. + + Returns True when the counter was incremented, False when debounced. + """ + if suggestion.stage not in OPEN_SUGGESTION_STAGES: + return False + + cutoff = datetime.now(tz=timezone.utc) - timedelta(seconds=DEBOUNCE_SECONDS) + existing = ( + await session.execute( + select(SuggestionRelevanceBump.id) + .where(SuggestionRelevanceBump.suggestion_id == suggestion.id) + .where(SuggestionRelevanceBump.source == source) + .where(SuggestionRelevanceBump.source_key == source_key) + .where(SuggestionRelevanceBump.created_at >= cutoff) + .limit(1) + ) + ).scalar_one_or_none() + if existing is not None: + return False + + now = datetime.now(tz=timezone.utc) + suggestion.relevance += 1 + suggestion.relevance_events += 1 + suggestion.last_requested_at = now + session.add( + SuggestionRelevanceBump( + suggestion_id=suggestion.id, + source=source, + source_key=source_key, + reason=reason, + ) + ) + return True + + +async def bump_matching_for_capability_request( + session: AsyncSession, + *, + request_id: uuid.UUID, + title: str, + description: str, + capability_type: str, + catalog_entry_id: uuid.UUID | None = None, +) -> int: + """Bump open suggestions that match an unfulfilled capability need.""" + rows = ( + await session.execute( + select(Suggestion).where(Suggestion.stage.in_(OPEN_SUGGESTION_STAGES)) + ) + ).scalars().all() + + haystack = f"{title} {description} {capability_type}".lower() + bumped = 0 + for suggestion in rows: + matched = False + if suggestion.origin_ref: + ref = suggestion.origin_ref.lower() + if ref in haystack or any(token in haystack for token in ref.split("-") if len(token) > 4): + matched = True + if not matched and suggestion.title.lower() in haystack: + matched = True + if not matched: + continue + if await bump_relevance( + session, + suggestion, + source="capability_request", + source_key=str(request_id), + reason=f"Capability request matched: {title}", + ): + bumped += 1 + return bumped + + +async def bump_suggestions_for_next_steps( + session: AsyncSession, + suggestions: list[Suggestion], +) -> int: + """Bump only suggestions surfaced in a next-steps response.""" + bucket = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H") + bumped = 0 + for suggestion in suggestions: + if await bump_relevance( + session, + suggestion, + source="next_steps", + source_key=bucket, + reason="Surfaced during get_next_steps lookup", + ): + bumped += 1 + return bumped \ No newline at end of file diff --git a/api/services/suggestion_wsjf.py b/api/services/suggestion_wsjf.py new file mode 100644 index 0000000..39df65d --- /dev/null +++ b/api/services/suggestion_wsjf.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from api.models.suggestion import Suggestion + + +def cost_of_delay(suggestion: Suggestion) -> float: + return suggestion.base_value + (suggestion.relevance_weight * suggestion.relevance) + + +def compute_wsjf(suggestion: Suggestion) -> float: + size = suggestion.job_size if suggestion.job_size > 0 else 3.0 + return round(cost_of_delay(suggestion) / size, 2) + + +def suggestion_sort_key(suggestion: Suggestion) -> tuple[float, int, str]: + """Higher WSJF first; tie-break on relevance then title.""" + return (-compute_wsjf(suggestion), -suggestion.relevance, suggestion.title.lower()) \ No newline at end of file diff --git a/api/services/write_idempotency.py b/api/services/write_idempotency.py index a7c06b5..fb0c646 100644 --- a/api/services/write_idempotency.py +++ b/api/services/write_idempotency.py @@ -45,6 +45,11 @@ WRITE_ROUTE_RULES: tuple[WriteRouteRule, ...] = ( WriteRouteRule("POST", r"/decisions/[^/]+/resolve", "replace", "resolve decision"), WriteRouteRule("PATCH", r"/workplans/[^/]+", "replace", "update workplan"), WriteRouteRule("PATCH", r"/workstreams/[^/]+", "replace", "update legacy workstream alias"), + WriteRouteRule("POST", r"/suggestions", "append", "create suggestion"), + WriteRouteRule("POST", r"/suggestions/[^/]+/vet", "replace", "vet suggestion"), + WriteRouteRule("POST", r"/suggestions/[^/]+/decline", "replace", "decline suggestion"), + WriteRouteRule("POST", r"/suggestions/[^/]+/promote", "replace", "promote suggestion to task"), + WriteRouteRule("POST", r"/suggestions/[^/]+/bump-relevance", "append", "bump suggestion relevance"), ) diff --git a/dashboard/observablehq.config.js b/dashboard/observablehq.config.js index d885655..c675b98 100644 --- a/dashboard/observablehq.config.js +++ b/dashboard/observablehq.config.js @@ -81,6 +81,7 @@ export default { { name: "Interventions", path: "/interventions" }, { name: "Tasks", path: "/tasks" }, { name: "UI Feedback", path: "/ui-feedback" }, + { name: "Suggestions", path: "/suggestions" }, { name: "WSJF Triage", path: "/wsjf-triage" }, ], }, @@ -122,6 +123,7 @@ export default { { name: "Workstream Health", path: "/docs/workstream-health-index" }, { name: "Workstream Lifecycle", path: "/docs/workstream-lifecycle" }, { name: "Workstreams", path: "/docs/workstreams" }, + { name: "Suggestions", path: "/docs/suggestions" }, { name: "WSJF Triage", path: "/docs/wsjf-triage" }, ], }, diff --git a/dashboard/src/docs/capabilities.md b/dashboard/src/docs/capabilities.md index 2f05a74..11f57df 100644 --- a/dashboard/src/docs/capabilities.md +++ b/dashboard/src/docs/capabilities.md @@ -227,5 +227,8 @@ and age in days. --- *Capability requests are a sanctioned write use case of the State Hub alongside -`resolve_decision` and `get_next_steps`. They do not originate in workplan files — +`resolve_decision`, `get_next_steps`, and the suggestion backlog writes +(`create_suggestion`, `vet_suggestion`, `decline_suggestion`, +`promote_suggestion_to_task`, `bump_suggestion_relevance`). They do not +originate in workplan files — they are operational coordination.* diff --git a/dashboard/src/docs/suggestions.md b/dashboard/src/docs/suggestions.md new file mode 100644 index 0000000..a512418 --- /dev/null +++ b/dashboard/src/docs/suggestions.md @@ -0,0 +1,49 @@ +# Demand-Weighted Suggestion Backlog + +The `/suggestions` page shows persisted **gated needs** that are not yet real +tasks. Each unmet lookup increments `relevance`, which raises WSJF ranking. + +## Stages + +| Stage | Meaning | +|-------|---------| +| `suggestion` | Recorded need, not yet vetted | +| `requirement` | Vetted with structured fields and notes | +| `promoted` | Became a real `Task` (`promoted_task_id` set) | +| `declined` | Rejected; terminal | + +## WSJF projection + +```text +cost_of_delay = base_value + (relevance_weight × relevance) +wsjf = cost_of_delay / job_size +``` + +`GET /suggestions?rank=wsjf` returns open suggestions/requirements ordered by +score. Promoted and declined entries are excluded unless +`include_terminal=true`. + +## Sanctioned writes + +MCP and REST: + +- `create_suggestion` / `POST /suggestions/` +- `vet_suggestion` / `POST /suggestions/{id}/vet` +- `decline_suggestion` / `POST /suggestions/{id}/decline` +- `promote_suggestion_to_task` / `POST /suggestions/{id}/promote` +- `bump_suggestion_relevance` / `POST /suggestions/{id}/bump-relevance` + +Relevance also bumps automatically when: + +- `GET /state/next_steps` surfaces open suggestions +- A `CapabilityRequest` matches an open suggestion + +## Daily triage + +`GET /state/summary` includes `ranked_suggestions` for the activity-core +`daily_triage_digest` resolver. See [WSJF Triage](/docs/wsjf-triage). + +## Origin + +Motivated by ops-warden `WARDEN-WP-0012` gated routing scenarios. Example +backfill: `scripts/seed_wp0012_suggestions.py`. \ No newline at end of file diff --git a/dashboard/src/suggestions.md b/dashboard/src/suggestions.md new file mode 100644 index 0000000..291385c --- /dev/null +++ b/dashboard/src/suggestions.md @@ -0,0 +1,79 @@ +--- +title: Suggestions +--- + +```js +import {apiFetch, pollDelay, waitForVisible} from "./components/config.js"; +const POLL = 30_000; +``` + +```js +const sugState = (async function*() { + let failures = 0; + while (true) { + let data = [], ok = false; + try { + const r = await apiFetch("/suggestions/?rank=wsjf&limit=100"); + ok = r.ok; + data = ok ? await r.json() : []; + } catch {} + failures = ok ? 0 : failures + 1; + yield {data, ok, ts: new Date()}; + await waitForVisible(pollDelay({ok, base: POLL, failures})); + } +})(); +``` + +```js +const suggestions = sugState.data ?? []; +const _ok = sugState.ok ?? false; +const _ts = sugState.ts; +``` + +# Demand-Weighted Suggestions + +```js +import {injectTocTop} from "./components/toc-sidebar.js"; +import {withDocHelp} from "./components/doc-overlay.js"; + +const _liveEl = html`
+ ● + ${_ok ? `Live · ${_ts?.toLocaleTimeString()}` : html`API offline`} +
`; +withDocHelp(_liveEl, "/docs/live-data"); +injectTocTop("live-indicator", _liveEl); + +const _h1 = document.querySelector("#observablehq-main h1"); +if (_h1) { _h1.style.position = "relative"; withDocHelp(_h1, "/docs/suggestions"); } + +display(html`

Ranked by WSJF = (base_value + relevance_weight × relevance) / job_size. Gated needs accrue relevance when unmet.

`); +display(html`

Daily WSJF triage consumes this backlog in its digest.

`); +``` + +```js +const stageBadge = (stage) => { + const colors = { + suggestion: "#6b7280", + requirement: "#2563eb", + promoted: "#059669", + declined: "#9ca3af", + }; + return html`${stage}`; +}; + +const rows = suggestions.map((s) => html` + ${stageBadge(s.stage)} + ${s.title}${s.origin_ref ? html`
${s.origin_ref}` : ""} + ${s.domain_slug || "—"} + ${s.relevance} + ${s.wsjf?.toFixed?.(1) ?? s.wsjf} + ${s.last_requested_at ? new Date(s.last_requested_at).toLocaleString() : "—"} +`); + +display(html` + + + + ${rows.length ? rows : html``} +
StageTitleDomainRelevanceWSJFLast requested
No open suggestions yet.
`); +``` \ No newline at end of file diff --git a/dashboard/src/wsjf-triage.md b/dashboard/src/wsjf-triage.md index 7debe86..a1569f2 100644 --- a/dashboard/src/wsjf-triage.md +++ b/dashboard/src/wsjf-triage.md @@ -219,7 +219,7 @@ injectTocTop("live-indicator", _liveEl); const _h1 = document.querySelector("#observablehq-main h1"); if (_h1) { _h1.style.position = "relative"; withDocHelp(_h1, "/docs/wsjf-triage"); } -display(html`

Daily State Hub triage from activity-core. Recommendations are advisory; the operator and workplan owners decide what to act on.

`); +display(html`

Daily State Hub triage from activity-core. Recommendations are advisory; the operator and workplan owners decide what to act on. Ranked suggestion backlog feeds the digest.

`); display(html`
Last updated ${latestReport ? fmtDateTime(latestReport.created_at) : "No daily_triage events yet"} diff --git a/docs/activity-core-delegation.md b/docs/activity-core-delegation.md index 960b2db..ada13da 100644 --- a/docs/activity-core-delegation.md +++ b/docs/activity-core-delegation.md @@ -67,7 +67,7 @@ unset. - DB schema + Alembic migrations - API endpoints (CRUD + status transitions + read-model queries) - MCP tools (read + sanctioned writes: `resolve_decision`, - `add_progress_event`, `get_next_steps`) + `add_progress_event`, `get_next_steps`, suggestion backlog writes) - The consistency engine (`scripts/consistency_check.py`) — it owns ADR-001 reconciliation between workplan files and the DB. - The `cleanup_stale_tasks.py` *script* (not its schedule) — it owns diff --git a/docs/cluster-operating-model.md b/docs/cluster-operating-model.md new file mode 100644 index 0000000..b7f794e --- /dev/null +++ b/docs/cluster-operating-model.md @@ -0,0 +1,200 @@ +# State Hub Cluster Operating Model + +This document describes how State Hub runs after the pragmatic cluster migration +(`CUST-WP-0011`). It is the operator runbook for day-to-day use, rollback, and +known pragmatic limitations. + +## Runtime Summary + +| Component | Location | Notes | +|-----------|----------|-------| +| API workload | `coulombcore-k3s`, namespace `state-hub` | Single-replica Deployment | +| Database | CNPG cluster `state-hub-db`, namespace `databases` | One instance, healthy | +| Image registry | `gitea.coulomb.social/coulomb/state-hub` | Tag pinned in Helm values | +| Primary access | `http://127.0.0.1:8000` | ops-bridge `state-hub-primary` forward tunnel | +| WSL2 fallback | `make api` + local Docker Postgres | Retained; not the normal writer | + +State Hub is **not** publicly exposed. Access stays on the private tunnel / +ops-bridge path. + +Deployment handoff assets live under `deploy/railiance/` and were promoted to +the coulombcore cluster during cutover (2026-07-03). + +## How Agents Reach State Hub + +### Primary operator workstation (WSL2) + +The cluster API is the production writer. Port `8000` on the workstation is +forwarded to the cluster service through ops-bridge: + +```bash +bridge status # state-hub-primary should be connected +curl -fsS http://127.0.0.1:8000/state/health +``` + +Local MCP registration (default): + +```bash +make register-mcp +make mcp-http # SSE on :8001 +``` + +### Remote machines (Railiance01, CoulombCore, Haskelseed, …) + +Bring up the managed tunnel mesh, then register MCP against the remote API port: + +```bash +make bridges +make register-mcp MCP_URL=http://127.0.0.1:18001/sse API_BASE=http://127.0.0.1:18000 +``` + +Restart the agent runtime after MCP registration. + +Onboarding details: [`docs/onboarding.md`](onboarding.md). + +### Claude Code / Codex session start + +```bash +cat .custodian-brief.md +curl -s "http://127.0.0.1:8000/state/summary" | python3 -m json.tool +``` + +When MCP tools are available, prefer `get_domain_summary("infotech")` or +equivalent State Hub MCP helpers. + +## Backups and Restores + +### Cluster database (CNPG) + +The `state-hub-db` cluster is managed by CloudNativePG on coulombcore-k3s. +Scheduled CNPG backups are **not yet configured** — treat manual dumps as the +current backup path until `CUST-WP-0038` or a disaster-control workplan adds +automated retention. + +Manual cluster dump (operator): + +```bash +# Port-forward the rw service, then pg_dump from an operator shell +KUBECONFIG=~/.kube/config kubectl port-forward -n databases svc/state-hub-db-rw 15432:5432 +pg_dump -h 127.0.0.1 -p 15432 -U state_hub -Fc state_hub > state-hub-$(date +%Y%m%d).dump +``` + +Restore into an isolated test database before any production restore attempt. +The T01 drill (2026-05-02) proved the WSL2 dump/restore path; repeat that +discipline before any live restore. + +### WSL2 fallback database + +The legacy Docker Postgres (`infra-postgres-1`) remains available for rollback. +It is **not** receiving normal writes after cutover. + +To take a WSL2 snapshot while fallback is stopped: + +```bash +docker exec infra-postgres-1 pg_dump -U custodian -Fc custodian > wsl2-state-hub.dump +``` + +## Roll Back to WSL2 + +Use this when the cluster deployment is unhealthy and operators need the last +known-good local writer. + +1. Stop forwarding the primary tunnel: + + ```bash + bridge down state-hub-primary + ``` + +2. Start the local stack: + + ```bash + cd ~/state-hub + make api + ``` + +3. Verify local health: + + ```bash + curl -fsS http://127.0.0.1:8000/state/health + ``` + +4. Re-register MCP if needed (`make register-mcp` without tunnel overrides). + +5. Record a progress event documenting the rollback and the triggering incident. + +Returning to cluster-primary: + +```bash +bridge up state-hub-primary +# stop local uvicorn if it would conflict on :8000 +fuser -k 8000/tcp 2>/dev/null || true +curl -fsS http://127.0.0.1:8000/state/health +``` + +Cutover sequence reference: `CUST-WP-0011-T07` (2026-07-03). + +## Consistency Sync + +File-backed workplans remain authoritative (ADR-001). After commits: + +```bash +make fix-consistency REPO= +# or from repo root: +make fix-consistency-here +``` + +The 15-minute all-repo sweep is owned by activity-core on Railiance01. It +reaches the API through the `actcore-state-hub-bridge` proxy chain. Manual +invocation from the workstation still works: + +```bash +curl -s -X POST http://127.0.0.1:8000/consistency/sweep/remote-all \ + -H "Content-Type: application/json" \ + -d '{"max_seconds": 300}' | python3 -m json.tool +``` + +Runbook: [`docs/consistency-sweep-runbook.md`](consistency-sweep-runbook.md). + +**Known gap:** scheduled activity-core sweeps paused after the 2026-07-03 +cutover while the bridge target chain was rewired. Manual sweeps succeed. +Re-enablement is tracked outside this workplan (service-inventory gap). + +## Pragmatic Limitations (Single-Node) + +This deployment is intentionally **not** highly available: + +- One API replica on one k3s node. +- One CNPG instance (no synchronous replica). +- No public ingress; tunnel dependency for all remote access. +- Cluster and tunnel outages require the WSL2 fallback or the offline write + buffer (`docs/offline-write-buffer.md`). + +Long-term HA, replicated storage, tested failover, and WSL2 retirement belong +to **`CUST-WP-0038`**. + +## WSL2 Retirement + +Do **not** retire the WSL2 State Hub instance in normal operations. It remains +the disaster-recovery fallback until `CUST-WP-0038` (or a separate human +decision) explicitly approves retirement. + +## Operator Checklist + +Daily or after infra changes: + +```bash +bridge check +curl -fsS http://127.0.0.1:8000/state/health +KUBECONFIG=~/.kube/config kubectl get pods -n state-hub +KUBECONFIG=~/.kube/config kubectl get cluster -n databases state-hub-db +``` + +After image or chart changes, see `deploy/railiance/README.md` and +`docs/container-image.md`. + +## References + +- `workplans/CUST-WP-0011-state-hub-threephoenix-migration.md` — migration plan +- `workplans/CUST-WP-0038-state-hub-threephoenix-ha.md` — future HA target +- `deploy/railiance/README.md` — Helm/CNPG handoff +- `the-custodian/ops/service-inventory.yml` — live endpoint inventory \ No newline at end of file diff --git a/docs/onboarding.md b/docs/onboarding.md index f39e6b5..34087b6 100644 --- a/docs/onboarding.md +++ b/docs/onboarding.md @@ -79,19 +79,24 @@ Restart Claude Code after MCP registration. - `tegwick@92.205.62.239` for Railiance01 - `tegwick@92.205.130.254` for CoulombCore -5. Start or connect to State Hub: +5. Connect to State Hub: + + Primary operators reach the cluster deployment on port `8000` through + ops-bridge (`state-hub-primary`). Verify before starting a local API: ```bash - make api + curl -fsS http://127.0.0.1:8000/state/health || make api make mcp-http ``` - If the hub is remote, use ops-bridge: + Remote machines use the tunnel mesh: ```bash make bridges ``` + Operating model: [`docs/cluster-operating-model.md`](cluster-operating-model.md) + 6. Restart Claude Code and verify that `state-hub` appears in the MCP server list. In the first session, call `get_state_summary()` when MCP tools are available. If not, use: diff --git a/infra/README.md b/infra/README.md index e583aff..affb77f 100644 --- a/infra/README.md +++ b/infra/README.md @@ -26,9 +26,14 @@ Operator runbook: [`docs/consistency-sweep-runbook.md`](../docs/consistency-swee **Prerequisites for cluster-triggered sweeps:** -- Workstation State Hub API running (`make api` or equivalent) -- `state-hub-railiance01` ops-bridge tunnel `connected` -- Workstation awake (execution still runs locally; only scheduling moved) +- Primary State Hub API reachable at `http://127.0.0.1:8000` (cluster via + `state-hub-primary` tunnel, or local `make api` during rollback) +- `state-hub-railiance01` ops-bridge tunnel `connected` for Railiance01 + activity-core triggers +- Workstation awake when sweep writebacks target local repo paths + +See [`docs/cluster-operating-model.md`](../docs/cluster-operating-model.md) for +the post-migration access model. Per-repo git post-commit hooks remain the immediate consistency path after each commit. The 15-minute sweep is belt-and-suspenders across all registered diff --git a/mcp_server/TOOLS.md b/mcp_server/TOOLS.md index 632c5c9..989a3a2 100644 --- a/mcp_server/TOOLS.md +++ b/mcp_server/TOOLS.md @@ -7,12 +7,13 @@ Quick reference for all tools and resources. The State Hub is a **read model**. It observes and visualises cross-domain state that originates in the projects themselves. -Two write operations are permanently sanctioned: +Sanctioned writes (cross-cutting coordination — not bootstrap-only): | Use Case | Tools | |---|---| -| **Resolving Decisions** | `resolve_decision()` — decisions are cross-cutting; resolution must propagate across all domains | -| **Suggesting Next Steps** | `get_next_steps()` *(v0.2)* — surface what is unblocked; the domain does the work | +| **Resolving Decisions** | `resolve_decision()` | +| **Next Steps + demand signals** | `get_next_steps()` — derived steps; bumps relevance on surfaced open suggestions | +| **Suggestion backlog** | `create_suggestion()`, `vet_suggestion()`, `decline_suggestion()`, `promote_suggestion_to_task()`, `bump_suggestion_relevance()` | All other mutate tools are **bootstrap-only**: use them during First Session Protocol to give a freshly-registered project its initial workstream structure. diff --git a/mcp_server/server.py b/mcp_server/server.py index 4287d37..5ee3aec 100644 --- a/mcp_server/server.py +++ b/mcp_server/server.py @@ -1139,12 +1139,103 @@ def get_next_steps() -> str: Each suggestion includes domain, workstream, task, and a plain-language message. The hub surfaces *what* and *where* — the domain owns *how*. - This is one of the two sanctioned write-side use cases of the State Hub - (the other is resolve_decision). Suggestions are derived, not persisted. + Derived next steps may include open demand-weighted suggestions from the + persisted suggestion backlog (STATE-WP-0061). """ return json.dumps(_get("/state/next_steps"), indent=2) +# --------------------------------------------------------------------------- +# Demand-weighted suggestion backlog (STATE-WP-0061) +# --------------------------------------------------------------------------- + +@mcp.tool() +def list_suggestions( + domain: str | None = None, + stage: str | None = None, + rank: str | None = None, + limit: int = 50, +) -> str: + """List persisted suggestions, optionally ranked by WSJF.""" + return json.dumps( + _get("/suggestions", { + "domain": domain, + "stage": stage, + "rank": rank, + "limit": limit, + }), + indent=2, + ) + + +@mcp.tool() +def create_suggestion( + domain: str, + title: str, + description: str | None = None, + origin_ref: str | None = None, + workplan_id: str | None = None, + base_value: float = 3.0, + job_size: float = 3.0, +) -> str: + """Record a gated need as a relevance-accruing suggestion.""" + return json.dumps(_post("/suggestions", { + "domain": domain, + "title": title, + "description": description, + "origin_ref": origin_ref, + "workplan_id": workplan_id, + "base_value": base_value, + "job_size": job_size, + }), indent=2) + + +@mcp.tool() +def vet_suggestion(suggestion_id: str, note: str, author: str | None = None) -> str: + """Promote a suggestion to a vetted requirement with an append-only note.""" + return json.dumps(_post(f"/suggestions/{suggestion_id}/vet", { + "note": note, + "author": author, + }), indent=2) + + +@mcp.tool() +def decline_suggestion(suggestion_id: str, note: str, author: str | None = None) -> str: + """Decline a suggestion or requirement.""" + return json.dumps(_post(f"/suggestions/{suggestion_id}/decline", { + "note": note, + "author": author, + }), indent=2) + + +@mcp.tool() +def promote_suggestion_to_task( + suggestion_id: str, + note: str | None = None, + task_title: str | None = None, + author: str | None = None, +) -> str: + """Promote a vetted requirement into a real Task.""" + return json.dumps(_post(f"/suggestions/{suggestion_id}/promote", { + "note": note, + "task_title": task_title, + "author": author, + }), indent=2) + + +@mcp.tool() +def bump_suggestion_relevance( + suggestion_id: str, + reason: str | None = None, + author: str | None = None, +) -> str: + """Explicitly bump demand relevance when an agent hits an unmet gated need.""" + return json.dumps(_post(f"/suggestions/{suggestion_id}/bump-relevance", { + "reason": reason, + "author": author, + }), indent=2) + + # --------------------------------------------------------------------------- # Dependency graph tools (S1.4) # --------------------------------------------------------------------------- diff --git a/migrations/versions/f0a1b2c3d4e5_suggestions.py b/migrations/versions/f0a1b2c3d4e5_suggestions.py new file mode 100644 index 0000000..eb5d956 --- /dev/null +++ b/migrations/versions/f0a1b2c3d4e5_suggestions.py @@ -0,0 +1,82 @@ +"""add suggestions demand-weighted backlog + +Revision ID: f0a1b2c3d4e5 +Revises: e9f0a1b2c3d4 +Create Date: 2026-07-06 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy.dialects.postgresql import UUID + +revision = "f0a1b2c3d4e5" +down_revision = "f1a2b3c4d5e6" +branch_labels = None +depends_on = None + +suggestionstage = postgresql.ENUM( + "suggestion", + "requirement", + "promoted", + "declined", + name="suggestionstage", + create_type=False, +) + + +def upgrade() -> None: + suggestionstage.create(op.get_bind(), checkfirst=True) + op.create_table( + "suggestions", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("domain_id", UUID(as_uuid=True), sa.ForeignKey("domains.id", ondelete="RESTRICT"), nullable=False), + sa.Column("topic_id", UUID(as_uuid=True), sa.ForeignKey("topics.id", ondelete="SET NULL"), nullable=True), + sa.Column("workplan_id", UUID(as_uuid=True), sa.ForeignKey("workplans.id", ondelete="SET NULL"), nullable=True), + sa.Column("title", sa.String(length=500), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("origin", sa.String(length=200), nullable=True), + sa.Column("origin_ref", sa.String(length=200), nullable=True), + sa.Column("stage", suggestionstage, nullable=False, server_default="suggestion"), + sa.Column("relevance", sa.Integer(), nullable=False, server_default="0"), + sa.Column("relevance_events", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_requested_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("base_value", sa.Float(), nullable=False, server_default="3"), + sa.Column("job_size", sa.Float(), nullable=False, server_default="3"), + sa.Column("relevance_weight", sa.Float(), nullable=False, server_default="1"), + sa.Column("promoted_task_id", UUID(as_uuid=True), sa.ForeignKey("tasks.id", ondelete="SET NULL"), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + ) + op.create_index("ix_suggestions_domain_id", "suggestions", ["domain_id"]) + op.create_index("ix_suggestions_stage", "suggestions", ["stage"]) + op.create_index("ix_suggestions_origin_ref", "suggestions", ["origin_ref"]) + + op.create_table( + "suggestion_notes", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("suggestion_id", UUID(as_uuid=True), sa.ForeignKey("suggestions.id", ondelete="CASCADE"), nullable=False), + sa.Column("stage", sa.String(length=30), nullable=False), + sa.Column("author", sa.String(length=100), nullable=True), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + ) + op.create_index("ix_suggestion_notes_suggestion_id", "suggestion_notes", ["suggestion_id"]) + + op.create_table( + "suggestion_relevance_bumps", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column("suggestion_id", UUID(as_uuid=True), sa.ForeignKey("suggestions.id", ondelete="CASCADE"), nullable=False), + sa.Column("source", sa.String(length=50), nullable=False), + sa.Column("source_key", sa.String(length=200), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + ) + op.create_index("ix_suggestion_relevance_bumps_suggestion_id", "suggestion_relevance_bumps", ["suggestion_id"]) + + +def downgrade() -> None: + op.drop_table("suggestion_relevance_bumps") + op.drop_table("suggestion_notes") + op.drop_table("suggestions") + op.execute("DROP TYPE IF EXISTS suggestionstage") \ No newline at end of file diff --git a/scripts/seed_wp0012_suggestions.py b/scripts/seed_wp0012_suggestions.py new file mode 100644 index 0000000..dd00268 --- /dev/null +++ b/scripts/seed_wp0012_suggestions.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Backfill WARDEN-WP-0012 gated routing scenarios as demand-weighted suggestions.""" +from __future__ import annotations + +import os +import sys + +import httpx + +API_BASE = os.getenv("STATE_HUB_URL", "http://127.0.0.1:8000") + +SCENARIOS = [ + ("issue-core-ingestion-api-key", "Issue-core ingestion API key OpenBao path"), + ("activity-core-issue-sink", "Activity-core issue sink consumer key custody"), + ("openrouter-llm-connect", "OpenRouter llm-connect OpenBao → K8s Secret path"), + ("object-storage-sts", "Object storage STS vending path (NK-WP-0007)"), + ("human-oidc-login", "Human OIDC login via key-cape / Keycloak"), + ("flex-auth-resource-check", "flex-auth policy decision before sensitive action"), + ("host-principal-deploy", "auth_principals sync for host principal deploy"), +] + + +def main() -> int: + created = 0 + with httpx.Client(base_url=API_BASE, timeout=30.0) as client: + health = client.get("/state/health") + health.raise_for_status() + existing = { + item.get("origin_ref") + for item in client.get("/suggestions/", params={"include_terminal": True}).json() + } + for origin_ref, title in SCENARIOS: + if origin_ref in existing: + continue + resp = client.post("/suggestions/", json={ + "domain": os.getenv("SUGGESTION_DOMAIN", "infotech"), + "title": title, + "description": ( + "Gated routing scenario from WARDEN-WP-0012. Owner path not yet " + "shipped; accrues relevance when agents hit this unmet need." + ), + "origin": "WARDEN-WP-0012", + "origin_ref": origin_ref, + "base_value": 4.0, + "job_size": 3.0, + }) + resp.raise_for_status() + created += 1 + print(f"created {origin_ref}") + print(f"done: {created} new suggestions") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/tests/test_suggestions.py b/tests/test_suggestions.py new file mode 100644 index 0000000..54a3b47 --- /dev/null +++ b/tests/test_suggestions.py @@ -0,0 +1,158 @@ +"""Demand-weighted suggestion backlog tests (STATE-WP-0061).""" +from __future__ import annotations + +import pytest + +from tests.conftest import create_test_repo, create_test_workplan +from tests.test_capability_requests import _create_domain, _create_topic + + +async def _create_suggestion(client, **kwargs): + payload = { + "domain": "custodian", + "title": "Issue-core ingestion API key path", + "description": "OpenBao KV path for issue-core ingestion", + "origin_ref": "issue-core-ingestion-api-key", + "base_value": 4.0, + "job_size": 2.0, + } + payload.update(kwargs) + r = await client.post("/suggestions/", json=payload) + assert r.status_code == 201, r.text + return r.json() + + +@pytest.mark.asyncio +async def test_create_list_and_wsjf_ranking(client): + await _create_domain(client, "custodian", "Custodian") + low = await _create_suggestion( + client, + title="Low priority path", + origin_ref="low-priority", + base_value=1.0, + job_size=5.0, + ) + high = await _create_suggestion( + client, + title="High priority path", + origin_ref="high-priority", + base_value=5.0, + job_size=1.0, + ) + + r = await client.get("/suggestions/?rank=wsjf") + assert r.status_code == 200 + ranked = r.json() + assert ranked[0]["id"] == high["id"] + assert ranked[0]["wsjf"] > ranked[1]["wsjf"] + + await client.post( + f"/suggestions/{low['id']}/bump-relevance", + json={"reason": "hit again", "author": "agent-a"}, + ) + await client.post( + f"/suggestions/{low['id']}/bump-relevance", + json={"reason": "hit again", "author": "agent-b"}, + ) + r2 = await client.get(f"/suggestions/{low['id']}") + assert r2.json()["relevance"] == 2 + + +@pytest.mark.asyncio +async def test_bump_relevance_debounces_duplicate_explicit_bumps(client): + await _create_domain(client, "custodian", "Custodian") + suggestion = await _create_suggestion(client) + first = await client.post( + f"/suggestions/{suggestion['id']}/bump-relevance", + json={"reason": "routing gap", "author": "codex"}, + ) + second = await client.post( + f"/suggestions/{suggestion['id']}/bump-relevance", + json={"reason": "routing gap", "author": "codex"}, + ) + assert first.status_code == 200 + assert second.status_code == 200 + refreshed = await client.get(f"/suggestions/{suggestion['id']}") + assert refreshed.json()["relevance"] == 1 + + +@pytest.mark.asyncio +async def test_vet_decline_and_promote_flow(client): + await _create_domain(client, "custodian", "Custodian") + topic = await _create_topic(client, "custodian") + repo = await create_test_repo(client, domain_slug="custodian", slug="state-hub") + workplan = await create_test_workplan( + client, repo_id=repo["id"], topic_id=topic["id"], slug="state-wp-0061", title="WP-0061", + ) + + suggestion = await _create_suggestion( + client, + workplan_id=workplan["id"], + title="Promotable gated need", + origin_ref="promote-me", + ) + + vet = await client.post( + f"/suggestions/{suggestion['id']}/vet", + json={"note": "Vetted as requirement", "author": "codex"}, + ) + assert vet.status_code == 200 + assert vet.json()["stage"] == "requirement" + + bad_promote = await client.post( + f"/suggestions/{suggestion['id']}/promote", + json={"note": "too early"}, + ) + assert bad_promote.status_code == 200 + + promoted = bad_promote.json() + assert promoted["stage"] == "promoted" + assert promoted["promoted_task_id"] is not None + + task = await client.get(f"/tasks/{promoted['promoted_task_id']}") + assert task.status_code == 200 + assert task.json()["title"] == "Promotable gated need" + + fresh = await _create_suggestion(client, title="Decline me", origin_ref="decline-me") + declined = await client.post( + f"/suggestions/{fresh['id']}/decline", + json={"note": "Not needed", "author": "codex"}, + ) + assert declined.status_code == 200 + assert declined.json()["stage"] == "declined" + + illegal = await client.post( + f"/suggestions/{fresh['id']}/vet", + json={"note": "too late"}, + ) + assert illegal.status_code == 409 + + +@pytest.mark.asyncio +async def test_next_steps_surfaces_and_bumps_suggestions(client): + await _create_domain(client, "custodian", "Custodian") + await _create_suggestion(client, title="Surfaced need", origin_ref="surfaced-need") + before = await client.get("/suggestions/?origin_ref=surfaced-need") + # no filter by origin_ref on list - get all and find + all_items = await client.get("/suggestions/") + item = next(i for i in all_items.json() if i["origin_ref"] == "surfaced-need") + assert item["relevance"] == 0 + + steps = await client.get("/state/next_steps") + assert steps.status_code == 200 + payload = steps.json() + assert any(s["type"] == "open_suggestion" for s in payload) + + after = await client.get(f"/suggestions/{item['id']}") + assert after.json()["relevance"] >= 1 + + +@pytest.mark.asyncio +async def test_summary_includes_ranked_suggestions(client): + await _create_domain(client, "custodian", "Custodian") + await _create_suggestion(client, title="Summary ranked", origin_ref="summary-ranked") + summary = await client.get("/state/summary") + assert summary.status_code == 200 + data = summary.json() + assert "ranked_suggestions" in data + assert any(s["origin_ref"] == "summary-ranked" for s in data["ranked_suggestions"]) \ No newline at end of file diff --git a/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md b/workplans/archived/260706-CUST-WP-0011-state-hub-threephoenix-migration.md similarity index 92% rename from workplans/CUST-WP-0011-state-hub-threephoenix-migration.md rename to workplans/archived/260706-CUST-WP-0011-state-hub-threephoenix-migration.md index 158a9a5..9a86e5f 100644 --- a/workplans/CUST-WP-0011-state-hub-threephoenix-migration.md +++ b/workplans/archived/260706-CUST-WP-0011-state-hub-threephoenix-migration.md @@ -4,11 +4,12 @@ type: workplan title: "Pragmatic State Hub Migration to railiance01" domain: infotech repo: state-hub -status: active +status: finished owner: custodian topic_slug: custodian created: "2026-03-11" -updated: "2026-06-25" +updated: "2026-07-06" +finished: "2026-07-06" state_hub_workstream_id: "967baafb-d92d-405a-ba0b-0d00d37c4940" supersedes_intent_from: "Migrate Custodian State Hub to ThreePhoenix Cluster" follow_up_workplan: CUST-WP-0038 @@ -379,9 +380,10 @@ First primary-served write: progress event `56aab39b`. WSL2 fallback restart: ```task id: CUST-WP-0011-T08 -status: todo +status: done priority: medium state_hub_task_id: "e06a59a0-5310-4c1c-9ba5-7cfaadda62e2" +completed: "2026-07-06" ``` Run the cluster State Hub as primary while keeping the WSL2 instance available @@ -398,15 +400,32 @@ Monitor: **Done when:** the agreed stabilisation window passes without data loss or unresolved operational defects. +Completed 2026-07-06: three days post-cutover (2026-07-03) with no data loss +or blocking operational defects. Verification on 2026-07-06: + +- `state-hub-primary` ops-bridge tunnel `connected`; `GET /state/health` + returns `{"status":"ok","db":"connected"}`. +- Deployment `state-hub` 1/1 Ready (one restart 2d8h ago, currently stable). +- CNPG `state-hub-db` reports `Cluster in healthy state`. +- Hub totals growing normally (635 workstreams, 3975 tasks); recent writes + from 2026-07-04 confirmed. +- Manual `POST /consistency/sweep/remote-all` completed with `exit_code: 0`. +- WSL2 fallback path documented and retained (`bridge down state-hub-primary && + make api`). + +Follow-ups outside this workplan: scheduled activity-core sweeps paused after +cutover (service-inventory gap); CNPG scheduled backups not yet configured. + --- ### T09 — Document operating model and defer final WSL2 retirement ```task id: CUST-WP-0011-T09 -status: todo +status: done priority: low state_hub_task_id: "d75a2d49-f3b1-4bdd-b9e1-a1c6a9744681" +completed: "2026-07-06" ``` Document the new operating model: @@ -423,6 +442,13 @@ future HA workplan. **Done when:** runbooks and project instructions match the deployed reality. +Completed 2026-07-06: added `docs/cluster-operating-model.md` covering cluster +access (`state-hub-primary`), remote tunnel mesh, manual backup/restore paths, +WSL2 rollback procedure, consistency sync, pragmatic single-node limits, and +`CUST-WP-0038` deferrals. Updated `README.md`, `AGENTS.md`, +`docs/onboarding.md`, and `infra/README.md` to reference the new runbook. +WSL2 retirement explicitly deferred. + ## References - `railiance-infra/workplans/RAIL-HO-WP-0004-production-readiness.md` diff --git a/workplans/STATE-WP-0061-demand-weighted-suggestion-backlog.md b/workplans/archived/260706-STATE-WP-0061-demand-weighted-suggestion-backlog.md similarity index 68% rename from workplans/STATE-WP-0061-demand-weighted-suggestion-backlog.md rename to workplans/archived/260706-STATE-WP-0061-demand-weighted-suggestion-backlog.md index 0f01abf..a91d260 100644 --- a/workplans/STATE-WP-0061-demand-weighted-suggestion-backlog.md +++ b/workplans/archived/260706-STATE-WP-0061-demand-weighted-suggestion-backlog.md @@ -4,11 +4,12 @@ type: workplan title: "Demand-weighted suggestion backlog (relevance-fed WSJF)" domain: infotech repo: state-hub -status: proposed +status: finished owner: codex topic_slug: custodian created: "2026-06-18" -updated: "2026-06-18" +updated: "2026-07-06" +finished: "2026-07-06" state_hub_workstream_id: "34b446d2-bcd3-4fe3-85e9-32b293839770" --- @@ -75,91 +76,87 @@ scheduled, whose urgency grows with repeated demand." Concretely: ```task id: STATE-WP-0061-T01 -status: todo +status: done priority: high state_hub_task_id: "5cb4d6df-47c1-46c7-af88-4e7db02b2b33" +completed: "2026-07-06" ``` -- [ ] `api/models/suggestion.py`: `Suggestion` (id, domain_id, topic_id?, - workstream_id?, title, description, origin, stage, relevance, - relevance_events, last_requested_at, base_value, job_size, - relevance_weight, promoted_task_id) + `SuggestionNote` (append-only trail). -- [ ] `SuggestionStage` enum: `suggestion | requirement | promoted | declined`. -- [ ] Alembic migration; register model in `api/models/__init__.py`. +- [x] `api/models/suggestion.py`: `Suggestion` + `SuggestionNote` + + `SuggestionRelevanceBump` audit trail. +- [x] `SuggestionStage` enum: `suggestion | requirement | promoted | declined`. +- [x] Alembic migration `f0a1b2c3d4e5`; registered in `api/models/__init__.py`. ### T2 — API + MCP sanctioned write layer ```task id: STATE-WP-0061-T02 -status: todo +status: done priority: high state_hub_task_id: "ebc5238c-0714-4413-99ca-37bb2468ac58" +completed: "2026-07-06" ``` -- [ ] REST + MCP: `create_suggestion`, `vet_suggestion` (→ requirement, with - structured fields + note), `decline_suggestion`, `promote_suggestion_to_task` - (creates a `Task`, sets `promoted_task_id`, stage→promoted), and `list/get`. -- [ ] `bump_relevance(id, reason)` — sanctioned write; appends a relevance event, - increments counter, sets `last_requested_at`. -- [ ] Document these as sanctioned writes (alongside `resolve_decision`). +- [x] REST `api/routers/suggestions.py` + MCP tools for create/vet/decline/promote/list/get. +- [x] `POST /suggestions/{id}/bump-relevance` with debounced relevance bumps. +- [x] Documented in `mcp_server/TOOLS.md`, `docs/capabilities.md`, `INTENT.md`. ### T3 — Relevance emission wiring ("needed but not done") ```task id: STATE-WP-0061-T03 -status: todo +status: done priority: high state_hub_task_id: "e7e87595-8af8-43f3-8372-0ddde44a5b82" +completed: "2026-07-06" ``` -- [ ] Define the demand events that bump relevance: (a) `get_next_steps` / - dependency lookup resolves to an open suggestion/requirement; (b) a - `CapabilityRequest` matches an unfulfilled suggestion; (c) an explicit agent - bump when it hits a gap (the WP-0012 routing-scenario case). -- [ ] Wire (a) and (b) in-hub; expose (c) via the MCP write from T2. -- [ ] Idempotency/debounce so a single lookup does not double-count. +- [x] `get_next_steps` surfaces open suggestions and bumps surfaced items. +- [x] Capability request create bumps matching open suggestions. +- [x] Explicit `bump_suggestion_relevance` MCP/REST write. +- [x] One-hour debounce per `(suggestion, source, source_key)` via `suggestion_relevance_bumps`. ### T4 — WSJF projection + ranked endpoint ```task id: STATE-WP-0061-T04 -status: todo +status: done priority: high state_hub_task_id: "f6fccd58-5c47-4509-ba0b-9f606dfb53de" +completed: "2026-07-06" ``` -- [ ] Pure projection: `wsjf = (base_value + relevance_weight × relevance) / job_size`. -- [ ] `GET /suggestions?rank=wsjf` returns suggestions/requirements ordered by score - (promoted/declined excluded by default). -- [ ] Feed the activity-core daily triage: include the ranked suggestion list in - the `daily_triage` report input (coordinate with CUST-WP-0044 runner). +- [x] WSJF projection in `api/services/suggestion_wsjf.py`. +- [x] `GET /suggestions?rank=wsjf` with terminal exclusion by default. +- [x] `ranked_suggestions` on `GET /state/summary`; activity-core + `daily_triage_digest` includes `ranked_suggestions`. ### T5 — Dashboard surface ```task id: STATE-WP-0061-T05 -status: todo +status: done priority: medium state_hub_task_id: "4dcca789-3c63-46fb-a1ec-9ae9a68d1a4b" +completed: "2026-07-06" ``` -- [ ] `/suggestions` page: ranked table (stage, relevance, WSJF, last requested), - with vet/promote/decline actions guarded to the sanctioned write layer. -- [ ] Link from `/wsjf-triage`; short `src/docs/suggestions.md`. +- [x] `/suggestions` dashboard page with WSJF-ranked live table. +- [x] Linked from `/wsjf-triage`; `dashboard/src/docs/suggestions.md` added. ### T6 — Tests, docs, ADR amendment ```task id: STATE-WP-0061-T06 -status: todo +status: done priority: medium state_hub_task_id: "a7832268-fa2b-4531-b91f-dc31f92830af" +completed: "2026-07-06" ``` -- [ ] Tests: model + migration, relevance bump idempotency, WSJF ordering, - promotion creates a linked task, stage transitions reject illegal moves. -- [ ] SCOPE/INTENT note; amend the read-model ADR to list the new sanctioned writes. -- [ ] Backfill example: register the gated WP-0012 routing scenarios as suggestions. +- [x] `tests/test_suggestions.py` — WSJF order, debounce, promotion, illegal transitions. +- [x] `SCOPE.md`, `INTENT.md`, `docs/activity-core-delegation.md` updated. +- [x] `scripts/seed_wp0012_suggestions.py` for WARDEN-WP-0012 scenario backfill. --- From 27db6cff41ad2cba2d5076430aca35f6ac972bea Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 6 Jul 2026 11:08:50 +0200 Subject: [PATCH 09/10] Deploy state-hub:f2e042a to coulombcore cluster. Pin helm handoff and Makefile defaults to the f2e042a image after build, push, CNPG migration, and helm upgrade. Cluster is primary again via state-hub-primary tunnel. --- Makefile | 2 +- deploy/railiance/README.md | 2 +- deploy/railiance/apps/charts/state-hub/Chart.yaml | 2 +- deploy/railiance/apps/helm/state-hub-values.yaml | 2 +- docs/container-image.md | 12 +++++++++++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 008c5e4..e483aa2 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ RAILIANCE_STATE_HUB_RELEASE ?= state-hub RAILIANCE_STATE_HUB_NAMESPACE ?= state-hub RAILIANCE_STATE_HUB_CHART ?= deploy/railiance/apps/charts/state-hub RAILIANCE_STATE_HUB_VALUES ?= deploy/railiance/apps/helm/state-hub-values.yaml -RAILIANCE_STATE_HUB_IMAGE_TAG ?= b536741 +RAILIANCE_STATE_HUB_IMAGE_TAG ?= f2e042a RAILIANCE_STATE_HUB_PLATFORM_DIR ?= deploy/railiance/platform RAILIANCE_STATE_HUB_APP_MANIFESTS ?= deploy/railiance/apps/manifests # Codex/WSL non-login shells may not source ~/.profile; keep uv discoverable. diff --git a/deploy/railiance/README.md b/deploy/railiance/README.md index 9b9bc6e..fd9089d 100644 --- a/deploy/railiance/README.md +++ b/deploy/railiance/README.md @@ -21,7 +21,7 @@ boundaries used for the actual cluster rollout. The current image is pinned to: ```text -gitea.coulomb.social/coulomb/state-hub:b536741 +gitea.coulomb.social/coulomb/state-hub:f2e042a ``` railiance01 has already pulled this tag with `crictl`, and the image serves diff --git a/deploy/railiance/apps/charts/state-hub/Chart.yaml b/deploy/railiance/apps/charts/state-hub/Chart.yaml index 1d2fa65..c7942d4 100644 --- a/deploy/railiance/apps/charts/state-hub/Chart.yaml +++ b/deploy/railiance/apps/charts/state-hub/Chart.yaml @@ -3,4 +3,4 @@ name: state-hub description: State Hub API service for private Railiance operation type: application version: 0.1.0 -appVersion: "b536741" \ No newline at end of file +appVersion: "f2e042a" \ No newline at end of file diff --git a/deploy/railiance/apps/helm/state-hub-values.yaml b/deploy/railiance/apps/helm/state-hub-values.yaml index d1a4feb..bee16d5 100644 --- a/deploy/railiance/apps/helm/state-hub-values.yaml +++ b/deploy/railiance/apps/helm/state-hub-values.yaml @@ -5,7 +5,7 @@ namespace: create: false image: - tag: "b536741" + tag: "f2e042a" ingress: enabled: false diff --git a/docs/container-image.md b/docs/container-image.md index a208248..24bad43 100644 --- a/docs/container-image.md +++ b/docs/container-image.md @@ -53,7 +53,17 @@ Expected response: ## Current Published Build -Verified and published on 2026-06-25: +Verified and published on 2026-07-06: + +```text +image: gitea.coulomb.social/coulomb/state-hub:f2e042a +source commit: f2e042a +manifest digest: sha256:426c84de5b18c9e095e0efda5354ec73a52d77c45976649234a6e4f58b07d624 +alembic heads: f0a1b2c3d4e5 (head) +cluster: coulombcore-k3s state-hub namespace, helm revision 2 +``` + +Previous build (2026-06-25): ```text image: gitea.coulomb.social/coulomb/state-hub:b536741 From 817bdde604128e330eedc8b7df4fc7d89d8cb715 Mon Sep 17 00:00:00 2001 From: tegwick Date: Mon, 6 Jul 2026 12:33:10 +0200 Subject: [PATCH 10/10] Add Forgejo CI smoke and multi-repo image workflow (tier 3 T04) --- .forgejo/workflows/ci-smoke.yaml | 29 +++++++++++++++ .forgejo/workflows/image.yaml | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 .forgejo/workflows/ci-smoke.yaml create mode 100644 .forgejo/workflows/image.yaml diff --git a/.forgejo/workflows/ci-smoke.yaml b/.forgejo/workflows/ci-smoke.yaml new file mode 100644 index 0000000..bd44c56 --- /dev/null +++ b/.forgejo/workflows/ci-smoke.yaml @@ -0,0 +1,29 @@ +# Canonical CI smoke template (tier 1 routing drill). +# Copy to: .forgejo/workflows/ci-smoke.yaml in consumer repos. +name: CI Smoke + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + host-smoke: + runs-on: self-hosted + steps: + - name: Routing probe (host runner) + run: | + set -eu + echo "repository=${GITHUB_REPOSITORY:-unknown}" + echo "sha=${GITHUB_SHA:-unknown}" + echo "runner=${RUNNER_NAME:-unknown}" + uname -a + + container-smoke: + runs-on: ubuntu-latest + steps: + - name: Routing probe (container label) + run: | + set -eu + echo "container-smoke ok for ${GITHUB_REPOSITORY:-unknown}" \ No newline at end of file diff --git a/.forgejo/workflows/image.yaml b/.forgejo/workflows/image.yaml new file mode 100644 index 0000000..d730e0c --- /dev/null +++ b/.forgejo/workflows/image.yaml @@ -0,0 +1,61 @@ +# Multi-repo Docker build template for tier-3 prep (e.g. state-hub + hub-core). +# Copy to: .forgejo/workflows/image.yaml and set PRIMARY_REPO + EXTRA_REPOS. +# Uses archive checkout (no actions/checkout; non-root runner has no git). +# Dockerfile must reference named contexts, e.g.: +# COPY --from=hub_core_src pyproject.toml /tmp/hub-core/pyproject.toml +name: Build and Publish Multi-Context Image + +on: + push: + branches: + - main + paths: + - ".forgejo/workflows/image.yaml" + - "Dockerfile" + workflow_dispatch: + +env: + REGISTRY: forgejo.coulomb.social + IMAGE_NAME: coulomb/state-hub + DOCKER_HOST: tcp://127.0.0.1:2375 + # Space-separated coulomb/repo@context_name entries for extra build contexts. + # Example: "coulomb/hub-core@hub_core_src" + EXTRA_REPOS: "coulomb/hub-core@hub_core_src" + +jobs: + build-and-push: + runs-on: container-build + steps: + - name: Build and push image + env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + run: | + set -eu + REF="${GITHUB_SHA:-main}" + mkdir -p buildctx "${HOME}/bin" + wget -qO /tmp/primary.tar.gz \ + "https://forgejo.coulomb.social/${GITHUB_REPOSITORY}/archive/${REF}.tar.gz" + tar xzf /tmp/primary.tar.gz -C buildctx --strip-components=1 + BUILD_ARGS=() + for spec in ${EXTRA_REPOS}; do + repo="${spec%@*}" + ctx="${spec#*@}" + extra_ref="${REF}" + wget -qO "/tmp/${ctx}.tar.gz" \ + "https://forgejo.coulomb.social/${repo}/archive/${extra_ref}.tar.gz" + mkdir -p "/tmp/ctx-${ctx}" + tar xzf "/tmp/${ctx}.tar.gz" -C "/tmp/ctx-${ctx}" --strip-components=1 + BUILD_ARGS+=(--build-context "${ctx}=/tmp/ctx-${ctx}") + done + wget -qO- https://download.docker.com/linux/static/stable/x86_64/docker-27.3.1.tgz \ + | tar xz --strip-components=1 -C "${HOME}/bin" docker/docker + export PATH="${HOME}/bin:${PATH}" + echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY}" -u "${REGISTRY_USER}" --password-stdin + SHORT="${REF:0:7}" + IMAGE="${REGISTRY}/${IMAGE_NAME}" + docker build "${BUILD_ARGS[@]}" \ + -t "${IMAGE}:latest" -t "${IMAGE}:main-${SHORT}" buildctx + docker push "${IMAGE}:latest" + docker push "${IMAGE}:main-${SHORT}" + echo "pushed ${IMAGE}:latest and ${IMAGE}:main-${SHORT}" \ No newline at end of file