From 0dc001bf53e8abeccf280a7ca94de7201cad5013 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 23 Jul 2026 11:04:39 +0200 Subject: [PATCH] QONTO-WP-0003-T06: MCP smoke script + operator runbook update Add scripts/smoke_mcp.py, same shape as smoke_rest_api.py (random port, subprocess-launch against QONTO_FIXTURE_DIR, wait on /v1/health, assert, clean teardown), but goes further: generates a fresh QONTO_ASSISTANT_MCP_TOKEN per run and connects through it with the mcp SDK's streamablehttp_client, so the smoke exercises T03's bearer-token gate instead of bypassing it. Lists tools, calls all four, and confirms an out-of-catalog tool name comes back as a normal isError result through the real wire protocol rather than a crash. Extend docs/operator-runbook.md with a "One-command MCP smoke" section next to the REST one; cross-link from docs/mcp-integration.md's manual smoke walkthrough so the two don't drift. Co-Authored-By: Claude Sonnet 5 --- docs/mcp-integration.md | 5 + docs/operator-runbook.md | 18 +++ scripts/smoke_mcp.py | 159 +++++++++++++++++++++++++ workplans/QONTO-WP-0003-mcp-surface.md | 27 ++++- 4 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 scripts/smoke_mcp.py diff --git a/docs/mcp-integration.md b/docs/mcp-integration.md index 71399a4..4421307 100644 --- a/docs/mcp-integration.md +++ b/docs/mcp-integration.md @@ -155,6 +155,11 @@ the harness allows, per the shared `PolicyEngine.decide()` path. ## Local smoke: start with auth enabled, connect with only this snippet +For a one-command version of everything below (random port, generated +token, all four tools, plus the out-of-catalog deny check), see +`scripts/smoke_mcp.py` in `docs/operator-runbook.md`. The manual walkthrough +here is for when you want to see each step explicitly. + ```bash export QONTO_FIXTURE_DIR=tests/fixtures/qonto export QONTO_ASSISTANT_MCP_TOKEN=dev-local-token diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index 4c20435..3266a43 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -109,6 +109,24 @@ This starts the service on a random local port against the fixture payloads, checks `/v1/health`, `/v1/accounts`, a recent `31`-day snapshot, and a wider `90`-day cost-review snapshot, then shuts the process down. +## One-command MCP smoke + +```bash +../state-hub/.venv/bin/python scripts/smoke_mcp.py \ + --python ../state-hub/.venv/bin/python +``` + +Same shape as the REST smoke: starts the service on a random local port +against the fixture payloads, this time with `QONTO_ASSISTANT_MCP_TOKEN` set +to a freshly generated token so the auth layer (`docs/mcp-integration.md`) is +exercised too, not bypassed. Connects with the `mcp` SDK's +`streamablehttp_client`, lists tools, calls `qonto_ping`, +`qonto_org_summary`, `qonto_list_transactions`, and +`qonto_cost_run_rate_hints`, then confirms an out-of-catalog tool name +(`qonto_transfer_funds`, never registered) comes back as a normal +`isError` result rather than a crash or a policy bypass. No real Qonto +credentials involved. + ## Example calls Minimal local call: diff --git a/scripts/smoke_mcp.py b/scripts/smoke_mcp.py new file mode 100644 index 0000000..e1dbc1f --- /dev/null +++ b/scripts/smoke_mcp.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import secrets +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run a local MCP smoke test for qonto-assistant.") + parser.add_argument( + "--python", + default=sys.executable, + help="Python interpreter used to start qonto_assistant.main", + ) + parser.add_argument( + "--fixture-dir", + default=str(Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "qonto"), + help="Fixture directory containing organization.json and transactions.json", + ) + parser.add_argument( + "--startup-timeout", + type=float, + default=15.0, + help="Seconds to wait for the local service to start", + ) + return parser.parse_args() + + +def free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def wait_for_health(base_url: str, *, timeout_seconds: float) -> None: + deadline = time.monotonic() + timeout_seconds + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + request = urllib.request.Request(f"{base_url}/v1/health") + with urllib.request.urlopen(request, timeout=5) as response: + json.load(response) + return + except Exception as exc: # noqa: BLE001 + last_error = exc + time.sleep(0.25) + raise RuntimeError(f"Timed out waiting for {base_url}/v1/health: {last_error}") + + +async def run_mcp_checks(base_url: str, *, token: str) -> dict[str, object]: + headers = { + "Authorization": f"Bearer {token}", + "X-Actor-ID": "smoke", + "X-Tenant-ID": "binky", + } + async with streamablehttp_client(f"{base_url}/mcp", headers=headers) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + + tools = {tool.name for tool in (await session.list_tools()).tools} + expected_tools = { + "qonto_ping", + "qonto_org_summary", + "qonto_list_transactions", + "qonto_cost_run_rate_hints", + } + assert tools == expected_tools, f"unexpected tool catalog: {tools}" + + ping = await session.call_tool("qonto_ping", {}) + assert ping.isError is False + assert ping.structuredContent["status"] == "ok" + + org_summary = await session.call_tool("qonto_org_summary", {}) + assert org_summary.isError is False + assert org_summary.structuredContent["organization"]["name"] == "Binky Hedgehog GmbH" + assert org_summary.structuredContent["accounts"][0]["iban_last4"] == "6810" + + transactions = await session.call_tool( + "qonto_list_transactions", {"window_days": 31, "page_size": 50} + ) + assert transactions.isError is False + + cost_hints = await session.call_tool( + "qonto_cost_run_rate_hints", {"window_days": 90, "page_size": 50} + ) + assert cost_hints.isError is False + assert cost_hints.structuredContent["cost_run_rate_hints"]["recurring_debits"][0]["label"] == "HUB31" + + # Deny behavior for an out-of-catalog / spend-shaped tool name: + # this tool was never registered, so the low-level protocol + # handler must reject it as a normal error result, never a + # crash and never a policy bypass. + denied = await session.call_tool("qonto_transfer_funds", {}) + assert denied.isError is True + + return { + "tools": sorted(tools), + "org_summary": org_summary.structuredContent, + "cost_hints": cost_hints.structuredContent, + "out_of_catalog_denied": denied.isError, + } + + +def main() -> int: + args = parse_args() + repo_root = Path(__file__).resolve().parents[1] + port = free_port() + base_url = f"http://127.0.0.1:{port}" + token = secrets.token_urlsafe(16) + + env = os.environ.copy() + env["PYTHONPATH"] = str(repo_root / "src") + env["QONTO_ASSISTANT_HOST"] = "127.0.0.1" + env["QONTO_ASSISTANT_PORT"] = str(port) + env["QONTO_FIXTURE_DIR"] = str(Path(args.fixture_dir).resolve()) + env["QONTO_ASSISTANT_MCP_TOKEN"] = token + + process = subprocess.Popen( # noqa: S603 + [args.python, "-m", "qonto_assistant.main"], + cwd=repo_root, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + try: + wait_for_health(base_url, timeout_seconds=args.startup_timeout) + result = asyncio.run(run_mcp_checks(base_url, token=token)) + print(json.dumps(result, indent=2)) + return 0 + finally: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + if process.stdout is not None: + output = process.stdout.read().strip() + if output: + sys.stderr.write(output + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/workplans/QONTO-WP-0003-mcp-surface.md b/workplans/QONTO-WP-0003-mcp-surface.md index 7f7825c..6ea02f3 100644 --- a/workplans/QONTO-WP-0003-mcp-surface.md +++ b/workplans/QONTO-WP-0003-mcp-surface.md @@ -256,7 +256,7 @@ write exists to accidentally wire up. `pytest` → `31 passed`. ```task id: QONTO-WP-0003-T06 -status: todo +status: done priority: medium state_hub_task_id: "1093ec79-eb16-4797-8c9c-5423a925db77" ``` @@ -270,6 +270,31 @@ alongside the existing REST section. Done when: smoke script passes locally against fixtures without real Qonto credentials, matching the Phase 1 REST smoke pattern. +**Done 2026-07-23:** Added `scripts/smoke_mcp.py`, same shape as +`scripts/smoke_rest_api.py` (random free port, subprocess-launch the +service against `QONTO_FIXTURE_DIR`, wait on `/v1/health`, assert, tear +down cleanly). Goes one step further than the REST smoke since MCP now has +an auth layer REST doesn't: generates a fresh `QONTO_ASSISTANT_MCP_TOKEN` +per run and connects with the `mcp` SDK's `streamablehttp_client` using it, +so the smoke exercises T03's bearer-token gate rather than bypassing it. +Lists tools and asserts the exact expected catalog; calls `qonto_ping`, +`qonto_org_summary`, `qonto_list_transactions`, and +`qonto_cost_run_rate_hints` against fixture data with the same balance/ +recurring-debit assertions the REST smoke makes; calls a never-registered +tool name (`qonto_transfer_funds`) and asserts it comes back as a normal +`isError` result through the real wire protocol, not a crash — confirming +the low-level MCP protocol handler rejects unknown tools cleanly rather +than falling through to anything policy-adjacent. + +Extended `docs/operator-runbook.md` with a "One-command MCP smoke" section +next to the existing REST one, and cross-linked it from +`docs/mcp-integration.md`'s manual smoke walkthrough so the two don't drift +out of sync. + +Verified: `python3 scripts/smoke_mcp.py --python ../state-hub/.venv/bin/python` +exits `0` against fixtures, no real Qonto credentials. `pytest` → `31 passed` +(unchanged); `python3 -m compileall src tests scripts`. + ## Task: Closure review ```task