feat(mcp): deploy the MCP layer on central instead of tunnelling to a workstation

No MCP server ran on central, so remote agents reached dev-hub only through a
reverse tunnel back to the workstation — routing a request for a service on
their own machine out to another host and back.

Adds a gated mcp Deployment and ClusterIP Service running the same image with
`-m mcp_server.server`, API_BASE defaulted to the in-cluster API Service, and
tcpSocket probes. No Ingress: the MCP layer proxies an unauthenticated API and
must not be reachable from outside the cluster.

Two fixes were needed before the manifests could work:

- server.py hardcoded host="127.0.0.1". A Service routes to the pod IP, so a
  loopback bind is unreachable. Now MCP_HOST, still defaulting to loopback so
  local runs do not silently expose an unauthenticated proxy.
- The container runs `-m mcp_server.server`, not the file path, so /app lands
  on sys.path rather than /app/mcp_server.

mcp.enabled stays false in the deploy values: the running image predates
MCP_HOST, so enabling it before the tag is bumped would ship a pod that never
becomes reachable.

Refs CUST-WP-0067-T08

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 2583210@bnt-lap001
Assistant-Session: f2bff2d5-e9b2-4338-92ca-10282a927006

custodian_cli.py

@@ -560,10 +560,16 @@ def cmd_status(_args: argparse.Namespace) -> None:
     print(f"API: {health.get('status', '?')}  DB: {health.get('db', '?')}")
     summary = _api_get("/state/summary")
     t = summary["totals"]
-    print(f"Topics: {t['topics']['active']} active")
+    topics = t.get("topics", {})
-    print(f"Workstreams: {t['workstreams']['active']} active, {t['workstreams']['blocked']} blocked")
+    workplans = t.get("workplans") or t.get("workstreams", {})
-    print(f"Tasks: {t['tasks']['in_progress']} in-progress, {t['tasks']['todo']} todo, {t['tasks']['blocked']} blocked")
+    tasks = t.get("tasks", {})
-    print(f"Decisions: {t['decisions']['open']} open, {t['decisions']['escalated']} escalated")
+    decisions = t.get("decisions", {})
+    print(f"API base: {API_BASE}")
+    print(f"Topics: {topics.get('active', 0)} active")
+    print(f"Workplans: {workplans.get('active', 0)} active, {workplans.get('blocked', 0)} blocked")
+    # Task statuses are wait|todo|progress|done|cancel (see workplan-convention.md).
+    print(f"Tasks: {tasks.get('progress', 0)} in-progress, {tasks.get('todo', 0)} todo, {tasks.get('wait', 0)} waiting")
+    print(f"Decisions: {decisions.get('open', 0)} open, {decisions.get('escalated', 0)} escalated")
     blocking = summary.get("blocking_decisions", [])
     if blocking:
         print(f"\nBlocking decisions ({len(blocking)}):")

deploy/railiance/apps/charts/state-hub/templates/_helpers.tpl

@@ -23,4 +23,32 @@ app: {{ include "statehub.fullname" . }}
 {{- fail "image.tag is required - pin it in deploy/railiance/apps/helm/state-hub-values.yaml or pass --set image.tag=<sha>" -}}
 {{- end -}}
 {{- printf "%s:%s" .Values.image.repository .Values.image.tag -}}
-{{- end -}}
+{{- end -}}
+
+{{- define "statehub.mcpFullname" -}}
+{{- printf "%s-mcp" (include "statehub.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "statehub.mcpLabels" -}}
+app: {{ include "statehub.mcpFullname" . }}
+app.kubernetes.io/name: {{ include "statehub.mcpFullname" . }}
+app.kubernetes.io/component: mcp
+app.kubernetes.io/instance: {{ .Release.Name }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+app.kubernetes.io/part-of: railiance-apps
+helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
+railiance.io/layer: s5-app
+{{- end -}}
+
+{{- define "statehub.mcpSelectorLabels" -}}
+app: {{ include "statehub.mcpFullname" . }}
+{{- end -}}
+
+{{- define "statehub.mcpApiBase" -}}
+{{- if .Values.mcp.apiBase -}}
+{{- .Values.mcp.apiBase -}}
+{{- else -}}
+{{- printf "http://%s:%v" (include "statehub.fullname" .) .Values.service.port -}}
+{{- end -}}
+{{- end -}}

deploy/railiance/apps/charts/state-hub/templates/mcp-deployment.yaml

@@ -0,0 +1,75 @@
+{{- if .Values.mcp.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "statehub.mcpFullname" . }}
+  labels: {{- include "statehub.mcpLabels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.mcp.replicaCount }}
+  selector:
+    matchLabels: {{- include "statehub.mcpSelectorLabels" . | nindent 6 }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  template:
+    metadata:
+      labels: {{- include "statehub.mcpLabels" . | nindent 8 }}
+    spec:
+      securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: state-hub-mcp
+          image: {{ include "statehub.image" . | quote }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          securityContext: {{- toYaml .Values.securityContext | nindent 12 }}
+          # The image CMD serves the API; this pod runs the MCP layer instead.
+          # Use -m so the WORKDIR (/app) lands on sys.path — invoking the file
+          # directly would put /app/mcp_server there and break its own imports.
+          command: ["/app/.venv/bin/python", "-m", "mcp_server.server"]
+          ports:
+            - name: sse
+              containerPort: {{ .Values.mcp.service.targetPort }}
+              protocol: TCP
+          env:
+            - name: MCP_TRANSPORT
+              value: {{ .Values.mcp.transport | quote }}
+            - name: MCP_PORT
+              value: {{ .Values.mcp.service.targetPort | quote }}
+            # A Service routes to the pod IP, so the default loopback bind
+            # would be unreachable.
+            - name: MCP_HOST
+              value: "0.0.0.0"
+            # Stateless HTTP client over the API — no database access.
+            - name: API_BASE
+              value: {{ include "statehub.mcpApiBase" . | quote }}
+          {{- if .Values.mcp.probes.enabled }}
+          readinessProbe:
+            tcpSocket:
+              port: {{ .Values.mcp.service.targetPort }}
+            initialDelaySeconds: {{ .Values.mcp.probes.readiness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.mcp.probes.readiness.periodSeconds }}
+            timeoutSeconds: {{ .Values.mcp.probes.readiness.timeoutSeconds }}
+            failureThreshold: {{ .Values.mcp.probes.readiness.failureThreshold }}
+          livenessProbe:
+            tcpSocket:
+              port: {{ .Values.mcp.service.targetPort }}
+            initialDelaySeconds: {{ .Values.mcp.probes.liveness.initialDelaySeconds }}
+            periodSeconds: {{ .Values.mcp.probes.liveness.periodSeconds }}
+            timeoutSeconds: {{ .Values.mcp.probes.liveness.timeoutSeconds }}
+            failureThreshold: {{ .Values.mcp.probes.liveness.failureThreshold }}
+          {{- end }}
+          resources: {{- toYaml .Values.mcp.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{- toYaml . | nindent 8 }}
+      {{- end }}
+{{- end }}

deploy/railiance/apps/charts/state-hub/templates/mcp-service.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.mcp.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "statehub.mcpFullname" . }}
+  labels: {{- include "statehub.mcpLabels" . | nindent 4 }}
+spec:
+  # ClusterIP only, and deliberately no Ingress: the MCP layer proxies an
+  # unauthenticated API and must not be reachable from outside the cluster.
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.mcp.service.port }}
+      targetPort: {{ .Values.mcp.service.targetPort }}
+      protocol: TCP
+      name: sse
+  selector: {{- include "statehub.mcpSelectorLabels" . | nindent 4 }}
+{{- end }}

deploy/railiance/apps/charts/state-hub/values.yaml

@@ -47,6 +47,38 @@ ingress:
     traefik.ingress.kubernetes.io/router.tls: "true"
     cert-manager.io/cluster-issuer: letsencrypt-prod
 
+# MCP layer (CUST-WP-0067-T08). Runs from the same image with a different
+# command, as a stateless HTTP client over the API service. ClusterIP only —
+# it proxies an unauthenticated API, so it must never gain an Ingress.
+mcp:
+  enabled: false
+  replicaCount: 1
+  transport: sse
+  # Defaults to the in-cluster API Service; override only to point elsewhere.
+  apiBase: ""
+  service:
+    port: 8001
+    targetPort: 8001
+  probes:
+    enabled: true
+    liveness:
+      initialDelaySeconds: 30
+      periodSeconds: 30
+      timeoutSeconds: 5
+      failureThreshold: 3
+    readiness:
+      initialDelaySeconds: 10
+      periodSeconds: 10
+      timeoutSeconds: 5
+      failureThreshold: 3
+  resources:
+    requests:
+      cpu: 50m
+      memory: 128Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
 probes:
   enabled: true
   path: /state/health

mcp_server/server.py

@@ -3212,4 +3212,9 @@ if __name__ == "__main__":
         mcp.run(transport="stdio")
     else:
         port = int(os.environ.get("MCP_PORT", "8001"))
-        mcp.run(transport=transport, host="127.0.0.1", port=port)
+        # Default to loopback: this server is an unauthenticated proxy over the
+        # API, so it must not land on every interface by accident. In-cluster
+        # deployment sets MCP_HOST=0.0.0.0 because a Service routes to the pod
+        # IP, and a loopback bind is unreachable there (CUST-WP-0067-T08).
+        host = os.environ.get("MCP_HOST", "127.0.0.1")
+        mcp.run(transport=transport, host=host, port=port)