Add deployment pin-drift check (make verify-pin)

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Makefile

@@ -4,13 +4,14 @@ PIP ?= $(PYTHON) -m pip
 PYTEST ?= $(PYTHON) -m pytest
 RUFF ?= $(VENV)/bin/ruff
 
-.PHONY: help install-dev test lint run
+.PHONY: help install-dev test lint run verify-pin
 
 help:
 	@echo "make install-dev  Create .venv and install runtime + dev dependencies"
 	@echo "make test         Run the test suite"
 	@echo "make lint         Run syntax and static checks"
 	@echo "make run          Start the local API on 127.0.0.1:8090"
+	@echo "make verify-pin   Check production against the repo's pinned digest"
 
 $(VENV)/bin/python:
 	python3 -m venv $(VENV)
@@ -31,3 +32,8 @@ lint: $(VENV)/.dev-installed
 
 run: $(VENV)/.dev-installed
 	$(PYTHON) -m tenant_engine.main
+
+# Needs cluster access. tenant-engine runs on railiance01:
+#   KUBECONFIG=~/.kube/config-railiance01 make verify-pin
+verify-pin:
+	@./deploy/verify-pin.sh

WORK-RECORDS.md

@@ -15,7 +15,7 @@
 | workplan | TEN-WP-0004 | finished | — | workplans/TEN-WP-0004-production-runtime.md |
 | workplan | TEN-WP-0005 | finished | — | workplans/TEN-WP-0005-tenant-update-and-retirement-api.md |
 | workplan | TEN-WP-0006 | finished | — | workplans/TEN-WP-0006-guardrail-quota-policy.md |
-| workplan | TEN-WP-0007 | ready | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
+| workplan | TEN-WP-0007 | finished | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
 | task | ADHOC-2026-07-24-T01 | done | — | workplans/ADHOC-2026-07-24.md |
 | task | TEN-WP-0001-T01 | done | — | workplans/TEN-WP-0001-statehub-bootstrap.md |
 | task | TEN-WP-0001-T02 | done | — | workplans/TEN-WP-0001-statehub-bootstrap.md |
@@ -46,5 +46,5 @@
 | task | TEN-WP-0006-T05 | done | — | workplans/TEN-WP-0006-guardrail-quota-policy.md |
 | task | TEN-WP-0007-T01 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
 | task | TEN-WP-0007-T02 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
-| task | TEN-WP-0007-T03 | wait | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
+| task | TEN-WP-0007-T03 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
-| task | TEN-WP-0007-T04 | wait | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |
+| task | TEN-WP-0007-T04 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md |

deploy/README.md

@@ -35,6 +35,40 @@ The Deployment uses `Recreate` because the SQLite PVC is `ReadWriteOnce`.
 A new pod applies the forward-only lifecycle migration on startup against
 the existing database.
 
+## Detecting pin drift
+
+```bash
+KUBECONFIG=~/.kube/config-railiance01 make verify-pin
+```
+
+Exit 0 in sync, 1 on drift, 2 if it could not tell. Run it after any rollout,
+and periodically — drift is not an event you get told about.
+
+It compares four things that are supposed to agree:
+
+1. the digest this repo intends to run (`deploy/tenant-engine.yaml`);
+2. the digest the Deployment's spec asks for;
+3. the digest the **running pod** actually resolved — a spec can be correct
+   while the pod answering traffic is an older ReplicaSet that never finished
+   rolling, and consumers talk to the pod;
+4. the routes the live service actually serves.
+
+**Why this exists.** Between 2026-08-13 and 2026-08-16 production was silently
+rolled back to the TEN-WP-0004 image, so the lifecycle routes verified live on
+2026-08-13 were not being served. Nothing alerted. `user-engine` saw `404`, and
+because its conformance evidence is contract-level, its tests passed throughout.
+flex-auth suffered a parallel rollback in the same window that surfaced as
+`403`. Two different symptoms, one cause, no signal either time.
+
+The route check is not redundant with the digest check. A digest comparison
+catches a changed pin; it is blind to whether the workload behind a correct
+digest still serves the contract. Both failures we have actually seen were
+invisible because they produced *ordinary-looking* responses rather than errors.
+
+If it reports `Unauthorized`, check the cluster before the credential —
+`KUBECONFIG` defaulting to another cluster produces an identical message, and
+that cost us a round trip.
+
 ## Rollback
 
 ```bash

deploy/verify-pin.sh

@@ -0,0 +1,150 @@
+#!/usr/bin/env bash
+# verify-pin.sh -- detect deployment pin drift for tenant-engine.
+#
+# Compares three things that are supposed to agree and silently did not
+# between 2026-08-13 and 2026-08-16:
+#
+#   1. the digest this repo intends to run  (deploy/tenant-engine.yaml)
+#   2. the digest the Deployment asks for   (spec.template.spec.containers)
+#   3. the digest the running pod actually  (status.containerStatuses.imageID)
+#      resolved
+#
+# and then asks the live service what it is actually serving, because a
+# correct digest with a broken rollout still fails the consumer.
+#
+# Why (3) separately from (2): a Deployment can carry the right digest in its
+# spec while the pod that answers traffic is an older ReplicaSet that never
+# finished rolling. The consumer talks to the pod, not to the spec.
+#
+# Why the route check at all: the drift that hit us was invisible precisely
+# because it produced a 404 rather than an error. A digest comparison alone
+# would have caught this one, but "the deployment was replaced by something
+# else entirely" is the same class of failure and the digest check is blind
+# to whether that something serves the contract.
+#
+# Usage:
+#   deploy/verify-pin.sh                 # check cluster against the repo
+#   deploy/verify-pin.sh --quiet         # exit code only, for CI/cron
+#
+# Exit codes: 0 in sync | 1 drift detected | 2 could not determine
+
+set -uo pipefail
+
+NAMESPACE="tenant-engine"
+DEPLOYMENT="tenant-engine"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+MANIFEST="$REPO_ROOT/deploy/tenant-engine.yaml"
+QUIET=0
+LOCAL_PORT="${VERIFY_PIN_PORT:-18131}"
+
+# Routes whose absence is what a silent rollback to a pre-lifecycle or
+# pre-guardrail image actually looks like from a consumer's side.
+REQUIRED_ROUTES=(
+  "/tenants/{tenant_id}"
+  "/tenants/{tenant_id}/retire"
+  "/tenants/{tenant_id}/reactivate"
+  "/tenants/{tenant_id}/guardrails"
+  "/tenants/{tenant_id}/guardrails/{limit_key}"
+)
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --quiet) QUIET=1 ;;
+    --namespace) NAMESPACE="$2"; shift ;;
+    -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+say() { [[ $QUIET -eq 1 ]] || echo "$@"; }
+fail() { echo "DRIFT: $*" >&2; }
+
+command -v kubectl >/dev/null || { echo "kubectl not found" >&2; exit 2; }
+[[ -f "$MANIFEST" ]] || { echo "manifest not found: $MANIFEST" >&2; exit 2; }
+
+# 1. What the repo intends. Deliberately a plain grep rather than a YAML
+#    parser: the manifest pins by digest on a single line by convention, and
+#    a drift check that needs its own dependency tree is a check that stops
+#    being run.
+DESIRED="$(grep -oE 'image: [^ ]+@sha256:[a-f0-9]{64}' "$MANIFEST" | head -1 | sed 's/^image: //')"
+if [[ -z "$DESIRED" ]]; then
+  echo "could not read a digest-pinned image from $MANIFEST" >&2
+  echo "(is it pinned by tag instead of digest? that is itself a finding)" >&2
+  exit 2
+fi
+
+if ! kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" >/dev/null 2>&1; then
+  echo "cannot reach deployment $DEPLOYMENT in namespace $NAMESPACE" >&2
+  echo "hint: KUBECONFIG is '${KUBECONFIG:-<unset, using ~/.kube/config>}'." >&2
+  echo "      tenant-engine runs on railiance01 -- pointing at the wrong" >&2
+  echo "      cluster reports Unauthorized, which reads exactly like a" >&2
+  echo "      missing credential. Check the cluster before the credential." >&2
+  exit 2
+fi
+
+# 2. What the Deployment asks for.
+SPEC="$(kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" \
+  -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)"
+
+# 3. What the running pod actually resolved.
+RUNNING="$(kubectl -n "$NAMESPACE" get pods \
+  -l app.kubernetes.io/name="$DEPLOYMENT" \
+  --field-selector=status.phase=Running \
+  -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null)"
+
+say "desired (repo)      : $DESIRED"
+say "deployment spec     : ${SPEC:-<none>}"
+say "running pod imageID : ${RUNNING:-<none>}"
+
+STATUS=0
+DESIRED_DIGEST="${DESIRED##*@}"
+
+if [[ "$SPEC" != "$DESIRED" ]]; then
+  fail "deployment spec does not match the repo pin."
+  fail "  repo wants : $DESIRED"
+  fail "  cluster has: ${SPEC:-<none>}"
+  fail "  Someone changed the cluster without changing the repo, or a commit"
+  fail "  was never applied. Both have happened here."
+  STATUS=1
+fi
+
+if [[ -n "$RUNNING" && "$RUNNING" != *"$DESIRED_DIGEST"* ]]; then
+  fail "the running pod is not the desired digest."
+  fail "  Even if the spec is correct, a rollout may not have completed."
+  STATUS=1
+fi
+
+# 4. Ask the live service what it serves. A digest can be right while the
+#    workload behind it is not the one the contract describes.
+if command -v curl >/dev/null; then
+  kubectl -n "$NAMESPACE" port-forward "deploy/$DEPLOYMENT" "$LOCAL_PORT:8090" \
+    >/dev/null 2>&1 &
+  PF_PID=$!
+  trap 'kill "$PF_PID" 2>/dev/null' EXIT
+  for _ in $(seq 1 20); do
+    curl -sf -m 2 "http://127.0.0.1:$LOCAL_PORT/health" >/dev/null 2>&1 && break
+    sleep 0.5
+  done
+
+  OPENAPI="$(curl -sf -m 5 "http://127.0.0.1:$LOCAL_PORT/openapi.json" 2>/dev/null)"
+  if [[ -z "$OPENAPI" ]]; then
+    fail "could not read /openapi.json from the live service."
+    STATUS=1
+  else
+    for route in "${REQUIRED_ROUTES[@]}"; do
+      if [[ "$OPENAPI" != *"\"$route\""* ]]; then
+        fail "live service does not serve $route"
+        fail "  This is the failure mode that hid for three days: a consumer"
+        fail "  sees 404, not an error, and no policy event is emitted."
+        STATUS=1
+      fi
+    done
+    say "live routes         : all ${#REQUIRED_ROUTES[@]} required routes present"
+  fi
+fi
+
+if [[ $STATUS -eq 0 ]]; then
+  say "OK: repo, deployment spec, running pod, and served routes all agree."
+fi
+exit $STATUS