diff --git a/Makefile b/Makefile index 2cdcab0..e94cea0 100644 --- a/Makefile +++ b/Makefile @@ -4,13 +4,14 @@ PIP ?= $(PYTHON) -m pip PYTEST ?= $(PYTHON) -m pytest RUFF ?= $(VENV)/bin/ruff -.PHONY: help install-dev test lint run +.PHONY: help install-dev test lint run verify-pin help: @echo "make install-dev Create .venv and install runtime + dev dependencies" @echo "make test Run the test suite" @echo "make lint Run syntax and static checks" @echo "make run Start the local API on 127.0.0.1:8090" + @echo "make verify-pin Check production against the repo's pinned digest" $(VENV)/bin/python: python3 -m venv $(VENV) @@ -31,3 +32,8 @@ lint: $(VENV)/.dev-installed run: $(VENV)/.dev-installed $(PYTHON) -m tenant_engine.main + +# Needs cluster access. tenant-engine runs on railiance01: +# KUBECONFIG=~/.kube/config-railiance01 make verify-pin +verify-pin: + @./deploy/verify-pin.sh diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 989bd99..4218ec1 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -15,7 +15,7 @@ | workplan | TEN-WP-0004 | finished | — | workplans/TEN-WP-0004-production-runtime.md | | workplan | TEN-WP-0005 | finished | — | workplans/TEN-WP-0005-tenant-update-and-retirement-api.md | | workplan | TEN-WP-0006 | finished | — | workplans/TEN-WP-0006-guardrail-quota-policy.md | -| workplan | TEN-WP-0007 | ready | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | +| workplan | TEN-WP-0007 | finished | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | | task | ADHOC-2026-07-24-T01 | done | — | workplans/ADHOC-2026-07-24.md | | task | TEN-WP-0001-T01 | done | — | workplans/TEN-WP-0001-statehub-bootstrap.md | | task | TEN-WP-0001-T02 | done | — | workplans/TEN-WP-0001-statehub-bootstrap.md | @@ -46,5 +46,5 @@ | task | TEN-WP-0006-T05 | done | — | workplans/TEN-WP-0006-guardrail-quota-policy.md | | task | TEN-WP-0007-T01 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | | task | TEN-WP-0007-T02 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | -| task | TEN-WP-0007-T03 | wait | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | -| task | TEN-WP-0007-T04 | wait | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | +| task | TEN-WP-0007-T03 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | +| task | TEN-WP-0007-T04 | done | — | workplans/TEN-WP-0007-guardrail-production-rollout.md | diff --git a/deploy/README.md b/deploy/README.md index 58e7ec0..286eab3 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -35,6 +35,40 @@ The Deployment uses `Recreate` because the SQLite PVC is `ReadWriteOnce`. A new pod applies the forward-only lifecycle migration on startup against the existing database. +## Detecting pin drift + +```bash +KUBECONFIG=~/.kube/config-railiance01 make verify-pin +``` + +Exit 0 in sync, 1 on drift, 2 if it could not tell. Run it after any rollout, +and periodically — drift is not an event you get told about. + +It compares four things that are supposed to agree: + +1. the digest this repo intends to run (`deploy/tenant-engine.yaml`); +2. the digest the Deployment's spec asks for; +3. the digest the **running pod** actually resolved — a spec can be correct + while the pod answering traffic is an older ReplicaSet that never finished + rolling, and consumers talk to the pod; +4. the routes the live service actually serves. + +**Why this exists.** Between 2026-08-13 and 2026-08-16 production was silently +rolled back to the TEN-WP-0004 image, so the lifecycle routes verified live on +2026-08-13 were not being served. Nothing alerted. `user-engine` saw `404`, and +because its conformance evidence is contract-level, its tests passed throughout. +flex-auth suffered a parallel rollback in the same window that surfaced as +`403`. Two different symptoms, one cause, no signal either time. + +The route check is not redundant with the digest check. A digest comparison +catches a changed pin; it is blind to whether the workload behind a correct +digest still serves the contract. Both failures we have actually seen were +invisible because they produced *ordinary-looking* responses rather than errors. + +If it reports `Unauthorized`, check the cluster before the credential — +`KUBECONFIG` defaulting to another cluster produces an identical message, and +that cost us a round trip. + ## Rollback ```bash diff --git a/deploy/verify-pin.sh b/deploy/verify-pin.sh new file mode 100755 index 0000000..aa69acf --- /dev/null +++ b/deploy/verify-pin.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# verify-pin.sh -- detect deployment pin drift for tenant-engine. +# +# Compares three things that are supposed to agree and silently did not +# between 2026-08-13 and 2026-08-16: +# +# 1. the digest this repo intends to run (deploy/tenant-engine.yaml) +# 2. the digest the Deployment asks for (spec.template.spec.containers) +# 3. the digest the running pod actually (status.containerStatuses.imageID) +# resolved +# +# and then asks the live service what it is actually serving, because a +# correct digest with a broken rollout still fails the consumer. +# +# Why (3) separately from (2): a Deployment can carry the right digest in its +# spec while the pod that answers traffic is an older ReplicaSet that never +# finished rolling. The consumer talks to the pod, not to the spec. +# +# Why the route check at all: the drift that hit us was invisible precisely +# because it produced a 404 rather than an error. A digest comparison alone +# would have caught this one, but "the deployment was replaced by something +# else entirely" is the same class of failure and the digest check is blind +# to whether that something serves the contract. +# +# Usage: +# deploy/verify-pin.sh # check cluster against the repo +# deploy/verify-pin.sh --quiet # exit code only, for CI/cron +# +# Exit codes: 0 in sync | 1 drift detected | 2 could not determine + +set -uo pipefail + +NAMESPACE="tenant-engine" +DEPLOYMENT="tenant-engine" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MANIFEST="$REPO_ROOT/deploy/tenant-engine.yaml" +QUIET=0 +LOCAL_PORT="${VERIFY_PIN_PORT:-18131}" + +# Routes whose absence is what a silent rollback to a pre-lifecycle or +# pre-guardrail image actually looks like from a consumer's side. +REQUIRED_ROUTES=( + "/tenants/{tenant_id}" + "/tenants/{tenant_id}/retire" + "/tenants/{tenant_id}/reactivate" + "/tenants/{tenant_id}/guardrails" + "/tenants/{tenant_id}/guardrails/{limit_key}" +) + +while [[ $# -gt 0 ]]; do + case "$1" in + --quiet) QUIET=1 ;; + --namespace) NAMESPACE="$2"; shift ;; + -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +say() { [[ $QUIET -eq 1 ]] || echo "$@"; } +fail() { echo "DRIFT: $*" >&2; } + +command -v kubectl >/dev/null || { echo "kubectl not found" >&2; exit 2; } +[[ -f "$MANIFEST" ]] || { echo "manifest not found: $MANIFEST" >&2; exit 2; } + +# 1. What the repo intends. Deliberately a plain grep rather than a YAML +# parser: the manifest pins by digest on a single line by convention, and +# a drift check that needs its own dependency tree is a check that stops +# being run. +DESIRED="$(grep -oE 'image: [^ ]+@sha256:[a-f0-9]{64}' "$MANIFEST" | head -1 | sed 's/^image: //')" +if [[ -z "$DESIRED" ]]; then + echo "could not read a digest-pinned image from $MANIFEST" >&2 + echo "(is it pinned by tag instead of digest? that is itself a finding)" >&2 + exit 2 +fi + +if ! kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" >/dev/null 2>&1; then + echo "cannot reach deployment $DEPLOYMENT in namespace $NAMESPACE" >&2 + echo "hint: KUBECONFIG is '${KUBECONFIG:-}'." >&2 + echo " tenant-engine runs on railiance01 -- pointing at the wrong" >&2 + echo " cluster reports Unauthorized, which reads exactly like a" >&2 + echo " missing credential. Check the cluster before the credential." >&2 + exit 2 +fi + +# 2. What the Deployment asks for. +SPEC="$(kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" \ + -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" + +# 3. What the running pod actually resolved. +RUNNING="$(kubectl -n "$NAMESPACE" get pods \ + -l app.kubernetes.io/name="$DEPLOYMENT" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null)" + +say "desired (repo) : $DESIRED" +say "deployment spec : ${SPEC:-}" +say "running pod imageID : ${RUNNING:-}" + +STATUS=0 +DESIRED_DIGEST="${DESIRED##*@}" + +if [[ "$SPEC" != "$DESIRED" ]]; then + fail "deployment spec does not match the repo pin." + fail " repo wants : $DESIRED" + fail " cluster has: ${SPEC:-}" + fail " Someone changed the cluster without changing the repo, or a commit" + fail " was never applied. Both have happened here." + STATUS=1 +fi + +if [[ -n "$RUNNING" && "$RUNNING" != *"$DESIRED_DIGEST"* ]]; then + fail "the running pod is not the desired digest." + fail " Even if the spec is correct, a rollout may not have completed." + STATUS=1 +fi + +# 4. Ask the live service what it serves. A digest can be right while the +# workload behind it is not the one the contract describes. +if command -v curl >/dev/null; then + kubectl -n "$NAMESPACE" port-forward "deploy/$DEPLOYMENT" "$LOCAL_PORT:8090" \ + >/dev/null 2>&1 & + PF_PID=$! + trap 'kill "$PF_PID" 2>/dev/null' EXIT + for _ in $(seq 1 20); do + curl -sf -m 2 "http://127.0.0.1:$LOCAL_PORT/health" >/dev/null 2>&1 && break + sleep 0.5 + done + + OPENAPI="$(curl -sf -m 5 "http://127.0.0.1:$LOCAL_PORT/openapi.json" 2>/dev/null)" + if [[ -z "$OPENAPI" ]]; then + fail "could not read /openapi.json from the live service." + STATUS=1 + else + for route in "${REQUIRED_ROUTES[@]}"; do + if [[ "$OPENAPI" != *"\"$route\""* ]]; then + fail "live service does not serve $route" + fail " This is the failure mode that hid for three days: a consumer" + fail " sees 404, not an error, and no policy event is emitted." + STATUS=1 + fi + done + say "live routes : all ${#REQUIRED_ROUTES[@]} required routes present" + fi +fi + +if [[ $STATUS -eq 0 ]]; then + say "OK: repo, deployment spec, running pod, and served routes all agree." +fi +exit $STATUS