#!/usr/bin/env bash # verify-pin.sh -- detect deployment pin drift for tenant-engine. # # Compares three things that are supposed to agree and silently did not # between 2026-08-13 and 2026-08-16: # # 1. the digest this repo intends to run (deploy/tenant-engine.yaml) # 2. the digest the Deployment asks for (spec.template.spec.containers) # 3. the digest the running pod actually (status.containerStatuses.imageID) # resolved # # and then asks the live service what it is actually serving, because a # correct digest with a broken rollout still fails the consumer. # # Why (3) separately from (2): a Deployment can carry the right digest in its # spec while the pod that answers traffic is an older ReplicaSet that never # finished rolling. The consumer talks to the pod, not to the spec. # # Why the route check at all: the drift that hit us was invisible precisely # because it produced a 404 rather than an error. A digest comparison alone # would have caught this one, but "the deployment was replaced by something # else entirely" is the same class of failure and the digest check is blind # to whether that something serves the contract. # # Usage: # deploy/verify-pin.sh # check cluster against the repo # deploy/verify-pin.sh --quiet # exit code only, for CI/cron # # Exit codes: 0 in sync | 1 drift detected | 2 could not determine set -uo pipefail NAMESPACE="tenant-engine" DEPLOYMENT="tenant-engine" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" MANIFEST="$REPO_ROOT/deploy/tenant-engine.yaml" QUIET=0 LOCAL_PORT="${VERIFY_PIN_PORT:-18131}" # Routes whose absence is what a silent rollback to a pre-lifecycle or # pre-guardrail image actually looks like from a consumer's side. REQUIRED_ROUTES=( "/tenants/{tenant_id}" "/tenants/{tenant_id}/retire" "/tenants/{tenant_id}/reactivate" "/tenants/{tenant_id}/guardrails" "/tenants/{tenant_id}/guardrails/{limit_key}" ) while [[ $# -gt 0 ]]; do case "$1" in --quiet) QUIET=1 ;; --namespace) NAMESPACE="$2"; shift ;; -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac shift done say() { [[ $QUIET -eq 1 ]] || echo "$@"; } fail() { echo "DRIFT: $*" >&2; } command -v kubectl >/dev/null || { echo "kubectl not found" >&2; exit 2; } [[ -f "$MANIFEST" ]] || { echo "manifest not found: $MANIFEST" >&2; exit 2; } # 1. What the repo intends. Deliberately a plain grep rather than a YAML # parser: the manifest pins by digest on a single line by convention, and # a drift check that needs its own dependency tree is a check that stops # being run. DESIRED="$(grep -oE 'image: [^ ]+@sha256:[a-f0-9]{64}' "$MANIFEST" | head -1 | sed 's/^image: //')" if [[ -z "$DESIRED" ]]; then echo "could not read a digest-pinned image from $MANIFEST" >&2 echo "(is it pinned by tag instead of digest? that is itself a finding)" >&2 exit 2 fi if ! kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" >/dev/null 2>&1; then echo "cannot reach deployment $DEPLOYMENT in namespace $NAMESPACE" >&2 echo "hint: KUBECONFIG is '${KUBECONFIG:-}'." >&2 echo " tenant-engine runs on railiance01 -- pointing at the wrong" >&2 echo " cluster reports Unauthorized, which reads exactly like a" >&2 echo " missing credential. Check the cluster before the credential." >&2 exit 2 fi # 2. What the Deployment asks for. SPEC="$(kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" \ -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)" # 3. What the running pod actually resolved. RUNNING="$(kubectl -n "$NAMESPACE" get pods \ -l app.kubernetes.io/name="$DEPLOYMENT" \ --field-selector=status.phase=Running \ -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null)" say "desired (repo) : $DESIRED" say "deployment spec : ${SPEC:-}" say "running pod imageID : ${RUNNING:-}" STATUS=0 DESIRED_DIGEST="${DESIRED##*@}" if [[ "$SPEC" != "$DESIRED" ]]; then fail "deployment spec does not match the repo pin." fail " repo wants : $DESIRED" fail " cluster has: ${SPEC:-}" fail " Someone changed the cluster without changing the repo, or a commit" fail " was never applied. Both have happened here." STATUS=1 fi if [[ -n "$RUNNING" && "$RUNNING" != *"$DESIRED_DIGEST"* ]]; then fail "the running pod is not the desired digest." fail " Even if the spec is correct, a rollout may not have completed." STATUS=1 fi # 4. Ask the live service what it serves. A digest can be right while the # workload behind it is not the one the contract describes. if command -v curl >/dev/null; then kubectl -n "$NAMESPACE" port-forward "deploy/$DEPLOYMENT" "$LOCAL_PORT:8090" \ >/dev/null 2>&1 & PF_PID=$! trap 'kill "$PF_PID" 2>/dev/null' EXIT for _ in $(seq 1 20); do curl -sf -m 2 "http://127.0.0.1:$LOCAL_PORT/health" >/dev/null 2>&1 && break sleep 0.5 done OPENAPI="$(curl -sf -m 5 "http://127.0.0.1:$LOCAL_PORT/openapi.json" 2>/dev/null)" if [[ -z "$OPENAPI" ]]; then fail "could not read /openapi.json from the live service." STATUS=1 else for route in "${REQUIRED_ROUTES[@]}"; do if [[ "$OPENAPI" != *"\"$route\""* ]]; then fail "live service does not serve $route" fail " This is the failure mode that hid for three days: a consumer" fail " sees 404, not an error, and no policy event is emitted." STATUS=1 fi done say "live routes : all ${#REQUIRED_ROUTES[@]} required routes present" fi fi if [[ $STATUS -eq 0 ]]; then say "OK: repo, deployment spec, running pod, and served routes all agree." fi exit $STATUS