150 lines
5.6 KiB
Bash
Executable file
150 lines
5.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# verify-pin.sh -- detect deployment pin drift for tenant-engine.
|
|
#
|
|
# Compares three things that are supposed to agree and silently did not
|
|
# between 2026-08-13 and 2026-08-16:
|
|
#
|
|
# 1. the digest this repo intends to run (deploy/tenant-engine.yaml)
|
|
# 2. the digest the Deployment asks for (spec.template.spec.containers)
|
|
# 3. the digest the running pod actually (status.containerStatuses.imageID)
|
|
# resolved
|
|
#
|
|
# and then asks the live service what it is actually serving, because a
|
|
# correct digest with a broken rollout still fails the consumer.
|
|
#
|
|
# Why (3) separately from (2): a Deployment can carry the right digest in its
|
|
# spec while the pod that answers traffic is an older ReplicaSet that never
|
|
# finished rolling. The consumer talks to the pod, not to the spec.
|
|
#
|
|
# Why the route check at all: the drift that hit us was invisible precisely
|
|
# because it produced a 404 rather than an error. A digest comparison alone
|
|
# would have caught this one, but "the deployment was replaced by something
|
|
# else entirely" is the same class of failure and the digest check is blind
|
|
# to whether that something serves the contract.
|
|
#
|
|
# Usage:
|
|
# deploy/verify-pin.sh # check cluster against the repo
|
|
# deploy/verify-pin.sh --quiet # exit code only, for CI/cron
|
|
#
|
|
# Exit codes: 0 in sync | 1 drift detected | 2 could not determine
|
|
|
|
set -uo pipefail
|
|
|
|
NAMESPACE="tenant-engine"
|
|
DEPLOYMENT="tenant-engine"
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
MANIFEST="$REPO_ROOT/deploy/tenant-engine.yaml"
|
|
QUIET=0
|
|
LOCAL_PORT="${VERIFY_PIN_PORT:-18131}"
|
|
|
|
# Routes whose absence is what a silent rollback to a pre-lifecycle or
|
|
# pre-guardrail image actually looks like from a consumer's side.
|
|
REQUIRED_ROUTES=(
|
|
"/tenants/{tenant_id}"
|
|
"/tenants/{tenant_id}/retire"
|
|
"/tenants/{tenant_id}/reactivate"
|
|
"/tenants/{tenant_id}/guardrails"
|
|
"/tenants/{tenant_id}/guardrails/{limit_key}"
|
|
)
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--quiet) QUIET=1 ;;
|
|
--namespace) NAMESPACE="$2"; shift ;;
|
|
-h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}"; exit 0 ;;
|
|
*) echo "unknown argument: $1" >&2; exit 2 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
say() { [[ $QUIET -eq 1 ]] || echo "$@"; }
|
|
fail() { echo "DRIFT: $*" >&2; }
|
|
|
|
command -v kubectl >/dev/null || { echo "kubectl not found" >&2; exit 2; }
|
|
[[ -f "$MANIFEST" ]] || { echo "manifest not found: $MANIFEST" >&2; exit 2; }
|
|
|
|
# 1. What the repo intends. Deliberately a plain grep rather than a YAML
|
|
# parser: the manifest pins by digest on a single line by convention, and
|
|
# a drift check that needs its own dependency tree is a check that stops
|
|
# being run.
|
|
DESIRED="$(grep -oE 'image: [^ ]+@sha256:[a-f0-9]{64}' "$MANIFEST" | head -1 | sed 's/^image: //')"
|
|
if [[ -z "$DESIRED" ]]; then
|
|
echo "could not read a digest-pinned image from $MANIFEST" >&2
|
|
echo "(is it pinned by tag instead of digest? that is itself a finding)" >&2
|
|
exit 2
|
|
fi
|
|
|
|
if ! kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" >/dev/null 2>&1; then
|
|
echo "cannot reach deployment $DEPLOYMENT in namespace $NAMESPACE" >&2
|
|
echo "hint: KUBECONFIG is '${KUBECONFIG:-<unset, using ~/.kube/config>}'." >&2
|
|
echo " tenant-engine runs on railiance01 -- pointing at the wrong" >&2
|
|
echo " cluster reports Unauthorized, which reads exactly like a" >&2
|
|
echo " missing credential. Check the cluster before the credential." >&2
|
|
exit 2
|
|
fi
|
|
|
|
# 2. What the Deployment asks for.
|
|
SPEC="$(kubectl -n "$NAMESPACE" get deploy "$DEPLOYMENT" \
|
|
-o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null)"
|
|
|
|
# 3. What the running pod actually resolved.
|
|
RUNNING="$(kubectl -n "$NAMESPACE" get pods \
|
|
-l app.kubernetes.io/name="$DEPLOYMENT" \
|
|
--field-selector=status.phase=Running \
|
|
-o jsonpath='{.items[0].status.containerStatuses[0].imageID}' 2>/dev/null)"
|
|
|
|
say "desired (repo) : $DESIRED"
|
|
say "deployment spec : ${SPEC:-<none>}"
|
|
say "running pod imageID : ${RUNNING:-<none>}"
|
|
|
|
STATUS=0
|
|
DESIRED_DIGEST="${DESIRED##*@}"
|
|
|
|
if [[ "$SPEC" != "$DESIRED" ]]; then
|
|
fail "deployment spec does not match the repo pin."
|
|
fail " repo wants : $DESIRED"
|
|
fail " cluster has: ${SPEC:-<none>}"
|
|
fail " Someone changed the cluster without changing the repo, or a commit"
|
|
fail " was never applied. Both have happened here."
|
|
STATUS=1
|
|
fi
|
|
|
|
if [[ -n "$RUNNING" && "$RUNNING" != *"$DESIRED_DIGEST"* ]]; then
|
|
fail "the running pod is not the desired digest."
|
|
fail " Even if the spec is correct, a rollout may not have completed."
|
|
STATUS=1
|
|
fi
|
|
|
|
# 4. Ask the live service what it serves. A digest can be right while the
|
|
# workload behind it is not the one the contract describes.
|
|
if command -v curl >/dev/null; then
|
|
kubectl -n "$NAMESPACE" port-forward "deploy/$DEPLOYMENT" "$LOCAL_PORT:8090" \
|
|
>/dev/null 2>&1 &
|
|
PF_PID=$!
|
|
trap 'kill "$PF_PID" 2>/dev/null' EXIT
|
|
for _ in $(seq 1 20); do
|
|
curl -sf -m 2 "http://127.0.0.1:$LOCAL_PORT/health" >/dev/null 2>&1 && break
|
|
sleep 0.5
|
|
done
|
|
|
|
OPENAPI="$(curl -sf -m 5 "http://127.0.0.1:$LOCAL_PORT/openapi.json" 2>/dev/null)"
|
|
if [[ -z "$OPENAPI" ]]; then
|
|
fail "could not read /openapi.json from the live service."
|
|
STATUS=1
|
|
else
|
|
for route in "${REQUIRED_ROUTES[@]}"; do
|
|
if [[ "$OPENAPI" != *"\"$route\""* ]]; then
|
|
fail "live service does not serve $route"
|
|
fail " This is the failure mode that hid for three days: a consumer"
|
|
fail " sees 404, not an error, and no policy event is emitted."
|
|
STATUS=1
|
|
fi
|
|
done
|
|
say "live routes : all ${#REQUIRED_ROUTES[@]} required routes present"
|
|
fi
|
|
fi
|
|
|
|
if [[ $STATUS -eq 0 ]]; then
|
|
say "OK: repo, deployment spec, running pod, and served routes all agree."
|
|
fi
|
|
exit $STATUS
|