tegwick
90ebdb7e7d
Version 2 measured everything correctly — not_ready=0, query_failed=0, zero 503s — and still printed FAILED. grep -c exits 1 when it counts zero matches, so the '|| echo "?"' guard appended a marker on top of the legitimate 0 and the verdict test could never match. Third defect in the same instrument. The verdict logic is now tested rather than assumed: the counter was fed a matching and a non-matching line and confirmed to return 1 and 0. Worth noting which direction each failure pointed. This one erred toward alarm, which is survivable. live-image-digest-match erred the other way and reported success it had not established — that is the one that would have shipped. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM Assistant: claude-code Assistant-Model: opus Assistant-Process: 388925@bnt-lap001 Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502
55 lines
2.5 KiB
Bash
Executable file
55 lines
2.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Verification for RCP-WP-0002-T04: stay ready across a credential lease
|
|
# rotation (runtime lease TTL 30m).
|
|
#
|
|
# v2. The first version treated an empty kubectl result as "not ready", so a
|
|
# transient API hiccup was recorded as a service failure and produced a FAILED
|
|
# verdict for a service that never returned a single 503. A check that cannot
|
|
# tell its own failure from the failure it watches for is worse than no check.
|
|
OUT="$1"; MINUTES="${2:-20}"
|
|
DEADLINE=$(( $(date +%s) + MINUTES * 60 ))
|
|
: > "$OUT"
|
|
echo "started $(date -Is) — ${MINUTES}m watch (runtime lease TTL 30m)" >> "$OUT"
|
|
NOTREADY=0; QUERYFAIL=0; SAMPLES=0
|
|
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
|
SAMPLES=$((SAMPLES+1))
|
|
if ! OUTJSON=$(kubectl -n canned-prompts get deploy canned-prompts \
|
|
-o jsonpath='{.status.readyReplicas}|{.status.replicas}' 2>/dev/null) || [ -z "${OUTJSON%%|*}${OUTJSON##*|}" ]; then
|
|
QUERYFAIL=$((QUERYFAIL+1))
|
|
echo "$(date -Is) QUERY-FAILED (instrument, not service)" >> "$OUT"
|
|
else
|
|
READY="${OUTJSON%%|*}"; WANT="${OUTJSON##*|}"
|
|
if [ "$READY" = "$WANT" ] && [ -n "$READY" ]; then
|
|
echo "$(date -Is) ready=$READY/$WANT" >> "$OUT"
|
|
else
|
|
NOTREADY=$((NOTREADY+1))
|
|
echo "$(date -Is) NOT READY ready=${READY:-0}/${WANT:-?}" >> "$OUT"
|
|
fi
|
|
fi
|
|
sleep 60
|
|
done
|
|
# Corroborate with the kubelet, which probes every 5s and is a far better
|
|
# instrument than this loop's 1/min sampling.
|
|
# grep -c exits 1 when the count is zero, so `|| echo "?"` appended a marker on
|
|
# top of a legitimate "0" and the verdict test never matched. Capture the logs
|
|
# first: a kubectl failure is then distinguishable from an absence of 503s,
|
|
# which is the whole point.
|
|
if LOGS=$(kubectl -n canned-prompts logs deploy/canned-prompts --since="${MINUTES}m" 2>/dev/null); then
|
|
P503=$(printf '%s' "$LOGS" | grep -c '" 503' || true)
|
|
P503=${P503:-0}
|
|
else
|
|
P503="unavailable"
|
|
fi
|
|
UP=$(kubectl -n canned-prompts get pod -l app.kubernetes.io/name=canned-prompts -o jsonpath='{.items[0].status.startTime}' 2>/dev/null)
|
|
R=$(kubectl -n canned-prompts get pod -l app.kubernetes.io/name=canned-prompts -o jsonpath='{.items[0].status.containerStatuses[0].restartCount}' 2>/dev/null)
|
|
{
|
|
echo "finished $(date -Is)"
|
|
echo "samples=$SAMPLES not_ready=$NOTREADY query_failed=$QUERYFAIL"
|
|
echo "kubelet readiness 503s in window: $P503 (probe every 5s)"
|
|
echo "pod started $UP, restarts=$R"
|
|
if [ "$NOTREADY" -eq 0 ] && [ "$P503" = "0" ]; then
|
|
echo "RESULT: survived — continuously ready past the lease TTL"
|
|
else
|
|
echo "RESULT: FAILED"
|
|
fi
|
|
} >> "$OUT"
|