railiance-platform/tools/apps-pg-isolation-probe.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
KUBECONFIG=${KUBECONFIG:-/home/worsch/.kube/config-railiance01}
KUBECTL_BIN=${KUBECTL_BIN:-kubectl}
EXPECTED_CLUSTER_UID=${RAILIANCE01_CLUSTER_UID:-a553c742-0115-43d4-99a4-a5ca56fe0786}
NAMESPACE=apps-pg-isolation-probe
DB_HOST=apps-pg-rw.databases.svc.cluster.local
K=("$KUBECTL_BIN" --kubeconfig="$KUBECONFIG")
pass=0
fail=0
namespace_created=0

cleanup() {
  if [[ "$namespace_created" == 1 && "${KEEP:-0}" != 1 ]]; then
    "${K[@]}" delete namespace "$NAMESPACE" --wait=false >/dev/null 2>&1 || true
  fi
}
trap cleanup EXIT

record() {
  local outcome=$1 name=$2 detail=${3:-}
  if [[ "$outcome" == pass ]]; then
    pass=$((pass + 1))
    printf 'PASS|%s%s\n' "$name" "${detail:+|$detail}"
  else
    fail=$((fail + 1))
    printf 'FAIL|%s%s\n' "$name" "${detail:+|$detail}"
  fi
}

psql_as() {
  local pod=$1 role=$2 database=$3 sql=$4
  "${K[@]}" exec -n "$NAMESPACE" "$pod" -- \
    psql -X -v ON_ERROR_STOP=1 -At -h "$DB_HOST" -U "$role" -d "$database" -c "$sql"
}

psql_super() {
  local sql=$1
  "${K[@]}" exec -n databases apps-pg-1 -c postgres -- \
    psql -X -v ON_ERROR_STOP=1 -U postgres -d postgres -At -c "$sql"
}

expect_value() {
  local name=$1 pod=$2 role=$3 database=$4 sql=$5 expected=$6 got
  if ! got=$(psql_as "$pod" "$role" "$database" "$sql" 2>&1); then
    record fail "$name" "query failed"
  elif [[ "$got" == "$expected" ]]; then
    record pass "$name" "$got"
  else
    record fail "$name" "expected=$expected got=$got"
  fi
}

expect_denied() {
  local name=$1 pod=$2 role=$3 database=$4 sql=$5
  if psql_as "$pod" "$role" "$database" "$sql" >/dev/null 2>&1; then
    record fail "$name" "attempt succeeded"
  else
    record pass "$name" "denied"
  fi
}

copy_secret() {
  local name=$1
  "${K[@]}" get secret "$name" -n databases -o json \
    | jq --arg ns "$NAMESPACE" \
      'del(.metadata.annotations, .metadata.creationTimestamp, .metadata.resourceVersion,
           .metadata.uid, .metadata.ownerReferences, .metadata.managedFields) |
       .metadata.namespace=$ns' \
    | "${K[@]}" apply -f - >/dev/null
}

measure_vergabe_ms() {
  local start end
  start=$(date +%s%3N)
  for _ in 1 2 3 4 5; do
    psql_as vergabe-probe vergabe vergabe_db 'SELECT 1' >/dev/null
  done
  end=$(date +%s%3N)
  echo $((end - start))
}

uid=$("${K[@]}" get namespace kube-system -o jsonpath='{.metadata.uid}')
if [[ "$uid" != "$EXPECTED_CLUSTER_UID" ]]; then
  echo "REFUSING: expected railiance01 UID $EXPECTED_CLUSTER_UID, got $uid" >&2
  exit 2
fi
if "${K[@]}" get namespace "$NAMESPACE" >/dev/null 2>&1; then
  echo "REFUSING: probe namespace already exists: $NAMESPACE" >&2
  exit 2
fi

"${K[@]}" apply -f "$ROOT/helm/apps-pg-isolation-probe.yaml" >/dev/null
namespace_created=1
copy_secret coulomb-social-app-credentials
copy_secret vergabe-app-credentials
"${K[@]}" wait --for=condition=Ready pod/coulomb-probe pod/vergabe-probe \
  -n "$NAMESPACE" --timeout=60s >/dev/null

echo 'SECTION|consumer-boundary'
expect_value 'coulomb own database connects' coulomb-probe coulomb_social coulomb_social_db 'SELECT 1' 1
expect_value 'vergabe own database connects' vergabe-probe vergabe vergabe_db 'SELECT 1' 1
expect_denied 'coulomb cannot connect to vergabe database' coulomb-probe coulomb_social vergabe_db 'SELECT 1'
expect_denied 'vergabe cannot connect to coulomb database' vergabe-probe vergabe coulomb_social_db 'SELECT 1'
expect_denied 'coulomb cannot connect to platform bootstrap database' coulomb-probe coulomb_social apps_meta 'SELECT 1'
expect_denied 'consumer cannot alter peer role' coulomb-probe coulomb_social coulomb_social_db 'ALTER ROLE vergabe CONNECTION LIMIT 999'
expect_value 'coulomb sees its own table' coulomb-probe coulomb_social coulomb_social_db 'SELECT count(*) FROM members_user' 1
expect_value 'coulomb statement timeout binds' coulomb-probe coulomb_social coulomb_social_db 'SHOW statement_timeout' 15s
expect_value 'coulomb idle transaction timeout binds' coulomb-probe coulomb_social coulomb_social_db 'SHOW idle_in_transaction_session_timeout' 15s
expect_value 'coulomb connection limit is 20' coulomb-probe coulomb_social coulomb_social_db "SELECT rolconnlimit FROM pg_roles WHERE rolname=current_user" 20
expect_denied 'overlong statement is terminated' coulomb-probe coulomb_social coulomb_social_db 'SELECT pg_sleep(20)'

echo 'SECTION|noisy-neighbour-connection-saturation'
baseline_ms=$(measure_vergabe_ms)
existing=$(psql_super "SELECT count(*) FROM pg_stat_activity WHERE usename='coulomb_social'")
hold=$((20 - existing))
if (( hold < 1 )); then
  record fail 'saturation setup has free role slots' "existing=$existing"
else
  "${K[@]}" exec -n "$NAMESPACE" coulomb-probe -- sh -c '
    count=$1
    i=1
    while [ "$i" -le "$count" ]; do
      psql -X -v ON_ERROR_STOP=1 -At -h "$2" -U coulomb_social -d coulomb_social_db -c "\! sleep 25" >/dev/null 2>&1 &
      i=$((i + 1))
    done
    wait
  ' sh "$hold" "$DB_HOST" &
  saturator_pid=$!

  saturated=0
  for _ in $(seq 1 50); do
    current=$(psql_super "SELECT count(*) FROM pg_stat_activity WHERE usename='coulomb_social'")
    if [[ "$current" == 20 ]]; then saturated=1; break; fi
    sleep 0.2
  done
  if [[ "$saturated" == 1 ]]; then
    record pass 'coulomb reaches its declared connection ceiling' 'connections=20'
  else
    record fail 'coulomb reaches its declared connection ceiling' "connections=$current"
  fi
  expect_denied 'coulomb excess connection is rejected' coulomb-probe coulomb_social coulomb_social_db 'SELECT 1'
  if during_ms=$(measure_vergabe_ms); then
    record pass 'vergabe remains available during peer saturation' "baseline_5q_ms=$baseline_ms saturated_5q_ms=$during_ms"
  else
    during_ms=unavailable
    record fail 'vergabe remains available during peer saturation'
  fi
  wait "$saturator_pid" || true
fi

echo "SUMMARY|passed=$pass|failed=$fail|baseline_5q_ms=$baseline_ms|saturated_5q_ms=${during_ms:-not-run}"
(( fail == 0 ))