activity-core/scripts/prod_automation_status.sh

#!/usr/bin/env bash
# ACTIVITY-WP-0021-T08: production automation evidence without workstation DB.
# Usage:
#   ./scripts/prod_automation_status.sh [since]
#   since: "sunday" | ISO timestamp (default: sunday Europe/Berlin as UTC floor)
set -euo pipefail

SINCE_ARG="${1:-sunday}"
SSH_HOST="${PROD_AUTOMATION_SSH_HOST:-railiance01}"
NS="${PROD_AUTOMATION_NS:-activity-core}"

# The Makefile target is `make prod-automation-status SINCE=sunday`, so operators
# reasonably type the same `SINCE=` form when calling the script directly. Accept
# it rather than passing the literal into timestamptz, where every section fails
# with a separate parse error while the run still looks like it produced a report.
if [[ "$SINCE_ARG" == SINCE=* ]]; then
  SINCE_ARG="${SINCE_ARG#SINCE=}"
fi

usage() {
  cat >&2 <<'USAGE'
Usage: ./scripts/prod_automation_status.sh [since]
  since: "sunday" (default) | an ISO-8601 timestamp, e.g. 2026-08-17T00:00:00Z
Read-only. Requires SSH to the production host.
USAGE
}

if [[ "$SINCE_ARG" == "sunday" ]]; then
  # Floor of most recent Sunday 00:00 Europe/Berlin in UTC (portable enough for ops).
  SINCE_UTC="$(python3 - <<'PY'
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo
tz = ZoneInfo("Europe/Berlin")
now = datetime.now(tz)
days_back = (now.weekday() + 1) % 7
sunday = (now - timedelta(days=days_back)).replace(hour=0, minute=0, second=0, microsecond=0)
print(sunday.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S%z"))
PY
)"
else
  # Validate before the value reaches five separate SQL/JSON consumers: an
  # unparseable `since` must fail fast and loudly, not produce a report whose
  # every section is an error the reader has to notice.
  if ! SINCE_UTC="$(python3 - "$SINCE_ARG" <<'PYVALIDATE'
import sys
from datetime import datetime, timezone

raw = sys.argv[1].strip()
try:
    parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
except ValueError:
    sys.exit(1)
if parsed.tzinfo is None:
    parsed = parsed.replace(tzinfo=timezone.utc)
print(parsed.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S%z"))
PYVALIDATE
  )"; then
    echo "error: could not parse since value: ${SINCE_ARG}" >&2
    usage
    exit 2
  fi
fi

echo "=== prod automation status (railiance01) since ${SINCE_UTC} ==="
echo "host=${SSH_HOST} namespace=${NS}"
echo

ssh -o BatchMode=yes -o ConnectTimeout=15 "${SSH_HOST}" "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
echo '--- API health ---'
kubectl -n ${NS} exec deploy/actcore-api -- /app/.venv/bin/python3 -c 'import urllib.request; print(urllib.request.urlopen(\"http://127.0.0.1:8010/health\").read().decode())'
echo
echo '--- runs by activity ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT d.name, d.enabled, count(*) AS runs,
       min(coalesce(r.fired_at,r.scheduled_for)) AS first_fire,
       max(coalesce(r.fired_at,r.scheduled_for)) AS last_fire,
       sum(r.tasks_spawned) AS tasks
FROM activity_runs r
JOIN activity_definitions d ON d.id = r.activity_id
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
GROUP BY d.name, d.enabled
ORDER BY runs DESC, d.name;
\"
echo
echo '--- execution outcomes by activity ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT d.name, o.state, count(*) AS runs,
       min(o.created_at) AS first_created,
       max(o.updated_at) AS last_updated
FROM ops_runs o
JOIN activity_definitions d ON d.id = o.activity_definition_id
WHERE o.created_at >= timestamptz '${SINCE_UTC}'
GROUP BY d.name, o.state
ORDER BY d.name, o.state;
\"
echo
echo '--- failed or unclaimed execution detail (bounded) ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT o.created_at, d.name, o.state, o.attempt, o.claim_owner,
       left(o.title, 72) AS title,
       left(coalesce(o.result->>'error', o.result->>'reason', ''), 240) AS failure
FROM ops_runs o
JOIN activity_definitions d ON d.id = o.activity_definition_id
WHERE o.created_at >= timestamptz '${SINCE_UTC}'
  AND o.state IN ('failed', 'open', 'claimed')
ORDER BY o.created_at DESC
LIMIT 30;
\"
echo
echo '--- daily triage validation evidence (bounded) ---'
kubectl -n ${NS} exec deploy/actcore-api -- /app/.venv/bin/python3 -c '
import json, urllib.parse, urllib.request
from datetime import datetime

since = datetime.fromisoformat(\"${SINCE_UTC}\")
url = \"http://actcore-statehub-edge-relay:8000/progress/?\" + urllib.parse.urlencode(
    {\"event_type\": \"daily_triage\", \"limit\": 100}
)
items = json.load(urllib.request.urlopen(url, timeout=10))
rows = []
for item in items:
    created_raw = item.get(\"created_at\")
    if not created_raw:
        continue
    created = datetime.fromisoformat(created_raw.replace(\"Z\", \"+00:00\"))
    if created < since:
        continue
    detail = item.get(\"detail\") or {}
    rows.append({
        \"created_at\": created_raw,
        \"run_id\": detail.get(\"activity_core_run_id\"),
        \"output_validated\": detail.get(\"output_validated\"),
        \"summary\": str(item.get(\"summary\") or \"\")[:200],
        \"validation_error\": str(detail.get(\"validation_error\") or \"\")[:240],
    })
print(json.dumps(rows[:14], indent=2))
'
echo
echo '--- non-high-frequency fires ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT d.name, r.fired_at, r.tasks_spawned, r.run_id
FROM activity_runs r
JOIN activity_definitions d ON d.id = r.activity_id
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
  AND d.name NOT IN ('State Hub Consistency Sweep', 'Hourly RecentlyOnScope Reports')
ORDER BY r.fired_at;
\"
"