112 lines
4.3 KiB
Bash
Executable file
112 lines
4.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# ACTIVITY-WP-0021-T08: production automation evidence without workstation DB.
|
|
# Usage:
|
|
# ./scripts/prod_automation_status.sh [since]
|
|
# since: "sunday" | ISO timestamp (default: sunday Europe/Berlin as UTC floor)
|
|
set -euo pipefail
|
|
|
|
SINCE_ARG="${1:-sunday}"
|
|
SSH_HOST="${PROD_AUTOMATION_SSH_HOST:-railiance01}"
|
|
NS="${PROD_AUTOMATION_NS:-activity-core}"
|
|
|
|
if [[ "$SINCE_ARG" == "sunday" ]]; then
|
|
# Floor of most recent Sunday 00:00 Europe/Berlin in UTC (portable enough for ops).
|
|
SINCE_UTC="$(python3 - <<'PY'
|
|
from datetime import datetime, timedelta, timezone
|
|
from zoneinfo import ZoneInfo
|
|
tz = ZoneInfo("Europe/Berlin")
|
|
now = datetime.now(tz)
|
|
days_back = (now.weekday() + 1) % 7
|
|
sunday = (now - timedelta(days=days_back)).replace(hour=0, minute=0, second=0, microsecond=0)
|
|
print(sunday.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S%z"))
|
|
PY
|
|
)"
|
|
else
|
|
SINCE_UTC="$SINCE_ARG"
|
|
fi
|
|
|
|
echo "=== prod automation status (railiance01) since ${SINCE_UTC} ==="
|
|
echo "host=${SSH_HOST} namespace=${NS}"
|
|
echo
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=15 "${SSH_HOST}" "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
|
|
echo '--- API health ---'
|
|
kubectl -n ${NS} exec deploy/actcore-api -- /app/.venv/bin/python3 -c 'import urllib.request; print(urllib.request.urlopen(\"http://127.0.0.1:8010/health\").read().decode())'
|
|
echo
|
|
echo '--- runs by activity ---'
|
|
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
|
|
SELECT d.name, d.enabled, count(*) AS runs,
|
|
min(coalesce(r.fired_at,r.scheduled_for)) AS first_fire,
|
|
max(coalesce(r.fired_at,r.scheduled_for)) AS last_fire,
|
|
sum(r.tasks_spawned) AS tasks
|
|
FROM activity_runs r
|
|
JOIN activity_definitions d ON d.id = r.activity_id
|
|
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
|
|
GROUP BY d.name, d.enabled
|
|
ORDER BY runs DESC, d.name;
|
|
\"
|
|
echo
|
|
echo '--- execution outcomes by activity ---'
|
|
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
|
|
SELECT d.name, o.state, count(*) AS runs,
|
|
min(o.created_at) AS first_created,
|
|
max(o.updated_at) AS last_updated
|
|
FROM ops_runs o
|
|
JOIN activity_definitions d ON d.id = o.activity_definition_id
|
|
WHERE o.created_at >= timestamptz '${SINCE_UTC}'
|
|
GROUP BY d.name, o.state
|
|
ORDER BY d.name, o.state;
|
|
\"
|
|
echo
|
|
echo '--- failed or unclaimed execution detail (bounded) ---'
|
|
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
|
|
SELECT o.created_at, d.name, o.state, o.attempt, o.claim_owner,
|
|
left(o.title, 72) AS title,
|
|
left(coalesce(o.result->>'error', o.result->>'reason', ''), 240) AS failure
|
|
FROM ops_runs o
|
|
JOIN activity_definitions d ON d.id = o.activity_definition_id
|
|
WHERE o.created_at >= timestamptz '${SINCE_UTC}'
|
|
AND o.state IN ('failed', 'open', 'claimed')
|
|
ORDER BY o.created_at DESC
|
|
LIMIT 30;
|
|
\"
|
|
echo
|
|
echo '--- daily triage validation evidence (bounded) ---'
|
|
kubectl -n ${NS} exec deploy/actcore-api -- /app/.venv/bin/python3 -c '
|
|
import json, urllib.parse, urllib.request
|
|
from datetime import datetime
|
|
|
|
since = datetime.fromisoformat(\"${SINCE_UTC}\")
|
|
url = \"http://actcore-statehub-edge-relay:8000/progress/?\" + urllib.parse.urlencode(
|
|
{\"event_type\": \"daily_triage\", \"limit\": 100}
|
|
)
|
|
items = json.load(urllib.request.urlopen(url, timeout=10))
|
|
rows = []
|
|
for item in items:
|
|
created_raw = item.get(\"created_at\")
|
|
if not created_raw:
|
|
continue
|
|
created = datetime.fromisoformat(created_raw.replace(\"Z\", \"+00:00\"))
|
|
if created < since:
|
|
continue
|
|
detail = item.get(\"detail\") or {}
|
|
rows.append({
|
|
\"created_at\": created_raw,
|
|
\"run_id\": detail.get(\"activity_core_run_id\"),
|
|
\"output_validated\": detail.get(\"output_validated\"),
|
|
\"summary\": str(item.get(\"summary\") or \"\")[:200],
|
|
\"validation_error\": str(detail.get(\"validation_error\") or \"\")[:240],
|
|
})
|
|
print(json.dumps(rows[:14], indent=2))
|
|
'
|
|
echo
|
|
echo '--- non-high-frequency fires ---'
|
|
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
|
|
SELECT d.name, r.fired_at, r.tasks_spawned, r.run_id
|
|
FROM activity_runs r
|
|
JOIN activity_definitions d ON d.id = r.activity_id
|
|
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
|
|
AND d.name NOT IN ('State Hub Consistency Sweep', 'Hourly RecentlyOnScope Reports')
|
|
ORDER BY r.fired_at;
|
|
\"
|
|
"
|