coulomb/activity-core
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement ACTIVITY-WP-0021 production automation reliability
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
Details
CI Smoke / container-smoke (push) Successful in 2s
Details
Build and Publish Container Image / build-and-push (push) Successful in 47s
Details

Browse source 
Root-cause IssueSink 503 (dead Forgejo PAT on issue-core), add state-hub
task sink path B, log runs before emit, harden sync_schedules, deterministic
SBOM/triage reports, DB probe thrash fix, and prod automation-status helper.
This commit is contained in:
tegwick 2026-07-21 04:21:55 +02:00
parent 1209ff6973
commit 98e8aa83bd
15 changed files with 638 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

58
scripts/prod_automation_status.sh Executable file
View file

@ -0,0 +1,58 @@
#!/usr/bin/env bash
# ACTIVITY-WP-0021-T08: production automation evidence without workstation DB.
# Usage:
# ./scripts/prod_automation_status.sh [since]
# since: "sunday" | ISO timestamp (default: sunday Europe/Berlin as UTC floor)
set -euo pipefail
SINCE_ARG="${1:-sunday}"
SSH_HOST="${PROD_AUTOMATION_SSH_HOST:-railiance01}"
NS="${PROD_AUTOMATION_NS:-activity-core}"
if [[ "$SINCE_ARG" == "sunday" ]]; then
# Floor of most recent Sunday 00:00 Europe/Berlin in UTC (portable enough for ops).
SINCE_UTC="$(python3 - <<'PY'
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo
tz = ZoneInfo("Europe/Berlin")
now = datetime.now(tz)
days_back = (now.weekday() + 1) % 7
sunday = (now - timedelta(days=days_back)).replace(hour=0, minute=0, second=0, microsecond=0)
print(sunday.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S%z"))
PY
)"
else
SINCE_UTC="$SINCE_ARG"
fi
echo "=== prod automation status (railiance01) since ${SINCE_UTC} ==="
echo "host=${SSH_HOST} namespace=${NS}"
echo
ssh -o BatchMode=yes -o ConnectTimeout=15 "${SSH_HOST}" "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
echo '--- API health ---'
kubectl -n ${NS} exec deploy/actcore-api -- /app/.venv/bin/python3 -c 'import urllib.request; print(urllib.request.urlopen(\"http://127.0.0.1:8010/health\").read().decode())'
echo
echo '--- runs by activity ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT d.name, d.enabled, count(*) AS runs,
min(coalesce(r.fired_at,r.scheduled_for)) AS first_fire,
max(coalesce(r.fired_at,r.scheduled_for)) AS last_fire,
sum(r.tasks_spawned) AS tasks
FROM activity_runs r
JOIN activity_definitions d ON d.id = r.activity_id
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
GROUP BY d.name, d.enabled
ORDER BY runs DESC, d.name;
\"
echo
echo '--- non-high-frequency fires ---'
kubectl -n ${NS} exec actcore-app-db-0 -- psql -U actcore -d actcore -c \"
SELECT d.name, r.fired_at, r.tasks_spawned, r.run_id
FROM activity_runs r
JOIN activity_definitions d ON d.id = r.activity_id
WHERE coalesce(r.scheduled_for, r.fired_at) >= timestamptz '${SINCE_UTC}'
AND d.name NOT IN ('State Hub Consistency Sweep', 'Hourly RecentlyOnScope Reports')
ORDER BY r.fired_at;
\"
"