Harden activity-core verifier evidence
This commit is contained in:
parent
45e57b0a11
commit
6f42bf114b
2 changed files with 206 additions and 16 deletions
|
|
@ -16,9 +16,17 @@ STATE_HUB_PROGRESS_POLL_SECONDS="${STATE_HUB_PROGRESS_POLL_SECONDS:-5}"
|
|||
ACTIVITY_CORE_REPO="${ACTIVITY_CORE_REPO:-/home/worsch/activity-core}"
|
||||
ACTIVITY_CORE_REMOTE_REPO="${ACTIVITY_CORE_REMOTE_REPO:-}"
|
||||
ACTIVITY_CORE_CLUSTER_HOST="${ACTIVITY_CORE_CLUSTER_HOST:-railiance01}"
|
||||
ACTIVITY_CORE_ALLOW_LOCAL_KUBECTL="${ACTIVITY_CORE_ALLOW_LOCAL_KUBECTL:-0}"
|
||||
ACTIVITY_CORE_SYNC_RUNTIME_BUNDLE="${ACTIVITY_CORE_SYNC_RUNTIME_BUNDLE:-auto}"
|
||||
ACTIVITY_CORE_RESTART_DEPLOYMENTS="${ACTIVITY_CORE_RESTART_DEPLOYMENTS:-0}"
|
||||
if [[ "$ACTIVITY_CORE_CLUSTER_HOST" == "local" ]]; then
|
||||
if [[ "$ACTIVITY_CORE_ALLOW_LOCAL_KUBECTL" != "1" ]]; then
|
||||
{
|
||||
echo "ACTIVITY_CORE_CLUSTER_HOST=local requires ACTIVITY_CORE_ALLOW_LOCAL_KUBECTL=1"
|
||||
echo "Default verifier execution is cluster-owned via railiance01/SSH."
|
||||
} >&2
|
||||
exit 2
|
||||
fi
|
||||
ACTIVITY_CORE_CLUSTER_HOST=""
|
||||
fi
|
||||
if [[ -z "$ACTIVITY_CORE_REMOTE_REPO" ]]; then
|
||||
|
|
@ -38,9 +46,12 @@ STARTED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
|
|||
CURRENT_GATE="startup"
|
||||
REMOTE_REVISION=""
|
||||
API_IMAGE=""
|
||||
API_IMAGE_ID=""
|
||||
SYNC_STATUS_JSON=""
|
||||
DEFINITION_JSON=""
|
||||
TRIGGER_JSON=""
|
||||
TRIGGER_KEY=""
|
||||
EXPECTED_RUN_ID=""
|
||||
PROGRESS_JSON=""
|
||||
EVIDENCE_NOTE_JSON=""
|
||||
|
||||
|
|
@ -49,8 +60,8 @@ export STATE_HUB_URL EVIDENCE_WORKSTREAM_ID EVIDENCE_TASK_ID
|
|||
export STATE_HUB_PROGRESS_TIMEOUT_SECONDS STATE_HUB_PROGRESS_POLL_SECONDS
|
||||
export INTER_HUB_SUBMISSION_STATUS INTER_HUB_DEFER_REASON STARTED_AT
|
||||
export ACTIVITY_CORE_CLUSTER_HOST ACTIVITY_CORE_REMOTE_REPO
|
||||
export ACTIVITY_CORE_SYNC_RUNTIME_BUNDLE ACTIVITY_CORE_RESTART_DEPLOYMENTS
|
||||
export REMOTE_REVISION API_IMAGE SYNC_STATUS_JSON DEFINITION_JSON TRIGGER_JSON PROGRESS_JSON
|
||||
export ACTIVITY_CORE_ALLOW_LOCAL_KUBECTL ACTIVITY_CORE_SYNC_RUNTIME_BUNDLE ACTIVITY_CORE_RESTART_DEPLOYMENTS
|
||||
export REMOTE_REVISION API_IMAGE API_IMAGE_ID SYNC_STATUS_JSON DEFINITION_JSON TRIGGER_JSON TRIGGER_KEY EXPECTED_RUN_ID PROGRESS_JSON
|
||||
|
||||
log() {
|
||||
printf '[activity-core-verify] %s\n' "$*"
|
||||
|
|
@ -121,10 +132,12 @@ detail = {
|
|||
"activity_core_repo": os.environ.get("ACTIVITY_CORE_REMOTE_REPO"),
|
||||
"activity_core_revision": os.environ.get("REMOTE_REVISION") or None,
|
||||
"api_image": os.environ.get("API_IMAGE") or None,
|
||||
"api_image_id": os.environ.get("API_IMAGE_ID") or None,
|
||||
"runtime_bundle": "k8s/railiance/20-runtime.yaml",
|
||||
"sync_job": sync_status,
|
||||
"definition": definition,
|
||||
"manual_trigger": trigger,
|
||||
"expected_activity_core_run_id": os.environ.get("EXPECTED_RUN_ID") or None,
|
||||
"state_hub_progress": progress,
|
||||
"inter_hub_submission": {
|
||||
"status": os.environ.get("INTER_HUB_SUBMISSION_STATUS"),
|
||||
|
|
@ -137,8 +150,9 @@ if status == "passed":
|
|||
summary = (
|
||||
"Railiance activity-core deploy/verify passed: runtime reconciled, "
|
||||
"actcore-sync completed, ops-service-inventory-probes remains disabled, "
|
||||
f"manual trigger {trigger.get('workflow_id') if isinstance(trigger, dict) else 'unknown'} ran, "
|
||||
f"and State Hub ops_inventory_probe progress {progress.get('id') if isinstance(progress, dict) else 'unknown'} exists."
|
||||
f"manual trigger {trigger.get('workflow_id') if isinstance(trigger, dict) else 'unknown'} ran as "
|
||||
f"{os.environ.get('EXPECTED_RUN_ID') or 'unknown run'}, and State Hub ops_inventory_probe progress "
|
||||
f"{progress.get('id') if isinstance(progress, dict) else 'unknown'} matched that run."
|
||||
)
|
||||
else:
|
||||
summary = (
|
||||
|
|
@ -189,6 +203,16 @@ set -euo pipefail
|
|||
command -v kubectl >/dev/null
|
||||
EOF
|
||||
)"
|
||||
if [[ -z "$ACTIVITY_CORE_CLUSTER_HOST" ]]; then
|
||||
LOCAL_CONTEXT="$(
|
||||
cluster_bash "$(cat <<'EOF'
|
||||
set -euo pipefail
|
||||
kubectl config current-context 2>/dev/null || true
|
||||
EOF
|
||||
)"
|
||||
)"
|
||||
log "local kubectl context: ${LOCAL_CONTEXT:-unknown}"
|
||||
fi
|
||||
|
||||
CURRENT_GATE="runtime bundle sync"
|
||||
if should_sync_runtime_bundle; then
|
||||
|
|
@ -255,6 +279,17 @@ kubectl -n $(quote "$NAMESPACE") get deploy actcore-api -o jsonpath='{.spec.temp
|
|||
EOF
|
||||
)"
|
||||
)"
|
||||
API_IMAGE_ID="$(
|
||||
cluster_bash "$(cat <<EOF
|
||||
set -euo pipefail
|
||||
kubectl -n $(quote "$NAMESPACE") get pod -l app.kubernetes.io/name=actcore-api -o jsonpath='{.items[0].status.containerStatuses[0].imageID}'
|
||||
EOF
|
||||
)"
|
||||
)"
|
||||
if [[ -z "$REMOTE_REVISION" && -z "$API_IMAGE_ID" ]]; then
|
||||
printf 'could not determine activity-core revision or actcore-api imageID\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
SYNC_STATUS_JSON="$(
|
||||
cluster_bash "$(cat <<EOF
|
||||
set -euo pipefail
|
||||
|
|
@ -262,7 +297,7 @@ kubectl -n $(quote "$NAMESPACE") get job actcore-sync -o json
|
|||
EOF
|
||||
)" | python3 -c 'import json,sys; j=json.load(sys.stdin); s=j.get("status",{}); print(json.dumps({"name": j["metadata"]["name"], "succeeded": s.get("succeeded", 0), "failed": s.get("failed", 0), "completion_time": s.get("completionTime")}))'
|
||||
)"
|
||||
export API_IMAGE SYNC_STATUS_JSON
|
||||
export API_IMAGE API_IMAGE_ID SYNC_STATUS_JSON
|
||||
|
||||
CURRENT_GATE="disabled definition check"
|
||||
log "checking ${DEFINITION_SLUG} is present and disabled"
|
||||
|
|
@ -331,7 +366,31 @@ if [[ -z "$TRIGGER_JSON" ]]; then
|
|||
printf 'manual trigger produced no output\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
export TRIGGER_JSON
|
||||
TRIGGER_KEY="$(
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
|
||||
trigger = json.loads(os.environ["TRIGGER_JSON"])
|
||||
trigger_key = trigger.get("trigger_key")
|
||||
if not trigger_key:
|
||||
raise SystemExit("manual trigger response did not include trigger_key")
|
||||
print(trigger_key)
|
||||
PY
|
||||
)"
|
||||
export TRIGGER_KEY
|
||||
EXPECTED_RUN_ID="$(
|
||||
python3 - <<'PY'
|
||||
import os
|
||||
import uuid
|
||||
|
||||
definition_id = os.environ["DEFINITION_ID"]
|
||||
trigger_key = os.environ["TRIGGER_KEY"]
|
||||
print(uuid.uuid5(uuid.NAMESPACE_URL, f"{definition_id}:{trigger_key}"))
|
||||
PY
|
||||
)"
|
||||
export TRIGGER_JSON EXPECTED_RUN_ID
|
||||
log "manual trigger run id: ${EXPECTED_RUN_ID}"
|
||||
|
||||
CURRENT_GATE="State Hub ops_inventory_probe evidence"
|
||||
log "polling State Hub for ops_inventory_probe progress"
|
||||
|
|
@ -348,6 +407,8 @@ base = os.environ["STATE_HUB_URL"].rstrip("/")
|
|||
started = datetime.fromisoformat(os.environ["STARTED_AT"].replace("Z", "+00:00"))
|
||||
timeout = int(os.environ["STATE_HUB_PROGRESS_TIMEOUT_SECONDS"])
|
||||
interval = int(os.environ["STATE_HUB_PROGRESS_POLL_SECONDS"])
|
||||
definition_id = os.environ["DEFINITION_ID"]
|
||||
expected_run_id = os.environ["EXPECTED_RUN_ID"]
|
||||
deadline = time.monotonic() + timeout
|
||||
url = base + "/progress/?" + urllib.parse.urlencode({"event_type": "ops_inventory_probe"})
|
||||
|
||||
|
|
@ -358,18 +419,27 @@ while time.monotonic() < deadline:
|
|||
created_at = datetime.fromisoformat(event["created_at"].replace("Z", "+00:00"))
|
||||
if created_at >= started:
|
||||
detail = event.get("detail") or {}
|
||||
print(json.dumps({
|
||||
"id": event["id"],
|
||||
"event_type": event.get("event_type"),
|
||||
"summary": event.get("summary"),
|
||||
"author": event.get("author"),
|
||||
"created_at": event.get("created_at"),
|
||||
"detail_keys": sorted(detail.keys()) if isinstance(detail, dict) else [],
|
||||
}))
|
||||
raise SystemExit(0)
|
||||
if (
|
||||
isinstance(detail, dict)
|
||||
and detail.get("activity_id") == definition_id
|
||||
and detail.get("activity_core_run_id") == expected_run_id
|
||||
):
|
||||
print(json.dumps({
|
||||
"id": event["id"],
|
||||
"event_type": event.get("event_type"),
|
||||
"summary": event.get("summary"),
|
||||
"author": event.get("author"),
|
||||
"created_at": event.get("created_at"),
|
||||
"activity_id": detail.get("activity_id"),
|
||||
"activity_core_run_id": detail.get("activity_core_run_id"),
|
||||
"expected_activity_core_run_id": expected_run_id,
|
||||
"idempotency_key": detail.get("idempotency_key"),
|
||||
"detail_keys": sorted(detail.keys()),
|
||||
}))
|
||||
raise SystemExit(0)
|
||||
time.sleep(interval)
|
||||
|
||||
raise SystemExit(f"no ops_inventory_probe progress found after {timeout}s")
|
||||
raise SystemExit(f"no ops_inventory_probe progress for manual run {expected_run_id} found after {timeout}s")
|
||||
PY
|
||||
)"
|
||||
export PROGRESS_JSON
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue