Bind multiple queue worker identities, one token each (WP-0039-T01)

ACTIVITY_CORE_WORKERS maps worker_id=ENV_NAME, where each token env must be
ACTIVITY_CORE_WORKER_TOKEN[_SUFFIX]. Without the map, the legacy single pair
behaves exactly as before. Duplicate identities, missing or shared tokens, a
token equal to the operator token, and an unlisted legacy identity all fail
worker mutations closed with 503. Operator/SSO reads keep working.

Declare per-identity OpenBao paths and an ExternalSecret, not yet applied.
The policy, seeding and rollout are waiting tasks T02-T04, answering
secrets-engine SECRETS-WP-0009-T03 and SECRETS-WP-0011-T04.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 151606@bnt-lap001
Assistant-Session: 3c0a4ad5-bb8b-4bf7-b9f0-fa5f29204e48

src/activity_core/ops_runs_api.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import hmac
 import os
+import re
 import uuid
 from collections.abc import Callable
 from datetime import datetime
@@ -38,6 +39,8 @@ _get_db: Callable[[], async_sessionmaker[AsyncSession]] | None = None
 
 WORKER_TOKEN_ENV = "ACTIVITY_CORE_WORKER_TOKEN"
 WORKER_ID_ENV = "ACTIVITY_CORE_WORKER_ID"
+WORKERS_ENV = "ACTIVITY_CORE_WORKERS"
+_WORKER_TOKEN_ENV_RE = re.compile(r"ACTIVITY_CORE_WORKER_TOKEN(?:_[A-Z0-9]+)*")
 
 
 def bind_ops_runs_deps(
@@ -54,6 +57,8 @@ def _db() -> async_sessionmaker[AsyncSession]:
 
 
 def _worker_token_configured() -> bool:
+    if (os.environ.get(WORKERS_ENV) or "").strip():
+        return True
     return bool((os.environ.get(WORKER_TOKEN_ENV) or "").strip())
 
 
@@ -62,6 +67,77 @@ def configured_worker_id() -> str | None:
     return value or None
 
 
+class WorkerConfigError(Exception):
+    """Worker credential configuration is incomplete or ambiguous."""
+
+
+def worker_credentials() -> list[tuple[str, str]]:
+    """Return ``(worker_id, token)`` pairs, each token bound to one identity.
+
+    ``ACTIVITY_CORE_WORKERS`` is a non-secret, comma-separated list of
+    ``worker_id=ENV_NAME`` entries. Each ENV_NAME must be
+    ``ACTIVITY_CORE_WORKER_TOKEN`` or ``ACTIVITY_CORE_WORKER_TOKEN_<SUFFIX>`` and
+    hold that worker's token. Without it, the legacy single pair
+    ``ACTIVITY_CORE_WORKER_ID`` + ``ACTIVITY_CORE_WORKER_TOKEN`` applies.
+    Duplicate identities, missing or shared tokens, and a token equal to the
+    operator token raise ``WorkerConfigError`` so callers fail closed.
+    """
+    raw = (os.environ.get(WORKERS_ENV) or "").strip()
+    if not raw:
+        token = (os.environ.get(WORKER_TOKEN_ENV) or "").strip()
+        if not token:
+            return []
+        worker_id = configured_worker_id()
+        if not worker_id:
+            raise WorkerConfigError(f"worker identity not configured ({WORKER_ID_ENV})")
+        pairs = [(worker_id, token)]
+    else:
+        pairs = []
+        for entry in raw.split(","):
+            entry = entry.strip()
+            if not entry:
+                continue
+            worker_id, sep, env_name = (part.strip() for part in entry.partition("="))
+            if not sep or not worker_id or _WORKER_TOKEN_ENV_RE.fullmatch(env_name) is None:
+                raise WorkerConfigError(f"invalid {WORKERS_ENV} entry")
+            token = (os.environ.get(env_name) or "").strip()
+            if not token:
+                raise WorkerConfigError(f"worker token not configured ({env_name})")
+            pairs.append((worker_id, token))
+        if not pairs:
+            raise WorkerConfigError(f"{WORKERS_ENV} lists no workers")
+        legacy_id = configured_worker_id()
+        if legacy_id and legacy_id not in {worker_id for worker_id, _ in pairs}:
+            raise WorkerConfigError(f"{WORKER_ID_ENV} is not listed in {WORKERS_ENV}")
+
+    ids = [worker_id for worker_id, _ in pairs]
+    tokens = [token for _, token in pairs]
+    if len(set(ids)) != len(ids):
+        raise WorkerConfigError("duplicate worker identity")
+    if len(set(tokens)) != len(tokens):
+        raise WorkerConfigError("worker tokens must not be shared")
+    operator = (os.environ.get("ACTIVITY_CORE_OPERATOR_TOKEN") or "").strip()
+    if operator and operator in tokens:
+        raise WorkerConfigError("worker token must differ from the operator token")
+    return pairs
+
+
+def _authenticate_worker_token(token: str | None) -> str | None:
+    """Return the identity bound to ``token``; raise 503 on bad configuration."""
+    try:
+        pairs = worker_credentials()
+    except WorkerConfigError as exc:
+        raise HTTPException(status_code=503, detail=str(exc)) from exc
+    if not token:
+        return None
+    matched = None
+    for worker_id, expected in pairs:
+        # Compare against every entry so timing does not reveal the position.
+        if hmac.compare_digest(token, expected):
+            matched = worker_id
+    return matched
+
+
 def _extract_worker_token(
     *,
     x_worker_token: str | None,
@@ -85,18 +161,15 @@ def require_worker_or_operator(
     worker_tok = _extract_worker_token(
         x_worker_token=x_worker_token, authorization=authorization
     )
-    expected_worker = (os.environ.get(WORKER_TOKEN_ENV) or "").strip()
-    if (
-        expected_worker
-        and worker_tok
-        and hmac.compare_digest(worker_tok, expected_worker)
-    ):
-        worker_id = configured_worker_id()
-        if not worker_id:
-            raise HTTPException(
-                status_code=503,
-                detail=f"worker identity not configured ({WORKER_ID_ENV})",
-            )
+    worker_id = None
+    if worker_tok:
+        try:
+            worker_id = _authenticate_worker_token(worker_tok)
+        except HTTPException:
+            # Broken worker config must not lock out operator/SSO reads;
+            # worker-only mutations still fail closed in require_worker.
+            worker_id = None
+    if worker_id:
         return f"worker:{worker_id}"
 
     sso = extract_sso_principal(request)
@@ -125,22 +198,16 @@ def require_worker(
     x_worker_token: str | None = None,
     authorization: str | None = None,
 ) -> str:
-    """Return the configured worker identity for a valid worker credential."""
+    """Return the identity bound to a valid worker credential."""
     del request  # reserved for future mTLS/proxy-bound worker principals
-    expected_worker = (os.environ.get(WORKER_TOKEN_ENV) or "").strip()
     worker_tok = _extract_worker_token(
         x_worker_token=x_worker_token, authorization=authorization
     )
 
-    if expected_worker:
-        if not worker_tok or not hmac.compare_digest(worker_tok, expected_worker):
-            raise HTTPException(status_code=401, detail="worker auth required")
-        worker_id = configured_worker_id()
+    if _worker_token_configured():
+        worker_id = _authenticate_worker_token(worker_tok)
         if not worker_id:
-            raise HTTPException(
-                status_code=503,
-                detail=f"worker identity not configured ({WORKER_ID_ENV})",
-            )
+            raise HTTPException(status_code=401, detail="worker auth required")
         return worker_id
 
     if allow_unauth_mutations() and not operator_token_configured():