tegwick
db56d8468e
operations.py: OpenBao KV v2 CAS, LLDAP self-rotation (forward and revert), ExternalSecret force-sync and status, rollout restart (RolloutFailed is a definite failure), pilot checks. synthetic_pilot.py on the T08 stack plus OpenBao 2.5.4 with CCR-2026-0058's policy: rotation completed; a one-off wrong delivery reverted in declared order with the earlier value restored. Runner: revert steps run in declared order (BEA-10), definite failures revert, unknown outcomes stop, receipts carry error types. Found and corrected (in railiance-platform) CCR-2026-0058's required_parameters refusing reads. Packaging moved to new T13. 29 executor tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 994874@bnt-lap001 Assistant-Session: 6017c416-d619-47eb-bd1a-f670a02d7d87
332 lines
15 KiB
Python
332 lines
15 KiB
Python
"""Pilot operations and checks for the bounded executor (NK-WP-0056-T12).
|
|
|
|
The T06 pilot rotates one directory bind (svc-authelia) under one admission:
|
|
|
|
openbao.kv.write-next-version read version N, write a fresh value as N+1 with cas=N
|
|
openbao.kv.write-previous-value-as-next-version revert: write the previous value as the next version (CAS)
|
|
lldap.user.change-own-password-from-kv set the bind's LLDAP password to the current KV value,
|
|
signing in as the bind with a value it still accepts
|
|
(self-rotation, founder decision 3d0c8f86); forward and revert
|
|
eso.force-sync ask the delivery to re-project the value (ExternalSecret)
|
|
k8s.rollout-restart restart the consumer and wait for the rollout
|
|
|
|
Checks: kv-advanced, bind-signs-in, eso-synced, synthetic-journey.
|
|
|
|
Values live only in Context.data (memory) and in the requests that carry them; nothing here prints, logs or
|
|
returns a value. Every adapter is stdlib HTTP so the image needs no client libraries.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import secrets
|
|
import ssl
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from typing import Callable
|
|
|
|
HERE = pathlib.Path(__file__).resolve().parent
|
|
REPO = HERE.parent
|
|
_spec = importlib.util.spec_from_file_location("journey", REPO / "sso-mfa/k8s/synthetic-journey/journey.py")
|
|
journey = importlib.util.module_from_spec(_spec)
|
|
_spec.loader.exec_module(journey)
|
|
|
|
FIELD = "LDAP_BIND_PASSWORD"
|
|
|
|
|
|
class OperationError(Exception):
|
|
"""A definite failure: the operation did not take effect."""
|
|
|
|
|
|
class RolloutFailed(Exception):
|
|
"""The restart was issued but the consumer never became ready: a definite, known-bad outcome that the
|
|
executor reverts (executor.StepFailed semantics)."""
|
|
definite_failure = True
|
|
|
|
|
|
def _opener(ca_file: str | None):
|
|
context = ssl.create_default_context(cafile=ca_file) if ca_file else ssl.create_default_context()
|
|
return urllib.request.build_opener(urllib.request.ProxyHandler({}), urllib.request.HTTPSHandler(context=context))
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------------
|
|
# OpenBao KV v2
|
|
# --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
class OpenBao:
|
|
def __init__(self, addr: str, *, token: str | None = None, k8s_role: str | None = None,
|
|
jwt_file: str | None = None, k8s_mount: str = "kubernetes", ca_file: str | None = None,
|
|
timeout: float = 15):
|
|
self.addr = addr.rstrip("/")
|
|
self.open = _opener(ca_file).open
|
|
self.timeout = timeout
|
|
self.token = token
|
|
if self.token is None:
|
|
jwt = pathlib.Path(jwt_file).read_text().strip()
|
|
auth = self._call("POST", f"/v1/auth/{k8s_mount}/login", {"role": k8s_role, "jwt": jwt}, auth=False)
|
|
self.token = auth["auth"]["client_token"]
|
|
|
|
def _call(self, method: str, path: str, body: dict | None = None, auth: bool = True) -> dict:
|
|
headers = {"Content-Type": "application/json"}
|
|
if auth:
|
|
headers["X-Vault-Token"] = self.token
|
|
request = urllib.request.Request(self.addr + path, method=method, headers=headers,
|
|
data=json.dumps(body).encode() if body is not None else None)
|
|
try:
|
|
with self.open(request, timeout=self.timeout) as response:
|
|
raw = response.read()
|
|
except urllib.error.HTTPError as error:
|
|
error.close()
|
|
raise OperationError(f"openbao {method} {error.code}") from None
|
|
return json.loads(raw) if raw else {}
|
|
|
|
@staticmethod
|
|
def _api(kv_path: str) -> str:
|
|
mount, _, rest = kv_path.partition("/")
|
|
return f"/v1/{mount}/data/{rest}"
|
|
|
|
def read(self, kv_path: str) -> tuple[dict, int]:
|
|
body = self._call("GET", self._api(kv_path))["data"]
|
|
return body["data"], int(body["metadata"]["version"])
|
|
|
|
def write(self, kv_path: str, data: dict, cas: int) -> int:
|
|
return int(self._call("POST", self._api(kv_path), {"options": {"cas": cas}, "data": data})["data"]["version"])
|
|
|
|
def revoke_self(self) -> None:
|
|
try:
|
|
self._call("POST", "/v1/auth/token/revoke-self", {})
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------------
|
|
# LLDAP
|
|
# --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
class Directory:
|
|
"""Sign-in over LLDAP's HTTP API; password changes through the pinned image's lldap_set_password (OPAQUE)."""
|
|
|
|
def __init__(self, base_url: str, set_password_command: list[str], *, ca_file: str | None = None,
|
|
tool_base_url: str | None = None, timeout: float = 15):
|
|
self.base = base_url.rstrip("/")
|
|
self.tool_base = (tool_base_url or base_url).rstrip("/")
|
|
self.command = set_password_command
|
|
self.open = _opener(ca_file).open
|
|
self.timeout = timeout
|
|
|
|
def login(self, user: str, password: str) -> str | None:
|
|
request = urllib.request.Request(self.base + "/auth/simple/login", method="POST",
|
|
headers={"Content-Type": "application/json"},
|
|
data=json.dumps({"username": user, "password": password}).encode())
|
|
try:
|
|
with self.open(request, timeout=self.timeout) as response:
|
|
return json.load(response).get("token")
|
|
except urllib.error.HTTPError as error:
|
|
error.close()
|
|
return None
|
|
|
|
def set_own_password(self, token: str, user: str, password: str) -> None:
|
|
done = subprocess.run([*self.command, "--base-url", self.tool_base, "--token", token, "--username", user],
|
|
env={"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "LLDAP_USER_PASSWORD": password},
|
|
capture_output=True, timeout=60, check=False)
|
|
if done.returncode:
|
|
raise OperationError("lldap_set_password refused")
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------------
|
|
# Kubernetes (stdlib client, in-cluster ServiceAccount)
|
|
# --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
class Kube:
|
|
def __init__(self, api: str = "https://kubernetes.default.svc",
|
|
token_file: str = "/var/run/secrets/kubernetes.io/serviceaccount/token",
|
|
ca_file: str | None = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", timeout: float = 15):
|
|
self.api = api.rstrip("/")
|
|
self.token_file = token_file
|
|
self.open = _opener(ca_file).open
|
|
self.timeout = timeout
|
|
|
|
def _call(self, method: str, path: str, body: dict | None = None) -> dict:
|
|
headers = {"Authorization": "Bearer " + pathlib.Path(self.token_file).read_text().strip(),
|
|
"Accept": "application/json"}
|
|
data = None
|
|
if body is not None:
|
|
headers["Content-Type"] = "application/merge-patch+json"
|
|
data = json.dumps(body).encode()
|
|
request = urllib.request.Request(self.api + path, method=method, headers=headers, data=data)
|
|
try:
|
|
with self.open(request, timeout=self.timeout) as response:
|
|
return json.load(response)
|
|
except urllib.error.HTTPError as error:
|
|
error.close()
|
|
raise OperationError(f"kube {method} {error.code}") from None
|
|
|
|
def get(self, path: str) -> dict:
|
|
return self._call("GET", path)
|
|
|
|
def patch(self, path: str, body: dict) -> dict:
|
|
return self._call("PATCH", path, body)
|
|
|
|
|
|
def now_utc() -> dt.datetime:
|
|
return dt.datetime.now(dt.UTC).replace(microsecond=0)
|
|
|
|
|
|
class KubeDelivery:
|
|
"""eso.force-sync and its eso-synced check, through the ExternalSecret API (no Secret read)."""
|
|
|
|
def __init__(self, kube: Kube, namespace: str, external_secret: str, clock: Callable[[], dt.datetime] = now_utc):
|
|
self.kube, self.clock = kube, clock
|
|
self.path = f"/apis/external-secrets.io/v1/namespaces/{namespace}/externalsecrets/{external_secret}"
|
|
|
|
def force_sync(self) -> dt.datetime:
|
|
at = self.clock()
|
|
self.kube.patch(self.path, {"metadata": {"annotations": {"force-sync": str(int(at.timestamp()))}}})
|
|
return at
|
|
|
|
def synced(self, since: dt.datetime) -> bool:
|
|
status = self.kube.get(self.path).get("status", {})
|
|
ready = any(c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions", []))
|
|
refreshed = status.get("refreshTime")
|
|
return ready and bool(refreshed) and dt.datetime.fromisoformat(refreshed.replace("Z", "+00:00")) >= since
|
|
|
|
|
|
class KubeRestart:
|
|
"""k8s.rollout-restart: patch the restartedAt annotation, then wait until the rollout is complete."""
|
|
|
|
def __init__(self, kube: Kube, namespace: str, deployment: str, timeout: float = 300, poll: float = 3,
|
|
clock: Callable[[], dt.datetime] = now_utc, sleep: Callable[[float], None] = time.sleep,
|
|
monotonic: Callable[[], float] = time.monotonic):
|
|
self.kube, self.timeout, self.poll, self.clock, self.sleep, self.monotonic = kube, timeout, poll, clock, sleep, monotonic
|
|
self.path = f"/apis/apps/v1/namespaces/{namespace}/deployments/{deployment}"
|
|
|
|
def restart(self) -> None:
|
|
stamp = self.clock().isoformat().replace("+00:00", "Z")
|
|
self.kube.patch(self.path, {"spec": {"template": {"metadata": {"annotations": {
|
|
"kubectl.kubernetes.io/restartedAt": stamp}}}}})
|
|
deadline = self.monotonic() + self.timeout
|
|
while self.monotonic() < deadline:
|
|
d = self.kube.get(self.path)
|
|
spec, status = d.get("spec", {}), d.get("status", {})
|
|
want = spec.get("replicas", 1)
|
|
if (status.get("observedGeneration", 0) >= d.get("metadata", {}).get("generation", 0)
|
|
and status.get("updatedReplicas") == want and status.get("availableReplicas") == want
|
|
and status.get("replicas") == want and not status.get("unavailableReplicas")):
|
|
return
|
|
self.sleep(self.poll)
|
|
raise RolloutFailed("rollout did not complete") # known-bad: the runner reverts
|
|
|
|
|
|
# --------------------------------------------------------------------------------------------------
|
|
# The pilot registry
|
|
# --------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def pilot_registry(*, bao: OpenBao, directory: Directory, delivery, restart, bind_kv: str, bind_user: str,
|
|
journey_kv: str, journey_config, journey_state: dict | None = None):
|
|
"""Return (operations, checks) for the bind-rotation admission. `delivery` has force_sync()/synced(since);
|
|
`restart` has restart(). Values stay in ctx.data. `journey_state` carries the last one-time-code time step
|
|
across runs (privacyIDEA accepts each 30 s step once)."""
|
|
shared = journey_state if journey_state is not None else {}
|
|
|
|
def kv_target(step: dict) -> str:
|
|
target = step["target"]
|
|
if not target.startswith("openbao:") or target[len("openbao:"):] != bind_kv:
|
|
raise OperationError("target outside the configured bind")
|
|
return bind_kv
|
|
|
|
def generate(step, ctx):
|
|
path = kv_target(step)
|
|
data, version = bao.read(path)
|
|
if set(data) != {FIELD}:
|
|
raise OperationError("kv shape")
|
|
fresh = secrets.token_urlsafe(32)
|
|
ctx.data.update(previous=data[FIELD], fresh=fresh)
|
|
ctx.data["expect"] = (bao.write(path, {FIELD: fresh}, cas=version), "fresh")
|
|
|
|
def restore_kv(step, ctx):
|
|
path = kv_target(step)
|
|
_, version = bao.read(path)
|
|
ctx.data["expect"] = (bao.write(path, {FIELD: ctx.data["previous"]}, cas=version), "previous")
|
|
|
|
def change_own_password(step, ctx):
|
|
if step["target"] != "lldap:" + bind_user:
|
|
raise OperationError("target outside the configured bind")
|
|
want = bao.read(bind_kv)[0][FIELD]
|
|
if directory.login(bind_user, want):
|
|
return # already in effect
|
|
for known in (ctx.data.get("previous"), ctx.data.get("fresh")):
|
|
if known and known != want:
|
|
token = directory.login(bind_user, known)
|
|
if token:
|
|
directory.set_own_password(token, bind_user, want)
|
|
if not directory.login(bind_user, want):
|
|
raise OperationError("password change not effective")
|
|
return
|
|
raise OperationError("no known value signs in as the bind")
|
|
|
|
def force_sync(step, ctx):
|
|
ctx.data["delivered_at"] = delivery.force_sync()
|
|
|
|
def rollout_restart(step, ctx):
|
|
restart.restart()
|
|
|
|
def check_kv_advanced(check, ctx):
|
|
version, which = ctx.data["expect"]
|
|
data, current = bao.read(bind_kv)
|
|
return current == version and data.get(FIELD) == ctx.data[which]
|
|
|
|
def check_bind_signs_in(check, ctx):
|
|
return directory.login(bind_user, bao.read(bind_kv)[0][FIELD]) is not None
|
|
|
|
def check_eso_synced(check, ctx):
|
|
deadline = time.monotonic() + check["timeout_seconds"] * 0.8
|
|
while True:
|
|
if delivery.synced(ctx.data["delivered_at"]):
|
|
return True
|
|
if time.monotonic() > deadline:
|
|
return False
|
|
time.sleep(2)
|
|
|
|
def check_journey(check, ctx):
|
|
creds, _ = bao.read(journey_kv)
|
|
cfg = _load_config(journey_config() if callable(journey_config) else journey_config)
|
|
last = shared.get("journey_step")
|
|
receipt = journey.run_journey(cfg, creds["PASSWORD"], journey.decode_seed(creds["TOTP_SEED"]),
|
|
min_time_step=None if last is None else last + 1)
|
|
if receipt.get("otp_time_step") is not None:
|
|
shared["journey_step"] = receipt["otp_time_step"]
|
|
ctx.data.setdefault("journeys", []).append({k: receipt.get(k) for k in ("result", "failed_step", "reason")})
|
|
return receipt["result"] == "pass"
|
|
|
|
operations = {
|
|
"openbao.kv.write-next-version": generate,
|
|
"openbao.kv.write-previous-value-as-next-version": restore_kv,
|
|
"lldap.user.change-own-password-from-kv": change_own_password,
|
|
"eso.force-sync": force_sync,
|
|
"k8s.rollout-restart": rollout_restart,
|
|
}
|
|
checks = {"kv-advanced": check_kv_advanced, "bind-signs-in": check_bind_signs_in,
|
|
"eso-synced": check_eso_synced, "synthetic-journey": check_journey}
|
|
return operations, checks
|
|
|
|
|
|
def _load_config(config: dict) -> dict:
|
|
import tempfile
|
|
fd, path = tempfile.mkstemp(suffix=".json", prefix="journey-")
|
|
try:
|
|
with os.fdopen(fd, "w") as out:
|
|
json.dump(config, out)
|
|
return journey.load_config(path)
|
|
finally:
|
|
os.unlink(path)
|