Enforce readiness tiers and reconcile blocked workplans

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a0e75a-fc5c-7913-9dba-9846210c766d

src/ops_mason/cli.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from pathlib import Path
 from collections.abc import Sequence
 
 from ops_mason.kubernetes_plane import (
@@ -26,6 +27,8 @@ def _parser() -> argparse.ArgumentParser:
     for name in ("render", "preflight", "verify", "rollback-plan"):
         command = commands.add_parser(name)
         command.add_argument("--bundle", required=True)
+        if name == "preflight":
+            command.add_argument("--readiness-repo", type=Path)
 
     apply_parser = commands.add_parser("apply")
     apply_parser.add_argument("--bundle", required=True)
@@ -33,6 +36,9 @@ def _parser() -> argparse.ArgumentParser:
     apply_parser.add_argument(
         "--expect-digest", required=True, help="exact digest returned by preflight"
     )
+    apply_parser.add_argument("--readiness-repo", type=Path)
+    apply_parser.add_argument("--activation", choices=("APPROVED", "BREAK_GLASS"), default="APPROVED")
+    apply_parser.add_argument("--break-glass-reason")
     return parser
 
 
@@ -43,7 +49,7 @@ def main(argv: Sequence[str] | None = None) -> int:
         if args.command == "render":
             result = bundle.render()
         elif args.command == "preflight":
-            result = preflight(bundle)
+            result = preflight(bundle, readiness_repo=args.readiness_repo)
         elif args.command == "verify":
             result = verify(bundle)
         elif args.command == "rollback-plan":
@@ -53,6 +59,9 @@ def main(argv: Sequence[str] | None = None) -> int:
                 bundle,
                 confirm_plan_id=args.confirm,
                 expected_digest=args.expect_digest,
+                readiness_repo=args.readiness_repo,
+                activation=args.activation,
+                break_glass_reason=args.break_glass_reason,
             )
         else:  # pragma: no cover - argparse enforces the command set
             raise AssertionError(args.command)

src/ops_mason/kubernetes_plane.py

@@ -8,18 +8,20 @@ and records metadata-only evidence.
 
 from __future__ import annotations
 
+import getpass
 import hashlib
 import json
 import subprocess
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import asdict, dataclass
-from datetime import UTC, datetime
+from datetime import UTC, date, datetime
 from pathlib import Path
 from typing import Any
 
 import yaml
 
 from ops_mason.plan import ConstructionPlan
+from ops_mason.readiness import inspect_readiness
 
 
 class PlaneError(RuntimeError):
@@ -93,6 +95,7 @@ class PlaneBundle:
     dependencies: tuple[Dependency, ...]
     evidence_path: Path
     documents: tuple[dict[str, Any], ...]
+    readiness: dict[str, Any] | None = None
 
     @classmethod
     def load(cls, path: str | Path) -> "PlaneBundle":
@@ -172,6 +175,7 @@ class PlaneBundle:
             dependencies=dependencies,
             evidence_path=local_path(str(raw["evidence_path"])),
             documents=tuple(documents),
+            readiness=raw.get("readiness"),
         )
         bundle.validate()
         return bundle
@@ -185,6 +189,12 @@ class PlaneBundle:
         return digest.hexdigest()
 
     def validate(self) -> None:
+        if self.readiness is not None and (
+            not isinstance(self.readiness, dict)
+            or not isinstance(self.readiness.get("target"), dict)
+            or not isinstance(self.readiness.get("source"), dict)
+        ):
+            raise PlaneError("readiness needs target and source mappings")
         actual_refs = tuple(_document_ref(doc, self.allowed_objects) for doc in self.documents)
         if len(set(actual_refs)) != len(actual_refs):
             raise PlaneRefused("bundle contains duplicate Kubernetes object identities")
@@ -233,6 +243,7 @@ class PlaneBundle:
             "source_revision": self.source_revision,
             "implementation_revision": self.implementation_revision,
             "expected_context": self.expected_context,
+            "readiness": self.readiness,
             "objects": [asdict(ref) | {"display": ref.display} for ref in self.allowed_objects],
             "forbidden_kinds": sorted(self.forbidden_kinds),
             "plan_id": self.plan().id,
@@ -363,7 +374,10 @@ def _check_inputs_clean(bundle: PlaneBundle, runner: Runner) -> None:
         raise PlaneRefused("repository must be committed and clean before Kubernetes mutation")
 
 
-def preflight(bundle: PlaneBundle, runner: Runner = subprocess_runner) -> dict[str, Any]:
+def preflight(
+    bundle: PlaneBundle, runner: Runner = subprocess_runner, *,
+    readiness_repo: Path | None = None, activation: str = "APPROVED", today: date | None = None,
+) -> dict[str, Any]:
     context = _run(runner, ["kubectl", "config", "current-context"]).stdout.strip()
     if context != bundle.expected_context:
         raise PlaneRefused(
@@ -464,6 +478,7 @@ def preflight(bundle: PlaneBundle, runner: Runner = subprocess_runner) -> dict[s
             if str(item.path.relative_to(bundle.repo_root)) not in server_validated
         ],
         "dependencies": dependency_evidence,
+        "readiness": inspect_readiness(bundle, runner, readiness_repo, activation, today),
     }
 
 
@@ -489,9 +504,9 @@ def verify(bundle: PlaneBundle, runner: Runner = subprocess_runner) -> dict[str,
     for resource in ("pods", "secrets"):
         result = _run(
             runner,
-            ["kubectl", "-n", bundle.expected_namespace, "get", resource, "-o", "json"],
+            ["kubectl", "-n", bundle.expected_namespace, "get", resource, "-o", "name"],
         )
-        count = len(json.loads(result.stdout).get("items", []))
+        count = len(result.stdout.splitlines())
         if count:
             raise PlaneError(
                 f"negative-scope check failed: {count} {resource} exist in "
@@ -538,6 +553,10 @@ def apply(
     confirm_plan_id: str,
     expected_digest: str,
     runner: Runner = subprocess_runner,
+    readiness_repo: Path | None = None,
+    activation: str = "APPROVED",
+    break_glass_reason: str | None = None,
+    today: date | None = None,
 ) -> dict[str, Any]:
     plan = bundle.plan()
     if not plan.is_approved():
@@ -552,8 +571,17 @@ def apply(
         raise PlaneRefused(
             f"bundle digest confirmation mismatch: expected {bundle.digest}"
         )
+    if activation not in {"APPROVED", "BREAK_GLASS"}:
+        raise PlaneRefused("unknown activation")
+    if activation == "BREAK_GLASS" and not (break_glass_reason or "").strip():
+        raise PlaneRefused("BREAK_GLASS requires a non-empty reason")
     _check_inputs_clean(bundle, runner)
-    before = preflight(bundle, runner)
+    readiness = inspect_readiness(bundle, runner, readiness_repo, activation, today)
+    if not readiness["direct_apply_allowed"]:
+        raise PlaneRefused(f"production tier requires GitOps or BREAK_GLASS: {readiness['reason']}")
+    before = preflight(bundle, runner, readiness_repo=readiness_repo, activation=activation, today=today)
+    if not before["readiness"]["direct_apply_allowed"]:
+        raise PlaneRefused("readiness changed during preflight; refusing mutation")
 
     server_validated = list(before["server_validated_manifests"])
     persisted: list[str] = []
@@ -607,6 +635,14 @@ def apply(
             "server_validated_manifests": server_validated,
             "persisted_manifests": persisted,
         },
+        "readiness": before["readiness"],
+        "activation": activation,
+        "break_glass": {
+            "reason": break_glass_reason.strip(),
+            "actor": getpass.getuser(),
+            "recorded_at": datetime.now(UTC).isoformat(),
+            "follow_up": "Commit the same change to the manifest repository that ArgoCD reconciles.",
+        } if activation == "BREAK_GLASS" else None,
         "preflight": before,
         "verification": verified,
         "rollback": rollback_plan(bundle),

src/ops_mason/readiness.py

@@ -0,0 +1,145 @@
+"""Readiness quality gate; no credential or Kubernetes access."""
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+from datetime import date
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+TRANSITION_END = date(2026, 12, 21)
+NON_PRODUCTION = {"declared", "installed", "verified"}
+
+
+def resolve_tier(
+    state: str | None, target: dict[str, Any], activation: str, today: date,
+) -> dict[str, Any]:
+    """Resolve already verified source facts. Unknown facts stay production."""
+    tier = "non-production" if state in NON_PRODUCTION or state == "explicit-whitehat" else "production"
+    transition = (
+        tier == "production" and state == "production-approved"
+        and target.get("kind") == "rapp"
+        and target.get("rapp_id") == "rapp-policy-nexus"
+        and today < TRANSITION_END and activation == "APPROVED"
+    )
+    return {
+        "tier": tier,
+        "direct_apply_allowed": activation in {"APPROVED", "BREAK_GLASS"}
+        and (tier == "non-production" or activation == "BREAK_GLASS" or transition),
+        "transition": transition,
+    }
+
+
+def inspect_readiness(
+    bundle, runner, repo: Path | None, activation: str = "APPROVED",
+    today: date | None = None,
+) -> dict[str, Any]:
+    """Verify pinned source, local freshness and history before trusting a tier.
+
+    A reviewed bundle supplies the namespace-to-rApp mapping. It must identify
+    the namespace explicitly; source mappings, when present, must agree.
+    """
+    today = today or date.today()
+    block = bundle.readiness or {}
+    target = block.get("target", {})
+    source = block.get("source", {})
+    evidence: dict[str, Any] = {
+        "target": target, "source": source, "activation": activation,
+        "state": None, "reason": "missing or unverifiable readiness",
+    }
+
+    def finish(state=None, reason="unverifiable readiness"):
+        evidence.update(state=state, reason=reason)
+        evidence.update(resolve_tier(state, target, activation, today))
+        return evidence
+
+    if not block or target.get("namespace") != bundle.expected_namespace:
+        return finish(reason="missing readiness or namespace mapping")
+    # A namespace placement never covers other cluster-scoped objects.
+    if any(not ref.namespace and (ref.kind != "Namespace" or ref.name != bundle.expected_namespace)
+           for ref in bundle.allowed_objects):
+        return finish(reason="target includes objects outside the namespace mapping")
+    if target.get("kind") not in {"rapp", "namespace"}:
+        return finish(reason="unmapped platform target")
+    if source.get("repo") != "reef-railiance" or source.get("path") != "bindings/rapps.yaml":
+        return finish(reason="unsupported readiness source")
+    revision = source.get("revision", "")
+    digest = source.get("sha256", "")
+    if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision):
+        return finish(reason="source needs a full commit id")
+    if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest):
+        return finish(reason="source needs a SHA-256 digest")
+    root = repo or bundle.repo_root.parent / "reef-railiance"
+    prefix = ["git", "-C", str(root)]
+    path = source["path"]
+
+    def git(*args):
+        result = runner([*prefix, *args])
+        if result.returncode:
+            raise ValueError("readiness git verification failed")
+        return result.stdout
+
+    def rows(content):
+        data = yaml.safe_load(content)
+        if not isinstance(data, dict) or not isinstance(data.get("bound_rapps"), list):
+            raise ValueError("invalid readiness document")
+        result = data["bound_rapps"]
+        if any(not isinstance(row, dict) or not isinstance(row.get("rapp_id"), str) for row in result):
+            raise ValueError("invalid readiness binding")
+        if len({row["rapp_id"] for row in result}) != len(result):
+            raise ValueError("duplicate readiness binding")
+        return result
+
+    try:
+        if git("rev-parse", "--is-shallow-repository").strip() != "false":
+            return finish(reason="complete readiness history is required")
+        content = git("show", f"{revision}:{path}")
+        if hashlib.sha256(content.encode()).hexdigest() != digest:
+            return finish(reason="readiness digest mismatch")
+        git("merge-base", "--is-ancestor", revision, "HEAD")
+        git("diff", "--exit-code", revision, "HEAD", "--", path)
+        if git("status", "--porcelain", "--", path).strip():
+            return finish(reason="readiness source has uncommitted changes")
+        bindings = rows(content)
+        if target["kind"] == "namespace":
+            # Only the founder's one named placement is compiled into this gate.
+            if bundle.id != "whitehat-foundational-plane" or bundle.expected_namespace != "whitehat":
+                return finish(reason="no explicit tier placement for target")
+            if any(row.get("namespace") == "whitehat" or "whitehat" in row.get("namespaces", [])
+                   or row["rapp_id"] == "rapp-whitehat" for row in bindings):
+                return finish(reason="whitehat has a binding; repin using the binding target")
+            return finish("explicit-whitehat", "founder placement 2026-09-21")
+        rapp_id = target.get("rapp_id")
+        matches = [row for row in bindings if row["rapp_id"] == rapp_id]
+        if len(matches) != 1:
+            return finish(reason="rApp target is not listed")
+        row = matches[0]
+        if (row.get("namespace") and row["namespace"] != bundle.expected_namespace) or (
+            "namespaces" in row and bundle.expected_namespace not in row["namespaces"]
+        ):
+            return finish(reason="source namespace mapping disagrees with bundle")
+        state = row.get("readiness_state")
+        # Scan all reachable history, including intervening promotions later
+        # reverted. An evidence lapse must never silently lower the tier.
+        history = git("log", "--format=%H", "HEAD", "--", path).splitlines()
+        if not history:
+            return finish(reason="readiness history is missing")
+        previous = []
+        for commit in history:
+            if not re.fullmatch(r"[0-9a-f]{40}", commit):
+                return finish(reason="invalid readiness history")
+            for old in rows(git("show", f"{commit}:{path}")):
+                if old["rapp_id"] == rapp_id:
+                    previous.append(old.get("readiness_state"))
+        if "production-approved" in previous:
+            return finish("production-approved", "production tier retained from binding history")
+        if state == "deprecated":
+            state = next((s for s in previous if s != "deprecated"), None)
+        if state not in NON_PRODUCTION | {"production-approved"}:
+            return finish(reason="unknown readiness state or prior tier")
+        return finish(state, "verified pinned binding and history")
+    except (OSError, ValueError, TypeError, subprocess.TimeoutExpired, yaml.YAMLError):
+        return finish(reason="readiness source or history unavailable or invalid")