diff --git a/argocd/platform-addons/secret-annotation-guard/README.md b/argocd/platform-addons/secret-annotation-guard/README.md index 8179d7d..d4fe9c0 100644 --- a/argocd/platform-addons/secret-annotation-guard/README.md +++ b/argocd/platform-addons/secret-annotation-guard/README.md @@ -21,3 +21,36 @@ Rollback is a manual Argo sync of a reviewed revision without the binding (with explicit resource-scoped pruning), or attended break-glass deletion of the binding followed by Git reconciliation. Removing the binding reopens this leak path. The policy does not rotate credentials or isolate agent accounts. + +## September 28 rollout and rollback + +Policy source `800cbfa` and Application declaration `54885ac` were deployed +through manual, resource-scoped Argo sync. Native type checking passed. Nine +synthetic checks passed: clean creation/update/server apply, rejection of +annotated create/update including empty values, client-side apply rejection, +and fixture cleanup. + +Enforcement was rolled back when ESO v0.16.1 targets stopped refreshing. +That version copies ExternalSecret metadata when no target template exists; +31 current ExternalSecrets have no template. Removing annotations from targets +alone cannot stop the controller adding them back. All 39 ExternalSecrets +recovered after binding deletion; failed controllers were explicitly refreshed. +The policy remains installed but UNBOUND. `binding.pending.yaml` is deliberately +absent from kustomization. No ordinary sync of this revision enables enforcement. + +Before enabling: add explicit metadata templates in the 31 owning declarations, +preserving intended labels/annotations except last-applied; apply through owner +paths; verify actual ESO refresh with annotation-free target Secrets. Retain +existing data templates and remote references. Do not waive ESO from the policy +or upgrade the controller as a shortcut. Rerun cleanup, native admission tests +and an ESO refresh integration test, then include the binding and update the pin. + +The absent `platform-pg-drill` namespace still has an orphan Secret `drill-minio`, +a Deployment referencing it, a PVC and Service. Metadata patch fails because the +namespace is absent. No orphan object was deleted or namespace recreated. Its +disposition stays in CUST-WP-0073-T03; do not report cluster-wide cleanup complete. + +Source for the diagnosed behavior: +https://github.com/external-secrets/external-secrets/blob/v0.16.1/pkg/controllers/externalsecret/externalsecret_controller_template.go + +Detailed receipts: the-custodian/docs/evidence/2026-09-28-secret-annotation-*.json. diff --git a/argocd/platform-addons/secret-annotation-guard/binding.pending.yaml b/argocd/platform-addons/secret-annotation-guard/binding.pending.yaml new file mode 100644 index 0000000..2665523 --- /dev/null +++ b/argocd/platform-addons/secret-annotation-guard/binding.pending.yaml @@ -0,0 +1,8 @@ +# NOT included in kustomization: ESO v0.16.1 propagation must be fixed first. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: reject-secret-last-applied +spec: + policyName: reject-secret-last-applied + validationActions: [Deny] diff --git a/argocd/platform-addons/secret-annotation-guard/policy.yaml b/argocd/platform-addons/secret-annotation-guard/policy.yaml index 22b8200..fda3fc9 100644 --- a/argocd/platform-addons/secret-annotation-guard/policy.yaml +++ b/argocd/platform-addons/secret-annotation-guard/policy.yaml @@ -17,11 +17,3 @@ spec: validations: - expression: '!has(object.metadata.annotations) || !("kubectl.kubernetes.io/last-applied-configuration" in object.metadata.annotations)' message: "Secret last-applied annotations are forbidden; use server-side apply or replace." ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingAdmissionPolicyBinding -metadata: - name: reject-secret-last-applied -spec: - policyName: reject-secret-last-applied - validationActions: [Deny] diff --git a/scripts/prove_secret_annotation_guard.py b/scripts/prove_secret_annotation_guard.py new file mode 100644 index 0000000..35066bf --- /dev/null +++ b/scripts/prove_secret_annotation_guard.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Positive/negative admission proof using only a uniquely named synthetic Secret.""" +import copy +import json +import subprocess +import uuid +from datetime import datetime, timezone + +KEY = "kubectl.kubernetes.io/last-applied-configuration" + + +def run(args, obj=None): + return subprocess.run(["kubectl", "-n", "whitehat", *args], + input=json.dumps(obj) if obj is not None else None, + capture_output=True, text=True, timeout=30) + + +def denied(result): + # Do not accept connectivity/RBAC failures as admission-policy success. + return result.returncode != 0 and "Secret last-applied annotations are forbidden" in result.stderr + + +def main(): + name = "cust-0073-proof-" + uuid.uuid4().hex[:12] + obj = {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, + "type": "Opaque", "data": {"fixture": "c3ludGhldGlj"}} + report = {"captured_at": datetime.now(timezone.utc).isoformat(), "namespace": "whitehat", + "fixture": name, "synthetic_only": True, "checks": {}} + created = False + try: + result = run(["create", "--field-manager=cust-0073-proof", "-f", "-"], obj) + created = result.returncode == 0 + report["checks"]["clean_create_allowed"] = created + if not created: + raise RuntimeError("synthetic create failed") + for label, value in [("empty", ""), ("populated", "synthetic")]: + annotated = copy.deepcopy(obj) + annotated["metadata"]["name"] = name + "-denied" + annotated["metadata"]["annotations"] = {KEY: value} + report["checks"][label + "_annotated_create_denied"] = denied(run(["create", "--dry-run=server", "-f", "-"], annotated)) + patch = {"metadata": {"annotations": {KEY: value}}} + report["checks"][label + "_annotated_update_denied"] = denied(run(["patch", "secret", name, "--dry-run=server", "--type=merge", "-p", json.dumps(patch)])) + report["checks"]["client_apply_denied"] = denied(run(["apply", "--dry-run=server", "-f", "-"], obj)) + report["checks"]["clean_server_apply_allowed"] = run(["apply", "--server-side", "--field-manager=cust-0073-proof", "-f", "-"], obj).returncode == 0 + report["checks"]["clean_update_allowed"] = run(["patch", "secret", name, "--type=merge", "-p", json.dumps({"data": {"fixture": "c3ludGhldGljLXVwZGF0ZQ=="}})]).returncode == 0 + except (RuntimeError, subprocess.SubprocessError, OSError): + report["error"] = "proof incomplete; raw output suppressed" + finally: + if created: + try: + report["checks"]["fixture_removed"] = run(["delete", "secret", name, "--wait=true"]).returncode == 0 + except (subprocess.SubprocessError, OSError): + report["checks"]["fixture_removed"] = False + report["passed"] = "error" not in report and len(report["checks"]) == 9 and all(report["checks"].values()) + print(json.dumps(report, indent=2)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/secret_annotation_maintenance.py b/scripts/secret_annotation_maintenance.py index 670c6d3..c837e62 100644 --- a/scripts/secret_annotation_maintenance.py +++ b/scripts/secret_annotation_maintenance.py @@ -43,11 +43,18 @@ def maintain(clean=False): if len(pair) != 2 or not all(NAME.fullmatch(value) for value in pair): raise RuntimeError("invalid Secret identity output; suppressed") rows.append(pair) + namespaces = run(["get", "namespaces", "-o", "name"]).splitlines() + if not all(value.startswith("namespace/") and NAME.fullmatch(value.split("/", 1)[1]) for value in namespaces): + raise RuntimeError("invalid namespace inventory; suppressed") + active_namespaces = {value.split("/", 1)[1] for value in namespaces} report = {"captured_at": datetime.now(timezone.utc).isoformat(), "mode": "clean" if clean else "inspect", "checked": 0, - "annotated": [], "cleaned": [], "complete": False} + "annotated": [], "cleaned": [], "orphaned_namespace": [], "complete": False} try: for namespace, name in rows: + if namespace not in active_namespaces: + report["orphaned_namespace"].append(namespace + "/" + name) + continue report["checked"] += 1 if not inspect(namespace, name): continue @@ -61,7 +68,8 @@ def maintain(clean=False): if inspect(namespace, name): raise RuntimeError("annotation still present") report["cleaned"].append(identity) - report["complete"] = True + report["active_namespace_scan_complete"] = True + report["complete"] = not report["orphaned_namespace"] except (RuntimeError, subprocess.SubprocessError, OSError): report["error"] = "maintenance incomplete; raw output suppressed; inspect before retry" return report