From db91818e844ff378ade960c0b92b36e290377431 Mon Sep 17 00:00:00 2001 From: codex Date: Mon, 28 Sep 2026 18:15:27 +0200 Subject: [PATCH] Advance supervised agent records and close verified Secret annotation guard --- .custodian-brief.md | 4 +- .../agents/custodian-codex/supervision.json | 34 + WORK-RECORDS.md | 14 +- docs/agent-autonomy-decision.md | 157 ++ docs/agent-environment-orientation.md | 23 +- docs/changes/CUST-WP-0073/README.md | 125 ++ .../CUST-WP-0073/orphan-secret-deletion.json | 27 + .../prove_secret_annotation_guard.py | 60 + .../reject-secret-last-applied.yaml | 27 + .../secret_annotation_maintenance.py | 91 + .../2026-09-28-allocation-provenance.json | 16 + .../2026-09-28-allocation-reconcile.json | 696 ++++++ .../2026-09-28-allocation-reconcile.md | 91 + .../2026-09-28-allocation-telemetry.json | 1922 +++++++++++++++++ .../2026-09-28-cluster-observation.json | 1387 ++++++++++++ .../2026-09-28-eso-metadata-changes.json | 319 +++ .../2026-09-28-eso-metadata-preflight.json | 167 ++ .../2026-09-28-eso-metadata-publication.json | 85 + .../2026-09-28-eso-refresh-after-binding.json | 284 +++ ...2026-09-28-eso-refresh-before-binding.json | 284 +++ .../2026-09-28-orphan-secret-deletion.json | 31 + ...cret-annotation-admission-proof-final.json | 18 + ...-annotation-admission-proof-reenabled.json | 18 + ...-28-secret-annotation-admission-proof.json | 17 + ...9-28-secret-annotation-cleanup-active.json | 68 + ...cret-annotation-cleanup-after-eso-fix.json | 12 + ...09-28-secret-annotation-cleanup-retry.json | 16 + .../2026-09-28-secret-annotation-cleanup.json | 70 + ...2026-09-28-secret-annotation-enforced.json | 18 + ...26-09-28-secret-annotation-final-scan.json | 10 + ...-09-28-secret-annotation-post-binding.json | 22 + ...2026-09-28-secret-annotation-rollback.json | 10 + .../2026-09-28-secret-annotation-rollout.md | 147 ++ ...09-28-secret-annotation-scan-enforced.json | 12 + docs/evidence/2026-09-28-sizing-review.md | 94 + ...09-28-supervised-runtime-coordination.json | 11 + .../2026-09-28-supervised-sandbox-proof.json | 20 + docs/evidence/namespace-ownership.yaml | 6 +- scripts/prove_supervised_sandbox.py | 84 + scripts/summarize_agent_supervision.py | 80 + tests/test_agent_supervision.py | 57 + tests/test_secret_annotation_maintenance.py | 61 + ...sured-workload-sizing-and-weekly-review.md | 53 +- ...UST-WP-0073-agent-credential-separation.md | 174 +- 44 files changed, 6868 insertions(+), 54 deletions(-) create mode 100644 .kaizen/agents/custodian-codex/supervision.json create mode 100644 docs/agent-autonomy-decision.md create mode 100644 docs/changes/CUST-WP-0073/README.md create mode 100644 docs/changes/CUST-WP-0073/orphan-secret-deletion.json create mode 100644 docs/changes/CUST-WP-0073/prove_secret_annotation_guard.py create mode 100644 docs/changes/CUST-WP-0073/reject-secret-last-applied.yaml create mode 100644 docs/changes/CUST-WP-0073/secret_annotation_maintenance.py create mode 100644 docs/evidence/2026-09-28-allocation-provenance.json create mode 100644 docs/evidence/2026-09-28-allocation-reconcile.json create mode 100644 docs/evidence/2026-09-28-allocation-reconcile.md create mode 100644 docs/evidence/2026-09-28-allocation-telemetry.json create mode 100644 docs/evidence/2026-09-28-cluster-observation.json create mode 100644 docs/evidence/2026-09-28-eso-metadata-changes.json create mode 100644 docs/evidence/2026-09-28-eso-metadata-preflight.json create mode 100644 docs/evidence/2026-09-28-eso-metadata-publication.json create mode 100644 docs/evidence/2026-09-28-eso-refresh-after-binding.json create mode 100644 docs/evidence/2026-09-28-eso-refresh-before-binding.json create mode 100644 docs/evidence/2026-09-28-orphan-secret-deletion.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-admission-proof-final.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-admission-proof-reenabled.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-admission-proof.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-cleanup-active.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-cleanup-after-eso-fix.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-cleanup-retry.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-cleanup.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-enforced.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-final-scan.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-post-binding.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-rollback.json create mode 100644 docs/evidence/2026-09-28-secret-annotation-rollout.md create mode 100644 docs/evidence/2026-09-28-secret-annotation-scan-enforced.json create mode 100644 docs/evidence/2026-09-28-sizing-review.md create mode 100644 docs/evidence/2026-09-28-supervised-runtime-coordination.json create mode 100644 docs/evidence/2026-09-28-supervised-sandbox-proof.json create mode 100644 scripts/prove_supervised_sandbox.py create mode 100644 scripts/summarize_agent_supervision.py create mode 100644 tests/test_agent_supervision.py create mode 100644 tests/test_secret_annotation_maintenance.py diff --git a/.custodian-brief.md b/.custodian-brief.md index 1f824df..fc3c3f4 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -2,7 +2,7 @@ # Custodian Brief — the-custodian **Domain:** infotech -**Last synced:** 2026-09-28 13:21 UTC +**Last synced:** 2026-09-28 14:34 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams @@ -11,9 +11,9 @@ Progress: 1/5 done | workplan_id: `a98a9f34-83b4-5c8c-8107-e05f7806d50d` **Open tasks:** +- ! Reject last-applied annotations on Secrets `5abbfcae` - ! Rotate what was exposed `1b41b532` - ► Build and hand out the agent identity `4b88b0b7` -- ► Reject last-applied annotations on Secrets `5abbfcae` - ► Fleet guidance and harness guards `90725511` ### Size Railiance workloads from actual demand and establish a weekly allocation review diff --git a/.kaizen/agents/custodian-codex/supervision.json b/.kaizen/agents/custodian-codex/supervision.json new file mode 100644 index 0000000..778d52f --- /dev/null +++ b/.kaizen/agents/custodian-codex/supervision.json @@ -0,0 +1,34 @@ +{ + "schema": "custodian.agent-supervision.v1", + "agent_id": "custodian-codex", + "identity_binding": "logical supervised coding-agent record; interactive runtime isolation not yet enforced", + "scope": "CUST-WP-0071 and CUST-WP-0073 preparation and founder-authorized Railiance changes", + "mode": "supervised", + "supervisor": "Bernd Worsch", + "policy_ref": "docs/agent-autonomy-decision.md", + "runtime_enforcement": "not_yet_migrated", + "autopilot_grant": null, + "cost_budget_eur": null, + "risk_limit_eur": null, + "proposals": [ + { + "proposal_id": "CUST-WP-0073-T03-annotation-guard", + "disposition": "unscored", + "outcome": "failed", + "reason": "Conversation authorization predates exact-revision scoring. Native admission tests passed, but ESO refresh failed under the guard. Binding rolled back and 39/39 ExternalSecrets recovered. Retain this adverse outcome; it is not promotion evidence.", + "proposal_ref": "docs/changes/CUST-WP-0073/README.md", + "execution_ref": "railiance-platform@54885ac1589074a68d9607d1be250c84c5c2307a", + "refinement_or_rescue": true, + "verification_ref": "docs/evidence/2026-09-28-secret-annotation-rollout.md", + "remediation": { + "authorization_ref": "Founder continuation: Good, go on, after the 31-declaration fix was described", + "source_changes_ref": "docs/evidence/2026-09-28-eso-metadata-publication.json", + "verification_ref": "docs/evidence/2026-09-28-secret-annotation-enforced.json", + "integration_ref": "docs/evidence/2026-09-28-eso-refresh-after-binding.json", + "outcome": "39/39 fresh successful refreshes under Deny; nine native admission checks pass", + "refinement_or_rescue": true, + "new_scored_trial": false + } + } + ] +} diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index e049670..dcf4544 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -73,7 +73,7 @@ | workplan | CUST-WP-0070 | finished | — | workplans/CUST-WP-0070-publication-repo-category.md | | workplan | CUST-WP-0071 | active | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | | workplan | CUST-WP-0072 | finished | — | workplans/CUST-WP-0072-fleet-flavor-and-depends-on-backfill.md | -| workplan | CUST-WP-0073 | proposed | — | workplans/CUST-WP-0073-agent-credential-separation.md | +| workplan | CUST-WP-0073 | active | — | workplans/CUST-WP-0073-agent-credential-separation.md | | workplan | THE-WP-0001 | finished | — | workplans/THE-WP-0001-federation-interface.md | | task | CUST-WP-ADHOC-2026-05-02-T01 | done | — | workplans/ADHOC-2026-05-02.md | | task | CUST-WP-ADHOC-2026-07-02-T01 | done | — | workplans/ADHOC-2026-07-02.md | @@ -455,18 +455,18 @@ | task | CUST-WP-0070-T02 | done | — | workplans/CUST-WP-0070-publication-repo-category.md | | task | CUST-WP-0070-T03 | done | — | workplans/CUST-WP-0070-publication-repo-category.md | | task | CUST-WP-0071-T01 | done | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | -| task | CUST-WP-0071-T02 | wait | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | -| task | CUST-WP-0071-T03 | wait | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | +| task | CUST-WP-0071-T02 | progress | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | +| task | CUST-WP-0071-T03 | progress | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | | task | CUST-WP-0071-T04 | wait | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | | task | CUST-WP-0071-T05 | wait | — | workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md | | task | CUST-WP-0072-T01 | done | — | workplans/CUST-WP-0072-fleet-flavor-and-depends-on-backfill.md | | task | CUST-WP-0072-T02 | done | — | workplans/CUST-WP-0072-fleet-flavor-and-depends-on-backfill.md | | task | CUST-WP-0072-T03 | done | — | workplans/CUST-WP-0072-fleet-flavor-and-depends-on-backfill.md | | task | CUST-WP-0072-T04 | done | — | workplans/CUST-WP-0072-fleet-flavor-and-depends-on-backfill.md | -| task | CUST-WP-0073-T01 | todo | — | workplans/CUST-WP-0073-agent-credential-separation.md | -| task | CUST-WP-0073-T02 | todo | — | workplans/CUST-WP-0073-agent-credential-separation.md | -| task | CUST-WP-0073-T03 | todo | — | workplans/CUST-WP-0073-agent-credential-separation.md | -| task | CUST-WP-0073-T04 | todo | — | workplans/CUST-WP-0073-agent-credential-separation.md | +| task | CUST-WP-0073-T01 | done | — | workplans/CUST-WP-0073-agent-credential-separation.md | +| task | CUST-WP-0073-T02 | progress | — | workplans/CUST-WP-0073-agent-credential-separation.md | +| task | CUST-WP-0073-T03 | wait | — | workplans/CUST-WP-0073-agent-credential-separation.md | +| task | CUST-WP-0073-T04 | progress | — | workplans/CUST-WP-0073-agent-credential-separation.md | | task | CUST-WP-0073-T05 | wait | — | workplans/CUST-WP-0073-agent-credential-separation.md | | task | THE-WP-0001-T01 | done | — | workplans/THE-WP-0001-federation-interface.md | | task | THE-WP-0001-T02 | done | — | workplans/THE-WP-0001-federation-interface.md | diff --git a/docs/agent-autonomy-decision.md b/docs/agent-autonomy-decision.md new file mode 100644 index 0000000..48b2f90 --- /dev/null +++ b/docs/agent-autonomy-decision.md @@ -0,0 +1,157 @@ +# Agent-specific supervised and autopilot modes + +Founder decision, 2026-09-28, `GOVERN @ estate`. +Recorded under CUST-WP-0073-T01. This supersedes the proposed permanent choice +between observation-only agents and unrestricted deployment access. + +## Accepted direction + +Autonomy is a characteristic of an identified agent, scoped to the work it is +trusted to perform. Start agents in **supervised-mode**. Record how often the +supervisor accepts privileged-action proposals unchanged and how well those +exact proposals work when executed. Only a demonstrated record of successful +operation without adaptation or refinement supports promotion to +**autopilot-mode**. Autopilot has both a cost budget and a risk limit in EUR, +which the supervisor can tune per agent and scope. + +This decision accepts the model. It does not promote an agent, choose numerical +limits, authorize a live credential cutover or claim runtime enforcement exists. + +## Minimal operating contract + +An agent instance/assignment carries its stable identity, mode, supervisor, +allowed action/target scope, execution-profile revision and reference to its +promotion/limit decision. Mode persists across sessions. Trust in one scope does +not automatically grant trust in another. New scopes start supervised; material +model, tool-profile or execution-environment changes require a supervisor review +of whether prior evidence still applies. + +In supervised-mode, ordinary already-authorized preparation, reads, local edits +and tests continue. For a privileged action the agent prepares the exact action, +target, expected result, verification/rollback, cost estimate and risk estimate. +The supervisor approves that proposal or performs it through the privileged +execution path. The receipt records whether approval or human execution was +needed. Approval is bound to the reviewed proposal revision; changing that +revision requires review again. The agent cannot approve itself. + +In autopilot-mode, a privileged action may run without per-action approval only +inside the agent's granted scope and both monetary limits. Missing estimates, +expired grants, insufficient remaining budget, unbounded risk, or actions outside +scope return that action to supervision. Existing human-only lanes remain +human-only unless explicitly changed by the governing authority. Mode and work +record `lane` are separate dimensions; effective authority is their intersection. + +## Evidence for promotion + +Use existing approval and execution receipts, keyed by agent, scope, profile +revision and a stable proposal ID/digest. Record: + +- the original proposal, supervisor disposition (unchanged / revised / + rejected), revisions and reason; pending/withdrawn proposals stay visible; +- approval and execution identities, timestamps and the exact executed revision; +- outcome verification, interventions, rollback/recovery, observed cost and + realized loss/incident evidence. Store references, not credentials. + +For a declared review window, report counts as well as rates: + +- **Unchanged acceptance rate:** proposals accepted without changes divided by + all adjudicated original proposals. Revised and rejected proposals remain in + the denominator; retries do not become fresh successes. +- **Unchanged execution success rate:** original proposals executed as accepted, + passing the agreed verification with no corrective refinement or rescue, + divided by all executed original proposals with completed verification. + Execution of a supervisor-revised proposal does not earn an unchanged success. +- Also report pending/unverified outcomes and supervisor interventions/time. + Approval without execution is not a successful outcome. No observations means + unknown, never 100%. + +The supervisor promotes explicitly for a named scope using an agreed minimum +sample, review window, acceptance/success thresholds and incident tolerance. +These values are not set by this decision; no default percentage grants access. +Successful harmless work alone does not establish competence for higher-risk +privileged actions. Promotion, revocation and limit changes retain history. +The supervisor can reduce autonomy immediately; failed verification or missing +enforcement stops further autonomous privileged actions pending review. + +## Two distinct EUR controls + +**Cost budget** bounds attributable spend and commitments over an explicit period, +including execution costs and resources/services the action commits to. Reserve +the estimated maximum cost before execution, reconcile actuals afterwards, and +count concurrent reservations against the same remaining budget. Unknown cost is +not free. Record the budget source, currency and reset period. Do not confuse a +token limit or provider subscription with a complete euro-denominated budget. + +**Risk limit** bounds potential loss, separately from normal spending. Proposed +operational interpretation for each grant: a conservatively assessed credible +loss bound per action plus aggregate outstanding exposure from concurrent or +dependent actions. Include recovery expense, service interruption and data loss +where applicable, with assumptions and uncertainty. An expected-loss average +alone must not hide a much larger credible downside. An unpriced or unbounded +consequence requires supervision. EUR limits do not price away human-only or +other non-monetary prohibitions. The supervisor accepts the valuation method +and the action/exposure limits when granting autopilot; the agent cannot raise +its own limit or declare its own estimate authoritative. + +Before enabling autopilot, verify that the execution path enforces reservations, +limits and revocation across concurrent actions. Recording fields in a manifest +does not enforce a budget. Until that proof exists, the mode remains supervised. + +## Credential boundary and existing owners + +Keep admin credentials out of the agent's direct reach in both modes. A scoped +privileged execution path performs approved actions in supervised-mode and +policy-authorized actions in autopilot-mode, returning sanitized outcome evidence. +Unrestricted sudo, admin kubeconfig access or unreviewed GitOps deployment would +bypass that path. Separate identities/isolation remain necessary, but the +observation-only profile is the supervised starting profile, not a permanent +limit on what an agent may accomplish. + +Reuse existing boundaries rather than build another supervisor service: + +| Concern | Existing surface / limit | +|---|---| +| Agent identity, mode and performance | Consumer-owned agent instance/assignment records; `agentic-resources` performance loop. Its broader workforce inventory/assignment contracts are still proposed. | +| Exact human approval and execution outcome | `approval-engine` approval object, `informed-decision` supervisor presentation, execution receipts. State Hub decisions record governance; they are not runtime approval tokens. | +| Runtime enforcement and credential custody | `glas-harness`, selected rein and sandbox; existing authorization and credential-owner paths. A prompt or mode label grants nothing. | +| Monetary authority | `fin-hub` budget source, with execution accounting supplied by the relevant runtime/service. | +| Risk judgement | Supervisor-approved estimates; `risk-nexus` can hold evidence but is not a live EUR risk gate. | +| Work and review evidence | Existing CUST-WP-0073 tasks and State Hub progress; no new task/workplan or service. | + +## Bounded application to CUST-WP-0073 + +T01's policy choice is resolved by this decision. T02 implements and proves the +supervised starting boundary and records the existing execution path selected; +T04 adds per-agent mode and proposal/outcome evidence to guidance and the chosen +existing receipts. First implementation may use reviewed file records and +existing receipts. No new dashboard, automatic promotion engine, monetary risk +estimator or general workforce system is required to finish credential separation. + +Autopilot is a promotion option requiring its own concrete grant and demonstrated +enforcement, not an activation promised by this workplan. No such grant exists +from this decision. T03's annotation policy and T05's deferred rotation remain +unchanged. CUST-WP-0071 retains its measurement and weekly-review scope. + +## Supervised record in use + +The initial consumer-owned record is +`.kaizen/agents/custodian-codex/supervision.json`. Generate its descriptive report: + +```bash +python3 scripts/summarize_agent_supervision.py .kaizen/agents/custodian-codex/supervision.json +``` + +For each future scored proposal, retain the original digest before submission, +the supervisor's approval reference and approved digest, then the executed digest +and verification receipt. Use one stable proposal ID across revisions. Record +`refinement_or_rescue` explicitly. Accepted revised proposals, rejections and +rescued executions cannot earn unchanged success. Pending/unverified outcomes +remain visible. These records contain references and outcomes, never credential +values or executable approval tokens. + +Earlier conversation authorization is retained as `unscored` where an exact +submitted revision was not recorded. It is not backfilled into promotion evidence. +The initial rates are unknown. The utility is descriptive and grants no authority; +autopilot remains disabled and the interactive runtime has not been migrated. +The existing sand-boxer isolation mechanism has a separate non-model proof in +`docs/evidence/2026-09-28-supervised-sandbox-proof.json`. diff --git a/docs/agent-environment-orientation.md b/docs/agent-environment-orientation.md index 6943375..403ef4d 100644 --- a/docs/agent-environment-orientation.md +++ b/docs/agent-environment-orientation.md @@ -1,7 +1,7 @@ # Agent environment orientation **Audience:** every coding agent working in this estate (Claude Code, Codex, Grok, custodian workers). It is tool-neutral. -**Owner:** the-custodian. **Last verified:** 2026-09-24. +**Owner:** the-custodian. **Last verified:** 2026-09-28 (§6); other sections retain their dated evidence. These are facts about the *environment*: where things run, how to reach them, and the traps that cost real time. Each section names its owner. When a fact changes, fix it here and in the owner's record. @@ -76,9 +76,24 @@ Owner: railiance-platform (OpenBao) and ops-warden (the `warden access` lane). ## 6. Secrets and credentials - Run the credential-routing check (`warden route find ""`) before requesting anything. See the "Credential and access routing" section in every repo's `AGENTS.md`. -- **Never read a Secret with `-o yaml`, `-o json`, `describe`-style tools, or even `-o jsonpath='{.metadata}'`.** A Secret created with `kubectl apply` carries its full data in the `kubectl.kubernetes.io/last-applied-configuration` annotation, so a metadata read prints the secret. To test for the annotation without printing a value: - `kubectl get secret -o go-template='{{ if index .metadata.annotations "kubectl.kubernetes.io/last-applied-configuration" }}HAS-ANNOTATION{{ else }}clean{{ end }}'` -- **Do not run any other go-template or jsonpath against a Secret.** On 2026-09-23 a template that only asked for `len .metadata.ownerReferences` failed because the field was absent, and kubectl printed the raw object, including `.data` and the last-applied annotation. A metadata-only template is not safe on that basis. The presence check above is the only template to use. +- **Never print Secret objects, annotations, or raw kubectl error output.** A + client-side apply annotation can contain a second copy of every value, and a + failing template can dump the object. Metadata-only intent does not make a + command safe. +- **Use the output-suppressing maintenance helper for annotation checks:** + `railiance-platform/scripts/secret_annotation_maintenance.py` (no arguments + inspects; `--clean` is the supervised mutation). It uses only validated + namespace/name fields and a presence template tested for absent, empty and + populated annotation maps and empty annotation values. It captures and discards + raw kubectl output/errors; receipts contain only names, counts and booleans. +- **The September 24 inline presence template is withdrawn.** On September 28, + `index .metadata.annotations` failed on an absent map. The wrapper suppressed + the resulting object dump; no values reached the agent transcript. Do not run + standalone go-template/jsonpath against real Secrets, including the old check. +- Agent autonomy is per identified agent and scope: start supervised, record + exact proposals and verified outcomes, promote explicitly only under bounded + EUR cost and risk grants. See `docs/agent-autonomy-decision.md`. A mode label or + approval rate does not isolate credentials or grant runtime permissions. - ExternalSecrets use ClusterSecretStores. The target pattern is **OpenBao Kubernetes auth**: one ServiceAccount per consumer, 15-minute tokens, and a policy scoped to exact paths. Five stores still use static `*-eso-token` Secrets, which expire. Do not re-run the old `*-eso-token-apply` scripts. ## 7. GitOps (ArgoCD) on railiance01 diff --git a/docs/changes/CUST-WP-0073/README.md b/docs/changes/CUST-WP-0073/README.md new file mode 100644 index 0000000..b9ee99c --- /dev/null +++ b/docs/changes/CUST-WP-0073/README.md @@ -0,0 +1,125 @@ +# Credential separation — reviewed change package + +2026-09-28. Prepared under the existing CUST-WP-0073 tasks. The initial admission rollout was rolled back, then corrected and re-enabled; see the +[rollout receipt](../../evidence/2026-09-28-secret-annotation-rollout.md). The founder selected agent-specific supervised/autopilot modes in +[the decision record](../../agent-autonomy-decision.md). The concrete supervised +execution path and account cutover remain unimplemented. + +## Verified current state + +`ssh railiance01` runs as `tegwick` (uid 1000), with `(ALL) NOPASSWD: ALL`. +`/etc/rancher/k3s/k3s.yaml` is `644 root root`. +`kubectl auth whoami` reports `system:admin`, `system:masters`. +No credential contents were read. The public principal inventory in +`railiance-infra/ansible/inventory/ssh_principals.yaml` maps both `agt-*` and +`adm-full` to `tegwick`. ops-warden issues certificates; railiance-infra owns +host principal mapping. A different principal on this same account does not +separate privilege. + +## Supervised starting identity and later scoped promotion (T01/T02) + +Use separate agent and admin OS identities on both the workstation and server. +Agents must not inherit the admin SSH key/certificate, SSH agent socket, sudo, +container-runtime socket, kubeconfig, OpenBao token or admin home directory. +Moving a file or changing the default context under the same unrestricted +account is insufficient. Keep an independently verified attended admin session +open during cutover; verify recovery before withdrawing the old agent path. + +The supervised starting Kubernetes identity is outside `system:masters`, with an +explicit reviewed observation allowlist. Do not simply bind built-in `view`: +ConfigMaps, pod specs and logs can themselves contain credentials. Do not grant +workload edits, arbitrary ConfigMap writes, exec/attach/portforward, proxy, +logs, Secret verbs, token issuance, impersonation, RBAC writes or CSR approval. +Broader action authority is a per-agent autopilot grant earned through evidence, +with cost and risk limits in EUR; it is not unrestricted credential access. + +Deployment create/patch authority allows code or mounts to extract credentials. +This is an upstream documented escalation route, not a missing deny rule: +[Kubernetes RBAC good practices](https://kubernetes.io/docs/concepts/security/rbac-good-practices/). +Agents prepare exact changes; in supervised-mode the supervisor approves or runs +them through the privileged path. An agent-controlled GitOps write path must obey +that same gate. Later autopilot may execute scoped actions without individual +approval only through verified policy and budget/risk enforcement. Otherwise it +recreates the bypass. Mode, supervisor, proposal dispositions and verified +outcomes belong to the identified agent's existing records and receipts. + +Set k3s's persistent kubeconfig mode to `600` in railiance-enablement and fix +the existing file, but do not mistake that step for OS-account separation. +The final selected launcher/account setup must prove agents cannot regain the +old admin account, including through workstation/Windows interoperability. +No new credential broker or harness implementation is proposed. + +Acceptance uses the actual agent process/account, not only admin impersonation: +whoami without masters; denied Secret get/list/watch; denied pod exec, workload +writes, logs, token issuance and impersonation; denied admin-file reads and +sudo; no accessible admin socket/key/token; approved observation succeeds; +attended admin recovery succeeds. Record authorization booleans and file access +results, never credential contents. Negative checks must not attempt to print +an actual secret if access unexpectedly succeeds. + +## Secret annotation policy (T03) + +`reject-secret-last-applied.yaml` contains a native v1 policy and Deny binding. +It rejects the annotation key even when its value is empty, on CREATE/UPDATE, +cluster-wide. It introduces no controller or workload. Server dry-run on the +verified v1.35.1+k3s1 cluster accepted both objects on September 28: + +```text +validatingadmissionpolicy.admissionregistration.k8s.io/reject-secret-last-applied serverside-applied (server dry run) +validatingadmissionpolicybinding.admissionregistration.k8s.io/reject-secret-last-applied serverside-applied (server dry run) +``` + +Subsequent live native tests passed, but actual ESO refresh failed and required +rollback. Enforcement is now enabled after explicit metadata fixes and successful fresh +refreshes of all 39 ExternalSecrets. The platform owner's pinned revision +`7daf7e9` is authoritative; the original proposal file is retained for history. +Reference: [ValidatingAdmissionPolicy](https://kubernetes.io/docs/reference/access-authn-authz/validating-admission-policy/). + +Before any retry, fix and verify the 31 ESO declarations described in the rollout +receipt, then in one attended railiance-platform window: remove existing last-applied +annotations without logging values; persist the manifest in that owner's +deployment path; dry-run and diff; install policy and binding. Use only a +synthetic, non-credential Secret in a scratch namespace to verify clean create +and update succeed, annotated create/update (including empty annotation) fail, +and client-side apply fails. Verify the policy type-check status, then remove +the fixture. Record only names, booleans and counts. Do not use dry-run output +of real Secret objects or print admission errors containing real values. + +If the policy interrupts a required write, the attended admin can delete its +binding, fix the writer to use server-side apply/replace, and rebind. This +temporarily reopens annotation leakage and must be recorded. The policy does +not revoke any credential or stop other ways of reading secrets. + +## Guidance and deferred rotation (T04/T05) + +The September 24 standing notice exists. The raw presence template is now +withdrawn: absent annotations can trigger a dump of the full Secret. Use only +the maintenance helper, which captures and suppresses kubectl output on errors. Claude's recorded guard remains a +stopgap. No equivalent Codex/Grok read-denial hook has been established by this +work; this session has broad kubectl/SSH permissions, so instructions are the +current protection. No claim is made that every Grok installation was inspected. +OpenBao/Vault secret reads belong inside the same attended boundary, regardless +of preapproved command prefixes. + +Rotation remains in existing T05 with the founder's September 24 trigger-based +deferral. Do not silently cancel it or open another plan. At final closure, +either execute the rotation in its attended window or explicitly resolve its +existing disposition under the work-record rules. No rotation was performed. + +## Bounded implementation evidence + +The existing sandbox mechanism passed the synthetic checks in +[the sandbox receipt](../../evidence/2026-09-28-supervised-sandbox-proof.json). +It does not prove interactive agent migration. The per-agent record at +`.kaizen/agents/custodian-codex/supervision.json` keeps this agent supervised, +with no autopilot grant or EUR limits assigned. The descriptive summarizer +`scripts/summarize_agent_supervision.py` cannot authorize or promote an agent. +No eligible scoring sample exists; the failed annotation rollout and recovery +are retained visibly rather than counted as unchanged success. + +Current T03 outcome: nine admission checks pass; all 39 ExternalSecrets refreshed +successfully under Deny; the guard is Synced/Healthy. Source fixes and deployment +receipts are in the linked rollout record. The absent-namespace orphan Secret was deleted after explicit founder approval, +using its exact UID precondition; absence and unchanged 3 GiB PVC were verified. +`orphan-secret-deletion.json` now contains the execution disposition. T03 is +complete. Agent-runtime migration remains a separate unfinished task. diff --git a/docs/changes/CUST-WP-0073/orphan-secret-deletion.json b/docs/changes/CUST-WP-0073/orphan-secret-deletion.json new file mode 100644 index 0000000..b591a4f --- /dev/null +++ b/docs/changes/CUST-WP-0073/orphan-secret-deletion.json @@ -0,0 +1,27 @@ +{ + "reviewed_at": "2026-09-28T14:30:16.548657+00:00", + "resource": "Secret", + "namespace": "platform-pg-drill", + "name": "drill-minio", + "uid": "2fcb66df-d90f-4776-8ea0-8ca1a04bd307", + "created_at": "2026-08-13T11:09:33Z", + "namespace_exists": false, + "pods_in_namespace": 0, + "delete_options": { + "apiVersion": "v1", + "kind": "DeleteOptions", + "preconditions": { + "uid": "2fcb66df-d90f-4776-8ea0-8ca1a04bd307" + } + }, + "proposal": "Delete only this orphan drill Secret, using the UID precondition. Do not recreate namespace, delete PVC or alter other resources.", + "reason": "Namespace is absent; API refuses annotation-only metadata update. Secret is an August 13 scratch drill artifact; referencing Deployment has zero Ready replicas and no pods.", + "risk": "Deletion removes the remaining credential copy in this orphan Secret. No Secret value has been inspected or archived.", + "storage": "Bound 3Gi platform-pg-drill-1 PVC and its PV retained unchanged.", + "approval": "Founder explicitly selected: Delete only the orphan Secret", + "executed": true, + "server_dry_run_passed": true, + "executed_at": "2026-09-28T16:07:36.040146+00:00", + "verified_absent": true, + "pvc_unchanged": true +} diff --git a/docs/changes/CUST-WP-0073/prove_secret_annotation_guard.py b/docs/changes/CUST-WP-0073/prove_secret_annotation_guard.py new file mode 100644 index 0000000..35066bf --- /dev/null +++ b/docs/changes/CUST-WP-0073/prove_secret_annotation_guard.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Positive/negative admission proof using only a uniquely named synthetic Secret.""" +import copy +import json +import subprocess +import uuid +from datetime import datetime, timezone + +KEY = "kubectl.kubernetes.io/last-applied-configuration" + + +def run(args, obj=None): + return subprocess.run(["kubectl", "-n", "whitehat", *args], + input=json.dumps(obj) if obj is not None else None, + capture_output=True, text=True, timeout=30) + + +def denied(result): + # Do not accept connectivity/RBAC failures as admission-policy success. + return result.returncode != 0 and "Secret last-applied annotations are forbidden" in result.stderr + + +def main(): + name = "cust-0073-proof-" + uuid.uuid4().hex[:12] + obj = {"apiVersion": "v1", "kind": "Secret", "metadata": {"name": name}, + "type": "Opaque", "data": {"fixture": "c3ludGhldGlj"}} + report = {"captured_at": datetime.now(timezone.utc).isoformat(), "namespace": "whitehat", + "fixture": name, "synthetic_only": True, "checks": {}} + created = False + try: + result = run(["create", "--field-manager=cust-0073-proof", "-f", "-"], obj) + created = result.returncode == 0 + report["checks"]["clean_create_allowed"] = created + if not created: + raise RuntimeError("synthetic create failed") + for label, value in [("empty", ""), ("populated", "synthetic")]: + annotated = copy.deepcopy(obj) + annotated["metadata"]["name"] = name + "-denied" + annotated["metadata"]["annotations"] = {KEY: value} + report["checks"][label + "_annotated_create_denied"] = denied(run(["create", "--dry-run=server", "-f", "-"], annotated)) + patch = {"metadata": {"annotations": {KEY: value}}} + report["checks"][label + "_annotated_update_denied"] = denied(run(["patch", "secret", name, "--dry-run=server", "--type=merge", "-p", json.dumps(patch)])) + report["checks"]["client_apply_denied"] = denied(run(["apply", "--dry-run=server", "-f", "-"], obj)) + report["checks"]["clean_server_apply_allowed"] = run(["apply", "--server-side", "--field-manager=cust-0073-proof", "-f", "-"], obj).returncode == 0 + report["checks"]["clean_update_allowed"] = run(["patch", "secret", name, "--type=merge", "-p", json.dumps({"data": {"fixture": "c3ludGhldGljLXVwZGF0ZQ=="}})]).returncode == 0 + except (RuntimeError, subprocess.SubprocessError, OSError): + report["error"] = "proof incomplete; raw output suppressed" + finally: + if created: + try: + report["checks"]["fixture_removed"] = run(["delete", "secret", name, "--wait=true"]).returncode == 0 + except (subprocess.SubprocessError, OSError): + report["checks"]["fixture_removed"] = False + report["passed"] = "error" not in report and len(report["checks"]) == 9 and all(report["checks"].values()) + print(json.dumps(report, indent=2)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/changes/CUST-WP-0073/reject-secret-last-applied.yaml b/docs/changes/CUST-WP-0073/reject-secret-last-applied.yaml new file mode 100644 index 0000000..2ab563a --- /dev/null +++ b/docs/changes/CUST-WP-0073/reject-secret-last-applied.yaml @@ -0,0 +1,27 @@ +# Prepared under CUST-WP-0073-T03; not installed. +# Owner: railiance-platform. Clean existing annotations in an attended session +# before binding; otherwise subsequent updates to those Secrets are rejected. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: reject-secret-last-applied +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE", "UPDATE"] + resources: ["secrets"] + scope: "*" + validations: + - expression: '!has(object.metadata.annotations) || !("kubectl.kubernetes.io/last-applied-configuration" in object.metadata.annotations)' + message: "Secret last-applied annotations are forbidden; use server-side apply or replace." +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: reject-secret-last-applied +spec: + policyName: reject-secret-last-applied + validationActions: [Deny] diff --git a/docs/changes/CUST-WP-0073/secret_annotation_maintenance.py b/docs/changes/CUST-WP-0073/secret_annotation_maintenance.py new file mode 100644 index 0000000..c837e62 --- /dev/null +++ b/docs/changes/CUST-WP-0073/secret_annotation_maintenance.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Bounded CUST-WP-0073-T03 maintenance. Never emit kubectl output/errors. + +Run on railiance01 through the supervised admin path. Default is inspection; +--clean removes only the last-applied annotation, leaving Secret data untouched. +""" +import argparse +import json +import re +import subprocess +from datetime import datetime, timezone + +KEY = "kubectl.kubernetes.io/last-applied-configuration" +PRESENCE = ('{{ $found := false }}{{ range $key, $_ := .metadata.annotations }}' + '{{ if eq $key "' + KEY + '" }}{{ $found = true }}{{ end }}{{ end }}' + '{{ if $found }}HAS-ANNOTATION{{ else }}clean{{ end }}') +NAME = re.compile(r"^[a-z0-9][a-z0-9.-]*$") + + +def run(args): + # Even a template error can contain the entire Secret. Never forward it. + result = subprocess.run(["kubectl", *args], capture_output=True, text=True, timeout=30) + if result.returncode: + raise RuntimeError("kubectl operation failed; output suppressed") + return result.stdout.strip() + + +def inspect(namespace, name): + value = run(["-n", namespace, "get", "secret", name, "-o", "go-template=" + PRESENCE]) + if value not in ("clean", "HAS-ANNOTATION"): + raise RuntimeError("unexpected presence result; output suppressed") + return value == "HAS-ANNOTATION" + + +def maintain(clean=False): + # Custom columns use fixed universally-present identity fields; no annotation + # or data output. Validate before using any returned text as an argument. + identities = run(["get", "secrets", "-A", "--no-headers", "-o", + "custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name"]) + rows = [] + for line in identities.splitlines(): + pair = line.split() + if len(pair) != 2 or not all(NAME.fullmatch(value) for value in pair): + raise RuntimeError("invalid Secret identity output; suppressed") + rows.append(pair) + namespaces = run(["get", "namespaces", "-o", "name"]).splitlines() + if not all(value.startswith("namespace/") and NAME.fullmatch(value.split("/", 1)[1]) for value in namespaces): + raise RuntimeError("invalid namespace inventory; suppressed") + active_namespaces = {value.split("/", 1)[1] for value in namespaces} + report = {"captured_at": datetime.now(timezone.utc).isoformat(), + "mode": "clean" if clean else "inspect", "checked": 0, + "annotated": [], "cleaned": [], "orphaned_namespace": [], "complete": False} + try: + for namespace, name in rows: + if namespace not in active_namespaces: + report["orphaned_namespace"].append(namespace + "/" + name) + continue + report["checked"] += 1 + if not inspect(namespace, name): + continue + identity = namespace + "/" + name + report["annotated"].append(identity) + if clean: + # A single JSON patch operation cannot modify credential data. + patch = [{"op": "remove", "path": "/metadata/annotations/" + KEY.replace("/", "~1")}] + run(["-n", namespace, "patch", "secret", name, "--type=json", + "-p", json.dumps(patch)]) + if inspect(namespace, name): + raise RuntimeError("annotation still present") + report["cleaned"].append(identity) + report["active_namespace_scan_complete"] = True + report["complete"] = not report["orphaned_namespace"] + except (RuntimeError, subprocess.SubprocessError, OSError): + report["error"] = "maintenance incomplete; raw output suppressed; inspect before retry" + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--clean", action="store_true") + args = parser.parse_args() + try: + report = maintain(args.clean) + except (RuntimeError, subprocess.SubprocessError, OSError): + report = {"complete": False, "error": "inventory failed; raw output suppressed"} + print(json.dumps(report, indent=2)) + return 0 if report["complete"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/evidence/2026-09-28-allocation-provenance.json b/docs/evidence/2026-09-28-allocation-provenance.json new file mode 100644 index 0000000..344275c --- /dev/null +++ b/docs/evidence/2026-09-28-allocation-provenance.json @@ -0,0 +1,16 @@ +{ + "source_observation": "2026-09-28T12:30:20Z", + "revisions": { + "/home/worsch/railiance-cluster": "550b50aa94ad0c10f68bbf7eac18d7c89f992c77", + "/home/worsch/state-hub": "e92471df3b415f9692a25eef9470f667c377b468", + "/home/worsch/rail-knative": "cd38dba3653e7fc85d3b2d3a074811b9a7216f25", + "/home/worsch/railiance-enablement": "28baf36ad6399543315288a9dd5038882899a3a5" + }, + "active_pod_unsupported_accounting_features": [], + "unsupported_features_checked": [ + "pod-level resources", + "overhead", + "restartable init containers" + ], + "scope": "Read-only observation; no Secret objects queried; metrics are samples, not performance acceptance." +} diff --git a/docs/evidence/2026-09-28-allocation-reconcile.json b/docs/evidence/2026-09-28-allocation-reconcile.json new file mode 100644 index 0000000..11f3854 --- /dev/null +++ b/docs/evidence/2026-09-28-allocation-reconcile.json @@ -0,0 +1,696 @@ +{ + "schema": "custodian.allocation-reconcile.v1", + "workplan_id": "CUST-WP-0071-T01", + "captured_at": "2026-09-28T12:30:20Z", + "cluster_observation_schema": "railiance.cluster-resource-observation.v1", + "count_host_and_cluster_cpus_once": true, + "host": { + "owner": "railiance-cluster", + "resource_id": "resource:hosteurope:railiance01", + "same_cpus_as_cluster": true, + "cluster_resource_id": "resource:railiance:reef-railiance:k3s" + }, + "capacity_cpu_m": 4000, + "scheduled_request_cpu_m": 3420, + "pending_unscheduled_cpu_m": 0, + "residual_cpu_m": 580, + "not_a_scheduling_guarantee": true, + "workloads": [ + { + "namespace": "state-hub", + "workload": "railiance-apps", + "pods": 2, + "cpu_request_m": 260, + "memory_request_bytes": 671088640, + "owner": "state-hub", + "service": "state-hub", + "tenant": "unknown" + }, + { + "namespace": "core-hub", + "workload": "rapp-core-hub", + "pods": 3, + "cpu_request_m": 200, + "memory_request_bytes": 939524096, + "owner": "core-hub", + "service": "core-hub", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "forgejo-db", + "pods": 1, + "cpu_request_m": 200, + "memory_request_bytes": 536870912, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "sso", + "workload": "net-kingdom-sso-mfa", + "pods": 4, + "cpu_request_m": 150, + "memory_request_bytes": 268435456, + "owner": "net-kingdom", + "service": "keycape-authelia", + "tenant": "unknown" + }, + { + "namespace": "knative-serving", + "workload": "knative-serving", + "pods": 4, + "cpu_request_m": 140, + "memory_request_bytes": 377487360, + "owner": "rail-knative", + "service": "knative-serving", + "tenant": "unknown" + }, + { + "namespace": "flex-auth", + "workload": "flex-auth", + "pods": 6, + "cpu_request_m": 110, + "memory_request_bytes": 201326592, + "owner": "flex-auth", + "service": "flex-auth", + "tenant": "unknown" + }, + { + "namespace": "mfa", + "workload": "net-kingdom-sso-mfa", + "pods": 1, + "cpu_request_m": 110, + "memory_request_bytes": 402653184, + "owner": "net-kingdom", + "service": "lldap-mfa", + "tenant": "unknown" + }, + { + "namespace": "reuse", + "workload": "reuse-surface", + "pods": 2, + "cpu_request_m": 110, + "memory_request_bytes": 301989888, + "owner": "reuse-surface", + "service": "reuse-surface", + "tenant": "unknown" + }, + { + "namespace": "activity-core", + "workload": "activity-core", + "pods": 10, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "activity-core", + "service": "activity-core", + "tenant": "unknown" + }, + { + "namespace": "cnpg-system", + "workload": "cloudnative-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 104857600, + "owner": "rapp-postgres", + "service": "cloudnative-pg", + "tenant": "unknown" + }, + { + "namespace": "coulomb-social", + "workload": "coulomb-social", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "coulomb-social", + "service": "coulomb-social", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "apps-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "net-kingdom-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "platform-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "platform-pg-2", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "forgejo", + "workload": "gitea", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 134217728, + "owner": "railiance-forge", + "service": "forgejo", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "coredns-7bdb54f89", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 73400320, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "metrics-server-786d997795", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 73400320, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "openbao", + "workload": "openbao", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "railiance-platform", + "service": "openbao", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "prometheus", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 536870912, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "user-engine", + "workload": "user-engine-pg", + "pods": 1, + "cpu_request_m": 100, + "memory_request_bytes": 268435456, + "owner": "user-engine", + "service": "user-engine", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "grafana", + "pods": 1, + "cpu_request_m": 70, + "memory_request_bytes": 201326592, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "vergabe-demo-company", + "workload": "vergabe-teilnahme", + "pods": 1, + "cpu_request_m": 60, + "memory_request_bytes": 268435456, + "owner": "railiance-apps", + "service": "vergabe-teilnahme", + "tenant": "unknown" + }, + { + "namespace": "argocd", + "workload": "argocd-application-controller", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 268435456, + "owner": "railiance-enablement", + "service": "argocd", + "tenant": "unknown" + }, + { + "namespace": "audit-core", + "workload": "audit-core", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 67108864, + "owner": "audit-core", + "service": "audit-core", + "tenant": "unknown" + }, + { + "namespace": "coulomb", + "workload": "ihp-railiance-probe", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 134217728, + "owner": "unknown", + "service": "ihp-railiance-probe", + "tenant": "unknown" + }, + { + "namespace": "issue-core", + "workload": "issue-core", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 134217728, + "owner": "issue-core", + "service": "issue-core", + "tenant": "unknown" + }, + { + "namespace": "kourier-system", + "workload": "3scale-kourier-gateway", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 209715200, + "owner": "rail-knative", + "service": "kourier", + "tenant": "unknown" + }, + { + "namespace": "target-revenue", + "workload": "target-revenue", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 268435456, + "owner": "target-revenue", + "service": "target-revenue", + "tenant": "unknown" + }, + { + "namespace": "user-engine", + "workload": "user-engine", + "pods": 1, + "cpu_request_m": 50, + "memory_request_bytes": 67108864, + "owner": "user-engine", + "service": "user-engine", + "tenant": "unknown" + }, + { + "namespace": "knative-serving", + "workload": "net-kourier-controller", + "pods": 1, + "cpu_request_m": 30, + "memory_request_bytes": 209715200, + "owner": "rail-knative", + "service": "knative-serving", + "tenant": "unknown" + }, + { + "namespace": "argocd", + "workload": "argocd-repo-server", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 134217728, + "owner": "railiance-enablement", + "service": "argocd", + "tenant": "unknown" + }, + { + "namespace": "canned-prompts", + "workload": "canned-prompts", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 100663296, + "owner": "canned-prompts", + "service": "canned-prompts", + "tenant": "unknown" + }, + { + "namespace": "email-connect", + "workload": "email-connect", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "email-connect", + "service": "email-connect", + "tenant": "unknown" + }, + { + "namespace": "openbao", + "workload": "rapp-openbao", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 33554432, + "owner": "railiance-platform", + "service": "openbao", + "tenant": "unknown" + }, + { + "namespace": "rein-aharness", + "workload": "rein-aharness", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rein-aharness", + "service": "rein-aharness", + "tenant": "unknown" + }, + { + "namespace": "sbom-nexus", + "workload": "sbom-nexus", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "sbom-nexus", + "service": "sbom-nexus", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "alertmanager", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "kube-state-metrics", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 67108864, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "telemetry", + "workload": "rapp-telemetry", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 134217728, + "owner": "rapp-telemetry", + "service": "prometheus-grafana", + "tenant": "unknown" + }, + { + "namespace": "tenant-engine", + "workload": "tenant-engine", + "pods": 1, + "cpu_request_m": 25, + "memory_request_bytes": 50331648, + "owner": "tenant-engine", + "service": "tenant-engine", + "tenant": "unknown" + }, + { + "namespace": "rapp-qonto-egress", + "workload": "qonto-egress-proxy", + "pods": 1, + "cpu_request_m": 20, + "memory_request_bytes": 67108864, + "owner": "rapp-qonto", + "service": "qonto-egress", + "tenant": "tenant:friendly:binky" + }, + { + "namespace": "activity-core", + "workload": "actcore-temporal-ui-tls-2-1888679036-1756117428", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 67108864, + "owner": "activity-core", + "service": "activity-core", + "tenant": "unknown" + }, + { + "namespace": "approval-engine", + "workload": "approval-engine", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 67108864, + "owner": "approval-engine", + "service": "approval-engine", + "tenant": "unknown" + }, + { + "namespace": "argocd", + "workload": "argocd-applicationset-controller", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 67108864, + "owner": "railiance-enablement", + "service": "argocd", + "tenant": "unknown" + }, + { + "namespace": "argocd", + "workload": "argocd-redis", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 33554432, + "owner": "railiance-enablement", + "service": "argocd", + "tenant": "unknown" + }, + { + "namespace": "policy-nexus", + "workload": "policy-nexus", + "pods": 1, + "cpu_request_m": 10, + "memory_request_bytes": 33554432, + "owner": "policy-nexus", + "service": "policy-nexus", + "tenant": "unknown" + }, + { + "namespace": "bao-notice", + "workload": "bao-notice", + "pods": 1, + "cpu_request_m": 5, + "memory_request_bytes": 16777216, + "owner": "railiance-platform", + "service": "bao-notice", + "tenant": "unknown" + }, + { + "namespace": "informed-decision", + "workload": "informed-decision", + "pods": 1, + "cpu_request_m": 5, + "memory_request_bytes": 67108864, + "owner": "informed-decision", + "service": "informed-decision", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "cainjector", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "cert-manager", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "cert-manager", + "workload": "webhook", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "cert-manager", + "tenant": "unknown" + }, + { + "namespace": "databases", + "workload": "state-hub-db", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "rapp-postgres", + "service": "apps-pg", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-cert-controller", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-webhook", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-platform", + "service": "external-secrets", + "tenant": "unknown" + }, + { + "namespace": "forgejo", + "workload": "forgejo-runner", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-forge", + "service": "forgejo", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "local-path-provisioner", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "svclb-traefik-0c8aecaf", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "kube-system", + "workload": "traefik", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "railiance-cluster", + "service": "k3s-control-plane", + "tenant": "unknown" + }, + { + "namespace": "target-revenue", + "workload": "target-revenue-pg", + "pods": 1, + "cpu_request_m": 0, + "memory_request_bytes": 0, + "owner": "target-revenue", + "service": "target-revenue", + "tenant": "unknown" + } + ], + "unknown_namespaces": [], + "zero_request_workloads": [ + "cert-manager/cainjector", + "cert-manager/cert-manager", + "cert-manager/webhook", + "databases/state-hub-db", + "external-secrets/external-secrets", + "external-secrets/external-secrets-cert-controller", + "external-secrets/external-secrets-webhook", + "forgejo/forgejo-runner", + "kube-system/local-path-provisioner", + "kube-system/svclb-traefik-0c8aecaf", + "kube-system/traefik", + "target-revenue/target-revenue-pg" + ], + "pending_unscheduled": [], + "measurement_gaps": [ + "node 239.62.205.92.host.secureserver.net lacks independent region/zone labels" + ], + "state_hub_preflight_observation": { + "schema": "state-hub.release-headroom-preflight.v1-input", + "observed_at": "2026-09-28T12:30:20Z", + "freshness_seconds": 0, + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16770076672, + "allocated_cpu_m": 3420, + "allocated_memory_bytes": 9806282752 + } + ], + "pending_unrelated": [] + }, + "signal_notes": [ + "Host resource:hosteurope:railiance01 and cluster resource:railiance:reef-railiance:k3s are the same 4 vCPU; do not add them.", + "Prometheus namespace_cpu:kube_pod_container_resource_requests:sum omitted Forgejo 100m on 2026-09-11; Kubernetes API is the scheduler-effective source.", + "node-exporter was disabled; host CPU usage is not a reservation and is not treated as spare capacity.", + "Zero-request pods are demand, not zero. Missing metrics are listed as gaps, not zeros.", + "STATE-WP-0091 preflight consumes state_hub_preflight_observation; residual millicores are not admission." + ], + "state_hub_preflight": { + "schema": "state-hub.release-headroom-preflight.v1", + "ok": true, + "observed_at": "2026-09-28T12:30:20Z", + "freshness_seconds": 0, + "remaining_cpu_m": 580, + "remaining_memory_bytes": 6963793920, + "pending_unrelated_cpu_m": 0, + "api_surge_cpu_m": 100, + "mcp_surge_cpu_m": 10, + "migrate_cpu_m": 50, + "atomic": true, + "reasons": [], + "notes": [ + "Aggregate remaining millicores is not a kube-scheduler guarantee.", + "Preflight does not lower requests and does not start Helm." + ], + "hook_order": [ + "pre-upgrade migrate job (helm.sh/hook-weight -5)", + "API RollingUpdate maxSurge=1 maxUnavailable=0", + "MCP RollingUpdate maxSurge=1 maxUnavailable=0" + ] + } +} diff --git a/docs/evidence/2026-09-28-allocation-reconcile.md b/docs/evidence/2026-09-28-allocation-reconcile.md new file mode 100644 index 0000000..a426289 --- /dev/null +++ b/docs/evidence/2026-09-28-allocation-reconcile.md @@ -0,0 +1,91 @@ +# Railiance allocation reconcile — 2026-09-28T12:30:20Z + +CUST-WP-0071-T01. Scheduler-effective requests from the Kubernetes API +via `railiance-cluster` observe (RCLUSTER-WP-0014 / RAIL-BS-WP-0014). +Host and cluster are the same 4 vCPU and are counted once. + +- Capacity: 4000m +- Scheduled requests: 3420m +- Pending unscheduled: 0m +- Residual (not a guarantee): 580m +- Unknown namespaces: none +- Zero-request workloads: 12 + +| Namespace | Workload | Owner | Tenant | CPU request (m) | Pods | +|---|---|---|---|---:|---:| +| state-hub | railiance-apps | state-hub | unknown | 260 | 2 | +| core-hub | rapp-core-hub | core-hub | unknown | 200 | 3 | +| databases | forgejo-db | rapp-postgres | unknown | 200 | 1 | +| sso | net-kingdom-sso-mfa | net-kingdom | unknown | 150 | 4 | +| knative-serving | knative-serving | rail-knative | unknown | 140 | 4 | +| flex-auth | flex-auth | flex-auth | unknown | 110 | 6 | +| mfa | net-kingdom-sso-mfa | net-kingdom | unknown | 110 | 1 | +| reuse | reuse-surface | reuse-surface | unknown | 110 | 2 | +| activity-core | activity-core | activity-core | unknown | 100 | 10 | +| cnpg-system | cloudnative-pg | rapp-postgres | unknown | 100 | 1 | +| coulomb-social | coulomb-social | coulomb-social | unknown | 100 | 1 | +| databases | apps-pg | rapp-postgres | unknown | 100 | 1 | +| databases | net-kingdom-pg | rapp-postgres | unknown | 100 | 1 | +| databases | platform-pg | rapp-postgres | unknown | 100 | 1 | +| databases | platform-pg-2 | rapp-postgres | unknown | 100 | 1 | +| forgejo | gitea | railiance-forge | unknown | 100 | 1 | +| kube-system | coredns-7bdb54f89 | railiance-cluster | unknown | 100 | 1 | +| kube-system | metrics-server-786d997795 | railiance-cluster | unknown | 100 | 1 | +| openbao | openbao | railiance-platform | unknown | 100 | 1 | +| telemetry | prometheus | rapp-telemetry | unknown | 100 | 1 | +| user-engine | user-engine-pg | user-engine | unknown | 100 | 1 | +| telemetry | grafana | rapp-telemetry | unknown | 70 | 1 | +| vergabe-demo-company | vergabe-teilnahme | railiance-apps | unknown | 60 | 1 | +| argocd | argocd-application-controller | railiance-enablement | unknown | 50 | 1 | +| audit-core | audit-core | audit-core | unknown | 50 | 1 | +| coulomb | ihp-railiance-probe | unknown | unknown | 50 | 1 | +| issue-core | issue-core | issue-core | unknown | 50 | 1 | +| kourier-system | 3scale-kourier-gateway | rail-knative | unknown | 50 | 1 | +| target-revenue | target-revenue | target-revenue | unknown | 50 | 1 | +| user-engine | user-engine | user-engine | unknown | 50 | 1 | +| knative-serving | net-kourier-controller | rail-knative | unknown | 30 | 1 | +| argocd | argocd-repo-server | railiance-enablement | unknown | 25 | 1 | +| canned-prompts | canned-prompts | canned-prompts | unknown | 25 | 1 | +| email-connect | email-connect | email-connect | unknown | 25 | 1 | +| openbao | rapp-openbao | railiance-platform | unknown | 25 | 1 | +| rein-aharness | rein-aharness | rein-aharness | unknown | 25 | 1 | +| sbom-nexus | sbom-nexus | sbom-nexus | unknown | 25 | 1 | +| telemetry | alertmanager | rapp-telemetry | unknown | 25 | 1 | +| telemetry | kube-state-metrics | rapp-telemetry | unknown | 25 | 1 | +| telemetry | rapp-telemetry | rapp-telemetry | unknown | 25 | 1 | +| tenant-engine | tenant-engine | tenant-engine | unknown | 25 | 1 | +| rapp-qonto-egress | qonto-egress-proxy | rapp-qonto | tenant:friendly:binky | 20 | 1 | +| activity-core | actcore-temporal-ui-tls-2-1888679036-1756117428 | activity-core | unknown | 10 | 1 | +| approval-engine | approval-engine | approval-engine | unknown | 10 | 1 | +| argocd | argocd-applicationset-controller | railiance-enablement | unknown | 10 | 1 | +| argocd | argocd-redis | railiance-enablement | unknown | 10 | 1 | +| policy-nexus | policy-nexus | policy-nexus | unknown | 10 | 1 | +| bao-notice | bao-notice | railiance-platform | unknown | 5 | 1 | +| informed-decision | informed-decision | informed-decision | unknown | 5 | 1 | +| cert-manager | cainjector | railiance-cluster | unknown | 0 | 1 | +| cert-manager | cert-manager | railiance-cluster | unknown | 0 | 1 | +| cert-manager | webhook | railiance-cluster | unknown | 0 | 1 | +| databases | state-hub-db | rapp-postgres | unknown | 0 | 1 | +| external-secrets | external-secrets | railiance-platform | unknown | 0 | 1 | +| external-secrets | external-secrets-cert-controller | railiance-platform | unknown | 0 | 1 | +| external-secrets | external-secrets-webhook | railiance-platform | unknown | 0 | 1 | +| forgejo | forgejo-runner | railiance-forge | unknown | 0 | 1 | +| kube-system | local-path-provisioner | railiance-cluster | unknown | 0 | 1 | +| kube-system | svclb-traefik-0c8aecaf | railiance-cluster | unknown | 0 | 1 | +| kube-system | traefik | railiance-cluster | unknown | 0 | 1 | +| target-revenue | target-revenue-pg | target-revenue | unknown | 0 | 1 | + +## STATE-WP-0091 preflight + +- ok: `True` +- remaining_cpu_m: 580 +- note: Aggregate remaining millicores is not a kube-scheduler guarantee. +- note: Preflight does not lower requests and does not start Helm. + +## Signal notes + +- Host resource:hosteurope:railiance01 and cluster resource:railiance:reef-railiance:k3s are the same 4 vCPU; do not add them. +- Prometheus namespace_cpu:kube_pod_container_resource_requests:sum omitted Forgejo 100m on 2026-09-11; Kubernetes API is the scheduler-effective source. +- node-exporter was disabled; host CPU usage is not a reservation and is not treated as spare capacity. +- Zero-request pods are demand, not zero. Missing metrics are listed as gaps, not zeros. +- STATE-WP-0091 preflight consumes state_hub_preflight_observation; residual millicores are not admission. diff --git a/docs/evidence/2026-09-28-allocation-telemetry.json b/docs/evidence/2026-09-28-allocation-telemetry.json new file mode 100644 index 0000000..7f4d74d --- /dev/null +++ b/docs/evidence/2026-09-28-allocation-telemetry.json @@ -0,0 +1,1922 @@ +{ + "captured_at": "2026-09-28T12:31:36.118774+00:00", + "window": "7d", + "step": "5m", + "source": "railiance01 telemetry/telemetry-prometheus", + "queries": { + "cpu_p95_m": { + "promql": "quantile_over_time(0.95, (sum by(namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",namespace!=\"\"}[5m])) * 1000)[7d:5m])", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598698.984, + "1.4681554906763745" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598698.984, + "0.07871011462286445" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598698.984, + "8.694578240605301" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598698.984, + "1.5182348889339834" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598698.984, + "2.6506969944038157" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598698.984, + "22.50139139435514" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598698.984, + "0.048018729655285294" + ] + }, + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598698.984, + "62.98789693528294" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598698.984, + "162.2019899603059" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598698.984, + "3.87858881344983" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598698.984, + "2.8308062071667996" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598698.984, + "10.03772352630247" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598698.984, + "30.84795570481625" + ] + }, + { + "metric": { + "namespace": "kube-system" + }, + "value": [ + 1790598698.984, + "50.77775253094598" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598698.984, + "22.193180649576938" + ] + }, + { + "metric": { + "namespace": "cert-manager" + }, + "value": [ + 1790598698.984, + "3.076799467612271" + ] + }, + { + "metric": { + "namespace": "forgejo" + }, + "value": [ + 1790598698.984, + "1454.3502378724597" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598698.984, + "0.5445829173922484" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598698.984, + "2.076307498264848" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598698.984, + "2.200816451394358" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598698.984, + "7.745583555327931" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598698.984, + "0.5233767198036381" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598698.984, + "0.2803617186119557" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598698.984, + "12.290185971151937" + ] + }, + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598698.984, + "12.237800326735258" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598698.984, + "0.46375880336754965" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598698.984, + "8.04068735186573" + ] + }, + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598698.984, + "238.08189359059205" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598698.984, + "0" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598698.984, + "4.248221048834943" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598698.984, + "0.3907803524112691" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598698.984, + "200.5016066767808" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598698.984, + "0.4060094179063722" + ] + }, + { + "metric": { + "namespace": "external-secrets" + }, + "value": [ + 1790598698.984, + "4.730820012727696" + ] + } + ] + } + } + }, + "cpu_peak_m": { + "promql": "max_over_time((sum by(namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",namespace!=\"\"}[5m])) * 1000)[7d:5m])", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598700.799, + "21.256119406763144" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598700.799, + "20.098675361207707" + ] + }, + { + "metric": { + "namespace": "forgejo" + }, + "value": [ + 1790598700.799, + "2208.2657696839487" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598700.799, + "13.57630688951853" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598700.799, + "0" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598700.799, + "2.7048240772554304" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598700.799, + "35.65445387158706" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598700.799, + "8.537043947327875" + ] + }, + { + "metric": { + "namespace": "external-secrets" + }, + "value": [ + 1790598700.799, + "5.488507380396456" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598700.799, + "3.8668882583743414" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598700.799, + "9.188389419784812" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598700.799, + "7.368359129557097" + ] + }, + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598700.799, + "124.26843467833817" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598700.799, + "212.84646933028958" + ] + }, + { + "metric": { + "namespace": "cert-manager" + }, + "value": [ + 1790598700.799, + "5.521991775024277" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598700.799, + "16.843981487140702" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598700.799, + "2.1657390775500183" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598700.799, + "0.4123837293968221" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598700.799, + "13.876895853412185" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598700.799, + "48.7531487779586" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598700.799, + "3.33800450829649" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598700.799, + "8.438956954634925" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598700.799, + "0.33664637559473465" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598700.799, + "34.54705625788541" + ] + }, + { + "metric": { + "namespace": "kube-system" + }, + "value": [ + 1790598700.799, + "61.47455600452856" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598700.799, + "4.061440411371207" + ] + }, + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598700.799, + "341.4287618816661" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598700.799, + "627.460597446356" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598700.799, + "26.05093096419507" + ] + }, + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598700.799, + "9.477878023898242" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598700.799, + "0.1973011769435795" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598700.799, + "14.001901527219845" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598700.799, + "11.119415977815748" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598700.799, + "0.3359550430393116" + ] + } + ] + } + } + }, + "memory_peak_bytes": { + "promql": "max_over_time((sum by(namespace) (container_memory_working_set_bytes{container!=\"\",container!=\"POD\",namespace!=\"\"}))[7d:5m])", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598702.039, + "867237888" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598702.039, + "27729920" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598702.039, + "286830592" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598702.039, + "1549639680" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598702.039, + "282542080" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598702.039, + "200421376" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598702.039, + "155594752" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598702.039, + "27607040" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598702.039, + "553553920" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598702.039, + "35319808" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598702.039, + "146567168" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598702.039, + "98258944" + ] + }, + { + "metric": { + "namespace": "external-secrets" + }, + "value": [ + 1790598702.039, + "132960256" + ] + }, + { + "metric": { + "namespace": "cert-manager" + }, + "value": [ + 1790598702.039, + "165998592" + ] + }, + { + "metric": { + "namespace": "kube-system" + }, + "value": [ + 1790598702.039, + "195194880" + ] + }, + { + "metric": { + "namespace": "forgejo" + }, + "value": [ + 1790598702.039, + "4731015168" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598702.039, + "13770752" + ] + }, + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598702.039, + "243916800" + ] + }, + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598702.039, + "60248064" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598702.039, + "42110976" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598702.039, + "40075264" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598702.039, + "61878272" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598702.039, + "54759424" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598702.039, + "79110144" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598702.039, + "170573824" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598702.039, + "5165056" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598702.039, + "153796608" + ] + }, + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598702.039, + "1214410752" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598702.039, + "120561664" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598702.039, + "458752" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598702.039, + "79978496" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598702.039, + "166182912" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598702.039, + "626089984" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598702.039, + "67678208" + ] + } + ] + } + } + }, + "cpu_sample_count": { + "promql": "count_over_time((sum by(namespace) (rate(container_cpu_usage_seconds_total{container!=\"\",container!=\"POD\",namespace!=\"\"}[5m])) * 1000)[7d:5m])", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "external-secrets" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "cert-manager" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598703.137, + "1999" + ] + }, + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "kube-system" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "forgejo" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598703.137, + "1951" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598703.137, + "1316" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598703.137, + "2016" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598703.137, + "2016" + ] + } + ] + } + } + }, + "cpu_throttle_ratio": { + "promql": "sum by(namespace) (increase(container_cpu_cfs_throttled_periods_total{container!=\"\",container!=\"POD\",namespace!=\"\"}[7d])) / sum by(namespace) (increase(container_cpu_cfs_periods_total{container!=\"\",container!=\"POD\",namespace!=\"\"}[7d]))", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598706.33, + "0.0000029508384047599103" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598706.33, + "0.00034924303710924385" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598706.33, + "0.24179063146141916" + ] + }, + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598706.33, + "0.07106864670295301" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598706.33, + "8.147083714659416e-07" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598706.33, + "0.0031751387037305206" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598706.33, + "0.0005176792369777579" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598706.33, + "0.00010566753561465879" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598706.33, + "0.000037403802096483105" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598706.33, + "0.0007955796762871699" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598706.33, + "0.0006548299182990454" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598706.33, + "0.000008707290291084663" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598706.33, + "0.0000889727693413045" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598706.33, + "0.0008797173221304966" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598706.33, + "0.000010525872943966402" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598706.33, + "0.005108302054962069" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598706.33, + "0.00009388990242380517" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598706.33, + "0.00009383451982060759" + ] + }, + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598706.33, + "0.0028285575358513826" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598706.33, + "NaN" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598706.33, + "0.0002545043937517724" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598706.33, + "0.00023899434835148128" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598706.33, + "0" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598706.33, + "0.01289720210571906" + ] + } + ] + } + } + }, + "restarts": { + "promql": "sum by(namespace) (increase(kube_pod_container_status_restarts_total[7d]))", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + { + "metric": { + "namespace": "activity-core" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "bao-notice" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "knative-serving" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "telemetry" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "core-hub" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "coulomb-social" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "vergabe-demo-company" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "approval-engine" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "argocd" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "audit-core" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "sso" + }, + "value": [ + 1790598708.549, + "6.0220958893284955" + ] + }, + { + "metric": { + "namespace": "mfa" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "user-engine" + }, + "value": [ + 1790598708.549, + "1.0075757575757576" + ] + }, + { + "metric": { + "namespace": "canned-prompts" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "external-secrets" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "cert-manager" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "kube-system" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "forgejo" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "email-connect" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "flex-auth" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "coulomb" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "informed-decision" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "issue-core" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "kourier-system" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "reuse" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "cnpg-system" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "openbao" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "policy-nexus" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "databases" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "platform-pg-drill" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "target-revenue" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "rein-aharness" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "sbom-nexus" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "rapp-qonto-egress" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "state-hub" + }, + "value": [ + 1790598708.549, + "0" + ] + }, + { + "metric": { + "namespace": "tenant-engine" + }, + "value": [ + 1790598708.549, + "742.0586746797406" + ] + } + ] + } + } + }, + "forgejo_recording": { + "promql": "namespace_cpu:kube_pod_container_resource_requests:sum{namespace=\"forgejo\"}", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [] + } + } + }, + "node_cpu_series": { + "promql": "count(node_cpu_seconds_total)", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [] + } + } + }, + "http_request_series": { + "promql": "count({__name__=~\"http_requests_total|http_request_duration_seconds_count\",namespace=\"vergabe-demo-company\"})", + "response": { + "status": "success", + "data": { + "resultType": "vector", + "result": [] + } + } + } + } +} diff --git a/docs/evidence/2026-09-28-cluster-observation.json b/docs/evidence/2026-09-28-cluster-observation.json new file mode 100644 index 0000000..a3691c2 --- /dev/null +++ b/docs/evidence/2026-09-28-cluster-observation.json @@ -0,0 +1,1387 @@ +{ + "schema_version": "railiance.cluster-resource-observation.v1", + "record_type": "usage_observation", + "resource_id": "resource:railiance:reef-railiance:k3s", + "source": "railiance-cluster", + "workplan_id": "RCLUSTER-WP-0014", + "reef": "reef-railiance", + "captured_at": "2026-09-28T12:30:20Z", + "capacity": { + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "allocatable": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "ephemeral_storage_bytes": 196555267123, + "pods": 110 + }, + "topology": { + "region": null, + "zone": null, + "hostname": "239.62.205.92.host.secureserver.net", + "provider_id": "k3s://239.62.205.92.host.secureserver.net" + }, + "observed": { + "cpu_millicores": 3963, + "memory_bytes": 12075401216 + } + } + ], + "totals": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "pods": 110, + "pv_bytes": 181529477120 + } + }, + "utilization": { + "workloads": [ + { + "namespace": "activity-core", + "workload": "actcore-temporal-ui-tls-2-1888679036-1756117428", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 67108864 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 3145728 + } + }, + { + "namespace": "activity-core", + "workload": "activity-core", + "pods": 10, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 52, + "memory_bytes": 781189120 + } + }, + { + "namespace": "approval-engine", + "workload": "approval-engine", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 20971520 + } + }, + { + "namespace": "argocd", + "workload": "argocd-application-controller", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 378535936 + } + }, + { + "namespace": "argocd", + "workload": "argocd-applicationset-controller", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 20971520 + } + }, + { + "namespace": "argocd", + "workload": "argocd-redis", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 5, + "memory_bytes": 4194304 + } + }, + { + "namespace": "argocd", + "workload": "argocd-repo-server", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 32505856 + } + }, + { + "namespace": "audit-core", + "workload": "audit-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 26214400 + } + }, + { + "namespace": "bao-notice", + "workload": "bao-notice", + "pods": 1, + "requests": { + "cpu_millicores": 5, + "memory_bytes": 16777216 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 67108864 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "canned-prompts", + "workload": "canned-prompts", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 402653184 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 62914560 + } + }, + { + "namespace": "cert-manager", + "workload": "cainjector", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 67108864 + } + }, + { + "namespace": "cert-manager", + "workload": "cert-manager", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 45088768 + } + }, + { + "namespace": "cert-manager", + "workload": "webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 19922944 + } + }, + { + "namespace": "cnpg-system", + "workload": "cloudnative-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 104857600 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 209715200 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 73400320 + } + }, + { + "namespace": "core-hub", + "workload": "rapp-core-hub", + "pods": 3, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 939524096 + }, + "limits": { + "cpu_millicores": 2500, + "memory_bytes": 2684354560 + }, + "observed": { + "cpu_millicores": 11, + "memory_bytes": 254803968 + } + }, + { + "namespace": "coulomb", + "workload": "ihp-railiance-probe", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "coulomb-social", + "workload": "coulomb-social", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 140509184 + } + }, + { + "namespace": "databases", + "workload": "apps-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 89128960 + } + }, + { + "namespace": "databases", + "workload": "forgejo-db", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 43, + "memory_bytes": 250609664 + } + }, + { + "namespace": "databases", + "workload": "net-kingdom-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 13, + "memory_bytes": 224395264 + } + }, + { + "namespace": "databases", + "workload": "platform-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 75497472 + } + }, + { + "namespace": "databases", + "workload": "platform-pg-2", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 142606336 + } + }, + { + "namespace": "databases", + "workload": "state-hub-db", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 34, + "memory_bytes": 332398592 + } + }, + { + "namespace": "email-connect", + "workload": "email-connect", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 12582912 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 47185920 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-cert-controller", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 44040192 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 30408704 + } + }, + { + "namespace": "flex-auth", + "workload": "flex-auth", + "pods": 6, + "requests": { + "cpu_millicores": 110, + "memory_bytes": 201326592 + }, + "limits": { + "cpu_millicores": 1800, + "memory_bytes": 1207959552 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 49283072 + } + }, + { + "namespace": "forgejo", + "workload": "forgejo-runner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 5, + "memory_bytes": 218103808 + } + }, + { + "namespace": "forgejo", + "workload": "gitea", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 2349, + "memory_bytes": 2495610880 + } + }, + { + "namespace": "informed-decision", + "workload": "informed-decision", + "pods": 1, + "requests": { + "cpu_millicores": 5, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 26214400 + } + }, + { + "namespace": "issue-core", + "workload": "issue-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 39845888 + } + }, + { + "namespace": "knative-serving", + "workload": "knative-serving", + "pods": 4, + "requests": { + "cpu_millicores": 140, + "memory_bytes": 377487360 + }, + "limits": { + "cpu_millicores": 3500, + "memory_bytes": 3250585600 + }, + "observed": { + "cpu_millicores": 9, + "memory_bytes": 85983232 + } + }, + { + "namespace": "knative-serving", + "workload": "net-kourier-controller", + "pods": 1, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 524288000 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 39845888 + } + }, + { + "namespace": "kourier-system", + "workload": "3scale-kourier-gateway", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 838860800 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 17825792 + } + }, + { + "namespace": "kube-system", + "workload": "coredns-7bdb54f89", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 178257920 + }, + "observed": { + "cpu_millicores": 10, + "memory_bytes": 30408704 + } + }, + { + "namespace": "kube-system", + "workload": "local-path-provisioner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 12582912 + } + }, + { + "namespace": "kube-system", + "workload": "metrics-server-786d997795", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 46137344 + } + }, + { + "namespace": "kube-system", + "workload": "svclb-traefik-0c8aecaf", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "kube-system", + "workload": "traefik", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 14, + "memory_bytes": 79691776 + } + }, + { + "namespace": "mfa", + "workload": "net-kingdom-sso-mfa", + "pods": 1, + "requests": { + "cpu_millicores": 110, + "memory_bytes": 402653184 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 939524096 + }, + "observed": { + "cpu_millicores": 57, + "memory_bytes": 210763776 + } + }, + { + "namespace": "openbao", + "workload": "openbao", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 15, + "memory_bytes": 101711872 + } + }, + { + "namespace": "openbao", + "workload": "rapp-openbao", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "policy-nexus", + "workload": "policy-nexus", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 67108864 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "rapp-qonto-egress", + "workload": "qonto-egress-proxy", + "pods": 1, + "requests": { + "cpu_millicores": 20, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 165675008 + } + }, + { + "namespace": "rein-aharness", + "workload": "rein-aharness", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "reuse", + "workload": "reuse-surface", + "pods": 2, + "requests": { + "cpu_millicores": 110, + "memory_bytes": 301989888 + }, + "limits": { + "cpu_millicores": 550, + "memory_bytes": 603979776 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 52428800 + } + }, + { + "namespace": "sbom-nexus", + "workload": "sbom-nexus", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 62914560 + } + }, + { + "namespace": "sso", + "workload": "net-kingdom-sso-mfa", + "pods": 4, + "requests": { + "cpu_millicores": 150, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1150, + "memory_bytes": 671088640 + }, + "observed": { + "cpu_millicores": 26, + "memory_bytes": 71303168 + } + }, + { + "namespace": "state-hub", + "workload": "railiance-apps", + "pods": 2, + "requests": { + "cpu_millicores": 260, + "memory_bytes": 671088640 + }, + "limits": { + "cpu_millicores": 1500, + "memory_bytes": 2684354560 + }, + "observed": { + "cpu_millicores": 310, + "memory_bytes": 588251136 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 54525952 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue-pg", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 54525952 + } + }, + { + "namespace": "telemetry", + "workload": "alertmanager", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 34603008 + } + }, + { + "namespace": "telemetry", + "workload": "grafana", + "pods": 1, + "requests": { + "cpu_millicores": 70, + "memory_bytes": 201326592 + }, + "limits": { + "cpu_millicores": 700, + "memory_bytes": 671088640 + }, + "observed": { + "cpu_millicores": 13, + "memory_bytes": 407896064 + } + }, + { + "namespace": "telemetry", + "workload": "kube-state-metrics", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 30408704 + } + }, + { + "namespace": "telemetry", + "workload": "prometheus", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1610612736 + }, + "observed": { + "cpu_millicores": 52, + "memory_bytes": 914358272 + } + }, + { + "namespace": "telemetry", + "workload": "rapp-telemetry", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 27262976 + } + }, + { + "namespace": "tenant-engine", + "workload": "tenant-engine", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 50331648 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 49283072 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 44040192 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 7, + "memory_bytes": 71303168 + } + }, + { + "namespace": "vergabe-demo-company", + "workload": "vergabe-teilnahme", + "pods": 1, + "requests": { + "cpu_millicores": 60, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 175112192 + } + } + ] + }, + "storage": { + "classes": [ + { + "name": "local-path", + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer", + "default": true + } + ], + "claims": [ + { + "namespace": "activity-core", + "name": "actcore-statehub-edge-outbox", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "actcore-working-memory", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-app-db-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-nats-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-temporal-db-0", + "storage_class": "local-path", + "requested_bytes": 8589934592, + "phase": "Bound" + }, + { + "namespace": "approval-engine", + "name": "data-approval-engine-0", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "apps-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "forgejo-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "net-kingdom-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "platform-pg-1", + "storage_class": "local-path", + "requested_bytes": 21474836480, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "platform-pg-2-1", + "storage_class": "local-path", + "requested_bytes": 21474836480, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "state-hub-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "email-connect", + "name": "email-connect-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "forgejo-runner-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "gitea-shared-storage", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "informed-decision", + "name": "informed-decision-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-logs", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "audit-openbao-0", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "data-openbao-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "platform-pg-drill", + "name": "platform-pg-drill-1", + "storage_class": "local-path", + "requested_bytes": 3221225472, + "phase": "Bound" + }, + { + "namespace": "reuse", + "name": "reuse-surface-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "authelia-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "keycape-authentication-policy", + "storage_class": "local-path", + "requested_bytes": 67108864, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "lldap-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "target-revenue", + "name": "target-revenue-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "telemetry", + "name": "alertmanager-telemetry-alertmanager-db-alertmanager-telemetry-alertmanager-0", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "telemetry", + "name": "prometheus-telemetry-prometheus-db-prometheus-telemetry-prometheus-0", + "storage_class": "local-path", + "requested_bytes": 12884901888, + "phase": "Bound" + }, + { + "namespace": "telemetry", + "name": "telemetry-grafana", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-backups", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "vergabe-demo-company", + "name": "vergabe-teilnahme-app-state", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "vergabe-demo-company", + "name": "vergabe-teilnahme-media", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + } + ] + }, + "failure_domain": { + "current_contract": "single Ready node on railiance01; local storage is correlated with node loss", + "independent_domains_claimed": 1, + "threephoenix_target_is_live": false + }, + "allocation": { + "method_version": "cluster-raw-drivers-v1", + "drivers": [ + "cpu_requests", + "memory_requests", + "pvc_requested_bytes", + "observed_usage" + ], + "denominators": { + "cpu_requested_millicores": 3420, + "memory_requested_bytes": 9806282752, + "pvc_requested_bytes": 181529477120 + }, + "pending_unscheduled": { + "cpu_millicores": 0, + "memory_bytes": 0, + "pods": [] + }, + "residual_capacity": { + "cpu_millicores": 580, + "memory_bytes": 6963793920, + "pv_bytes": 0 + }, + "not_a_scheduling_guarantee": true, + "selection_owner": "resource-control" + }, + "state_hub_preflight_observation": { + "schema": "state-hub.release-headroom-preflight.v1-input", + "observed_at": "2026-09-28T12:30:20Z", + "freshness_seconds": 0, + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "unschedulable": false, + "allocatable_cpu_m": 4000, + "allocatable_memory_bytes": 16770076672, + "allocated_cpu_m": 3420, + "allocated_memory_bytes": 9806282752 + } + ], + "pending_unrelated": [] + }, + "measurement_gaps": [ + "node 239.62.205.92.host.secureserver.net lacks independent region/zone labels" + ], + "provenance": { + "commands": [ + "kubectl get nodes -o json", + "kubectl get pods -A -o json", + "kubectl get pvc -A -o json", + "kubectl get pv -o json", + "kubectl get storageclass -o json", + "kubectl top nodes --no-headers", + "kubectl top pods -A --no-headers" + ], + "secret_surfaces_read": false + } +} diff --git a/docs/evidence/2026-09-28-eso-metadata-changes.json b/docs/evidence/2026-09-28-eso-metadata-changes.json new file mode 100644 index 0000000..8e6c614 --- /dev/null +++ b/docs/evidence/2026-09-28-eso-metadata-changes.json @@ -0,0 +1,319 @@ +[ + { + "id": "activity-core/actcore-backup-offsite", + "source": "/home/worsch/activity-core/k8s/railiance/15-externalsecret-backup-offsite.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "activity-core", + "app.kubernetes.io/part-of": "activity-core" + }, + "annotations": { + "argocd.argoproj.io/sync-wave": "0" + } + } + } + }, + { + "id": "activity-core/actcore-forgejo-admin", + "source": "/home/worsch/activity-core/k8s/railiance/15-externalsecret-forgejo-admin.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "activity-core", + "app.kubernetes.io/part-of": "activity-core" + }, + "annotations": { + "argocd.argoproj.io/sync-wave": "0" + } + } + } + }, + { + "id": "activity-core/actcore-issue-core-runtime", + "source": "/home/worsch/activity-core/k8s/railiance/15-externalsecret-issue-core.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "activity-core", + "app.kubernetes.io/part-of": "activity-core" + }, + "annotations": { + "argocd.argoproj.io/sync-wave": "0" + } + } + } + }, + { + "id": "activity-core/actcore-ops-run-worker-tokens", + "source": "/home/worsch/activity-core/k8s/railiance/15-externalsecret-worker-tokens.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "activity-core", + "app.kubernetes.io/part-of": "activity-core" + } + } + } + }, + { + "id": "audit-core/audit-core-senders", + "source": "/home/worsch/audit-core/deploy/externalsecret-senders.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "email-connect/email-connect-runtime", + "source": "/home/worsch/email-connect/deploy/k8s/railiance/externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "email-connect", + "app.kubernetes.io/part-of": "email-connect" + }, + "annotations": { + "argocd.argoproj.io/sync-wave": "0" + } + } + } + }, + { + "id": "activity-core/llm-connect-provider-secrets", + "source": "/home/worsch/llm-connect/deploy/k8s/activity-core-llm-connect/externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "llm-connect", + "app.kubernetes.io/part-of": "railiance-gitops" + } + } + } + }, + { + "id": "forgejo/forgejo-mailer", + "source": "/home/worsch/railiance-apps/manifests/forgejo-mailer-externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "forgejo", + "app.kubernetes.io/part-of": "railiance-apps" + } + } + } + }, + { + "id": "reuse/reuse-surface-runtime", + "source": "/home/worsch/railiance-apps/manifests/reuse-surface-runtime-externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "reuse-surface", + "app.kubernetes.io/part-of": "railiance-apps" + } + } + } + }, + { + "id": "core-hub/core-hub-api-token", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/core-hub.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "core-hub/core-hub-runtime-database", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/core-hub.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "core-hub/core-hub-migration-database", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/core-hub.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "core-hub/hub-core-runtime-database", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/core-hub.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "core-hub/hub-core-migration-database", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/core-hub.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "sso/keycape-secrets-engine-approval-client", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/keycape-approval-clients.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "sso/keycape-approval-engine-operator-client", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/keycape-approval-clients.externalsecrets.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "sso/keycape-informed-decision-sitting-requester-client", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/sitting-requester.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "sso/keycape-secrets-engine-requester-client", + "source": "/home/worsch/railiance-platform/argocd/platform-addons/openbao-secretstore/t03-requester.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "approval-engine/approval-engine-audit", + "source": "/home/worsch/railiance-platform/manifests/factory-audit-senders.yaml", + "template": { + "metadata": { + "annotations": { + "railiance.io/credential-change": "CCR-2026-0021", + "railiance.io/admission": "approved" + } + } + } + }, + { + "id": "informed-decision/informed-decision-audit", + "source": "/home/worsch/railiance-platform/manifests/factory-audit-senders.yaml", + "template": { + "metadata": { + "annotations": { + "railiance.io/credential-change": "CCR-2026-0022", + "railiance.io/admission": "approved" + } + } + } + }, + { + "id": "sso/keycape-factor-read", + "source": "/home/worsch/railiance-platform/manifests/keycape-factor-custody.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/part-of": "net-kingdom-sso-mfa" + } + } + } + }, + { + "id": "state-hub/state-hub-rename-preflight", + "source": "/home/worsch/railiance-platform/openbao/state-hub-preflight/delivery.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "issue-core/issue-core-runtime", + "source": "/home/worsch/rapp-issue-core/manifests/20-secret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "issue-core", + "app.kubernetes.io/part-of": "issue-core" + } + } + } + }, + { + "id": "databases/platform-pg-backup-s3", + "source": "/home/worsch/rapp-postgres/helm/platform-pg-backup-s3.externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "platform-pg", + "app.kubernetes.io/part-of": "railiance-gitops", + "railiance.io/layer": "s3-platform" + } + } + } + }, + { + "id": "rapp-qonto/rapp-qonto", + "source": "/home/worsch/rapp-qonto/runtime/knative/externalsecret.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "telemetry/telemetry-alert-smtp", + "source": "/home/worsch/rapp-telemetry/acknowledgment/smtp-custody.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "telemetry/telemetry-grafana-admin", + "source": "/home/worsch/rapp-telemetry/manifests/custody.yaml", + "template": { + "metadata": {} + } + }, + { + "id": "user-engine/user-engine-runtime", + "source": "/home/worsch/rapp-user-engine/manifests/openbao-runtime.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "user-engine", + "app.kubernetes.io/part-of": "user-engine" + } + } + } + }, + { + "id": "user-engine/identity-provisioner-client", + "source": "/home/worsch/rapp-user-engine/manifests/openbao-runtime.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "user-engine", + "app.kubernetes.io/part-of": "user-engine" + } + } + } + }, + { + "id": "sso/identity-provisioner-token", + "source": "/home/worsch/rapp-user-engine/manifests/openbao-runtime.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "identity-provisioner", + "app.kubernetes.io/part-of": "net-kingdom-sso-mfa" + } + } + } + }, + { + "id": "target-revenue/target-revenue-runtime", + "source": "/home/worsch/target-revenue/k8s/railiance/externalsecret.yaml", + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "target-revenue", + "app.kubernetes.io/part-of": "target-revenue" + }, + "annotations": { + "argocd.argoproj.io/sync-wave": "0" + } + } + } + } +] diff --git a/docs/evidence/2026-09-28-eso-metadata-preflight.json b/docs/evidence/2026-09-28-eso-metadata-preflight.json new file mode 100644 index 0000000..d0198a1 --- /dev/null +++ b/docs/evidence/2026-09-28-eso-metadata-preflight.json @@ -0,0 +1,167 @@ +{ + "observed_at": "2026-09-28T13:54:07.234385+00:00", + "declarations": 31, + "server_dry_run": "passed", + "server_diff_exit": 1, + "checks": [ + { + "id": "activity-core/actcore-backup-offsite", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "activity-core/actcore-forgejo-admin", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "activity-core/actcore-issue-core-runtime", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "activity-core/actcore-ops-run-worker-tokens", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "audit-core/audit-core-senders", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "email-connect/email-connect-runtime", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "activity-core/llm-connect-provider-secrets", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "forgejo/forgejo-mailer", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "reuse/reuse-surface-runtime", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "core-hub/core-hub-api-token", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "core-hub/core-hub-runtime-database", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "core-hub/core-hub-migration-database", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "core-hub/hub-core-runtime-database", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "core-hub/hub-core-migration-database", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/keycape-secrets-engine-approval-client", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/keycape-approval-engine-operator-client", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/keycape-informed-decision-sitting-requester-client", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/keycape-secrets-engine-requester-client", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "approval-engine/approval-engine-audit", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "informed-decision/informed-decision-audit", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/keycape-factor-read", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "state-hub/state-hub-rename-preflight", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "issue-core/issue-core-runtime", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "databases/platform-pg-backup-s3", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "rapp-qonto/rapp-qonto", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "telemetry/telemetry-alert-smtp", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "telemetry/telemetry-grafana-admin", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "user-engine/user-engine-runtime", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "user-engine/identity-provisioner-client", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "sso/identity-provisioner-token", + "only_template_changed": true, + "template_metadata_matches": true + }, + { + "id": "target-revenue/target-revenue-runtime", + "only_template_changed": true, + "template_metadata_matches": true + } + ], + "no_credential_values_read": true, + "activation": "APPROVED", + "authority": "ADMINISTER @ realm:kubernetes/railiance01", + "authorization": "Founder: Good, go on, after 31-declaration remediation described." +} diff --git a/docs/evidence/2026-09-28-eso-metadata-publication.json b/docs/evidence/2026-09-28-eso-metadata-publication.json new file mode 100644 index 0000000..ae8a6d9 --- /dev/null +++ b/docs/evidence/2026-09-28-eso-metadata-publication.json @@ -0,0 +1,85 @@ +[ + { + "repo": "audit-core", + "commit": "f0dff91eb53cf4f29bc28ff6804a41aea07abe88", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "email-connect", + "commit": "73702b7e1a1671948b0545ef32d6e0de9cca9c65", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "llm-connect", + "commit": "08850d0aafc82bed457bdbfdb6a050f6f532320c", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "railiance-apps", + "commit": "53dcd0121925d9bc13edc3f07efc7a1775c14917", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "railiance-platform", + "commit": "80e053988fcd7f45c4e297a416e4915d7a73804a", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "rapp-issue-core", + "commit": "171545d42a710491a55b28ba7499d23c5ba657d2", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "rapp-postgres", + "commit": "11c1d489b580c45c612768fc6091c796bde8d77f", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "rapp-qonto", + "commit": "28acdd11e689464057127e51924ee4153195ae34", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "rapp-telemetry", + "commit": "34cfa03de5c298f0b3bbc6413e0f72e30d51d4c4", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "rapp-user-engine", + "commit": "76ca9dbf9d3c53266c15676f8fb4d83dae8a5aa1", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "target-revenue", + "commit": "a3a8a27a8cda32ae3c7b55353ee16a131c50a0a0", + "status": "applied", + "instance": "railiance01", + "exact_commit_verified": true + }, + { + "repo": "activity-core", + "commit": "19fc7e4597649b44581dca75bfb46227c552b7fd", + "status": "pushed via documented statehub fix-consistency; PASS with legacy warnings", + "exact_commit_verified": true + } +] diff --git a/docs/evidence/2026-09-28-eso-refresh-after-binding.json b/docs/evidence/2026-09-28-eso-refresh-after-binding.json new file mode 100644 index 0000000..fa3581e --- /dev/null +++ b/docs/evidence/2026-09-28-eso-refresh-after-binding.json @@ -0,0 +1,284 @@ +{ + "phase": "after-binding", + "started_at": "2026-09-28T14:29:01.530820+00:00", + "checked_at": "2026-09-28T14:29:47.150368+00:00", + "total": 39, + "ready": 39, + "fresh": 39, + "complete": true, + "rows": [ + { + "id": "activity-core/actcore-backup-offsite", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:02Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-forgejo-admin", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:03Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-issue-core-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:03Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-ops-run-worker-tokens", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:03Z", + "fresh_refresh": true + }, + { + "id": "activity-core/llm-connect-provider-secrets", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:04Z", + "fresh_refresh": true + }, + { + "id": "approval-engine/approval-engine-audit", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:04Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:04Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-database-migrate", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:04Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-senders", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:05Z", + "fresh_refresh": true + }, + { + "id": "canned-prompts/canned-prompts-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:05Z", + "fresh_refresh": true + }, + { + "id": "canned-prompts/canned-prompts-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-api-token", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:07Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-migration-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:07Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-runtime-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:08Z", + "fresh_refresh": true + }, + { + "id": "core-hub/hub-core-migration-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:08Z", + "fresh_refresh": true + }, + { + "id": "core-hub/hub-core-runtime-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:08Z", + "fresh_refresh": true + }, + { + "id": "databases/platform-pg-backup-s3", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:09Z", + "fresh_refresh": true + }, + { + "id": "email-connect/email-connect-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:09Z", + "fresh_refresh": true + }, + { + "id": "forgejo/forgejo-mailer", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:09Z", + "fresh_refresh": true + }, + { + "id": "informed-decision/informed-decision-audit", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:09Z", + "fresh_refresh": true + }, + { + "id": "issue-core/issue-core-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "rapp-qonto/rapp-qonto", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "reuse/reuse-surface-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "sbom-nexus/sbom-nexus-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "sbom-nexus/sbom-nexus-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "sso/identity-provisioner-token", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:10Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-approval-engine-operator-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:11Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-factor-read", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:11Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-informed-decision-sitting-requester-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:11Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-secrets-engine-approval-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:11Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-secrets-engine-requester-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:11Z", + "fresh_refresh": true + }, + { + "id": "state-hub/state-hub-rename-preflight", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:12Z", + "fresh_refresh": true + }, + { + "id": "target-revenue/target-revenue-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:12Z", + "fresh_refresh": true + }, + { + "id": "telemetry/telemetry-alert-smtp", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:12Z", + "fresh_refresh": true + }, + { + "id": "telemetry/telemetry-grafana-admin", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:12Z", + "fresh_refresh": true + }, + { + "id": "tenant-engine/tenant-engine-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:12Z", + "fresh_refresh": true + }, + { + "id": "tenant-engine/tenant-engine-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:13Z", + "fresh_refresh": true + }, + { + "id": "user-engine/identity-provisioner-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:13Z", + "fresh_refresh": true + }, + { + "id": "user-engine/user-engine-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:29:14Z", + "fresh_refresh": true + } + ] +} diff --git a/docs/evidence/2026-09-28-eso-refresh-before-binding.json b/docs/evidence/2026-09-28-eso-refresh-before-binding.json new file mode 100644 index 0000000..0100f3f --- /dev/null +++ b/docs/evidence/2026-09-28-eso-refresh-before-binding.json @@ -0,0 +1,284 @@ +{ + "phase": "before-binding", + "started_at": "2026-09-28T14:24:02.452772+00:00", + "checked_at": "2026-09-28T14:24:31.597651+00:00", + "total": 39, + "ready": 39, + "fresh": 39, + "complete": true, + "rows": [ + { + "id": "activity-core/actcore-backup-offsite", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-forgejo-admin", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-issue-core-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "activity-core/actcore-ops-run-worker-tokens", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "activity-core/llm-connect-provider-secrets", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "approval-engine/approval-engine-audit", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-database-migrate", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "audit-core/audit-core-senders", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:05Z", + "fresh_refresh": true + }, + { + "id": "canned-prompts/canned-prompts-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "canned-prompts/canned-prompts-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-api-token", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-migration-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/core-hub-runtime-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/hub-core-migration-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "core-hub/hub-core-runtime-database", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "databases/platform-pg-backup-s3", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "email-connect/email-connect-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "forgejo/forgejo-mailer", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:06Z", + "fresh_refresh": true + }, + { + "id": "informed-decision/informed-decision-audit", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "issue-core/issue-core-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "rapp-qonto/rapp-qonto", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "reuse/reuse-surface-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "sbom-nexus/sbom-nexus-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "sbom-nexus/sbom-nexus-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "sso/identity-provisioner-token", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-approval-engine-operator-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:07Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-factor-read", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-informed-decision-sitting-requester-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-secrets-engine-approval-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "sso/keycape-secrets-engine-requester-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "state-hub/state-hub-rename-preflight", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "target-revenue/target-revenue-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "telemetry/telemetry-alert-smtp", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "telemetry/telemetry-grafana-admin", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "tenant-engine/tenant-engine-postgres-migration", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:08Z", + "fresh_refresh": true + }, + { + "id": "tenant-engine/tenant-engine-postgres-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:09Z", + "fresh_refresh": true + }, + { + "id": "user-engine/identity-provisioner-client", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:09Z", + "fresh_refresh": true + }, + { + "id": "user-engine/user-engine-runtime", + "ready": true, + "has_template": true, + "refresh_time": "2026-09-28T14:24:09Z", + "fresh_refresh": true + } + ] +} diff --git a/docs/evidence/2026-09-28-orphan-secret-deletion.json b/docs/evidence/2026-09-28-orphan-secret-deletion.json new file mode 100644 index 0000000..022bb36 --- /dev/null +++ b/docs/evidence/2026-09-28-orphan-secret-deletion.json @@ -0,0 +1,31 @@ +{ + "executed_at": "2026-09-28T16:07:36.040146+00:00", + "authority": "ADMINISTER @ realm:kubernetes/railiance01", + "activation": "APPROVED", + "authorization": "Founder explicitly selected: Delete only the orphan Secret", + "deleted": { + "kind": "Secret", + "namespace": "platform-pg-drill", + "name": "drill-minio", + "uid": "2fcb66df-d90f-4776-8ea0-8ca1a04bd307" + }, + "uid_precondition_used": true, + "verified_absent": true, + "pvc_before": { + "uid": "f94df968-92d6-4f52-8e3a-3684ff7b064b", + "resource_version": "46926933", + "volume": "pvc-f94df968-92d6-4f52-8e3a-3684ff7b064b", + "storage": "3Gi", + "phase": "Bound" + }, + "pvc_after": { + "uid": "f94df968-92d6-4f52-8e3a-3684ff7b064b", + "resource_version": "46926933", + "volume": "pvc-f94df968-92d6-4f52-8e3a-3684ff7b064b", + "storage": "3Gi", + "phase": "Bound" + }, + "pvc_unchanged": true, + "other_resources_mutated": false, + "secret_values_read_or_archived": false +} diff --git a/docs/evidence/2026-09-28-secret-annotation-admission-proof-final.json b/docs/evidence/2026-09-28-secret-annotation-admission-proof-final.json new file mode 100644 index 0000000..07fd86e --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-admission-proof-final.json @@ -0,0 +1,18 @@ +{ + "captured_at": "2026-09-28T13:01:46.140808+00:00", + "namespace": "whitehat", + "fixture": "cust-0073-proof-7525cc730079", + "synthetic_only": true, + "checks": { + "clean_create_allowed": true, + "empty_annotated_create_denied": true, + "empty_annotated_update_denied": true, + "populated_annotated_create_denied": true, + "populated_annotated_update_denied": true, + "client_apply_denied": true, + "clean_server_apply_allowed": true, + "clean_update_allowed": true, + "fixture_removed": true + }, + "passed": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-admission-proof-reenabled.json b/docs/evidence/2026-09-28-secret-annotation-admission-proof-reenabled.json new file mode 100644 index 0000000..5056922 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-admission-proof-reenabled.json @@ -0,0 +1,18 @@ +{ + "captured_at": "2026-09-28T14:28:57.199281+00:00", + "namespace": "whitehat", + "fixture": "cust-0073-proof-e199ed6810eb", + "synthetic_only": true, + "checks": { + "clean_create_allowed": true, + "empty_annotated_create_denied": true, + "empty_annotated_update_denied": true, + "populated_annotated_create_denied": true, + "populated_annotated_update_denied": true, + "client_apply_denied": true, + "clean_server_apply_allowed": true, + "clean_update_allowed": true, + "fixture_removed": true + }, + "passed": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-admission-proof.json b/docs/evidence/2026-09-28-secret-annotation-admission-proof.json new file mode 100644 index 0000000..487fe97 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-admission-proof.json @@ -0,0 +1,17 @@ +{ + "captured_at": "2026-09-28T13:00:29.360819+00:00", + "namespace": "whitehat", + "fixture": "cust-0073-proof-3063da4fd6ff", + "synthetic_only": true, + "checks": { + "clean_create_allowed": true, + "empty_annotated_create_denied": true, + "empty_annotated_update_denied": true, + "populated_annotated_create_denied": true, + "populated_annotated_update_denied": true, + "client_apply_denied": true, + "clean_server_apply_allowed": false, + "fixture_removed": true + }, + "passed": false +} diff --git a/docs/evidence/2026-09-28-secret-annotation-cleanup-active.json b/docs/evidence/2026-09-28-secret-annotation-cleanup-active.json new file mode 100644 index 0000000..178778e --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-cleanup-active.json @@ -0,0 +1,68 @@ +{ + "captured_at": "2026-09-28T12:58:10.819938+00:00", + "mode": "clean", + "checked": 257, + "annotated": [ + "approval-engine/approval-engine-audit", + "core-hub/core-hub-api-token", + "core-hub/core-hub-migration-database", + "core-hub/core-hub-runtime-database", + "core-hub/hub-core-migration-database", + "core-hub/hub-core-runtime-database", + "informed-decision/informed-decision-audit", + "rapp-qonto/rapp-qonto-runtime", + "reuse/reuse-surface-env", + "sso/authelia-secrets", + "sso/keycape-approval-engine-operator-client", + "sso/keycape-config", + "sso/keycape-factor-read", + "sso/keycape-informed-decision-sitting-requester-client", + "sso/keycape-pi-token", + "sso/keycape-rapp-qonto-client", + "sso/keycape-secrets-engine-approval-client", + "sso/keycape-secrets-engine-requester-client", + "sso/lldap-secrets", + "state-hub/state-hub-env", + "state-hub/state-hub-rename-preflight", + "target-revenue/target-revenue-pg-credentials", + "target-revenue/target-revenue-runtime", + "target-revenue/target-revenue-trf-app-credentials", + "telemetry/telemetry-alert-smtp", + "telemetry/telemetry-grafana-admin", + "user-engine/user-engine-delivery" + ], + "cleaned": [ + "approval-engine/approval-engine-audit", + "core-hub/core-hub-api-token", + "core-hub/core-hub-migration-database", + "core-hub/core-hub-runtime-database", + "core-hub/hub-core-migration-database", + "core-hub/hub-core-runtime-database", + "informed-decision/informed-decision-audit", + "rapp-qonto/rapp-qonto-runtime", + "reuse/reuse-surface-env", + "sso/authelia-secrets", + "sso/keycape-approval-engine-operator-client", + "sso/keycape-config", + "sso/keycape-factor-read", + "sso/keycape-informed-decision-sitting-requester-client", + "sso/keycape-pi-token", + "sso/keycape-rapp-qonto-client", + "sso/keycape-secrets-engine-approval-client", + "sso/keycape-secrets-engine-requester-client", + "sso/lldap-secrets", + "state-hub/state-hub-env", + "state-hub/state-hub-rename-preflight", + "target-revenue/target-revenue-pg-credentials", + "target-revenue/target-revenue-runtime", + "target-revenue/target-revenue-trf-app-credentials", + "telemetry/telemetry-alert-smtp", + "telemetry/telemetry-grafana-admin", + "user-engine/user-engine-delivery" + ], + "orphaned_namespace": [ + "platform-pg-drill/drill-minio" + ], + "complete": false, + "active_namespace_scan_complete": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-cleanup-after-eso-fix.json b/docs/evidence/2026-09-28-secret-annotation-cleanup-after-eso-fix.json new file mode 100644 index 0000000..38bc555 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-cleanup-after-eso-fix.json @@ -0,0 +1,12 @@ +{ + "captured_at": "2026-09-28T14:24:09.996656+00:00", + "mode": "clean", + "checked": 257, + "annotated": [], + "cleaned": [], + "orphaned_namespace": [ + "platform-pg-drill/drill-minio" + ], + "complete": false, + "active_namespace_scan_complete": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-cleanup-retry.json b/docs/evidence/2026-09-28-secret-annotation-cleanup-retry.json new file mode 100644 index 0000000..8def117 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-cleanup-retry.json @@ -0,0 +1,16 @@ +{ + "captured_at": "2026-09-28T12:55:20.775101+00:00", + "mode": "clean", + "checked": 168, + "annotated": [ + "approval-engine/approval-engine-audit", + "core-hub/core-hub-api-token", + "platform-pg-drill/drill-minio" + ], + "cleaned": [ + "approval-engine/approval-engine-audit", + "core-hub/core-hub-api-token" + ], + "complete": false, + "error": "maintenance incomplete; raw output suppressed; inspect before retry" +} diff --git a/docs/evidence/2026-09-28-secret-annotation-cleanup.json b/docs/evidence/2026-09-28-secret-annotation-cleanup.json new file mode 100644 index 0000000..0ef2413 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-cleanup.json @@ -0,0 +1,70 @@ +{ + "captured_at": "2026-09-28T12:52:14.833450+00:00", + "mode": "clean", + "checked": 168, + "annotated": [ + "activity-core/actcore-runtime-secret", + "activity-core/llm-connect-provider-secrets", + "approval-engine/approval-engine-audit", + "audit-core/audit-core-senders", + "core-hub/core-hub-api-token", + "core-hub/core-hub-migration-database", + "core-hub/core-hub-runtime-database", + "core-hub/hub-core-migration-database", + "core-hub/hub-core-runtime-database", + "coulomb/ihp-railiance-probe-env", + "databases/net-kingdom-pg-privacyidea-app", + "databases/platform-pg-backup-s3", + "databases/platform-pg-bootstrap", + "databases/state-hub-db-credentials", + "email-connect/email-connect-runtime", + "external-secrets/openbao-audit-core-approle", + "external-secrets/openbao-backup-object-storage-approle", + "external-secrets/openbao-rapp-qonto-approle", + "external-secrets/openbao-sso-user-engine-runtime-approle", + "external-secrets/openbao-user-engine-runtime-approle", + "forgejo/forgejo-mailer", + "forgejo/forgejo-runner-registration", + "informed-decision/informed-decision-audit", + "knative-serving/webhook-certs", + "mfa/privacyidea-auditkeys", + "mfa/privacyidea-config", + "mfa/privacyidea-enckey", + "mfa/privacyidea-trigger-admin", + "openbao/bao-tls", + "platform-pg-drill/drill-minio" + ], + "cleaned": [ + "activity-core/actcore-runtime-secret", + "activity-core/llm-connect-provider-secrets", + "approval-engine/approval-engine-audit", + "audit-core/audit-core-senders", + "core-hub/core-hub-api-token", + "core-hub/core-hub-migration-database", + "core-hub/core-hub-runtime-database", + "core-hub/hub-core-migration-database", + "core-hub/hub-core-runtime-database", + "coulomb/ihp-railiance-probe-env", + "databases/net-kingdom-pg-privacyidea-app", + "databases/platform-pg-backup-s3", + "databases/platform-pg-bootstrap", + "databases/state-hub-db-credentials", + "email-connect/email-connect-runtime", + "external-secrets/openbao-audit-core-approle", + "external-secrets/openbao-backup-object-storage-approle", + "external-secrets/openbao-rapp-qonto-approle", + "external-secrets/openbao-sso-user-engine-runtime-approle", + "external-secrets/openbao-user-engine-runtime-approle", + "forgejo/forgejo-mailer", + "forgejo/forgejo-runner-registration", + "informed-decision/informed-decision-audit", + "knative-serving/webhook-certs", + "mfa/privacyidea-auditkeys", + "mfa/privacyidea-config", + "mfa/privacyidea-enckey", + "mfa/privacyidea-trigger-admin", + "openbao/bao-tls" + ], + "complete": false, + "error": "maintenance incomplete; raw output suppressed; inspect before retry" +} diff --git a/docs/evidence/2026-09-28-secret-annotation-enforced.json b/docs/evidence/2026-09-28-secret-annotation-enforced.json new file mode 100644 index 0000000..77967b5 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-enforced.json @@ -0,0 +1,18 @@ +{ + "observed_at": "2026-09-28T14:30:57.341819+00:00", + "application_revision": "7daf7e90675b21c8f9556028e54aa8c0a13fd9f0", + "application_declaration_commit": "db51ec802801ddf85a80bf81980865c9f9239839", + "sync": "Synced", + "health": "Healthy", + "operation_phase": "Succeeded", + "binding_present": true, + "validation_actions": [ + "Deny" + ], + "policy_type_checking": {}, + "policy_observed_generation": 1, + "policy_generation": 1, + "externalsecrets_total": 39, + "externalsecrets_ready": 39, + "externalsecrets_with_template": 39 +} diff --git a/docs/evidence/2026-09-28-secret-annotation-final-scan.json b/docs/evidence/2026-09-28-secret-annotation-final-scan.json new file mode 100644 index 0000000..aa0abd5 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-final-scan.json @@ -0,0 +1,10 @@ +{ + "captured_at": "2026-09-28T16:08:03.695223+00:00", + "mode": "inspect", + "checked": 257, + "annotated": [], + "cleaned": [], + "orphaned_namespace": [], + "complete": true, + "active_namespace_scan_complete": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-post-binding.json b/docs/evidence/2026-09-28-secret-annotation-post-binding.json new file mode 100644 index 0000000..d4defb9 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-post-binding.json @@ -0,0 +1,22 @@ +{ + "captured_at": "2026-09-28T13:00:47.979858+00:00", + "mode": "clean", + "checked": 257, + "annotated": [ + "sso/keycape-approval-engine-operator-client", + "sso/keycape-factor-read", + "sso/keycape-secrets-engine-approval-client", + "state-hub/state-hub-rename-preflight" + ], + "cleaned": [ + "sso/keycape-approval-engine-operator-client", + "sso/keycape-factor-read", + "sso/keycape-secrets-engine-approval-client", + "state-hub/state-hub-rename-preflight" + ], + "orphaned_namespace": [ + "platform-pg-drill/drill-minio" + ], + "complete": false, + "active_namespace_scan_complete": true +} diff --git a/docs/evidence/2026-09-28-secret-annotation-rollback.json b/docs/evidence/2026-09-28-secret-annotation-rollback.json new file mode 100644 index 0000000..da9b962 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-rollback.json @@ -0,0 +1,10 @@ +{ + "observed_at": "2026-09-28T13:17:32.061629+00:00", + "application_revision": "6016f72a8db26df1eed7b7b9f3b36c8a0eb9f3b7", + "sync": "Synced", + "health": "Healthy", + "operation_phase": "Succeeded", + "binding_present": false, + "externalsecrets_total": 39, + "externalsecrets_ready": 39 +} diff --git a/docs/evidence/2026-09-28-secret-annotation-rollout.md b/docs/evidence/2026-09-28-secret-annotation-rollout.md new file mode 100644 index 0000000..2618579 --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-rollout.md @@ -0,0 +1,147 @@ +# Secret annotation guard: rollout, failed integration and rollback + +CUST-WP-0073-T03, September 28, 2026. The founder authorized continuing the +existing workplans. This receipt records an unsuccessful rollout with recovery; +it is not evidence for promoting the agent to autopilot. + +## Source and deployment + +The platform owner source added the native policy/binding and safe maintenance +helper in `railiance-platform@800cbfa870661d47bee34c747f04f6145875adf1`. +Application commit `54885ac1589074a68d9607d1be250c84c5c2307a` pinned that revision. +The AppProject allowlist gained only the two admission kinds. Publication used +repo-manager; deployment used manual Argo resource-scoped sync, without syncing +unrelated root changes. The application has no automated sync or finalizer. + +Policy type checking passed. The first synthetic proof failed on an SSA field +ownership conflict; its failed receipt is retained. The corrected proof tests +SSA of the same value followed by a clean update. All nine native checks passed: +clean create/update/SSA; annotated create/update rejected, including empty +values; client-side apply rejected; synthetic fixture removed. These checks did +not establish compatibility with existing controllers. + +## Actual integration failure + +ESO v0.16.1 copied the ExternalSecret source last-applied annotation back onto +its target when no target template existed. Under Deny enforcement, required +Secret refreshes failed (ten ExternalSecrets observed in SecretSyncedError). +31 live ExternalSecrets lacked an explicit template. The implementation is +visible in the [installed-version upstream source](https://github.com/external-secrets/external-secrets/blob/v0.16.1/pkg/controllers/externalsecret/externalsecret_controller_template.go): +without a target template the controller copies source metadata; with one it +uses template metadata. Merely removing annotations from the targets does not +fix this writer behavior. + +The binding was deleted promptly to restore refreshes. Failed ExternalSecrets +were explicitly force-refreshed. All 39 became Ready. Source rollback commit +`6016f72a8db26df1eed7b7b9f3b36c8a0eb9f3b7` removes the binding from kustomization +and keeps it in `binding.pending.yaml`. Application commit +`c5d65b0b405be9ce5624f49c6f6df54c12b38735` pins that policy-only revision. +Both were published using repo-manager; the root was synced only for this +Application, then the child synced to its policy-only revision. + +Final read-back at 13:17:32 UTC: application Synced/Healthy, operation Succeeded, +binding absent, 39/39 ExternalSecrets Ready. See +`2026-09-28-secret-annotation-rollback.json`. The policy remains installed but +UNBOUND. A normal sync of the pinned source cannot re-enable enforcement. + +## Cleanup and limits + +The recorded passes removed the annotation from 49 distinct Secrets in active +namespaces. Some were cleaned again after ESO recreated the annotation. This is +not a claim that all remain annotation-free after rollback. The helper changed +only that metadata key, never Secret data. ESO recovery performs its normal +refresh behavior; no credential rotation was performed by this work. + +`platform-pg-drill/drill-minio` is orphaned: its namespace is absent, so the API +refuses the metadata patch. A referencing Deployment, a PVC and Service also +remain without that namespace. No orphan was deleted or namespace recreated. +Cluster-wide cleanup is incomplete. Disposition remains under existing T03. + +Kubectl subprocess output was captured and suppressed throughout the helper. +The old raw presence template failed on absent annotations; its error was +suppressed rather than exposing a Secret dump. The replacement iterates keys +and handles absent/empty maps. Orientation §6 now requires the safe helper. + +## Remaining work in T03 + +Before re-enabling the strict guard, explicitly set target metadata in the 31 +owning ExternalSecret declarations while preserving intended labels/annotations +and all existing data templates. Verify actual controller refresh and clean +resulting target metadata, repeat cleanup and synthetic checks, then verify +ESO refresh with enforcement enabled. No ESO exemption or controller upgrade +is proposed. The owner source changes and orphan disposition remain unfinished; +no additional workplan or task has been created. + +Receipts alongside this file: `secret-annotation-cleanup.json`, +`secret-annotation-cleanup-retry.json`, `secret-annotation-cleanup-active.json`, +`secret-annotation-post-binding.json`, `secret-annotation-admission-proof.json`, +`secret-annotation-admission-proof-final.json`, and +`secret-annotation-rollback.json`, all prefixed `2026-09-28-`. + +## Corrected rollout — September 28 follow-up + +The founder instructed “Good, go on” after the 31-declaration remediation was +identified. Explicit target metadata was added to 31 ExternalSecrets in 23 files +across 12 owning repositories. Source labels and intentional annotations remain; +controller bookkeeping and last-applied are not inherited. Data mappings, data +templates, store references, creation/deletion policies and refresh intervals +were unchanged. Server-side dry-run and semantic comparison verified this for +all 31. A canary refreshed successfully and removed its copied annotation. + +All source changes were committed and published. Eleven repositories have exact +primary repo-manager receipts. activity-core's repo-manager registration refused +pre-existing historical workplan IDs; its documented `statehub fix-consistency` +path passed with warnings and pushed the exact metadata commit, without changing +those historical file IDs. See `2026-09-28-eso-metadata-publication.json` and +`2026-09-28-eso-metadata-changes.json`. Required user-engine checks passed (four +tests); telemetry pinned-chart fetch/check and family validation passed (one +pre-existing declaration warning). + +Thirty non-Argo-managed ExternalSecrets received metadata-only server-side +apply through the existing SSH admin path. Target Revenue's ExternalSecret used +a selective Argo sync at `a3a8a27a8cda32ae3c7b55353ee16a131c50a0a0`, declared by +platform commit `d2631f6112fca8f374b37aa52bc34400c12c95e1`. The operation result +lists only that ExternalSecret; no migration/bootstrap hook or workload rollout +was run. Its application is Synced and Healthy. + +All 39 ExternalSecrets completed fresh successful refreshes before enforcement. +A complete active-namespace scan checked 257 Secrets and found no last-applied +annotations; ESO had removed the formerly inherited metadata. The absent-namespace +orphan remained separately reported. + +Guard source `7daf7e90675b21c8f9556028e54aa8c0a13fd9f0` includes `binding.yaml`. +Application commit `db51ec802801ddf85a80bf81980865c9f9239839` pins that source. +Both were published through repo-manager. Selective root/child Argo sync enabled +the binding without unrelated app changes. At 14:30:57 UTC the guard was +Synced/Healthy, the binding was present with Deny, and policy type checking was +clear at observed generation 1. Nine native admission checks passed again. +**All 39 ExternalSecrets then completed fresh successful refreshes under Deny.** +Receipts: `2026-09-28-secret-annotation-enforced.json`, +`2026-09-28-secret-annotation-admission-proof-reenabled.json`, and +`2026-09-28-eso-refresh-{before,after}-binding.json`. + +The old rollout failure remains a failed original proposal requiring refinement +and recovery. Successful remediation does not retroactively earn unchanged +execution credit. No agent promotion, credential rotation or interactive-runtime +cutover is claimed. + +The remaining orphan is the August 13 Secret +`platform-pg-drill/drill-minio`, UID `2fcb66df-d90f-4776-8ea0-8ca1a04bd307`. +The namespace is absent and has no pods. A reviewable UID-bound proposal to +delete only that Secret is in `docs/changes/CUST-WP-0073/orphan-secret-deletion.json`. +Explicit deletion approval is pending; the bound 3 GiB PVC and all other resources +are outside that proposal. No deletion has occurred. + +### Approved orphan cleanup + +The founder explicitly approved deleting only the orphan Secret. The API accepted +the DELETE with UID precondition `2fcb66df-d90f-4776-8ea0-8ca1a04bd307`; a fresh +identity inventory verified absence. The 3 GiB PVC retained its exact UID, +resourceVersion, volume and Bound status. No other resources or the namespace +were changed, and no Secret values were read or archived. Receipt: +`2026-09-28-orphan-secret-deletion.json`. This resolves the cleanup exception +and completes CUST-WP-0073-T03; the earlier pending-deletion text is historical. + +Final complete cluster-wide scan after deletion: 257 Secrets checked, zero +forbidden annotations, zero orphan exceptions, helper exit 0. Receipt: +`2026-09-28-secret-annotation-final-scan.json`. diff --git a/docs/evidence/2026-09-28-secret-annotation-scan-enforced.json b/docs/evidence/2026-09-28-secret-annotation-scan-enforced.json new file mode 100644 index 0000000..d05c29f --- /dev/null +++ b/docs/evidence/2026-09-28-secret-annotation-scan-enforced.json @@ -0,0 +1,12 @@ +{ + "captured_at": "2026-09-28T14:29:51.628307+00:00", + "mode": "inspect", + "checked": 257, + "annotated": [], + "cleaned": [], + "orphaned_namespace": [ + "platform-pg-drill/drill-minio" + ], + "complete": false, + "active_namespace_scan_complete": true +} diff --git a/docs/evidence/2026-09-28-sizing-review.md b/docs/evidence/2026-09-28-sizing-review.md new file mode 100644 index 0000000..e8376b9 --- /dev/null +++ b/docs/evidence/2026-09-28-sizing-review.md @@ -0,0 +1,94 @@ +# CUST-WP-0071 — allocation review, 2026-09-28 + +## Recommendation + +Keep the accepted Vergabe pilot at 60m CPU / 256Mi memory requests and +1 CPU / 1Gi limits. Keep the already reduced Knative allocations. Propose no +new live allocation change from this sample. This is a provisional pilot +recommendation, not acceptance of representative user performance. + +The node has reservation room but little measured processing room at the +snapshot: 3,420m requested of 4,000m, zero pending requests, **3,963m observed +CPU** and 12,075,401,216 bytes observed memory. The 580m reservation residual +must not be called spare processing capacity. Avoid admitting concurrent builds +on the strength of that residual alone. + +## Evidence and coverage + +- Node verified as `92.205.62.239`, k3s v1.35.1+k3s1. +- `2026-09-28-allocation-reconcile.json` and `.md`: existing collector plus + updated namespace owners; same hardware counted once. State Hub release + preflight passes at this observation, not as a standing admission grant. +- `2026-09-28-cluster-observation.json`: retained source observation, including + instantaneous demand. Collector revisions are recorded in the companion + provenance file. No Secret objects were queried. +- Collector limitation checked against active pods: no pod-level resources, + overhead or restartable init containers were present. Its simplified init + accounting therefore did not encounter these unsupported features today. + This does not certify its accounting for future workloads. +- `2026-09-28-allocation-telemetry.json`: exact PromQL and responses for seven + days, evaluated at five-minute resolution, namespace aggregates. The five + namespaces below each have 2,016 CPU evaluation points. These are evaluation + points, not proof of complete raw scrape coverage or representative traffic. + +| Namespace | CPU p95 (m) | CPU peak (m) | Memory peak (MiB) | +|---|---:|---:|---:| +| vergabe-demo-company | 0.52 | 20.10 | 191.14 | +| knative-serving | 7.75 | 8.54 | 273.54 | +| kourier-system | 3.88 | 4.06 | 59.00 | +| forgejo | 1454.35 | 2208.27 | 4511.85 | +| databases | 238.08 | 341.43 | 1155.20 | + +Namespace peaks need not be simultaneous and cannot be added into a node peak. +Forgejo includes its runner; the databases row is shared demand, not an estimate +of Vergabe's incremental database cost. Namespace aggregates can hide missing +individual pod series. Peak means the maximum sampled value, not every burst. + +Vergabe's throttled-period ratio is 0.000106 (about 0.0106%); the restart counter +query reports zero increase for the namespaces above. Counter evidence is not +proof that deleted or recreated pods never restarted. Forgejo's throttle ratio +is absent; it is not zero. Host CPU history, Vergabe HTTP request/latency series, +and the Forgejo namespace CPU-request recording remain absent in these queries. + +Live Vergabe image: +`forgejo.coulomb.social/coulomb/vergabe-teilnahme@sha256:a26444f59c259698159c69ccb96f73dc648a261ece4c86bb2037a9d977870d91`. +One ready replica, 60m/1 CPU and 256Mi/1Gi. No customer data was written. + +## Existing work replaces duplicate changes + +`RAIL-KNATIVE-WP-0002` finished on September 27. Its T03 verifies that the +railiance-cluster installer now preserves the reduced requests. The stale inbox +warning about the installer reverting them is superseded by that file evidence. +`RAIL-EN-WP-0002` is also finished: ArgoCD resources are already declared and +applied. Neither warrants another task here. + +T03 retains review of Forgejo/runner demand, twelve zero-request workloads and +the distinction between release reservations and real CPU contention. There is +no approved new sizing change for T04 to deploy today. T04 must verify the +accepted final recommendation, including an explicit keep decision if supported, +rather than manufacture a resize to satisfy its title. + +## Smallest remaining execution + +Use the existing T02 for one bounded synthetic pilot session, with product-owner +response/error targets, two concurrent users, document round-trip, edits and +restart evidence. Reuse the invited-pilot fixture; do not create a load-test +service. Link the existing RAPPS-WP-0014-T03 acceptance work rather than duplicate +its data-recovery tasks. The seven-day idle/light-use sample is useful input but +does not replace this session. + +T03/T04 then resolve a single allocation recommendation and verify only accepted +changes. Preserve a 160m reservation envelope for the current State Hub +100m API + 10m MCP + 50m migration requests, and account separately for scheduled +maintenance and competing releases. This is a review assumption, not a global +admission policy or evidence that the node has 160m spare execution capacity. + +T05 reuses activity-core's durable scheduler: Monday 08:00 Europe/Berlin, +Custodian review ownership, retained reports and State Hub progress delivery. +Retain the exact queries with every report; missing signals stay unknown. +The minimum first report covers keep/investigate decisions above, allocation, +sample coverage and links to these existing tasks. The first scheduled run, +acknowledgment and missed-run/recovery evidence are still required. No weekly +schedule was installed by this review; no unattended receipt is claimed. + +No new task, workplan, intake, monitoring service or resource mutation was made. diff --git a/docs/evidence/2026-09-28-supervised-runtime-coordination.json b/docs/evidence/2026-09-28-supervised-runtime-coordination.json new file mode 100644 index 0000000..27a1966 --- /dev/null +++ b/docs/evidence/2026-09-28-supervised-runtime-coordination.json @@ -0,0 +1,11 @@ +{ + "scenario_type": "cross_owner_wait", + "case_id": "CUST-WP-0073-T02--GLAS-WP-0012", + "obligor_workplan_id": "GLAS-WP-0012", + "beneficiary_workplan_id": "CUST-WP-0073", + "state": "waiting", + "reason": "Hub confirms GLAS-WP-0012 blocked. The existing local profile lacks end-to-end production acceptance; standalone bwrap process proof is insufficient for interactive agent migration. No worker injected or owner task reassigned.", + "flavor": "implementation", + "observer": "the-custodian", + "next_action": "Observe existing GLAS-WP-0012 acceptance receipt before relying on its runtime; verify actual agent admin-path denial under CUST-WP-0073-T02." +} diff --git a/docs/evidence/2026-09-28-supervised-sandbox-proof.json b/docs/evidence/2026-09-28-supervised-sandbox-proof.json new file mode 100644 index 0000000..88f77d4 --- /dev/null +++ b/docs/evidence/2026-09-28-supervised-sandbox-proof.json @@ -0,0 +1,20 @@ +{ + "captured_at": "2026-09-28T12:50:24.073685+00:00", + "workplan_task": "CUST-WP-0073-T02", + "profile": "profile.bwrap-local", + "model_called": false, + "interactive_agent_migrated": false, + "sandbox_id": "0e96c810", + "checks": { + "admin_paths_absent": true, + "admin_environment_absent": true, + "only_loopback_interface": true, + "approved_observation_readable": true, + "proposal_preparation_works": true, + "wrong_consumer_denied": true, + "workspace_removed": true, + "destroyed": true, + "host_source_unchanged": true + }, + "passed": true +} diff --git a/docs/evidence/namespace-ownership.yaml b/docs/evidence/namespace-ownership.yaml index bb9c636..4b20004 100644 --- a/docs/evidence/namespace-ownership.yaml +++ b/docs/evidence/namespace-ownership.yaml @@ -1,8 +1,10 @@ # Namespace → owner/service mapping for CUST-WP-0071-T01. # Unknown namespaces stay unknown; do not invent tenants. namespaces: - knative-serving: {owner: rail-kubernetes, service: knative-serving, tenant: unknown} - kourier-system: {owner: rail-kubernetes, service: kourier, tenant: unknown} + knative-serving: {owner: rail-knative, service: knative-serving, tenant: unknown} + kourier-system: {owner: rail-knative, service: kourier, tenant: unknown} + argocd: {owner: railiance-enablement, service: argocd, tenant: unknown} + bao-notice: {owner: railiance-platform, service: bao-notice, tenant: unknown} kube-system: {owner: railiance-cluster, service: k3s-control-plane, tenant: unknown} cert-manager: {owner: railiance-cluster, service: cert-manager, tenant: unknown} external-secrets: {owner: railiance-platform, service: external-secrets, tenant: unknown} diff --git a/scripts/prove_supervised_sandbox.py b/scripts/prove_supervised_sandbox.py new file mode 100644 index 0000000..ce496b7 --- /dev/null +++ b/scripts/prove_supervised_sandbox.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""CUST-WP-0073-T02: exercise existing sand-boxer isolation without a model call. + +Run with ~/glas-harness/.venv/bin/python. This does not switch the current +interactive agent into the sandbox or certify a complete agent runtime. +""" +import json +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from sandboxer.core.manager import SandboxManager +from sandboxer.lifecycle.store import SandboxStore +from sandboxer.models import Consumer, SandboxCreateRequest, SandboxExecRequest +from sandboxer.payments.credits import CreditsStore +from sandboxer.snapshots.store import SnapshotStore + + +PROBE = r''' +import json, os, socket +from pathlib import Path +paths = ['/home/worsch', '/home/tegwick', '/root', '/etc/rancher/k3s', + '/run/docker.sock', '/var/run/docker.sock', '/run/containerd', + '/run/user/1000', '/mnt/c'] +checks = {'admin_paths_absent': all(not Path(p).exists() for p in paths), + 'admin_environment_absent': not any(os.environ.get(k) for k in + ['SSH_AUTH_SOCK', 'KUBECONFIG', 'BAO_TOKEN', 'VAULT_TOKEN']), + 'only_loopback_interface': socket.if_nameindex() == [(1, 'lo')], + 'approved_observation_readable': Path('observation.txt').read_text() == 'synthetic observation\n'} +Path('proposal.txt').write_text('synthetic privileged action proposal; not executed\n') +checks['proposal_preparation_works'] = Path('proposal.txt').is_file() +print(json.dumps(checks)) +''' + + +def main(): + report = {"captured_at": datetime.now(timezone.utc).isoformat(), + "workplan_task": "CUST-WP-0073-T02", "profile": "profile.bwrap-local", + "model_called": False, "interactive_agent_migrated": False} + with tempfile.TemporaryDirectory(prefix="cust-supervised-proof-") as temp: + root = Path(temp) + source = root / "source" + source.mkdir() + (source / "observation.txt").write_text("synthetic observation\n") + manager = SandboxManager( + store=SandboxStore(path=root / "sandboxes.json"), + credits=CreditsStore(path=root / "credits.json"), + snapshots=SnapshotStore(path=root / "snapshots.json"), + ) + consumer = Consumer(actor="agt", project="the-custodian", + run_id="cust-wp-0073-supervised-proof") + status = manager.create(SandboxCreateRequest( + profile="profile.bwrap-local", inputs={"repo": str(source)}, + consumer=consumer, ttl="5m", + )) + report["sandbox_id"] = status.sandbox_id + try: + result = manager.execute(status.sandbox_id, SandboxExecRequest( + command=["/usr/bin/python3", "-c", PROBE], consumer=consumer, + timeout_seconds=15, + )) + if result.exit_code or result.timed_out or result.output_truncated: + raise RuntimeError("sandbox probe failed; child output suppressed") + report["checks"] = json.loads(result.stdout) + wrong = Consumer(actor="agt", project="the-custodian", run_id="wrong-run") + try: + manager.execute(status.sandbox_id, SandboxExecRequest( + command=["/bin/true"], consumer=wrong, timeout_seconds=5)) + except (ValueError, PermissionError): + report["checks"]["wrong_consumer_denied"] = True + else: + report["checks"]["wrong_consumer_denied"] = False + finally: + destroyed = manager.destroy(status.sandbox_id) + report["checks"]["workspace_removed"] = not Path(status.reachability.workspace_dir).exists() + report["checks"]["destroyed"] = destroyed.state.value == "destroyed" + report["checks"]["host_source_unchanged"] = not (source / "proposal.txt").exists() + report["passed"] = all(report["checks"].values()) + print(json.dumps(report, indent=2)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/summarize_agent_supervision.py b/scripts/summarize_agent_supervision.py new file mode 100644 index 0000000..bf6a102 --- /dev/null +++ b/scripts/summarize_agent_supervision.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Summarize per-agent supervised proposals; never grant or execute authority.""" +import argparse +import json +from collections import Counter +from pathlib import Path + + +def summarize(record): + if record.get("mode") not in {"supervised", "autopilot"}: + raise ValueError("invalid agent mode") + if not record.get("agent_id") or not record.get("scope"): + raise ValueError("agent_id and scope required") + seen = set() + counts = Counter() + for proposal in record["proposals"]: + identity = proposal["proposal_id"] + if identity in seen: + raise ValueError("duplicate original proposal; revisions are not new trials") + seen.add(identity) + decision = proposal["disposition"] + outcome = proposal["outcome"] + if decision not in {"pending", "withdrawn", "accepted_unchanged", "accepted_revised", "rejected", "unscored"}: + raise ValueError("invalid disposition") + if outcome not in {"not_executed", "unverified", "succeeded", "failed"}: + raise ValueError("invalid outcome") + counts[decision] += 1 + counts["total"] += 1 + counts["outcome_" + outcome] += 1 + if proposal.get("refinement_or_rescue") is True: + counts["refinement_or_rescue"] += 1 + if decision == "unscored": + # Historic/broad conversation approval lacks an exact submitted + # revision. Preserve it without fabricating promotion evidence. + continue + if decision in {"accepted_unchanged", "accepted_revised", "rejected"}: + counts["adjudicated"] += 1 + if outcome != "not_executed" and decision not in {"accepted_unchanged", "accepted_revised"}: + raise ValueError("executed trial requires an accepted proposal") + if decision in {"accepted_unchanged", "accepted_revised"}: + if not proposal.get("approval_ref") or not proposal.get("original_digest") or not proposal.get("approved_digest"): + raise ValueError("scored acceptance requires exact proposal and approval references") + equal = proposal["original_digest"] == proposal["approved_digest"] + if equal != (decision == "accepted_unchanged"): + raise ValueError("disposition disagrees with approved revision") + if outcome == "unverified": + counts["unverified"] += 1 + if outcome in {"succeeded", "failed"}: + if not proposal.get("verification_ref") or not proposal.get("executed_digest"): + raise ValueError("verified execution requires receipt and exact executed revision") + if proposal["executed_digest"] != proposal["approved_digest"]: + raise ValueError("execution differs from approved revision") + if type(proposal.get("refinement_or_rescue")) is not bool: + raise ValueError("execution refinement/rescue must be explicit") + counts["verified_executions"] += 1 + if outcome == "succeeded" and decision == "accepted_unchanged" and not proposal["refinement_or_rescue"]: + counts["unchanged_successes"] += 1 + def rate(numerator, denominator): + return counts[numerator] / counts[denominator] if counts[denominator] else None + return {"agent_id": record["agent_id"], "scope": record["scope"], + "mode": record["mode"], "counts": dict(counts), + "unchanged_acceptance_rate": rate("accepted_unchanged", "adjudicated"), + "unchanged_execution_success_rate": rate("unchanged_successes", "verified_executions"), + "authority_granted": False, + "note": "Descriptive evidence only; null rates mean no eligible sample. Never promotes an agent."} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("record", type=Path) + args = parser.parse_args() + try: + report = summarize(json.loads(args.record.read_text())) + except (ValueError, KeyError, TypeError, OSError): + parser.exit(2, "Invalid supervision record; no report produced.\n") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_agent_supervision.py b/tests/test_agent_supervision.py new file mode 100644 index 0000000..8982860 --- /dev/null +++ b/tests/test_agent_supervision.py @@ -0,0 +1,57 @@ +import importlib.util +from pathlib import Path + +import pytest + +spec = importlib.util.spec_from_file_location("supervision", Path(__file__).resolve().parents[1] / "scripts/summarize_agent_supervision.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def record(*proposals): + return {"agent_id": "fixture", "scope": "synthetic", "mode": "supervised", "proposals": list(proposals)} + + +def proposal(identity, disposition="accepted_unchanged", outcome="succeeded", refinement=False): + approved = "original" if disposition == "accepted_unchanged" else "revised" + return dict(proposal_id=identity, disposition=disposition, outcome=outcome, + original_digest="original", approved_digest=approved, executed_digest=approved, + approval_ref="synthetic-approval", verification_ref="synthetic-verification", + refinement_or_rescue=refinement) + + +def test_no_sample_is_unknown_and_cannot_grant_authority(): + report = module.summarize(record()) + assert report["unchanged_acceptance_rate"] is None + assert report["unchanged_execution_success_rate"] is None + assert report["authority_granted"] is False + + +def test_revisions_rejections_and_rescue_do_not_earn_unchanged_success(): + report = module.summarize(record(proposal("one"), proposal("two", "accepted_revised"), + proposal("three", refinement=True), proposal("four", "rejected", "not_executed"))) + assert report["unchanged_acceptance_rate"] == 2 / 4 + assert report["unchanged_execution_success_rate"] == 1 / 3 + + +def test_approval_alone_is_not_execution_success(): + report = module.summarize(record(proposal("one", outcome="not_executed"))) + assert report["unchanged_acceptance_rate"] == 1 + assert report["unchanged_execution_success_rate"] is None + + +def test_duplicate_trials_and_unapproved_revisions_are_rejected(): + with pytest.raises(ValueError): + module.summarize(record(proposal("same"), proposal("same"))) + wrong = proposal("one"); wrong["executed_digest"] = "unapproved" + with pytest.raises(ValueError): + module.summarize(record(wrong)) + + +def test_unscored_history_remains_visible_without_manufacturing_a_rate(): + report = module.summarize(record(proposal("historic", "unscored", "failed", True))) + assert report["counts"]["unscored"] == 1 + assert report["counts"]["outcome_failed"] == 1 + assert report["counts"]["refinement_or_rescue"] == 1 + assert report["unchanged_acceptance_rate"] is None + assert report["unchanged_execution_success_rate"] is None diff --git a/tests/test_secret_annotation_maintenance.py b/tests/test_secret_annotation_maintenance.py new file mode 100644 index 0000000..29574b0 --- /dev/null +++ b/tests/test_secret_annotation_maintenance.py @@ -0,0 +1,61 @@ +"""Ensure maintenance cannot echo credentials or change Secret data.""" +import importlib.util +import json +import subprocess +from pathlib import Path +from unittest.mock import patch + +PATH = Path(__file__).resolve().parents[1] / "docs/changes/CUST-WP-0073/secret_annotation_maintenance.py" +spec = importlib.util.spec_from_file_location("maintenance", PATH) +maintenance = importlib.util.module_from_spec(spec) +spec.loader.exec_module(maintenance) + + +def test_failure_does_not_return_raw_secret_output(): + responses = [subprocess.CompletedProcess([], 0, "sso example\n", ""), + subprocess.CompletedProcess([], 0, "namespace/sso\n", ""), + subprocess.CompletedProcess([], 1, "SENSITIVE-STDOUT", "SENSITIVE-STDERR")] + with patch.object(maintenance.subprocess, "run", side_effect=responses): + report = maintenance.maintain() + assert report["complete"] is False + assert "SENSITIVE" not in json.dumps(report) + + +def test_clean_only_removes_annotation_and_checks_result(): + responses = ["sso example", "namespace/sso", "HAS-ANNOTATION", "secret/example patched", "clean"] + calls = [] + def fake(args): + calls.append(args) + return responses.pop(0) + with patch.object(maintenance, "run", side_effect=fake): + report = maintenance.maintain(clean=True) + assert report["complete"] and report["cleaned"] == ["sso/example"] + operation = json.loads(calls[3][-1]) + assert operation == [{"op": "remove", "path": "/metadata/annotations/kubectl.kubernetes.io~1last-applied-configuration"}] + assert calls[2] == calls[4] + + +def test_inspect_never_patches_and_rejects_unexpected_template_output(): + with patch.object(maintenance, "run", side_effect=["sso example", "namespace/sso", "SENSITIVE-DUMP"]): + report = maintenance.maintain() + assert not report["complete"] + assert "SENSITIVE" not in json.dumps(report) + + +def test_inventory_rejects_untrusted_arguments(): + with patch.object(maintenance, "run", return_value="sso --help"): + try: + maintenance.maintain(clean=True) + except RuntimeError: + pass + else: + raise AssertionError("unsafe identity accepted") + + +def test_orphan_namespace_is_explicit_and_never_deleted_or_claimed_clean(): + with patch.object(maintenance, "run", side_effect=["gone orphan\nsso normal", "namespace/sso", "clean"]) as mocked: + report = maintenance.maintain(clean=True) + assert report["orphaned_namespace"] == ["gone/orphan"] + assert not report["complete"] + assert report["active_namespace_scan_complete"] + assert mocked.call_count == 3 diff --git a/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md b/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md index 4520a79..397ca71 100644 --- a/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md +++ b/workplans/CUST-WP-0071-measured-workload-sizing-and-weekly-review.md @@ -9,7 +9,7 @@ flavor: planning owner: the-custodian topic_slug: custodian created: "2026-09-11" -updated: "2026-09-14" +updated: "2026-09-28" related: [STATE-WP-0091, RCLUSTER-WP-0014, RESOURCE-WP-0003, RAPP-TELEMETRY-WP-0001, RAIL-FAB-WP-0028, RAPPS-WP-0014, VERGABE-WP-0019, HFACT-WP-0001] state_hub_workstream_id: "2249bddb-7524-5add-bd5c-c4163a6ca0f3" --- @@ -18,7 +18,7 @@ state_hub_workstream_id: "2249bddb-7524-5add-bd5c-c4163a6ca0f3" User instruction, 2026-09-11: persist and register this work for later follow-up, then continue the invited Vergabe pilot at an explicitly accepted 60m CPU -request. This ready workplan is not a prerequisite to deploying that prototype. +request. This workplan is not a prerequisite to deploying that prototype. It is not an assertion that 60m or the inherited 100m is a measured requirement. The objective is an explainable, repeatable allocation process across Railiance @@ -94,11 +94,10 @@ updated reef-railiance-k3s owner evidence. ```task id: CUST-WP-0071-T02 -status: wait +status: progress priority: high assignee: the-custodian depends_on: [CUST-WP-0071-T01] -blocking_reason: "Await reliable measurements and the current invited-pilot deployment or an equivalent isolated fixture." state_hub_task_id: "82192370-2fd7-5363-88d2-3c67889d3d68" ``` @@ -112,8 +111,10 @@ database demand and request volume. Distinguish container CPU from incremental database/shared-service demand. No benchmark writes to existing customer data. Record workload sizes, concurrency, hardware/image/workers, duration, coverage -and limitations so results are reproducible. Agree response-time/error targets -with the product owner before claiming adequacy. Recommend request/limit and +and limitations so results are reproducible. Founder acceptance target, 2026-09-28: with two simultaneous users, p95 of +ordinary operations must be at most 2 seconds and there must be no failed +operations. Report document transfer time separately. This resolves the target +choice; obtain representative evidence before claiming adequacy. Recommend request/limit and memory values with a stated margin and revisit trigger; label an incomplete pilot sample provisional. A successful smoke test alone is not sizing proof. @@ -121,11 +122,10 @@ pilot sample provisional. A successful smoke test alone is not sizing proof. ```task id: CUST-WP-0071-T03 -status: wait +status: progress priority: high assignee: the-custodian depends_on: [CUST-WP-0071-T01, CUST-WP-0071-T02] -blocking_reason: "Await reconciled demand evidence and a measured pilot recommendation." state_hub_task_id: "6dc67558-eb1e-5bb6-a667-986f884dd495" ``` @@ -221,3 +221,40 @@ updated the mounted credential, and KeyCape recovered. This is immediate recover not a fleet sizing conclusion. T01 must include recurring maintenance-job demand and reliable scheduling headroom, not only resident pod allocations. Evidence: informed-decision/docs/evidence/2026-09-14-keycape-renewal-capacity-recovery.json. + +## September 28 bounded completion review + +The founder asks to finish with minimal additional tasks, workplans and +functionality. Keep all remaining work in T02–T05; do not spawn a monitoring +service, benchmark framework or replacement coordination plan. + +Evidence: `docs/evidence/2026-09-28-sizing-review.md`, allocation reconcile, +retained cluster observation, source revisions and exact seven-day PromQL +responses alongside it. T01 refreshed: 3420m requested / 4000m, no pending +requests, 580m reservation residual; instantaneous node CPU was 3963m, so this +is not spare processing capacity. No unsupported pod accounting features were +present in this snapshot. Namespace ownership refreshed. + +T02 is now in progress: the exact deployed pilot and seven days of measurements +are recorded. CPU p95 0.52m, sampled peak 20.10m, memory peak 191.14Mi; retain +60m/256Mi provisionally. Representative two-user activity, response/error +acceptance and incremental database attribution remain unproven. Use existing +RAPPS-WP-0014-T03 fixture/acceptance work; do not duplicate its recovery scope. + +T03 is now in progress: retain current pilot/Knative allocations, investigate +Forgejo/runner demand (namespace CPU p95 1454m), and account for twelve +zero-request workloads. RAIL-KNATIVE-WP-0002 and RAIL-EN-WP-0002 are finished; +their declaration/deployment work must not be repeated. No new resource change +is proposed from the incomplete sample. T04 can verify a justified keep decision; +it must not create an unnecessary resize. Its useful-operation acceptance stays. + +T05 still requires a durable activity-core schedule, retained report, owner +receipt and missed-run recovery proof. Monday 08:00 Europe/Berlin remains the +proposed cadence. None is represented as installed by this session. Existing +T02–T05 retain the remaining evidence and execution, with no new work records. + +September 28 follow-up: the founder selected the two-user p95 ≤ 2 seconds, +zero-failed-operations target (document transfer excluded from that latency +threshold). The current seven-day telemetry remains provisional until the +representative workflow runs. This is an acceptance criterion, not a claim that +it has passed. Keep the run and its evidence under existing T02. diff --git a/workplans/CUST-WP-0073-agent-credential-separation.md b/workplans/CUST-WP-0073-agent-credential-separation.md index 3ef6cd4..0315192 100644 --- a/workplans/CUST-WP-0073-agent-credential-separation.md +++ b/workplans/CUST-WP-0073-agent-credential-separation.md @@ -1,15 +1,15 @@ --- id: CUST-WP-0073 type: workplan -title: "Agents cannot read secret values: separate agent and admin credentials" +title: "Separate agent credentials and establish supervised privileged execution" domain: infotech repo: the-custodian -status: proposed -owner: claude-code +status: active +owner: the-custodian topic_slug: custodian flavor: implementation created: "2026-09-24" -updated: "2026-09-24" +updated: "2026-09-28" related: - KEY-WP-0033 - RPF-WP-0044 @@ -18,7 +18,7 @@ origin_ref: key-cape/docs/operations.md#before-any-live-change state_hub_workstream_id: "a98a9f34-83b4-5c8c-8107-e05f7806d50d" --- -# Agents cannot read secret values: separate agent and admin credentials +# Separate agent credentials and establish supervised privileged execution ## Why @@ -41,8 +41,12 @@ The root cause is the credentials, not the command: A command denylist chases them one by one, and it only binds the harness that enforces it. -**Goal:** the identity an agent uses cannot read a secret value by any command. -Then a leak needs a human's attended credential, not a mistake. +**Goal:** keep privileged credentials outside the agent's direct reach, and +mediate privileged actions according to the identified agent's autonomy mode. +Agents start supervised; proven agents may later receive scoped autopilot +permission with a cost budget and risk limit in EUR. Neither mode implies +unrestricted admin credentials. Founder decision, September 28: +`docs/agent-autonomy-decision.md`. **Scope:** builder mode, founder decision 2026-09-24. Rotating the exposed Secrets is deferred, not dropped (T05). This plan removes the problem class. @@ -51,42 +55,47 @@ Secrets is deferred, not dropped (T05). This plan removes the problem class. ```task id: CUST-WP-0073-T01 -status: todo +status: done priority: high state_hub_task_id: "4781bb99-020f-59f6-be13-e3b8777d3fd8" ``` -Decide, with railiance-platform and ops-warden: +**Done 2026-09-28 — policy decision.** The founder chooses autonomy as a +characteristic of the agent: supervised-mode initially, promotion only on +successful proposals without adaptation/refinement, and autopilot bounded by +cost and risk limits in EUR. Decision: `docs/agent-autonomy-decision.md`. -- **Agent identity:** a dedicated kube identity outside `system:masters`, - bound to the built-in `view` role plus the specific write verbs agents need - (for example patch and rollout restart on Deployments, ConfigMap updates). - No `secrets` verbs at all, since `list` and `watch` return data too. No - `pods/exec`, `pods/attach`, `pods/portforward` or `nodes/proxy`. No `helm`, - which stores its releases in Secrets. -- **Admin identity:** stays `system:admin`, reachable only by an attended step, - never readable from the agent's Unix account. -- **Where the agent credential lives** on the workstation and on railiance01. - ops-warden already distinguishes `adm`/`agt`/`atm` SSH principals. Mapping - `agt` to a restricted account on railiance01 is the obvious candidate; how - warden provisions those principals is still to be verified. -- **Paths that stay attended:** Secret writes, helm releases and break-glass. +The supervised starting identity is outside `system:masters` and has a reviewed +observation allowlist. Exact privileged actions go through supervisor approval +or supervisor execution, with unchanged-acceptance and verified unchanged-success +recorded separately. Admin credentials remain in the privileged execution path, +outside the agent's direct reach. Later autopilot removes per-action approval +only for an explicit grant within scope, cost and risk constraints; it does not +hand out unrestricted sudo or an admin kubeconfig. -Output: a decision record in the-custodian, resolved by the founder -(`GOVERN @ estate`). +Built-in `view` plus Deployment/ConfigMap writes cannot establish that boundary: +workload writes can extract credentials, and ConfigMaps/pod specs/logs can contain +values. Agent-visible results must be sanitized. T02 selects and proves the +actual account/profile/execution path. This decision resolves the identity and +autonomy policy, not the implementation, numeric promotion thresholds, euro +limits or authorization of a live cutover. Existing human-only lanes still apply. ## Build and hand out the agent identity ```task id: CUST-WP-0073-T02 -status: todo +status: progress priority: high state_hub_task_id: "4b88b0b7-dc7e-5612-aa5d-1a08f0530c35" ``` Owner: railiance-platform (RBAC), railiance-enablement (k3s install), -ops-warden (principal mapping). +railiance-infra (host principal mapping), ops-warden (certificate issuance). +- Bind the agent's stable identity and `supervised-mode` to its existing + instance/assignment record and a restricted runtime profile. Name its + supervisor and privileged execution path. No raw admin credentials in the + agent process/account; no unrestricted sudo, socket or GitOps bypass. - Create the ServiceAccount or client certificate, the ClusterRole and the binding in git, applied by the owner's documented path. - Set `write-kubeconfig-mode` to `600` in the k3s install config. Attended admin @@ -96,19 +105,34 @@ ops-warden (principal mapping). issuance through an attended login). - Proof: as the agent identity, `kubectl auth can-i get secrets -A` and `can-i create pods/exec -A` both answer `no`, and `kubectl auth whoami` - shows no `system:masters`. Record the output as evidence. + shows no `system:masters`. Record the output as evidence. Also verify that + an exact supervisor-approved action can execute through the selected privileged + path and return sanitized outcome evidence, while an unapproved/revised action + cannot use that approval. Test from the actual agent account, including denial + of the old admin SSH/sudo/credential paths. +- Autopilot remains disabled without a scoped promotion decision, concrete EUR + cost/risk limits and verified enforcement. Implementing a general promotion or + risk-scoring service is not required for this supervised credential boundary. ## Reject last-applied annotations on Secrets ```task id: CUST-WP-0073-T03 -status: todo +status: done priority: medium +needs_human: false state_hub_task_id: "5abbfcae-eb65-5b62-a7fa-f4f698f95312" ``` Owner: railiance-platform. +**Done 2026-09-28.** Explicit metadata fixes landed in all 31 affected ESO +declarations. The guard is active and Synced/Healthy; all nine native admission +checks and 39/39 fresh ESO refreshes under Deny pass. All 257 active-namespace +Secrets were annotation-free. After explicit founder approval, the one orphan +drill Secret was deleted with its UID precondition; absence verified and the +3 GiB PVC unchanged. Full evidence: `docs/evidence/2026-09-28-secret-annotation-rollout.md`. + - Add a `ValidatingAdmissionPolicy` (v1.35 is available) with its binding. It rejects any Secret carrying `kubectl.kubernetes.io/last-applied-configuration`. - Strip the annotation from existing Secrets first, cluster-wide, using the @@ -122,7 +146,7 @@ Owner: railiance-platform. ```task id: CUST-WP-0073-T04 -status: todo +status: progress priority: medium state_hub_task_id: "90725511-4e31-549f-b567-47feff1a9ca4" ``` @@ -140,8 +164,13 @@ state_hub_task_id: "90725511-4e31-549f-b567-47feff1a9ca4" acceptable for a stopgap. - Offer the same guard to the Codex and Grok harnesses, or record that they have none. Until T02 lands, those agents are protected by instructions only. -- Open question for the founder: `Bash(bao read *)`, `vault kv get` and - `vault read` are still pre-approved and print secret values the same way. +- OpenBao/Vault secret-reading permissions also belong behind the privileged + boundary; preapproved command prefixes do not implement supervision. +- Document the agent-specific mode and supervisor. Reference exact proposal and + execution receipts; track unchanged acceptance separately from verified + unchanged success, including refinements, rejections and interventions. Reuse + existing records and receipts per `docs/agent-autonomy-decision.md`; no new + dashboard, supervisor service or automatic promotion machinery. ## Rotate what was exposed @@ -165,3 +194,86 @@ Reopen on the first of: - a planned key rotation The passage of time alone reopens nothing. + +## September 28 implementation review + +The founder asks for minimal additional tasks/workplans/functionality. Keep +execution and all unresolved evidence in T01–T05. No new plan or task was opened. + +Reviewable package: `docs/changes/CUST-WP-0073/README.md` and +`reject-secret-last-applied.yaml` beside it. Live read-only inspection confirms +`tegwick` has unrestricted passwordless sudo, k3s kubeconfig is still 644, and +Kubernetes uses `system:admin` / `system:masters`. The public host inventory maps +agent and admin principals to this same account. A kubeconfig switch or chmod +alone is insufficient; do not claim the agent boundary has landed. + +T01's initial permanent observation-only proposal is superseded by the founder's +agent-specific supervised/autopilot decision in `docs/agent-autonomy-decision.md`. +T01 is done; T02 must +verify the actual agent execution environment has no route back through admin +SSH/sudo, tokens, sockets or automated deployment. This adds no new broker. + +T02 in progress: the existing sand-boxer `profile.bwrap-local` passed a +synthetic supervised-process proof: admin homes, Kubernetes/container socket +paths and privileged environment variables absent; only loopback networking; +observation readable; proposal writable; wrong consumer identity rejected; +workspace destroyed. Receipt: `docs/evidence/2026-09-28-supervised-sandbox-proof.json`. +This was not an interactive agent or a credential migration. Existing GLAS +local-profile acceptance and actual admin-path denial remain required in T02. + +T03 in progress: policy source `railiance-platform@800cbfa`, application `54885ac` +and nine passing native admission checks were followed by ESO refresh failures. +ESO v0.16.1 copies source metadata when an ExternalSecret has no target template; +31 declarations need explicit metadata before the strict guard is compatible. +The binding was removed and all 39 ExternalSecrets recovered. GitOps now pins +policy-only `6016f72` via application commit `c5d65b0`; the application is Synced +and Healthy, and enforcement is disabled. Detailed rollout and recovery receipt: +`docs/evidence/2026-09-28-secret-annotation-rollout.md`. + +Cleanup removed the duplicate annotation from 49 distinct active-namespace +Secrets across the recorded passes, but ESO can regenerate it while enforcement +is off. An orphan `platform-pg-drill/drill-minio` Secret cannot be patched because +its namespace is absent; a referencing Deployment, PVC and Service remain. No +orphan was deleted. Neither stable cluster-wide cleanup nor T03 completion is +claimed. Keep remediation and integration proof in this existing task. + +T04 in progress: orientation §6 withdraws the unsafe raw presence template; +only the capturing/sanitizing maintenance helper is allowed. A logical +per-agent supervised record lives at `.kaizen/agents/custodian-codex/supervision.json`. +Its summary separates unchanged acceptance from verified unchanged execution, +retains failed outcomes and rescue, and grants no authority. The rollout is an +unscored historical approval with a failed outcome and recovery, not promotion +evidence. There is no eligible acceptance-rate sample yet, no autopilot grant, +and no enforced interactive-runtime migration. Codex/Grok have no established +equivalent read-denial hook. Final guidance still needs the actual T02 path. +T05 remains the original trigger-based founder deferral, not cancelled or done. + +## Corrected admission rollout — September 28 continuation + +T03: all 31 affected ExternalSecrets now have explicit target metadata in their +owner sources (23 files, 12 repositories, committed and published). Server +dry-run verified only the target template changes; credential data mappings and +policies remain unchanged. All 39 ExternalSecrets refreshed successfully before +and after re-enabling Deny enforcement. All nine native admission checks pass. +The guard is Synced/Healthy at platform source `7daf7e9`, pinned by `db51ec8`. +The earlier rollback is historical, not the current live state. Detailed evidence: +`docs/evidence/2026-09-28-secret-annotation-rollout.md`. + +A complete scan of all 257 Secrets in existing namespaces found no forbidden +annotation after the writer fixes. T03 now waits only for the orphan +`platform-pg-drill/drill-minio`: its namespace is absent, so an annotation patch +is refused. UID-bound deletion of that one Secret is prepared and awaits +explicit authorization; no PVC deletion or namespace recreation is proposed. +All remaining work stays in existing tasks; no new task, workplan, controller +or service was introduced. T02 still needs actual supervised-runtime admission; +T05 keeps its founder-deferred rotation triggers. + +Post-enforcement scan: all 257 active-namespace Secrets remain annotation-free. +The exact orphan deletion also passed server-side dry-run; execution awaits +the founder response. Receipt: `docs/evidence/2026-09-28-secret-annotation-scan-enforced.json`. + +Final orphan disposition: the founder explicitly selected “Delete only the +orphan Secret.” The UID-bound deletion succeeded and absence was verified; +the bound 3 GiB PVC retained the same UID, resourceVersion, volume and status. +No other resources were changed. T03 is done and its human-needed flag cleared. +Receipt: `docs/evidence/2026-09-28-orphan-secret-deletion.json`.