diff --git a/argocd/railiance01/drafts/external-secrets.application.yaml b/argocd/railiance01/drafts/external-secrets.application.yaml index 347278d..eebc9e1 100644 --- a/argocd/railiance01/drafts/external-secrets.application.yaml +++ b/argocd/railiance01/drafts/external-secrets.application.yaml @@ -1,6 +1,6 @@ # DRAFT for railiance01 (RPF-WP-0044). Not synced by any root: move to # ../applications/ only in this app's adoption task, with the founder's go-ahead. -# No automated sync, no finalizer. targetRevision: chart version; T06 adds CRD Prune=false,Delete=false before merge. +# No automated sync, no finalizer. CRDs survive Application pruning/deletion. apiVersion: argoproj.io/v1alpha1 kind: Application metadata: @@ -21,6 +21,9 @@ spec: releaseName: external-secrets values: | installCRDs: true + crds: + annotations: + argocd.argoproj.io/sync-options: Prune=false,Delete=false serviceAccount: create: true name: external-secrets diff --git a/docs/argocd-coulombcore-retirement.md b/docs/argocd-coulombcore-retirement.md new file mode 100644 index 0000000..99fc628 --- /dev/null +++ b/docs/argocd-coulombcore-retirement.md @@ -0,0 +1,62 @@ +# Coulombcore ArgoCD retirement preparation + +Existing owner task: RPF-WP-0044-T08. Prepared September 27, 2026; blocked +pending a readable live inventory. No controller, application or workload was +changed. This document is the phase C preparation within the existing workplan. + +The host SSH lane works. Kubernetes returned Unauthorized for both the normal +kubectl context and `sudo -n k3s kubectl --kubeconfig +/etc/rancher/k3s/k3s.yaml get nodes`. The infrastructure/cluster owner must +restore accepted read access; do not rotate cluster credentials or restart k3s +as an incidental inventory fix. Current workload ownership cannot be inferred +from the old manifests. + +## Inventory and decision inputs + +Capture node/cluster identity, ArgoCD deployments/statefulsets and installed +version, Applications and AppProjects, each Application's destination, pinned +revision, tracked resource set, hooks, automated sync and finalizers. Read only +repository Secret metadata, never data. Identify external cluster destinations: +an old controller can still manage a remote cluster. Search platform and tenant +source for references to `argocd/applications/` and `argocd/bootstrap/`, including +Make entry points, and match each live application to its accepting owner. + +The railiance01 root uses `argocd/railiance01/applications`; the legacy root +uses `argocd/applications`. Do not remove the legacy tree while the old controller +can reconcile it with prune enabled. Keep evidence of each workload's current +replicas, readiness and image before any retirement execution. + +## Ordered execution after inventory and owner acceptance + +1. Have the cluster owner pin the installed railiance01 ArgoCD version and + reviewed resource requests in its canonical source. Inspect current requests; + the original phase A BestEffort observation is historical, not a fresh check. +2. Classify each old tracked resource: already adopted on railiance01, retained + on coulombcore with another owner, or separately approved for retirement. + An application name match does not prove matching cluster/resource identity. +3. Freeze the old root and child reconciliation through the cluster owner's + reviewed procedure. Confirm no running sync operation and no second writer. + Preserve the old Application specs and controller configuration in protected + recovery storage; repository credentials stay under existing custody. +4. Detach only inventory-approved Application objects without cascading workload + deletion. Review resource finalizers first; never delete the ArgoCD namespace + or CRDs as a shortcut. Verify every retained workload is still healthy and + each replacement owner can reconcile its accepted resource set. +5. Disable the old controller through its actual installation owner. Prove it + no longer reconciles and that no unrelated system uses its repository/auth + resources. Revoke retired credentials only through their custody owners. +6. Once the old controller is inert, remove the legacy source directories and + retire or repoint their callers in one reviewed platform change. Render the + railiance01 bootstrap and children and verify no legacy reference remains. + +Stop for missing inventory, ambiguous tracking, active operations, unknown +finalizers, missing acceptance or degraded retained workloads. Before detachment, +rollback restores the recorded sync configuration. After replacement ownership, +keep the old controller stopped until the replacement writer is explicitly +suspended; rollback must never run two reconcilers against one workload. Package +or data deletion needs its own exact approved disposition. + +Completion evidence must include the inventory, accepting owners, source/caller +changes, controller shutdown and retained-workload checks. T08's planning closure +still requires the live read-only inventory. Execution remains with the existing +cluster/infra owners; no new task or workplan is created here. diff --git a/docs/evidence/2026-09-27-argocd-loose-end-status.json b/docs/evidence/2026-09-27-argocd-loose-end-status.json new file mode 100644 index 0000000..8fe470b --- /dev/null +++ b/docs/evidence/2026-09-27-argocd-loose-end-status.json @@ -0,0 +1,492 @@ +{ + "observed_at": "2026-09-27T16:53:28.268968+00:00", + "applications": [ + { + "name": "activity-core", + "sync": { + "comparedTo": { + "destination": { + "namespace": "activity-core", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "k8s/gitops", + "repoURL": "https://forgejo.coulomb.social/coulomb/activity-core.git", + "targetRevision": "a12f1169f9d130058ce767f5b26de0606997916c" + } + }, + "revision": "a12f1169f9d130058ce767f5b26de0606997916c", + "status": "Synced" + }, + "health": { + "lastTransitionTime": "2026-09-27T14:06:22Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:50:21Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-27T14:06:19Z", + "finishedAt": "2026-09-27T14:06:20Z" + }, + "automated": null, + "conditions": [], + "resources": [ + { + "kind": "ConfigMap", + "name": "actcore-external-activity-definitions", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "kind": "ConfigMap", + "name": "actcore-ops-service-inventory", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "kind": "ConfigMap", + "name": "actcore-report-schemas", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "kind": "ConfigMap", + "name": "actcore-runtime-config", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "kind": "Service", + "name": "actcore-api", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "kind": "Service", + "name": "actcore-worker-metrics", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "group": "apps", + "kind": "Deployment", + "name": "actcore-api", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "group": "apps", + "kind": "Deployment", + "name": "actcore-event-router", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + }, + { + "group": "apps", + "kind": "Deployment", + "name": "actcore-worker", + "namespace": "activity-core", + "status": "Synced", + "version": "v1" + } + ] + }, + { + "name": "bao-notice", + "sync": { + "comparedTo": { + "destination": { + "namespace": "bao-notice", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "argocd/platform-addons/bao-notice", + "repoURL": "https://forgejo.coulomb.social/coulomb/railiance-platform.git", + "targetRevision": "9ee20967ca7b50a8926c99ae86fa027f8bfb0fa3" + } + }, + "revision": "9ee20967ca7b50a8926c99ae86fa027f8bfb0fa3", + "status": "OutOfSync" + }, + "health": { + "lastTransitionTime": "2026-09-23T22:56:40Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:52:12Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-23T22:56:34Z", + "finishedAt": "2026-09-23T22:56:36Z" + }, + "automated": null, + "conditions": [ + { + "lastTransitionTime": "2026-09-23T22:55:19Z", + "message": "Application has 1 orphaned resources", + "type": "OrphanedResourceWarning" + } + ], + "resources": [ + { + "kind": "ConfigMap", + "name": "bao-notice-conf-4f5g9kc7c2", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "kind": "ConfigMap", + "name": "bao-notice-conf-dg4hmf2dg4", + "namespace": "bao-notice", + "requiresPruning": true, + "status": "OutOfSync", + "version": "v1" + }, + { + "kind": "ConfigMap", + "name": "bao-notice-html-6mm6h6mgkf", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "kind": "Namespace", + "name": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "kind": "Service", + "name": "bao-notice", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "apps", + "kind": "Deployment", + "name": "bao-notice", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "networking.k8s.io", + "kind": "Ingress", + "name": "bao-notice", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "networking.k8s.io", + "kind": "Ingress", + "name": "bao-notice-http-redirect", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "networking.k8s.io", + "kind": "NetworkPolicy", + "name": "bao-notice-acme-solver", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "networking.k8s.io", + "kind": "NetworkPolicy", + "name": "bao-notice-isolation", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1" + }, + { + "group": "traefik.io", + "kind": "Middleware", + "name": "redirect-https", + "namespace": "bao-notice", + "status": "Synced", + "version": "v1alpha1" + } + ] + }, + { + "name": "eso-token-renewer", + "sync": { + "comparedTo": { + "destination": { + "namespace": "external-secrets", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "argocd/platform-addons/eso-token-renewer", + "repoURL": "https://forgejo.coulomb.social/coulomb/railiance-platform.git", + "targetRevision": "b2ebe108495c4d06ebb6ca33893ea4a2a23c9725" + } + }, + "revision": "b2ebe108495c4d06ebb6ca33893ea4a2a23c9725", + "status": "Synced" + }, + "health": { + "lastTransitionTime": "2026-09-23T22:38:19Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:50:17Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-23T22:38:19Z", + "finishedAt": "2026-09-23T22:38:19Z" + }, + "automated": null, + "conditions": [ + { + "lastTransitionTime": "2026-09-23T22:37:58Z", + "message": "Application has 28 orphaned resources", + "type": "OrphanedResourceWarning" + } + ], + "resources": [ + { + "kind": "ConfigMap", + "name": "eso-token-renewer-worker-5c86dt7fm7", + "namespace": "external-secrets", + "status": "Synced", + "version": "v1" + }, + { + "kind": "ServiceAccount", + "name": "eso-token-renewer", + "namespace": "external-secrets", + "status": "Synced", + "version": "v1" + }, + { + "group": "batch", + "kind": "CronJob", + "name": "eso-token-renewer", + "namespace": "external-secrets", + "status": "Synced", + "version": "v1" + } + ] + }, + { + "name": "openbao-secretstore", + "sync": { + "comparedTo": { + "destination": { + "namespace": "external-secrets", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "argocd/platform-addons/openbao-secretstore", + "repoURL": "https://forgejo.coulomb.social/coulomb/railiance-platform.git", + "targetRevision": "d2dbc19c254247652c49fda8721c80d53bca206a" + } + }, + "revision": "d2dbc19c254247652c49fda8721c80d53bca206a", + "status": "Synced" + }, + "health": { + "lastTransitionTime": "2026-09-21T17:09:00Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:52:14Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-21T17:09:16Z", + "finishedAt": "2026-09-21T17:09:18Z" + }, + "automated": null, + "conditions": [ + { + "lastTransitionTime": "2026-09-23T18:14:58Z", + "message": "Application has 28 orphaned resources", + "type": "OrphanedResourceWarning" + } + ], + "resources": [ + { + "group": "external-secrets.io", + "kind": "ClusterSecretStore", + "name": "openbao", + "status": "Synced", + "version": "v1beta1" + } + ] + }, + { + "name": "railiance-apps-root", + "sync": { + "comparedTo": { + "destination": { + "namespace": "argocd", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "argocd/railiance01/applications", + "repoURL": "https://forgejo.coulomb.social/coulomb/railiance-platform.git", + "targetRevision": "main" + } + }, + "revision": "5781d34b3b9a6e3779b913de200f0eaf63cabacd", + "status": "Synced" + }, + "health": { + "lastTransitionTime": "2026-09-21T17:08:13Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:51:31Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-27T14:05:47Z", + "finishedAt": "2026-09-27T14:05:49Z" + }, + "automated": null, + "conditions": [ + { + "lastTransitionTime": "2026-09-21T17:08:13Z", + "message": "Application has 1 orphaned resources", + "type": "OrphanedResourceWarning" + } + ], + "resources": [ + { + "group": "argoproj.io", + "kind": "Application", + "name": "activity-core", + "namespace": "argocd", + "status": "Synced", + "version": "v1alpha1" + }, + { + "group": "argoproj.io", + "kind": "Application", + "name": "bao-notice", + "namespace": "argocd", + "status": "Synced", + "syncWave": 10, + "version": "v1alpha1" + }, + { + "group": "argoproj.io", + "kind": "Application", + "name": "eso-token-renewer", + "namespace": "argocd", + "status": "Synced", + "syncWave": 2, + "version": "v1alpha1" + }, + { + "group": "argoproj.io", + "kind": "Application", + "name": "openbao-secretstore", + "namespace": "argocd", + "status": "Synced", + "syncWave": 1, + "version": "v1alpha1" + }, + { + "group": "argoproj.io", + "kind": "Application", + "name": "target-revenue", + "namespace": "argocd", + "status": "Synced", + "syncWave": 10, + "version": "v1alpha1" + } + ] + }, + { + "name": "target-revenue", + "sync": { + "comparedTo": { + "destination": { + "namespace": "target-revenue", + "server": "https://kubernetes.default.svc" + }, + "source": { + "path": "k8s/railiance", + "repoURL": "https://forgejo.coulomb.social/coulomb/target-revenue.git", + "targetRevision": "f1109d54eeda9f187daa215cf1c7163610d35d0a" + } + }, + "revision": "f1109d54eeda9f187daa215cf1c7163610d35d0a", + "status": "Synced" + }, + "health": { + "lastTransitionTime": "2026-09-21T17:13:49Z", + "status": "Healthy" + }, + "reconciledAt": "2026-09-27T16:50:17Z", + "operation": { + "phase": "Succeeded", + "startedAt": "2026-09-21T17:14:01Z", + "finishedAt": "2026-09-21T17:14:10Z" + }, + "automated": null, + "conditions": [ + { + "lastTransitionTime": "2026-09-21T17:13:49Z", + "message": "Application has 5 orphaned resources", + "type": "OrphanedResourceWarning" + } + ], + "resources": [ + { + "kind": "Service", + "name": "target-revenue", + "namespace": "target-revenue", + "status": "Synced", + "version": "v1" + }, + { + "group": "apps", + "kind": "Deployment", + "name": "target-revenue", + "namespace": "target-revenue", + "status": "Synced", + "syncWave": 3, + "version": "v1" + }, + { + "group": "external-secrets.io", + "kind": "ExternalSecret", + "name": "target-revenue-runtime", + "namespace": "target-revenue", + "status": "Synced", + "version": "v1beta1" + }, + { + "group": "networking.k8s.io", + "kind": "Ingress", + "name": "target-revenue", + "namespace": "target-revenue", + "status": "Synced", + "syncWave": 4, + "version": "v1" + }, + { + "group": "postgresql.cnpg.io", + "kind": "Cluster", + "name": "target-revenue-pg", + "namespace": "target-revenue", + "status": "Synced", + "syncWave": -2, + "version": "v1" + } + ] + } + ], + "coulombcore_inventory": "unavailable: Kubernetes Unauthorized, including explicit local k3s kubeconfig; no credential or controller mutation" +} diff --git a/docs/evidence/2026-09-27-eso-adoption-preparation.json b/docs/evidence/2026-09-27-eso-adoption-preparation.json new file mode 100644 index 0000000..3a7ff16 --- /dev/null +++ b/docs/evidence/2026-09-27-eso-adoption-preparation.json @@ -0,0 +1,12 @@ +{ + "observed_at": "2026-09-27T16:59:10.112128+00:00", + "chart": "0.16.1", + "draft_sha256": "a0b349cddd90aa707f080efd8556c8379d5b178d93b4c9e408f3b7f439bbf064", + "render_sha256": "55236645afc9f2d9d376c28e73aed70bd31727d0e6f166664b3ebd568d5f136f", + "objects": 39, + "protected_crds": 20, + "server_diff_exit_code": 1, + "applied": false, + "diff_summary": "Exactly 20 CRD metadata annotation additions; no spec, workload or other object changes", + "diff_sha256": "3c34ad673fc374442adc13eeaf17c5216fce7b904103e63014d10e4ba6272965" +} diff --git a/docs/evidence/2026-09-27-keycape-incident-owner-return.json b/docs/evidence/2026-09-27-keycape-incident-owner-return.json new file mode 100644 index 0000000..0e84d8d --- /dev/null +++ b/docs/evidence/2026-09-27-keycape-incident-owner-return.json @@ -0,0 +1,33 @@ +{ + "schema": "platform.incident-owner-return.v1", + "reviewed_at": "2026-09-27T17:01:20.402024+00:00", + "task": "RPF-WP-0027-T05", + "sources": [ + { + "repository": "net-kingdom", + "path": "workplans/NK-WP-0033-keycape-secret-exposure-rotation.md", + "commit": "1da6e5457ad86fff1173bd2ff6174d70719b487d", + "sha256": "8268aa86f7fa9f2a3848adfe2dc3022bcf2e9dc9197328993445f4138a40e513" + }, + { + "repository": "key-cape", + "path": "workplans/KEY-WP-0011-live-secret-exposure-recovery.md", + "commit": "6a996bd71e5e36d7483e7a79b3301bd895907644", + "sha256": "3c5458180fe81a04e357f8db737e4287f79640b5ab02682bcadc595b73f846cd" + } + ], + "operator_ruling_date": "2026-09-23", + "operator": "Bernd Worsch", + "owner_check": "reconcile-lldap-resolver-live.sh --check --predecessor-unavailable", + "owner_receipt": { + "resolver_lookup": "PASS", + "privacyidea_mfa": "PASS", + "predecessor_denial": "NOT-PROVEN", + "readiness": "PASS", + "health": "PASS", + "cleanup": "PASS" + }, + "disposition": "Operator explicitly accepted the August 27 observations; unrecoverable predecessor is not a required new test. NetKingdom marks incident closed.", + "live_action_performed_in_this_review": false, + "custody_handoff_complete": false +} diff --git a/docs/evidence/2026-09-27-service-assurance.json b/docs/evidence/2026-09-27-service-assurance.json new file mode 100644 index 0000000..fa10a6d --- /dev/null +++ b/docs/evidence/2026-09-27-service-assurance.json @@ -0,0 +1,192 @@ +{ + "observation": { + "schema": "railiance-platform.observation.v1", + "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786", + "captured_at": "2026-09-27T16:53:44.173824+00:00", + "signals": { + "apps-pg.ready": { + "result": "pass", + "observed_at": "2026-09-27T16:53:28.888062+00:00" + }, + "apps-pg.backup": { + "result": "pass", + "observed_at": "2026-09-27T02:15:16Z" + }, + "apps-pg.wal": { + "result": "pass", + "observed_at": "2026-09-27T16:53:28.888098+00:00" + }, + "apps-pg.headroom": { + "result": "pass", + "observed_at": "2026-09-27T16:53:16Z" + }, + "platform-pg.ready": { + "result": "pass", + "observed_at": "2026-09-27T16:53:32.163508+00:00" + }, + "platform-pg.backup": { + "result": "pass", + "observed_at": "2026-09-27T02:15:18Z" + }, + "platform-pg.wal": { + "result": "pass", + "observed_at": "2026-09-27T16:53:32.163550+00:00" + }, + "platform-pg.headroom": { + "result": "pass", + "observed_at": "2026-09-27T16:53:32Z" + }, + "platform-pg-2.ready": { + "result": "pass", + "observed_at": "2026-09-27T16:53:35.169803+00:00" + }, + "platform-pg-2.backup": { + "result": "pass", + "observed_at": "2026-09-27T02:15:14Z" + }, + "platform-pg-2.wal": { + "result": "pass", + "observed_at": "2026-09-27T16:53:35.169827+00:00" + }, + "platform-pg-2.headroom": { + "result": "pass", + "observed_at": "2026-09-27T16:53:22Z" + }, + "openbao.seal": { + "result": "pass", + "observed_at": "2026-09-27T16:53:42.333471+00:00" + }, + "eso.ready": { + "result": "pass", + "observed_at": "2026-09-27T16:53:43.311315+00:00" + }, + "eso.refresh": { + "result": "pass", + "observed_at": "2026-09-27T16:06:45Z" + }, + "eso.token-renewal": { + "result": "pass", + "observed_at": "2026-09-27T02:40:09Z" + }, + "apps-pg.restore": { + "result": "pass", + "observed_at": "2026-09-05T22:30:45.208512+00:00" + }, + "forgejo-db.restore": { + "result": "pass", + "observed_at": "2026-09-05T22:55:54.893587+00:00" + }, + "openbao.snapshot": { + "result": "pass", + "observed_at": "2026-08-22T22:29:21Z" + } + } + }, + "evaluation": { + "schema": "railiance-platform.assurance-signal.v1", + "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786", + "evaluated_at": "2026-09-27T16:53:44.173824+00:00", + "signals": { + "apps-pg.ready": { + "state": "healthy", + "owner": "railiance-platform" + }, + "apps-pg.backup": { + "state": "healthy", + "owner": "railiance-platform" + }, + "apps-pg.wal": { + "state": "healthy", + "owner": "railiance-platform" + }, + "apps-pg.restore": { + "state": "healthy", + "owner": "railiance-platform" + }, + "apps-pg.headroom": { + "state": "healthy", + "owner": "railiance-platform" + }, + "platform-pg.ready": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg.backup": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg.wal": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg.restore": { + "state": "missing", + "owner": "rapp-postgres" + }, + "platform-pg.headroom": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg-2.ready": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg-2.backup": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg-2.wal": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "platform-pg-2.restore": { + "state": "missing", + "owner": "rapp-postgres" + }, + "platform-pg-2.headroom": { + "state": "healthy", + "owner": "rapp-postgres" + }, + "openbao.seal": { + "state": "healthy", + "owner": "railiance-platform" + }, + "openbao.snapshot": { + "state": "stale", + "owner": "railiance-platform" + }, + "openbao.restore": { + "state": "missing", + "owner": "railiance-platform" + }, + "offsite.upload": { + "state": "missing", + "owner": "railiance-platform" + }, + "offsite.restore": { + "state": "missing", + "owner": "railiance-platform" + }, + "eso.ready": { + "state": "healthy", + "owner": "railiance-platform" + }, + "eso.refresh": { + "state": "healthy", + "owner": "railiance-platform" + }, + "forgejo-db.restore": { + "state": "healthy", + "owner": "railiance-platform" + }, + "eso.token-renewal": { + "state": "healthy", + "owner": "railiance-platform" + } + }, + "transport": "unmonitored", + "guarantees": "unsupported", + "threshold_status": "local-diagnostic-only", + "healthy": false + } +} diff --git a/docs/net-kingdom-credential-custody-contract.md b/docs/net-kingdom-credential-custody-contract.md index 45240a9..d7e4812 100644 --- a/docs/net-kingdom-credential-custody-contract.md +++ b/docs/net-kingdom-credential-custody-contract.md @@ -3,8 +3,8 @@ Historical resolver lanes: **draft / blocked**. KeyCape factor service lane: **active**, established and verified 2026-09-13 (below). Incident: `KEYCAPE-EXPOSURE-20260823-01` -Consumer procedure: NetKingdom `NK-WP-0033`, resolver reconciliation revision -`eec7007` / checkout `f2e578c` +Consumer procedure: NetKingdom `NK-WP-0033`; latest attended check used +checkout `6096c395` (script `4a38511`) on 2026-09-23. This document defines the Railiance-side contract without containing or deriving any credential value. It is not an authorization to fetch, export, @@ -49,6 +49,17 @@ Operator decision 2026-09-15: leave both historical resolver lanes blocked. Do not invent mount, path, or field names. The KeyCape factor service lane below is separate and does not close this gate. +September 27 evidence review: NetKingdom's September 23 operator ruling and +green attended `--check --predecessor-unavailable` receipt close the incident +verification obligation (RPF-WP-0027-T05). The predecessor remains NOT-PROVEN; +the operator explicitly accepted its unavailable disposition. The owner records +human custody at `operators/lldap/admin` and `operators/privacyidea/pi-admin`, +including KV v2 LLDAP version 1 and withheld delete under `operator-custody`. +These are confirmed owner coordinates, but they do not by themselves establish +either historical route's complete field, auth, expiry and handoff contract. +T03/T06 and the non-resolvable historical routes therefore remain blocked. +See `docs/evidence/2026-09-27-keycape-incident-owner-return.json`. + ## KeyCape factor service lane — authorized setup, 2026-09-13 RPF-WP-0040 / CCR-2026-0023 establish a new dedicated service lane. The user, diff --git a/docs/service-assurance.md b/docs/service-assurance.md index d96aec0..fc2df2a 100644 --- a/docs/service-assurance.md +++ b/docs/service-assurance.md @@ -61,13 +61,12 @@ CronJob status timestamps only. It fails when a newer scheduled run has not succeeded within an hour, and it goes stale after 36h. The tokens lapse after 7 days without renewal. -The following remain missing until a native value-safe adapter and acceptance -exist: validated isolated restore receipts, -OpenBao snapshot/restore proof, and offsite upload/restore receipts. Missing -adapters are not inferred healthy from pod readiness. The local producer is -not the Q2 standard; railiance-telemetry has no implemented receiving contract -in the reviewed checkout. Integration, routing and scheduled delivery remain -T04, and no notification was sent during implementation. +Current gaps include accepted platform-pg/platform-pg-2 and OpenBao isolated +restore samples, fresh OpenBao snapshots, and dated full-archive receipts. +Missing evidence is not inferred healthy from pod readiness. RTEL-WP-0002 +implements the Q2 reference contract and local delivery tests; its T04 still +owns production mapping, recipient and controlled failure/absence acceptance. +Integration, routing and scheduled delivery remain T04 here. ## Service records and evidence inventory @@ -87,7 +86,7 @@ provider invalidation/replacement recovery for the shared offsite lane. | OpenBao snapshot | WARDEN-WP-0027 preparation receipt, 2026-08-23 | Snapshot/encrypted off-host preparation, not isolated restore | | OpenBao restore | Existing `openbao-validate-restore-evidence.sh` and package procedure | Example receipt cannot pass as a fresh execution | | Recovery exercise | RPF-WP-0015-T02/T03 | Separate windows, synthetic driver/quorum and abort operator required | -| Logical/Forgejo offsite | activity-core backup definitions, existing helper/runbooks | No new upload/restore performed; RPF-WP-0029 remains open | +| Logical/Forgejo offsite | activity-core backup definitions, existing helper/runbooks | WP-0029 is finished; WP-0038 retains recurring primary/secondary activation | ## Admission and disclosure drift @@ -134,8 +133,26 @@ Decryption now retains `platform.forgejo-primary-decryption.v1` and its own operation times; older decryption receipts used the transfer schema through an overwrite bug. The restore tool explicitly accepts both forms, with verified hash/decryption flags. Existing historical receipts are unchanged. These producer -fixes enable future dated archive evidence; automatic archive adapters, fresh -end-to-end receipts and recurring execution are still pending. +fixes enable dated archive evidence. The full primary archive adapter is now +implemented; fresh end-to-end receipts and recurring execution remain pending. + +For `offsite.upload`, add a reviewed entry with `signal`, `path` and `sha256` +to the recovery index. It must name a full-profile Scaleway archive transfer +with completed multipart upload, version-pinned GET, matching byte counts/hash, +the application-archive destination and ordered timezone-aware timestamps. +For `offsite.restore`, the entry additionally names `decryption` and `transfer`, +each with `path` and `sha256`. The restore must attest database import, application +health, repository verification, all package blobs and successful scratch cleanup. +The receipt hashes, ciphertext identity, destination, profile and operation order +must match across all three receipts. Only the distinct decryption schema is +accepted for automatic assurance. Essentials and Nextcloud-only proofs cannot +substitute for this primary full-application recovery signal. Other supported +services still need their own evidence; one Forgejo receipt does not close T03. + +No historical index entry was added: the September 6 archive receipts lack +operation timestamps. They remain manual evidence. Tests use synthetic receipt +chains and prove expiry, provenance rejection and rejection of the real undated +receipt; those fixtures do not assert a new live recovery. The OpenBao snapshot adapter also accepts the reviewed, hash-pinned receipt in `reviews/`. It requires the expected source cluster identity, encrypted off-host diff --git a/history/2026-09-27-loose-end-review.md b/history/2026-09-27-loose-end-review.md new file mode 100644 index 0000000..39477b4 --- /dev/null +++ b/history/2026-09-27-loose-end-review.md @@ -0,0 +1,73 @@ +# Loose-end review — September 27, 2026 + +Reviewed every unfinished source workplan, terminal/archived task states, +the empty unread inbox and human-needed Hub records, with current owner sources +and read-only cluster evidence. No new workplan or task was created. Historical +`cancelled` task statuses in the superseded baseline are terminal, not open work. +Retired Hub aliases are not additional source obligations. + +## Completed work + +- Closed RPF-WP-0027-T05 from the newer NK-WP-0033 operator ruling and attended + September 23 receipt. KEY-WP-0011 already records the other credential classes. + Predecessor denial remains NOT-PROVEN; the operator accepted the unavailable + predecessor's disposition explicitly. Source commits and hashes are in + `docs/evidence/2026-09-27-keycape-incident-owner-return.json`. No rotation was + repeated. Historical custody routing still needs its complete accepted contract. +- Implemented the WP-0036-T03 full-primary archive assurance adapter. It requires + an ordered, hash-bound transfer/decryption/recovery chain, verified versioned + download, full profile, application/database/repository/package proof and cleanup. + New primary receipts preserve the archive profile. Undated historical receipts, + essentials-only restores, failures and mismatched chains cannot pass. No fresh + live archive receipt or recurring cadence is claimed by these fixture tests. +- Completed WP-0044-T06 repository preparation: all 20 ESO CRDs receive + `Prune=false,Delete=false` in the inactive draft. Chart 0.16.1 renders 39 objects; + server-side diff contains exactly those 20 annotation additions and no spec + changes. Evidence: `docs/evidence/2026-09-27-eso-adoption-preparation.json`. +- Prepared the concrete phase C retirement sequence under WP-0044-T08 at + `docs/argocd-coulombcore-retirement.md`. Live inventory still cannot finish: + coulombcore SSH succeeds but Kubernetes returns Unauthorized, including with + the explicit host-local k3s kubeconfig. No credential rotation or restart was + attempted as a workaround. +- Corrected the current index, custody incident status and stale assurance + documentation. Platform accepts policy-nexus's proposed chart/values ownership + split in principle; railiance-apps still must accept and publish its values + path/revision. No cross-owner agreement is fabricated. + +## Live observations and remaining gates + +`docs/evidence/2026-09-27-argocd-loose-end-status.json` records six Applications. +Openbao-secretstore and target-revenue are Synced/Healthy, with September 21 +healthy transition times; their original observation periods have elapsed. +Their remaining promotion/owner gates now show `wait` instead of `progress`. +All 25 ClusterSecretStores are Valid and issue-core-runtime is SecretSynced. +Activity-core is Synced/Healthy; its required observation cannot end before +September 28 at 16:06:22 Berlin. Its scoped broker/admission remains separate. +Bao-notice is Healthy but OutOfSync; this review did not sync unrelated apps. + +`docs/evidence/2026-09-27-service-assurance.json` preserves the current observation +and evaluation: 18 healthy, five missing and one stale signal. The stale receipt +is the old OpenBao snapshot. Missing signals are platform-pg/platform-pg-2 restore, +OpenBao restore and offsite upload/restore. The archive adapter is ready for fresh +reviewed receipts; the existing undated September 6 evidence is not re-dated. +Transport remains unmonitored. Q2 has a local contract and selected rapp-telemetry +package; RTEL-WP-0002-T04 still owns actual mapping/delivery/absence acceptance. + +All eight unfinished workplans now explicitly say `blocked`; their 23 remaining +tasks say `wait`. They retain the concrete unblocks in their September 27 entries. +The open work is dominated by native owner acceptance, protected attended actions, +fresh recovery evidence, inventory access and the time-bound observation gate. +No blanket approval request, duplicate task, new schedule, credential read, +backup expiration, production rollout or owner message was introduced. + +## Validation + +The repository suite passed 418 tests with one skip before the additional three +profile-propagation cases. The final focused archive suite passed all 70 tests, +including those three cases. Admission matches the reviewed baseline; live metadata capture succeeds. +The assurance evaluator correctly exits nonzero for the disclosed missing/stale +evidence. ESO render assertions and the exact metadata-only diff passed. + +Commit and Repo Manager synchronization receipts are recorded in the session +close progress event. The generated legacy aliases remain WP-0036-T06's scoped +projection-owner dependency and are not silently repaired by changing UUIDs. diff --git a/scripts/recovery_evidence.py b/scripts/recovery_evidence.py index 819063a..cc72869 100644 --- a/scripts/recovery_evidence.py +++ b/scripts/recovery_evidence.py @@ -8,6 +8,82 @@ from service_assurance import timestamp ROOT = Path(__file__).resolve().parents[1] +def pinned_receipt(entry, root): + path = (root / entry['path']).resolve() + if not path.is_relative_to((root / 'docs/evidence').resolve()): + raise ValueError('receipt outside evidence directory') + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != entry['sha256']: + raise ValueError('receipt drift') + return json.loads(raw) + + +def operation_times(receipt, now): + start, finish = (timestamp(receipt[key]) for key in ('started_at', 'finished_at')) + if not start <= finish <= now: + raise ValueError('invalid receipt chronology') + return start, finish + + +def primary_archive(receipt, now): + """Accept only explicit full primary transfers, including a verified versioned GET.""" + operation_times(receipt, now) + if (receipt['schema'] != 'platform.forgejo-primary-archive.v1' + or receipt['status'] != 'primary_fetched_pending_application_restore' + or receipt['stage'] != 'transfer_verified' + or receipt['archive_profile'] != 'full' + or not receipt['destination'].startswith( + 's3://railiance-platform-pg-backup/platform-pg/application-archives/forgejo/') + or not re.fullmatch(r'[0-9a-f]{64}', receipt['ciphertext_sha256']) + or not all(receipt.get(key) is True for key in + ('multipart_completed', 'version_pinned', 'download_hash_matches')) + or type(receipt['ciphertext_bytes']) is not int + or receipt['ciphertext_bytes'] <= 0 + or any(type(receipt[key]) is not int or receipt[key] != receipt['ciphertext_bytes'] + for key in ('uploaded_bytes', 'downloaded_bytes'))): + raise ValueError('primary archive not accepted') + + +def archive_signal(entry, receipt, now, root): + if entry['signal'] == 'offsite.upload': + primary_archive(receipt, now) + else: + start, _ = operation_times(receipt, now) + if (receipt['schema'] != 'platform.forgejo-isolated-restore.v1' + or receipt['status'] != 'restored' + or receipt['archive_profile'] != 'full' + or receipt['source_provider'] != 'Scaleway' + or receipt['stage'] != 'package_blob_recovery' + or not all(receipt.get(key) is True for key in + ('database_import', 'application_health', 'cleanup')) + or not receipt['repositories_verified'] + or type(receipt['package_blobs_verified']) is not int + or receipt['package_blobs_verified'] < 0 + or type(receipt['database_counts']['package_blobs']) is not int + or receipt['package_blobs_verified'] != receipt['database_counts']['package_blobs']): + raise ValueError('full application recovery not accepted') + decryption = pinned_receipt(entry['decryption'], root) + transfer = pinned_receipt(entry['transfer'], root) + primary_archive(transfer, now) + decrypt_start, decrypt_finish = operation_times(decryption, now) + if (decryption['schema'] != 'platform.forgejo-primary-decryption.v1' + or decryption['status'] != 'primary_fetched_pending_application_restore' + or decryption['archive_profile'] != 'full' + or decryption['decrypted'] is not True + or decryption['download_hash_matches'] is not True + or not re.fullmatch(r'[0-9a-f]{64}', decryption['plaintext_sha256']) + or receipt['transfer_receipt_sha256'] != entry['decryption']['sha256'] + or decryption['transfer_receipt_sha256'] != entry['transfer']['sha256'] + or not timestamp(transfer['finished_at']) <= decrypt_start <= decrypt_finish <= start + or receipt['offsite_artifact'] != transfer['destination'] + or decryption['destination'] != transfer['destination'] + or decryption['ciphertext_bytes'] != transfer['ciphertext_bytes'] + or any(r['ciphertext_sha256'] != transfer['ciphertext_sha256'] + for r in (receipt, decryption))): + raise ValueError('recovery provenance mismatch') + return {'result': 'pass', 'observed_at': receipt['finished_at']} + + def recovery_signals(now, root=ROOT): index = json.loads((root / 'assurance/recovery-evidence.json').read_text()) if index['schema'] != 'railiance-platform.recovery-evidence.v1': @@ -15,10 +91,14 @@ def recovery_signals(now, root=ROOT): signals = {} for entry in index['receipts']: signal = entry['signal'] - if signal in signals or signal not in ('apps-pg.restore', 'forgejo-db.restore', 'openbao.snapshot'): + if signal in signals or signal not in ('apps-pg.restore', 'forgejo-db.restore', + 'openbao.snapshot', 'offsite.upload', 'offsite.restore'): raise ValueError('unexpected recovery signal') sample = {'result': 'unavailable', 'observed_at': now.isoformat()} try: + if signal.startswith('offsite.'): + signals[signal] = archive_signal(entry, pinned_receipt(entry, root), now, root) + continue path = (root / entry['path']).resolve() allowed = [root / 'docs/evidence'] if signal == 'openbao.snapshot': @@ -61,7 +141,7 @@ def recovery_signals(now, root=ROOT): if not timestamp(receipt['started_at']) <= completed <= now: raise ValueError('invalid receipt chronology') sample = {'result': 'pass', 'observed_at': receipt['finished_at']} - except (OSError, ValueError, KeyError, TypeError): + except (OSError, ValueError, KeyError, TypeError, AttributeError): pass signals[signal] = sample return signals diff --git a/scripts/scaleway_forgejo_archive.py b/scripts/scaleway_forgejo_archive.py index 4cd87da..14c2593 100644 --- a/scripts/scaleway_forgejo_archive.py +++ b/scripts/scaleway_forgejo_archive.py @@ -77,10 +77,12 @@ def main(): receipt['stage']='source_validation';checkpoint() source=json.loads(a.source_receipt.read_text()) if (source.get('status')!='offsite_fetched_pending_isolated_restore' + or source.get('archive_profile', 'full') not in ('full', 'essentials') or not a.source.name.endswith('.zip.age') or a.output.exists() or not 0