diff --git a/Makefile b/Makefile index 523feda..c958ede 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,14 @@ k3s-install: ## Install k3s and Helm on all inventory hosts smoke: ## Run Kubernetes smoke tests bash tests/smoke_kube.sh +cluster-observe: ## Capture non-secret cluster capacity and allocation evidence + python3 tools/observe_cluster_resources.py \ + $(if $(REMOTE),--remote $(REMOTE),) \ + --output-dir docs/evidence/resource-reef-railiance-k3s/observations + +test-unit: ## Run non-live unit tests + python3 -m unittest discover -s tests -p 'test_*.py' -v + test-ha-failover: ## Run HA failover test (D3) — kills primary PG pod, asserts recovery bash tests/test_ha_failover.sh $(if $(GITEA_URL),$(GITEA_URL),) @@ -43,4 +51,4 @@ help: ## Show this help /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } \ /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) }' $(MAKEFILE_LIST) -.PHONY: backup restore preflight k3s-install smoke test-ha-failover verify-activity-core reconcile-activity-core-llm-connect deploy-activity-core-triage-robustness admin-sync-smoke help +.PHONY: backup restore preflight k3s-install smoke cluster-observe test-unit test-ha-failover verify-activity-core reconcile-activity-core-llm-connect deploy-activity-core-triage-robustness admin-sync-smoke help diff --git a/SCOPE.md b/SCOPE.md index f66250e..d4b3ea4 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -90,8 +90,8 @@ future `rail-kubernetes` repo and must not be treated as long-term S2 scope. `platform-pg`, `state-hub-db`, `target-revenue-pg`, `user-engine-pg` - No monitoring stack is deployed on either cluster — observability is Q2 and homed in `railiance-telemetry`, which is seeded but not implemented -- k3s API access is source-restricted by UFW; see `railiance-infra` - `RAIL-HO-WP-0009` +- k3s API access is tunnel-only on Railiance01; see `railiance-infra` + `RAIL-HO-WP-0009` and ADR-005. It is not a public/source-allowlisted surface. --- @@ -130,15 +130,15 @@ future `rail-kubernetes` repo and must not be treated as long-term S2 scope. ```capability type: infrastructure title: Kubernetes cluster provisioning (k3s) -description: Install and configure a production k3s cluster including Helm, ingress controller, CNI plugin, and kubeconfig management on Railiance servers. +description: Install and configure a production k3s cluster including Helm, Traefik ingress, networking, and kubeconfig management on Railiance servers. keywords: [kubernetes, k3s, cluster, helm, ingress, cni, k8s, provisioning] ``` ```capability type: infrastructure title: Cluster operators and addon management -description: Deploy and manage cluster-wide operators and addons (cert-manager, CloudNative PG operator, ArgoCD, nginx ingress) on the running Railiance Kubernetes cluster. -keywords: [operator, addon, cert-manager, cnpg, argocd, admission, kubernetes, cluster] +description: Deploy and manage cluster-wide operators and addons, including cert-manager, CloudNative PG, Traefik, Knative Serving, External Secrets, and OpenBao integration, on the running Kubernetes cluster. +keywords: [operator, addon, cert-manager, cnpg, traefik, knative, admission, kubernetes, cluster] ``` ```capability @@ -152,10 +152,10 @@ keywords: [backup, restore, age, encryption, k3s, state, helm, kubeconfig, disas ## Notes -Runs on COULOMBCORE (92.205.130.254). State Hub access via ops-bridge reverse -tunnel - `bridge up state-hub-coulombcore` from the workstation (see ADR-004). -Gitea Helm values were migrated to S5 (railiance-apps) in RAIL-HO-WP-0004-T06 - -boundary violation resolved. +The live primary is Railiance01 (`92.205.62.239`). CoulombCore +(`92.205.130.254`) is the older frozen cluster and remains a legacy recovery +surface, not the default deployment target. State Hub and Kubernetes API access +use the corresponding ops-bridge tunnels (see ADR-004/ADR-005). `bin/railiance` lifecycle subcommands, `railiance/app.toml`, and overlay/wrapper scaffolding stay in this repo only as a migration bridge. They are not the diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 10cd887..d4720b8 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -16,7 +16,7 @@ | workplan | RAIL-BS-WP-0012 | finished | — | workplans/RAIL-BS-WP-0012-rail-kubernetes-extraction.md | | workplan | RAIL-BS-WP-0013 | finished | — | workplans/RAIL-BS-WP-0013-knative-substrate.md | | workplan | RCLUSTER-WP-0007 | backlog | — | workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md | -| workplan | RCLUSTER-WP-0014 | ready | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| workplan | RCLUSTER-WP-0014 | finished | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | | task | RAIL-BS-ADHOC-2026-07-27-T01 | done | — | workplans/ADHOC-2026-07-27.md | | task | RAIL-BS-WP-0008-T01 | done | — | workplans/RAIL-BS-WP-0008-activity-core-wp0016-triage-output-deploy.md | | task | RAIL-BS-WP-0008-T02 | done | — | workplans/RAIL-BS-WP-0008-activity-core-wp0016-triage-output-deploy.md | @@ -48,6 +48,6 @@ | task | RCLUSTER-WP-0007-T05 | todo | — | workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md | | task | RCLUSTER-WP-0007-T06 | todo | — | workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md | | task | RCLUSTER-WP-0007-T07 | todo | — | workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md | -| task | RCLUSTER-WP-0014-T01 | todo | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | -| task | RCLUSTER-WP-0014-T02 | todo | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | -| task | RCLUSTER-WP-0014-T03 | todo | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| task | RCLUSTER-WP-0014-T01 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| task | RCLUSTER-WP-0014-T02 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | +| task | RCLUSTER-WP-0014-T03 | done | — | workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md | diff --git a/docs/cluster-resource-evidence.md b/docs/cluster-resource-evidence.md new file mode 100644 index 0000000..69c965d --- /dev/null +++ b/docs/cluster-resource-evidence.md @@ -0,0 +1,14 @@ +# Cluster resource evidence + +`make cluster-observe` captures a non-secret, timestamped observation for +`resource:railiance:reef-railiance:k3s`. It separates allocatable capacity, +requests, limits, optional observed usage, PVC demand, storage-class behavior, +and measurement gaps. It never reads Secret objects or pod environments. + +The allocation section publishes raw numerators, denominators, residual +capacity, and `cluster-raw-drivers-v1`. It does not select a financial +allocation formula or write booked cost; those decisions remain with +resource-control. + +Current failure-domain evidence remains explicit: one Ready node on +`railiance01` with local storage. ThreePhoenix is a target, not live capacity. diff --git a/docs/evidence/resource-reef-railiance-k3s/observations/20260822T100349Z.json b/docs/evidence/resource-reef-railiance-k3s/observations/20260822T100349Z.json new file mode 100644 index 0000000..2a82cc9 --- /dev/null +++ b/docs/evidence/resource-reef-railiance-k3s/observations/20260822T100349Z.json @@ -0,0 +1,1172 @@ +{ + "schema_version": "railiance.cluster-resource-observation.v1", + "record_type": "usage_observation", + "resource_id": "resource:railiance:reef-railiance:k3s", + "source": "railiance-cluster", + "workplan_id": "RCLUSTER-WP-0014", + "reef": "reef-railiance", + "captured_at": "2026-08-22T10:03:49Z", + "capacity": { + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "allocatable": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "ephemeral_storage_bytes": 196555267123, + "pods": 110 + }, + "topology": { + "region": null, + "zone": null, + "hostname": "239.62.205.92.host.secureserver.net", + "provider_id": "k3s://239.62.205.92.host.secureserver.net" + }, + "observed": { + "cpu_millicores": 535, + "memory_bytes": 6272581632 + } + } + ], + "totals": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "pods": 110, + "pv_bytes": 137438953472 + } + }, + "utilization": { + "workloads": [ + { + "namespace": "activity-core", + "workload": "activity-core", + "pods": 12, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 60, + "memory_bytes": 788529152 + } + }, + { + "namespace": "audit-core", + "workload": "audit-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 26214400 + } + }, + { + "namespace": "cert-manager", + "workload": "cainjector", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 48234496 + } + }, + { + "namespace": "cert-manager", + "workload": "cert-manager", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "cert-manager", + "workload": "webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 22020096 + } + }, + { + "namespace": "cnpg-system", + "workload": "cloudnative-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 104857600 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 209715200 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "core-hub", + "workload": "rapp-core-hub", + "pods": 4, + "requests": { + "cpu_millicores": 300, + "memory_bytes": 805306368 + }, + "limits": { + "cpu_millicores": 2000, + "memory_bytes": 2147483648 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 177209344 + } + }, + { + "namespace": "coulomb", + "workload": "ihp-railiance-probe", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "coulomb-social", + "workload": "coulomb-social", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 137363456 + } + }, + { + "namespace": "databases", + "workload": "apps-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 7, + "memory_bytes": 101711872 + } + }, + { + "namespace": "databases", + "workload": "forgejo-db", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 14, + "memory_bytes": 145752064 + } + }, + { + "namespace": "databases", + "workload": "net-kingdom-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 7, + "memory_bytes": 82837504 + } + }, + { + "namespace": "databases", + "workload": "platform-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 9, + "memory_bytes": 132120576 + } + }, + { + "namespace": "databases", + "workload": "state-hub-db", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 143654912 + } + }, + { + "namespace": "email-connect", + "workload": "email-connect", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 12582912 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 35651584 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-cert-controller", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 35651584 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 32505856 + } + }, + { + "namespace": "flex-auth", + "workload": "flex-auth", + "pods": 3, + "requests": { + "cpu_millicores": 75, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 900, + "memory_bytes": 603979776 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 15728640 + } + }, + { + "namespace": "forgejo", + "workload": "forgejo-runner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 564133888 + } + }, + { + "namespace": "forgejo", + "workload": "gitea", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 126, + "memory_bytes": 639631360 + } + }, + { + "namespace": "issue-core", + "workload": "issue-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "knative-serving", + "workload": "knative-serving", + "pods": 4, + "requests": { + "cpu_millicores": 600, + "memory_bytes": 377487360 + }, + "limits": { + "cpu_millicores": 3500, + "memory_bytes": 3250585600 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 96468992 + } + }, + { + "namespace": "knative-serving", + "workload": "net-kourier-controller", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 524288000 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 41943040 + } + }, + { + "namespace": "kourier-system", + "workload": "3scale-kourier-gateway", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 838860800 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 22020096 + } + }, + { + "namespace": "kube-system", + "workload": "coredns-7bdb54f89", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 178257920 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 28311552 + } + }, + { + "namespace": "kube-system", + "workload": "helm-install-traefik", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "kube-system", + "workload": "helm-install-traefik-crd", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "kube-system", + "workload": "local-path-provisioner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 15728640 + } + }, + { + "namespace": "kube-system", + "workload": "metrics-server-786d997795", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 39845888 + } + }, + { + "namespace": "kube-system", + "workload": "svclb-traefik-0c8aecaf", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 1048576 + } + }, + { + "namespace": "kube-system", + "workload": "traefik", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 9, + "memory_bytes": 66060288 + } + }, + { + "namespace": "mfa", + "workload": "net-kingdom-sso-mfa", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 114294784 + } + }, + { + "namespace": "mfa", + "workload": "privacyidea-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 50331648 + }, + "limits": { + "cpu_millicores": 150, + "memory_bytes": 100663296 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "openbao", + "workload": "openbao", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 10, + "memory_bytes": 102760448 + } + }, + { + "namespace": "openbao", + "workload": "openbao-server-test", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "openbao", + "workload": "rapp-openbao", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "platform-pg-drill", + "workload": "platform-pg-drill", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "policy-nexus", + "workload": "policy-nexus", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 67108864 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "rapp-qonto-egress", + "workload": "qonto-egress-proxy", + "pods": 1, + "requests": { + "cpu_millicores": 20, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 170917888 + } + }, + { + "namespace": "rein-aharness", + "workload": "rein-aharness", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "reuse", + "workload": "reuse-surface", + "pods": 2, + "requests": { + "cpu_millicores": 110, + "memory_bytes": 301989888 + }, + "limits": { + "cpu_millicores": 550, + "memory_bytes": 603979776 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 50331648 + } + }, + { + "namespace": "sso", + "workload": "authelia-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "sso", + "workload": "lldap-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "sso", + "workload": "net-kingdom-sso-mfa", + "pods": 4, + "requests": { + "cpu_millicores": 150, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1150, + "memory_bytes": 671088640 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 59768832 + } + }, + { + "namespace": "sso", + "workload": "nk-wp21-auth-test", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "state-hub", + "workload": "railiance-apps", + "pods": 1, + "requests": { + "cpu_millicores": 250, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 2147483648 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 171966464 + } + }, + { + "namespace": "state-hub", + "workload": "state-hub-alembic-init", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 53477376 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue-pg", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 52428800 + } + }, + { + "namespace": "tenant-engine", + "workload": "tenant-engine", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 50331648 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 50331648 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine", + "pods": 5, + "requests": { + "cpu_millicores": 175, + "memory_bytes": 402653184 + }, + "limits": { + "cpu_millicores": 3000, + "memory_bytes": 1610612736 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 29360128 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 57671680 + } + } + ] + }, + "storage": { + "classes": [ + { + "name": "local-path", + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer", + "default": true + } + ], + "claims": [ + { + "namespace": "activity-core", + "name": "actcore-statehub-edge-outbox", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "actcore-working-memory", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-app-db-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-nats-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-temporal-db-0", + "storage_class": "local-path", + "requested_bytes": 8589934592, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "apps-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "forgejo-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "net-kingdom-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "platform-pg-1", + "storage_class": "local-path", + "requested_bytes": 21474836480, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "state-hub-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "email-connect", + "name": "email-connect-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "forgejo-runner-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "gitea-shared-storage", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-logs", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "audit-openbao-0", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "data-openbao-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "platform-pg-drill", + "name": "platform-pg-drill-1", + "storage_class": "local-path", + "requested_bytes": 3221225472, + "phase": "Bound" + }, + { + "namespace": "reuse", + "name": "reuse-surface-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "authelia-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "lldap-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "target-revenue", + "name": "target-revenue-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "tenant-engine", + "name": "tenant-engine-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-backups", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + } + ] + }, + "failure_domain": { + "current_contract": "single Ready node on railiance01; local storage is correlated with node loss", + "independent_domains_claimed": 1, + "threephoenix_target_is_live": false + }, + "allocation": { + "method_version": "cluster-raw-drivers-v1", + "drivers": [ + "cpu_requests", + "memory_requests", + "pvc_requested_bytes", + "observed_usage" + ], + "denominators": { + "cpu_requested_millicores": 3830, + "memory_requested_bytes": 7457472512, + "pvc_requested_bytes": 137438953472 + }, + "residual_capacity": { + "cpu_millicores": 170, + "memory_bytes": 9312604160, + "pv_bytes": 0 + }, + "selection_owner": "resource-control" + }, + "measurement_gaps": [ + "node 239.62.205.92.host.secureserver.net lacks independent region/zone labels" + ], + "provenance": { + "commands": [ + "kubectl get nodes -o json", + "kubectl get pods -A -o json", + "kubectl get pvc -A -o json", + "kubectl get pv -o json", + "kubectl get storageclass -o json", + "kubectl top nodes --no-headers", + "kubectl top pods -A --no-headers" + ], + "secret_surfaces_read": false + } +} diff --git a/docs/evidence/resource-reef-railiance-k3s/observations/latest.json b/docs/evidence/resource-reef-railiance-k3s/observations/latest.json new file mode 100644 index 0000000..2a82cc9 --- /dev/null +++ b/docs/evidence/resource-reef-railiance-k3s/observations/latest.json @@ -0,0 +1,1172 @@ +{ + "schema_version": "railiance.cluster-resource-observation.v1", + "record_type": "usage_observation", + "resource_id": "resource:railiance:reef-railiance:k3s", + "source": "railiance-cluster", + "workplan_id": "RCLUSTER-WP-0014", + "reef": "reef-railiance", + "captured_at": "2026-08-22T10:03:49Z", + "capacity": { + "nodes": [ + { + "name": "239.62.205.92.host.secureserver.net", + "ready": true, + "allocatable": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "ephemeral_storage_bytes": 196555267123, + "pods": 110 + }, + "topology": { + "region": null, + "zone": null, + "hostname": "239.62.205.92.host.secureserver.net", + "provider_id": "k3s://239.62.205.92.host.secureserver.net" + }, + "observed": { + "cpu_millicores": 535, + "memory_bytes": 6272581632 + } + } + ], + "totals": { + "cpu_millicores": 4000, + "memory_bytes": 16770076672, + "pods": 110, + "pv_bytes": 137438953472 + } + }, + "utilization": { + "workloads": [ + { + "namespace": "activity-core", + "workload": "activity-core", + "pods": 12, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 60, + "memory_bytes": 788529152 + } + }, + { + "namespace": "audit-core", + "workload": "audit-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 26214400 + } + }, + { + "namespace": "cert-manager", + "workload": "cainjector", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 48234496 + } + }, + { + "namespace": "cert-manager", + "workload": "cert-manager", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "cert-manager", + "workload": "webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 22020096 + } + }, + { + "namespace": "cnpg-system", + "workload": "cloudnative-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 104857600 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 209715200 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "core-hub", + "workload": "rapp-core-hub", + "pods": 4, + "requests": { + "cpu_millicores": 300, + "memory_bytes": 805306368 + }, + "limits": { + "cpu_millicores": 2000, + "memory_bytes": 2147483648 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 177209344 + } + }, + { + "namespace": "coulomb", + "workload": "ihp-railiance-probe", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "coulomb-social", + "workload": "coulomb-social", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 137363456 + } + }, + { + "namespace": "databases", + "workload": "apps-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 7, + "memory_bytes": 101711872 + } + }, + { + "namespace": "databases", + "workload": "forgejo-db", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 14, + "memory_bytes": 145752064 + } + }, + { + "namespace": "databases", + "workload": "net-kingdom-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 7, + "memory_bytes": 82837504 + } + }, + { + "namespace": "databases", + "workload": "platform-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 9, + "memory_bytes": 132120576 + } + }, + { + "namespace": "databases", + "workload": "state-hub-db", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 143654912 + } + }, + { + "namespace": "email-connect", + "workload": "email-connect", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 250, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 12582912 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 35651584 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-cert-controller", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 35651584 + } + }, + { + "namespace": "external-secrets", + "workload": "external-secrets-webhook", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 32505856 + } + }, + { + "namespace": "flex-auth", + "workload": "flex-auth", + "pods": 3, + "requests": { + "cpu_millicores": 75, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 900, + "memory_bytes": 603979776 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 15728640 + } + }, + { + "namespace": "forgejo", + "workload": "forgejo-runner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 564133888 + } + }, + { + "namespace": "forgejo", + "workload": "gitea", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 126, + "memory_bytes": 639631360 + } + }, + { + "namespace": "issue-core", + "workload": "issue-core", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 40894464 + } + }, + { + "namespace": "knative-serving", + "workload": "knative-serving", + "pods": 4, + "requests": { + "cpu_millicores": 600, + "memory_bytes": 377487360 + }, + "limits": { + "cpu_millicores": 3500, + "memory_bytes": 3250585600 + }, + "observed": { + "cpu_millicores": 8, + "memory_bytes": 96468992 + } + }, + { + "namespace": "knative-serving", + "workload": "net-kourier-controller", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 524288000 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 41943040 + } + }, + { + "namespace": "kourier-system", + "workload": "3scale-kourier-gateway", + "pods": 1, + "requests": { + "cpu_millicores": 200, + "memory_bytes": 209715200 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 838860800 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 22020096 + } + }, + { + "namespace": "kube-system", + "workload": "coredns-7bdb54f89", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 178257920 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 28311552 + } + }, + { + "namespace": "kube-system", + "workload": "helm-install-traefik", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "kube-system", + "workload": "helm-install-traefik-crd", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "kube-system", + "workload": "local-path-provisioner", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 15728640 + } + }, + { + "namespace": "kube-system", + "workload": "metrics-server-786d997795", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 73400320 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 39845888 + } + }, + { + "namespace": "kube-system", + "workload": "svclb-traefik-0c8aecaf", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 1048576 + } + }, + { + "namespace": "kube-system", + "workload": "traefik", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 9, + "memory_bytes": 66060288 + } + }, + { + "namespace": "mfa", + "workload": "net-kingdom-sso-mfa", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 114294784 + } + }, + { + "namespace": "mfa", + "workload": "privacyidea-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 50331648 + }, + "limits": { + "cpu_millicores": 150, + "memory_bytes": 100663296 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "openbao", + "workload": "openbao", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 10, + "memory_bytes": 102760448 + } + }, + { + "namespace": "openbao", + "workload": "openbao-server-test", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "openbao", + "workload": "rapp-openbao", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 134217728 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "platform-pg-drill", + "workload": "platform-pg-drill", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 134217728 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "policy-nexus", + "workload": "policy-nexus", + "pods": 1, + "requests": { + "cpu_millicores": 10, + "memory_bytes": 33554432 + }, + "limits": { + "cpu_millicores": 100, + "memory_bytes": 67108864 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 4194304 + } + }, + { + "namespace": "rapp-qonto-egress", + "workload": "qonto-egress-proxy", + "pods": 1, + "requests": { + "cpu_millicores": 20, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 200, + "memory_bytes": 268435456 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 170917888 + } + }, + { + "namespace": "rein-aharness", + "workload": "rein-aharness", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 67108864 + }, + "limits": { + "cpu_millicores": 500, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "reuse", + "workload": "reuse-surface", + "pods": 2, + "requests": { + "cpu_millicores": 110, + "memory_bytes": 301989888 + }, + "limits": { + "cpu_millicores": 550, + "memory_bytes": 603979776 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 50331648 + } + }, + { + "namespace": "sso", + "workload": "authelia-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "sso", + "workload": "lldap-backup", + "pods": 3, + "requests": { + "cpu_millicores": 30, + "memory_bytes": 100663296 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "sso", + "workload": "net-kingdom-sso-mfa", + "pods": 4, + "requests": { + "cpu_millicores": 150, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1150, + "memory_bytes": 671088640 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 59768832 + } + }, + { + "namespace": "sso", + "workload": "nk-wp21-auth-test", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "state-hub", + "workload": "railiance-apps", + "pods": 1, + "requests": { + "cpu_millicores": 250, + "memory_bytes": 536870912 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 2147483648 + }, + "observed": { + "cpu_millicores": 4, + "memory_bytes": 171966464 + } + }, + { + "namespace": "state-hub", + "workload": "state-hub-alembic-init", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 0, + "memory_bytes": 0 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue", + "pods": 1, + "requests": { + "cpu_millicores": 50, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 536870912 + }, + "observed": { + "cpu_millicores": 2, + "memory_bytes": 53477376 + } + }, + { + "namespace": "target-revenue", + "workload": "target-revenue-pg", + "pods": 1, + "requests": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "limits": { + "cpu_millicores": 0, + "memory_bytes": 0 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 52428800 + } + }, + { + "namespace": "tenant-engine", + "workload": "tenant-engine", + "pods": 1, + "requests": { + "cpu_millicores": 25, + "memory_bytes": 50331648 + }, + "limits": { + "cpu_millicores": 300, + "memory_bytes": 201326592 + }, + "observed": { + "cpu_millicores": 3, + "memory_bytes": 50331648 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine", + "pods": 5, + "requests": { + "cpu_millicores": 175, + "memory_bytes": 402653184 + }, + "limits": { + "cpu_millicores": 3000, + "memory_bytes": 1610612736 + }, + "observed": { + "cpu_millicores": 1, + "memory_bytes": 29360128 + } + }, + { + "namespace": "user-engine", + "workload": "user-engine-pg", + "pods": 1, + "requests": { + "cpu_millicores": 100, + "memory_bytes": 268435456 + }, + "limits": { + "cpu_millicores": 1000, + "memory_bytes": 1073741824 + }, + "observed": { + "cpu_millicores": 6, + "memory_bytes": 57671680 + } + } + ] + }, + "storage": { + "classes": [ + { + "name": "local-path", + "provisioner": "rancher.io/local-path", + "reclaim_policy": "Delete", + "volume_binding_mode": "WaitForFirstConsumer", + "default": true + } + ], + "claims": [ + { + "namespace": "activity-core", + "name": "actcore-statehub-edge-outbox", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "actcore-working-memory", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-app-db-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-nats-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "activity-core", + "name": "data-actcore-temporal-db-0", + "storage_class": "local-path", + "requested_bytes": 8589934592, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "apps-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "forgejo-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "net-kingdom-pg-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "platform-pg-1", + "storage_class": "local-path", + "requested_bytes": 21474836480, + "phase": "Bound" + }, + { + "namespace": "databases", + "name": "state-hub-db-1", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "email-connect", + "name": "email-connect-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "forgejo-runner-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "forgejo", + "name": "gitea-shared-storage", + "storage_class": "local-path", + "requested_bytes": 10737418240, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-data", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "mfa", + "name": "privacyidea-logs", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "audit-openbao-0", + "storage_class": "local-path", + "requested_bytes": 2147483648, + "phase": "Bound" + }, + { + "namespace": "openbao", + "name": "data-openbao-0", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "platform-pg-drill", + "name": "platform-pg-drill-1", + "storage_class": "local-path", + "requested_bytes": 3221225472, + "phase": "Bound" + }, + { + "namespace": "reuse", + "name": "reuse-surface-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "authelia-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "sso", + "name": "lldap-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "target-revenue", + "name": "target-revenue-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + }, + { + "namespace": "tenant-engine", + "name": "tenant-engine-data", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-backups", + "storage_class": "local-path", + "requested_bytes": 1073741824, + "phase": "Bound" + }, + { + "namespace": "user-engine", + "name": "user-engine-pg-1", + "storage_class": "local-path", + "requested_bytes": 5368709120, + "phase": "Bound" + } + ] + }, + "failure_domain": { + "current_contract": "single Ready node on railiance01; local storage is correlated with node loss", + "independent_domains_claimed": 1, + "threephoenix_target_is_live": false + }, + "allocation": { + "method_version": "cluster-raw-drivers-v1", + "drivers": [ + "cpu_requests", + "memory_requests", + "pvc_requested_bytes", + "observed_usage" + ], + "denominators": { + "cpu_requested_millicores": 3830, + "memory_requested_bytes": 7457472512, + "pvc_requested_bytes": 137438953472 + }, + "residual_capacity": { + "cpu_millicores": 170, + "memory_bytes": 9312604160, + "pv_bytes": 0 + }, + "selection_owner": "resource-control" + }, + "measurement_gaps": [ + "node 239.62.205.92.host.secureserver.net lacks independent region/zone labels" + ], + "provenance": { + "commands": [ + "kubectl get nodes -o json", + "kubectl get pods -A -o json", + "kubectl get pvc -A -o json", + "kubectl get pv -o json", + "kubectl get storageclass -o json", + "kubectl top nodes --no-headers", + "kubectl top pods -A --no-headers" + ], + "secret_surfaces_read": false + } +} diff --git a/docs/threephoenix-implementation-gate.md b/docs/threephoenix-implementation-gate.md new file mode 100644 index 0000000..dbba0a8 --- /dev/null +++ b/docs/threephoenix-implementation-gate.md @@ -0,0 +1,37 @@ +# ThreePhoenix implementation gate + +`RCLUSTER-WP-0007` stays in backlog until every gate below is backed by +reviewable source or evidence. This prevents a target architecture from being +reported as live capacity and prevents destructive work on the current +single-node production cluster. + +## Required preflight + +- Three reef members have stable source-backed identities and map to three + independently provisioned, S1-converged hosts. +- The failure-domain record covers physical host, provider/site, power, + storage, and network dependencies. Co-located guests count as one domain. +- Private inter-node addresses, routes, and firewall rules are approved. The + Kubernetes API remains operator-only and no public 6443 grant is possible. +- k3s server roles, embedded-etcd join order, version pin, rollback path, and + quorum checks are reviewed. +- Join-token creation, delivery, rotation, and revocation use the approved + platform custody path; no token value is committed or copied into evidence. +- Storage prerequisites and the Longhorn-or-alternative decision are approved + against the actual host/storage failure domains. +- Backups and restore points exist before any migration or destructive test. +- An operator explicitly approves each drain, rebuild, detach, or failure + injection against a named host and maintenance window. + +## Evidence required to open implementation + +Record a dated preflight artifact that identifies the three reef-member and +host records, links the network and custody decisions, records current backup +checks, and names the approving operator. After that evidence is reviewed, the +workplan may move from `backlog` to `ready`; individual tasks still follow +their own `todo` → `progress` → `done` lifecycle. + +Until then, allowed work is limited to source-only design, validation tooling, +and dependency preparation. Provider purchase, live cluster joins, firewall +mutation, Longhorn installation, and failure injection remain outside the +authorized stage. diff --git a/tests/test_cluster_resources.py b/tests/test_cluster_resources.py new file mode 100644 index 0000000..a6a495e --- /dev/null +++ b/tests/test_cluster_resources.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import importlib.util +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("cluster_resources", ROOT / "tools" / "observe_cluster_resources.py") +assert SPEC and SPEC.loader +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +class ClusterResourceTests(unittest.TestCase): + def test_quantity_conversion(self) -> None: + self.assertEqual(250, module.cpu_millicores("250m")) + self.assertEqual(2000, module.cpu_millicores("2")) + self.assertEqual(1024**3, module.bytes_value("1Gi")) + + def test_aggregation_and_missing_metrics(self) -> None: + snapshots = { + "nodes": {"items": [{"metadata": {"name": "node-1", "labels": {"kubernetes.io/hostname": "node-1"}}, "spec": {}, "status": {"allocatable": {"cpu": "2", "memory": "4Gi", "ephemeral-storage": "20Gi", "pods": "110"}, "conditions": [{"type": "Ready", "status": "True"}]}}]}, + "pods": {"items": [{"metadata": {"name": "api-abc123", "namespace": "apps", "labels": {"app.kubernetes.io/name": "api"}}, "spec": {"containers": [{"resources": {"requests": {"cpu": "250m", "memory": "128Mi"}, "limits": {"cpu": "1", "memory": "512Mi"}}}]}}]}, + "pvcs": {"items": [{"metadata": {"name": "data", "namespace": "apps"}, "spec": {"storageClassName": "local-path", "resources": {"requests": {"storage": "1Gi"}}}, "status": {"phase": "Bound"}}]}, + "pvs": {"items": [{"spec": {"capacity": {"storage": "2Gi"}}}]}, + "storageclasses": {"items": [{"metadata": {"name": "local-path", "annotations": {"storageclass.kubernetes.io/is-default-class": "true"}}, "provisioner": "local-path", "reclaimPolicy": "Delete", "volumeBindingMode": "WaitForFirstConsumer"}]}, + "node_top": None, + "pod_top": None, + } + result = module.build_observation(snapshots, "2026-08-22T00:00:00Z") + self.assertEqual(250, result["allocation"]["denominators"]["cpu_requested_millicores"]) + self.assertEqual(1750, result["allocation"]["residual_capacity"]["cpu_millicores"]) + self.assertIn("pod metrics unavailable", result["measurement_gaps"]) + self.assertFalse(result["failure_domain"]["threephoenix_target_is_live"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/observe_cluster_resources.py b/tools/observe_cluster_resources.py new file mode 100755 index 0000000..e0b08a0 --- /dev/null +++ b/tools/observe_cluster_resources.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Capture non-secret cluster capacity, utilization, and allocation evidence.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +CPU_FACTORS = {"n": 0.000001, "u": 0.001, "m": 1.0, "": 1000.0} +BYTE_FACTORS = { + "": 1, + "Ki": 1024, + "Mi": 1024**2, + "Gi": 1024**3, + "Ti": 1024**4, + "K": 1000, + "M": 1000**2, + "G": 1000**3, + "T": 1000**4, +} + + +def cpu_millicores(value: str | None) -> int: + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(n|u|m)?", str(value or "0")) + if not match: + raise ValueError(f"unsupported CPU quantity: {value}") + return round(float(match.group(1)) * CPU_FACTORS[match.group(2) or ""]) + + +def bytes_value(value: str | None) -> int: + match = re.fullmatch(r"([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?", str(value or "0")) + if not match: + raise ValueError(f"unsupported byte quantity: {value}") + return round(float(match.group(1)) * BYTE_FACTORS[match.group(2) or ""]) + + +def workload_name(pod: dict[str, Any]) -> str: + metadata = pod.get("metadata") or {} + labels = metadata.get("labels") or {} + for key in ("app.kubernetes.io/part-of", "app.kubernetes.io/name", "app"): + if labels.get(key): + return str(labels[key]) + owners = metadata.get("ownerReferences") or [] + if owners and owners[0].get("name"): + return str(owners[0]["name"]) + return re.sub(r"-[a-f0-9]{6,}(?:-[a-z0-9]{5})?$", "", str(metadata.get("name", "unknown"))) + + +def parse_top(lines: str | None, *, pods: bool) -> dict[tuple[str, str] | str, dict[str, int]]: + result: dict[tuple[str, str] | str, dict[str, int]] = {} + for line in (lines or "").splitlines(): + fields = line.split() + if pods and len(fields) >= 4: + result[(fields[0], fields[1])] = { + "cpu_millicores": cpu_millicores(fields[2]), + "memory_bytes": bytes_value(fields[3]), + } + elif not pods and len(fields) >= 4: + result[fields[0]] = { + "cpu_millicores": cpu_millicores(fields[1]), + # kubectl top nodes: NAME CPU CPU% MEMORY MEMORY% + "memory_bytes": bytes_value(fields[3]), + } + return result + + +def build_observation(snapshots: dict[str, Any], captured_at: str) -> dict[str, Any]: + gaps: list[str] = [] + node_top = parse_top(snapshots.get("node_top", ""), pods=False) + pod_top = parse_top(snapshots.get("pod_top", ""), pods=True) + if snapshots.get("node_top") is None: + gaps.append("node metrics unavailable") + if snapshots.get("pod_top") is None: + gaps.append("pod metrics unavailable") + + nodes = [] + alloc_cpu = alloc_memory = alloc_pods = 0 + for node in snapshots["nodes"].get("items") or []: + meta = node.get("metadata") or {} + status = node.get("status") or {} + alloc = status.get("allocatable") or {} + labels = meta.get("labels") or {} + name = str(meta.get("name")) + topology = { + "region": labels.get("topology.kubernetes.io/region"), + "zone": labels.get("topology.kubernetes.io/zone"), + "hostname": labels.get("kubernetes.io/hostname"), + "provider_id": (node.get("spec") or {}).get("providerID"), + } + if not topology["region"] or not topology["zone"]: + gaps.append(f"node {name} lacks independent region/zone labels") + row = { + "name": name, + "ready": any(c.get("type") == "Ready" and c.get("status") == "True" for c in status.get("conditions") or []), + "allocatable": { + "cpu_millicores": cpu_millicores(alloc.get("cpu")), + "memory_bytes": bytes_value(alloc.get("memory")), + "ephemeral_storage_bytes": bytes_value(alloc.get("ephemeral-storage")), + "pods": int(alloc.get("pods") or 0), + }, + "topology": topology, + "observed": node_top.get(name), + } + nodes.append(row) + alloc_cpu += row["allocatable"]["cpu_millicores"] + alloc_memory += row["allocatable"]["memory_bytes"] + alloc_pods += row["allocatable"]["pods"] + + workloads: dict[tuple[str, str], dict[str, Any]] = {} + for pod in snapshots["pods"].get("items") or []: + meta = pod.get("metadata") or {} + namespace = str(meta.get("namespace", "default")) + name = workload_name(pod) + key = (namespace, name) + row = workloads.setdefault(key, { + "namespace": namespace, + "workload": name, + "pods": 0, + "requests": {"cpu_millicores": 0, "memory_bytes": 0}, + "limits": {"cpu_millicores": 0, "memory_bytes": 0}, + "observed": {"cpu_millicores": 0, "memory_bytes": 0} if snapshots.get("pod_top") is not None else None, + }) + row["pods"] += 1 + for container in (pod.get("spec") or {}).get("containers") or []: + resources = container.get("resources") or {} + requests = resources.get("requests") or {} + limits = resources.get("limits") or {} + row["requests"]["cpu_millicores"] += cpu_millicores(requests.get("cpu")) + row["requests"]["memory_bytes"] += bytes_value(requests.get("memory")) + row["limits"]["cpu_millicores"] += cpu_millicores(limits.get("cpu")) + row["limits"]["memory_bytes"] += bytes_value(limits.get("memory")) + metric = pod_top.get((namespace, str(meta.get("name")))) + if metric and row["observed"] is not None: + row["observed"]["cpu_millicores"] += metric["cpu_millicores"] + row["observed"]["memory_bytes"] += metric["memory_bytes"] + + storage_classes = [] + for item in snapshots["storageclasses"].get("items") or []: + storage_classes.append({ + "name": (item.get("metadata") or {}).get("name"), + "provisioner": item.get("provisioner"), + "reclaim_policy": item.get("reclaimPolicy"), + "volume_binding_mode": item.get("volumeBindingMode"), + "default": (item.get("metadata") or {}).get("annotations", {}).get("storageclass.kubernetes.io/is-default-class") == "true", + }) + pv_capacity = sum(bytes_value(((item.get("spec") or {}).get("capacity") or {}).get("storage")) for item in snapshots["pvs"].get("items") or []) + pvc_rows = [] + pvc_requested = 0 + for item in snapshots["pvcs"].get("items") or []: + meta = item.get("metadata") or {} + spec = item.get("spec") or {} + requested = bytes_value(((spec.get("resources") or {}).get("requests") or {}).get("storage")) + pvc_requested += requested + pvc_rows.append({ + "namespace": meta.get("namespace"), + "name": meta.get("name"), + "storage_class": spec.get("storageClassName"), + "requested_bytes": requested, + "phase": (item.get("status") or {}).get("phase"), + }) + + workload_rows = sorted(workloads.values(), key=lambda row: (row["namespace"], row["workload"])) + requested_cpu = sum(row["requests"]["cpu_millicores"] for row in workload_rows) + requested_memory = sum(row["requests"]["memory_bytes"] for row in workload_rows) + return { + "schema_version": "railiance.cluster-resource-observation.v1", + "record_type": "usage_observation", + "resource_id": "resource:railiance:reef-railiance:k3s", + "source": "railiance-cluster", + "workplan_id": "RCLUSTER-WP-0014", + "reef": "reef-railiance", + "captured_at": captured_at, + "capacity": { + "nodes": nodes, + "totals": {"cpu_millicores": alloc_cpu, "memory_bytes": alloc_memory, "pods": alloc_pods, "pv_bytes": pv_capacity}, + }, + "utilization": {"workloads": workload_rows}, + "storage": {"classes": storage_classes, "claims": pvc_rows}, + "failure_domain": { + "current_contract": "single Ready node on railiance01; local storage is correlated with node loss", + "independent_domains_claimed": 1 if nodes else 0, + "threephoenix_target_is_live": False, + }, + "allocation": { + "method_version": "cluster-raw-drivers-v1", + "drivers": ["cpu_requests", "memory_requests", "pvc_requested_bytes", "observed_usage"], + "denominators": {"cpu_requested_millicores": requested_cpu, "memory_requested_bytes": requested_memory, "pvc_requested_bytes": pvc_requested}, + "residual_capacity": {"cpu_millicores": max(alloc_cpu - requested_cpu, 0), "memory_bytes": max(alloc_memory - requested_memory, 0), "pv_bytes": max(pv_capacity - pvc_requested, 0)}, + "selection_owner": "resource-control", + }, + "measurement_gaps": sorted(set(gaps)), + "provenance": { + "commands": [ + "kubectl get nodes -o json", "kubectl get pods -A -o json", + "kubectl get pvc -A -o json", "kubectl get pv -o json", + "kubectl get storageclass -o json", "kubectl top nodes --no-headers", + "kubectl top pods -A --no-headers", + ], + "secret_surfaces_read": False, + }, + } + + +def run_json(command: list[str]) -> dict[str, Any]: + return json.loads(subprocess.check_output(command, text=True)) + + +def run_optional(command: list[str]) -> str | None: + completed = subprocess.run(command, text=True, capture_output=True, check=False) + return completed.stdout if completed.returncode == 0 else None + + +def collect(kubectl: str, remote: str | None = None) -> dict[str, Any]: + base = ["ssh", "-o", "BatchMode=yes", remote, kubectl] if remote else [kubectl] + snapshots = { + "nodes": run_json(base + ["get", "nodes", "-o", "json"]), + "pods": run_json(base + ["get", "pods", "-A", "-o", "json"]), + "pvcs": run_json(base + ["get", "pvc", "-A", "-o", "json"]), + "pvs": run_json(base + ["get", "pv", "-o", "json"]), + "storageclasses": run_json(base + ["get", "storageclass", "-o", "json"]), + "node_top": run_optional(base + ["top", "nodes", "--no-headers"]), + "pod_top": run_optional(base + ["top", "pods", "-A", "--no-headers"]), + } + captured_at = datetime.now(UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + return build_observation(snapshots, captured_at) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--kubectl", default="kubectl") + parser.add_argument("--remote", help="Optional SSH host on which kubectl runs") + parser.add_argument("--output-dir", type=Path) + args = parser.parse_args() + try: + observation = collect(args.kubectl, args.remote) + except (OSError, subprocess.CalledProcessError, json.JSONDecodeError, ValueError) as exc: + print(f"cluster observation failed: {exc}", file=sys.stderr) + return 1 + rendered = json.dumps(observation, indent=2) + "\n" + if args.output_dir: + args.output_dir.mkdir(parents=True, exist_ok=True) + stamp = observation["captured_at"].replace(":", "").replace("-", "") + destination = args.output_dir / f"{stamp}.json" + destination.write_text(rendered, encoding="utf-8") + (args.output_dir / "latest.json").write_text(rendered, encoding="utf-8") + print(destination) + else: + sys.stdout.write(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md b/workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md index b1830f0..276f6c5 100644 --- a/workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md +++ b/workplans/RCLUSTER-WP-0007-threephoenix-ha-cluster.md @@ -10,7 +10,7 @@ topic_slug: railiance repo_goal_id: "6ea441f7-7fe3-4598-922b-38baf20c0580" state_hub_workstream_id: "9e208376-23f1-40c7-9813-fac1f7d6ad3b" created: "2026-02-25" -updated: "2026-08-15" +updated: "2026-08-22" related: - REEF-RAILIANCE-WP-0003 - RAIL-BS-WP-0014 @@ -48,6 +48,21 @@ grant. The bootstrap copy of this file is retired. Do not re-open `RCLUSTER-WP-0007` in `railiance-bootstrap`. +## Implementation gate + +This workplan remains `backlog`; its tasks must not move to `progress` until +the preflight in `docs/threephoenix-implementation-gate.md` passes. In +particular, three independent source-backed reef members must be approved, +provisioned, and S1-converged, with their shared power, storage, host, and +network dependencies recorded. The current Railiance01 node plus guests on one +physical host do not satisfy that gate. + +Passing the gate also requires an approved private inter-node network design, +join-token custody/rotation through the platform custody path, and explicit +operator approval for destructive node actions. This repository must not buy +provider capacity, open public listeners, install Longhorn, or mutate the live +single-node cluster merely to make the target appear ready. + ## Why This Belongs Before Forgejo Forgejo will be the source forge, package base, and Actions surface for the @@ -267,6 +282,10 @@ This workplan should precede the Forgejo production cutover. It should also shape the Stage 2 and Stage 3 gates in `RAIL-BS-WP-0006` so canaries and promotions operate against the real HA substrate. +The first executable dependency is the machine-checkable/manual preflight in +`docs/threephoenix-implementation-gate.md`; until it passes, this plan is design +and dependency preparation only. + It is the substrate dependency for: - `reef-railiance/REEF-RAILIANCE-WP-0003` T04. After the S2 drill passes, the diff --git a/workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md b/workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md index d35119c..d068850 100644 --- a/workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md +++ b/workplans/RCLUSTER-WP-0014-capacity-utilization-and-allocation-evidence.md @@ -4,11 +4,11 @@ type: workplan title: "Publish cluster capacity, utilization, and allocation evidence" domain: financials repo: railiance-cluster -status: ready +status: finished owner: codex topic_slug: railiance created: "2026-08-11" -updated: "2026-08-15" +updated: "2026-08-22" related: - RAIL-BS-WP-0007 - RESOURCE-WP-0003 @@ -42,7 +42,7 @@ fin-hub allocate. No booked EUR in this repo. ```task id: RCLUSTER-WP-0014-T01 -status: todo +status: done priority: high state_hub_task_id: "a144bf1d-0402-4526-9bda-17846390270c" ``` @@ -56,11 +56,17 @@ metrics, and source commands. Done when repeated observations use a stable machine-readable interface and resource-control can distinguish headroom, reservations, and actual usage. +**Outcome (2026-08-22):** `make cluster-observe REMOTE=railiance01` now emits +versioned, timestamped JSON with node allocatable/observed resources, +namespace/workload requests and limits, PVC/PV/storage-class state, stable +units, source commands, and explicit gaps without reading Secrets or pod +environments. + ## T02 — Publish the failure-domain and storage contract ```task id: RCLUSTER-WP-0014-T02 -status: todo +status: done priority: high state_hub_task_id: "6460d9aa-3ded-420a-bf01-f0ccdeee0a02" ``` @@ -73,11 +79,16 @@ current capacity. Done when consumers can calculate correlated-failure risk and recognize which state remains local to `railiance01`. +**Outcome (2026-08-22):** the evidence contract reports one Ready node, +`local-path` storage, `threephoenix_ready: false`, and the missing independent +region/zone labels. Target HA and Longhorn capacity are not represented as +current facts. + ## T03 — Define technical allocation-driver evidence ```task id: RCLUSTER-WP-0014-T03 -status: todo +status: done priority: medium state_hub_task_id: "e5ff9b6c-e883-4356-9eca-b31757661d2f" ``` @@ -90,8 +101,22 @@ do not convert the result into booked spend. Done when resource-control can create an explainable allocation and fin-hub can reconcile it to referenced financial facts. +**Outcome (2026-08-22):** `cluster-raw-drivers-v1` publishes CPU requests, +memory requests, PVC bytes, observed usage, totals, residual capacity, and +uncertainty, while leaving formula selection and booked cost to +`resource-control`. + ## Acceptance -- [ ] Capacity evidence separates allocatable, requested, limited, and observed. -- [ ] Current single-host/local-storage risk is machine-readable. -- [ ] Allocation drivers reconcile to explicit totals and residuals. +- [x] Capacity evidence separates allocatable, requested, limited, and observed. +- [x] Current single-host/local-storage risk is machine-readable. +- [x] Allocation drivers reconcile to explicit totals and residuals. + +## Completion evidence + +- `python3 -m unittest discover -s tests -p 'test_*.py' -v`: 2 tests passed. +- Live read-only capture: + `docs/evidence/resource-reef-railiance-k3s/observations/20260822T100349Z.json`. +- Observed totals: 1 node, 53 workloads, 25 claims; 4,000m allocatable CPU, + 3,830m requested CPU, 16,770,076,672 allocatable memory bytes, and + 137,438,953,472 requested PVC bytes.