diff --git a/docs/credential-lane-designs/telemetry-grafana.md b/docs/credential-lane-designs/telemetry-grafana.md new file mode 100644 index 0000000..9ba320a --- /dev/null +++ b/docs/credential-lane-designs/telemetry-grafana.md @@ -0,0 +1,44 @@ +# Grafana administrator custody + +This is the attended bootstrap lane for rapp-telemetry on railiance01, not a +coding-agent credential vending interface or an accepted operator OIDC design. +User authorized service activation on September 6, 2026. + +- KV: platform/workloads/telemetry/grafana-admin; ADMIN_USERNAME and ADMIN_PASSWORD. +- ESO identity: telemetry/telemetry-grafana-eso, automount disabled. +- Kubernetes auth role: telemetry-grafana-eso; policy telemetry-grafana-admin-eso. +- Exact data-only read, self-lookup/revoke, audience openbao, 15-minute maximum; + no default policy, wildcard namespace, sibling grant or secret write. +- ClusterSecretStore openbao-telemetry-grafana restricts consumers to telemetry; + ExternalSecret telemetry-grafana-admin refreshes every 5 minutes and retains + the delivered Secret. Grafana references admin-user and admin-password. +- Coding-agent high-risk boundary denies both data and metadata, including when + combined with the ESO read policy. Cluster administration/ESO remain trusted. + +Source manifests belong to ../rapp-telemetry/manifests/. The platform writer +scripts/telemetry_grafana_custody.py uses CAS zero, refuses existing role/policy +or credential shape drift, preserves existing boundary policy text, and checks +positive and negative login/access before verifying ESO delivery. Run only under +scripts/openbao-attended-exec.py after warden route show +openbao-platform-admin-login. A unique --receipt path is required; receipts contain +metadata only. The bootstrap does not rotate existing credentials. + +scripts/telemetry_grafana_access.py checks anonymous and forged proxy-header +denials, positive native-admin API access, datasource and dashboards inside the +same attended envelope. It requires an operator-created localhost-only port +forward on 13001 to the known Grafana service. Do not substitute an untrusted +endpoint. It emits only a metadata receipt and never logs passwords or headers. + +Rotation must update the existing Grafana database administrator credential as +well as OpenBao; changing ESO/environment alone does not reset an initialized +Grafana admin password. Coordinate and test old-login rejection/new-login success +before calling rotation complete. Disable the ESO role to revoke future delivery +but remember the existing Kubernetes Secret and application password persist; +revoke application access and remove the binding deliberately in a compromise. +Do not delete retained recovery material as an implicit rollback. + +Accepted September 6 evidence: exact read and delivery match; wrong identity, +namespace and audience denied; sibling/write denied; coding-agent deny wins; +native login succeeds; anonymous/proxy-header requests denied. See +../rapp-telemetry/evidence/live/2026-09-06-railiance01.json. Independent backup and +isolated restore, operator OIDC and public production admission remain open. diff --git a/docs/evidence/telemetry-grafana-access-2026-09-06.json b/docs/evidence/telemetry-grafana-access-2026-09-06.json new file mode 100644 index 0000000..85b6e45 --- /dev/null +++ b/docs/evidence/telemetry-grafana-access-2026-09-06.json @@ -0,0 +1,9 @@ +{ + "schema": "rapp-telemetry.access.v1", + "status": "verified", + "anonymous_denied": true, + "forged_proxy_denied": true, + "native_admin_authenticated": true, + "prometheus_datasource_present": true, + "dashboard_count": 20 +} \ No newline at end of file diff --git a/docs/evidence/telemetry-grafana-custody-2026-09-06.json b/docs/evidence/telemetry-grafana-custody-2026-09-06.json new file mode 100644 index 0000000..1f9b58c --- /dev/null +++ b/docs/evidence/telemetry-grafana-custody-2026-09-06.json @@ -0,0 +1,14 @@ +{ + "schema": "rapp-telemetry.custody.v1", + "status": "verified", + "coding_agent_deny_wins": true, + "exact_read": true, + "secret_write_and_sibling_denied": true, + "wrong_sa_denied": true, + "wrong_namespace_denied": true, + "wrong_audience_denied": true, + "bounded_ttl": true, + "kv_version": 1, + "custody": "platform/workloads/telemetry/grafana-admin", + "delivery_matches": true +} \ No newline at end of file diff --git a/history/2026-09-06-telemetry-private-activation.md b/history/2026-09-06-telemetry-private-activation.md new file mode 100644 index 0000000..944a527 --- /dev/null +++ b/history/2026-09-06-telemetry-private-activation.md @@ -0,0 +1,35 @@ +# Telemetry private activation — 2026-09-06 + +User supplied telemetry.coulomb.social and authorized continuing on railiance01. +The separately owned rapp-telemetry package is now installed privately (Helm +revision 4, pinned kube-prometheus-stack 89.2.3). All five workloads are ready, +12/12 configured targets are up, and three PVCs are bound. DNS resolves to the +node and cert-manager issued the hostname certificate. Public ingress is staged, +not applied; Master ADR-0006/0008 admission remains incomplete. + +Platform established platform/workloads/telemetry/grafana-admin with CAS-zero +protected generation and dedicated ESO identity telemetry/telemetry-grafana-eso. +The exact role/policy is telemetry-grafana-eso / telemetry-grafana-admin-eso; +audience openbao, 15-minute maximum, no default policy. Native credential values +were compared with ESO delivery only inside the attended envelope. The coding +agent boundary denies both data and metadata, preserving existing live policy. +Positive read, sibling/write denial, wrong SA/namespace/audience rejection and +deny-over-read policy union checks passed. Attended sessions self-revoked. + +Grafana native admin authentication works; anonymous and forged proxy headers +return 401. Grafana retained its PVC through an update and still has 20 provisioned +dashboards plus its Prometheus datasource. Same-namespace service/pod access +works; an unrelated namespace is rejected. A chart default route referenced an +undefined Alertmanager receiver; explicit route replacement fixed it and a +rendered-config check now guards against recurrence. Grafana uses Recreate +updates to keep one SQLite writer. + +Custody details and lifecycle limitations: docs/credential-lane-designs/telemetry-grafana.md. +Deployment evidence: ../rapp-telemetry/evidence/live/2026-09-06-railiance01.json. +Detailed handoff: ../rapp-telemetry/docs/activation-2026-09-06.md. + +RAPP-TELEMETRY-WP-0001-T03 remains progress for isolated restore and independent +custody. T04 still waits for operator OIDC, actual S3 signal delivery/receipt, +an outside-node watchdog and public admission. RPF-WP-0036 is not closed by this +installation. Primary backup remains Scaleway; do not fill the 10GB Nextcloud +Backup account with telemetry time series. It is for the agreed essential set. diff --git a/openbao/policies/agent-high-risk-boundary.hcl b/openbao/policies/agent-high-risk-boundary.hcl index 1fc6ab6..18c532a 100644 --- a/openbao/policies/agent-high-risk-boundary.hcl +++ b/openbao/policies/agent-high-risk-boundary.hcl @@ -115,3 +115,11 @@ path "platform/data/workloads/state-hub/repository-rename-preflight" { path "platform/metadata/workloads/state-hub/repository-rename-preflight" { capabilities = ["deny"] } + +# Grafana administrator custody: attended operator and ESO only. +path "platform/data/workloads/telemetry/grafana-admin" { + capabilities = ["deny"] +} +path "platform/metadata/workloads/telemetry/grafana-admin" { + capabilities = ["deny"] +} diff --git a/scripts/telemetry_grafana_access.py b/scripts/telemetry_grafana_access.py new file mode 100644 index 0000000..633f4d1 --- /dev/null +++ b/scripts/telemetry_grafana_access.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Silent attended Grafana API probe through a fixed localhost tunnel.""" +import argparse +import base64 +import json +import os +import urllib.error +import urllib.request + +from state_hub_preflight_lane import LaneError, bao, data + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--receipt', required=True) + args = parser.parse_args() + fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + receipt = {'schema': 'rapp-telemetry.access.v1', 'status': 'failed'} + try: + policies = data(bao(['token', 'lookup', '-format=json']))['data']['policies'] + if 'platform-admin' not in policies or 'root' in policies: + raise LaneError('attended_platform_admin_required') + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + def request(path, headers=None): + req = urllib.request.Request('http://127.0.0.1:13001' + path, headers=headers or {}) + try: + with opener.open(req, timeout=10) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, None + for label, headers in [('anonymous_denied', {}), ('forged_proxy_denied', { + 'X-WEBAUTH-USER': 'admin', 'Remote-User': 'admin', 'Remote-Groups': 'admins'})]: + if request('/api/user', headers)[0] != 401: + raise LaneError('negative_access_failed') + receipt[label] = True + native = data(bao(['read', '-format=json', 'platform/data/workloads/telemetry/grafana-admin']))['data']['data'] + credential = base64.b64encode((native['ADMIN_USERNAME'] + ':' + native['ADMIN_PASSWORD']).encode()).decode() + headers = {'Authorization': 'Basic ' + credential} + status, user = request('/api/user', headers) + if status != 200 or user.get('login') != 'admin' or not user.get('isGrafanaAdmin'): + raise LaneError('positive_access_failed') + receipt['native_admin_authenticated'] = True + status, sources = request('/api/datasources', headers) + if status != 200 or not any(s['type'] == 'prometheus' for s in sources): + raise LaneError('datasource_missing') + receipt['prometheus_datasource_present'] = True + status, dashboards = request('/api/search?type=dash-db', headers) + if status != 200 or not dashboards: + raise LaneError('dashboards_missing') + receipt.update(status='verified', dashboard_count=len(dashboards)) + except Exception: + receipt['error'] = 'access_verification_failed' + finally: + with os.fdopen(fd, 'w') as output: + json.dump(receipt, output, indent=2) + return 0 if receipt['status'] == 'verified' else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/telemetry_grafana_custody.py b/scripts/telemetry_grafana_custody.py new file mode 100644 index 0000000..7dbfc4b --- /dev/null +++ b/scripts/telemetry_grafana_custody.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Silent attended bootstrap of an exact Grafana-admin ESO lane.""" +import argparse +import base64 +import json +import os +import re +import secrets +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from state_hub_preflight_lane import bao, command, data, assert_cluster, LaneError, capabilities, revoke +from repair_eso_kubernetes_auth import role_payload, check_role, verify_login +k = ['kubectl', '--kubeconfig', '/home/worsch/.kube/config-railiance01'] +lane = dict(service_account='telemetry-grafana-eso', namespace='telemetry', role='telemetry-grafana-eso', policy='telemetry-grafana-admin-eso', kv_path='platform/data/workloads/telemetry/grafana-admin') +policy = 'path "' + lane['kv_path'] + '" { capabilities = ["read"] }\npath "auth/token/lookup-self" { capabilities = ["read"] }\npath "auth/token/revoke-self" { capabilities = ["update"] }\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--receipt', required=True) + args = parser.parse_args() + fd = os.open(args.receipt, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + receipt = {'schema': 'rapp-telemetry.custody.v1', 'status': 'failed'} + + def absent(r): + return b'404' in r.stderr or b'No value found' in r.stderr + try: + assert_cluster(k) + identity = data(bao(['token', 'lookup', '-format=json']))['data']['policies'] + if 'platform-admin' not in identity or 'root' in identity: + raise LaneError('wrong_identity') + boundary_path = 'sys/policies/acl/agent-high-risk-boundary' + current = data(bao(['read', '-format=json', boundary_path]))['data']['policy'] + overlay = '' + for path in [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/', 1)]: + stanza = 'path "' + path + '" { capabilities = ["deny"] }\n' + if '"' + path + '"' not in current: + overlay += stanza + elif not re.search(r'path\s+"' + re.escape(path) + r'"\s*\{\s*capabilities\s*=\s*\["deny"\]\s*\}', current): + raise LaneError('boundary_path_drift') + if overlay: + if data(bao(['read', '-format=json', boundary_path]))['data']['policy'] != current: + raise LaneError('boundary_drift') + bao(['write', boundary_path, '-'], payload={'policy': current + '\n' + overlay}) + if data(bao(['read', '-format=json', boundary_path]))['data']['policy'] != current + '\n' + overlay: + raise LaneError('boundary_readback_failed') + r = bao(['read', '-format=json', lane['kv_path']], allow_failure=True) + if r.returncode: + if not absent(r): + raise LaneError('custody_absence_unproven') + bao(['write', lane['kv_path'], '-'], payload={'options': {'cas': 0}, 'data': {'ADMIN_USERNAME': 'admin', 'ADMIN_PASSWORD': secrets.token_urlsafe(48)}}) + native = data(bao(['read', '-format=json', lane['kv_path']]))['data'] + if set(native['data']) != {'ADMIN_USERNAME', 'ADMIN_PASSWORD'} or native['data']['ADMIN_USERNAME'] != 'admin' or len(native['data']['ADMIN_PASSWORD']) < 40: + raise LaneError('custody_drift') + for path, payload, checker in [('sys/policies/acl/' + lane['policy'], {'policy': policy}, lambda a: a['policy'] == policy), ('auth/kubernetes/role/' + lane['role'], role_payload(lane), None)]: + old = bao(['read', '-format=json', path], allow_failure=True) + if old.returncode: + if not absent(old): + raise LaneError('object_absence_unproven') + bao(['write', path, '-'], payload=payload) + elif checker: + if not checker(data(old)['data']): + raise LaneError('policy_drift') + else: + check_role(data(old)['data'], lane) + child = data(bao(['token', 'create', '-format=json', '-policy=' + lane['policy'], '-policy=agent-high-risk-boundary', '-no-default-policy', '-ttl=60s']))['auth']['client_token'] + try: + paths = [lane['kv_path'], lane['kv_path'].replace('/data/', '/metadata/', 1)] + caps = capabilities(child, paths) + if any((caps[p] != ['deny'] for p in paths)): + raise LaneError('coding_agent_boundary_failed') + receipt['coding_agent_deny_wins'] = True + finally: + revoke(child) + verify_login(k, lane, receipt) + command(k + ['apply', '-f', '/home/worsch/rapp-telemetry/manifests/custody.yaml']) + command(k + ['wait', '--for=condition=Ready', 'clustersecretstore/openbao-telemetry-grafana', '--timeout=45s']) + command(k + ['-n', 'telemetry', 'wait', '--for=condition=Ready', 'externalsecret/telemetry-grafana-admin', '--timeout=45s']) + live = data(command(k + ['-n', 'telemetry', 'get', 'secret', 'telemetry-grafana-admin', '-o', 'json']))['data'] + if base64.b64decode(live['admin-user']).decode() != native['data']['ADMIN_USERNAME'] or base64.b64decode(live['admin-password']).decode() != native['data']['ADMIN_PASSWORD']: + raise LaneError('delivery_mismatch') + receipt.update(status='verified', kv_version=native['metadata']['version'], custody='platform/workloads/telemetry/grafana-admin', delivery_matches=True) + except Exception: + receipt['error'] = 'custody_bootstrap_failed' + finally: + with os.fdopen(fd, 'w') as f: + json.dump(receipt, f, indent=2) + sys.exit(0 if receipt['status'] == 'verified' else 1) + + +if __name__ == "__main__": + main()