diff --git a/activity-definitions/daily-cnpg-option-a-backup.md b/activity-definitions/daily-cnpg-option-a-backup.md new file mode 100644 index 0000000..ac4b3d1 --- /dev/null +++ b/activity-definitions/daily-cnpg-option-a-backup.md @@ -0,0 +1,50 @@ +--- +id: daily-cnpg-option-a-backup +name: Daily CNPG Option A Backup +enabled: false +owner: custodian-agent +governance: custodian +status: active +trigger: + type: cron + cron_expression: "30 2 * * *" + timezone: UTC + misfire_policy: skip +context_sources: + - type: shell + query: cnpg_option_a_backup + required: true + params: + backup_script: /opt/railiance-platform/tools/cmd/cnpg-option-a-backup + dry_run: false + timeout_seconds: 7200 + kubeconfig_r01: /kube/config-hosteurope + kubeconfig_core: /kube/config + evidence_sinks: + - type: state-hub-progress + event_type: cnpg_option_a_backup + author: activity-core + bind_to: context.backup +--- + +# Daily CNPG Option A Backup + +Runs daily at **02:30 UTC** (RPO 24h). Invokes +`railiance-platform/tools/cmd/cnpg-option-a-backup` on the railiance01 +activity-core worker to age-encrypt logical dumps and upload to Nextcloud. + +**Workplan:** `RAILIANCE-WP-0016` (railiance-apps). + +## Enable checklist + +1. Vendor tools on railiance01 hostPath tree: + `tools/cmd/install-cnpg-backup-vendor-tools` (age + kubectl). +2. ExternalSecret `actcore-backup-offsite` synced into `actcore-runtime-secret` + (`NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`, `AGE_PUBLIC_KEY`). +3. Worker mounts: `/opt/railiance-platform`, `/kube` (host kubeconfigs). +4. CoulombCore kubeconfig present at `/kube/config` for Core clusters; R01 at + `/kube/config-hosteurope`. +5. `enabled: true` + `make sync` / schedule reconcile on railiance01. +6. Forced run succeeds; `make cnpg-backup-status` stays healthy. + +**Starts disabled** until T02–T04 wiring is verified on railiance01. diff --git a/k8s/railiance/15-externalsecret-backup-offsite.yaml b/k8s/railiance/15-externalsecret-backup-offsite.yaml new file mode 100644 index 0000000..d36f74d --- /dev/null +++ b/k8s/railiance/15-externalsecret-backup-offsite.yaml @@ -0,0 +1,41 @@ +# Sync Option A offsite backup credentials into actcore-runtime-secret. +# +# Prereqs on railiance01: +# - ClusterSecretStore openbao-activity-core +# - ESO token includes policy workload-kv-read-railiance-backup-offsite-lane +# (scripts/openbao-eso-token-apply.sh defaults updated in RAILIANCE-WP-0016) +# +# OpenBao path: platform/workloads/railiance/backup/offsite-lane +# Fields synced (never AGE_PRIVATE_KEY): +# NC_WEBDAV_TOKEN, NC_WEBDAV_URL +# AGE_PUBLIC_KEY is not secret material; injected via ConfigMap/env default in CLI. +# +# Catalog: warden route show railiance-backup-offsite-lane +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: actcore-backup-offsite + namespace: activity-core + labels: + app.kubernetes.io/name: activity-core + app.kubernetes.io/part-of: activity-core + annotations: + argocd.argoproj.io/sync-wave: "0" +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: openbao-activity-core + target: + name: actcore-runtime-secret + creationPolicy: Merge + deletionPolicy: Retain + data: + - secretKey: NC_WEBDAV_TOKEN + remoteRef: + key: platform/workloads/railiance/backup/offsite-lane + property: NC_WEBDAV_TOKEN + - secretKey: NC_WEBDAV_URL + remoteRef: + key: platform/workloads/railiance/backup/offsite-lane + property: NC_WEBDAV_URL diff --git a/k8s/railiance/20-runtime.yaml b/k8s/railiance/20-runtime.yaml index 784b5c1..755447c 100644 --- a/k8s/railiance/20-runtime.yaml +++ b/k8s/railiance/20-runtime.yaml @@ -1244,6 +1244,15 @@ spec: ports: - name: metrics containerPort: 9090 + env: + - name: ISSUE_SINK_TYPE + value: state-hub + - name: KUBECONFIG_R01 + value: /kube/config-hosteurope + - name: KUBECONFIG_CORE + value: /kube/config + - name: PATH + value: /opt/railiance-platform/tools/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin envFrom: - configMapRef: name: actcore-runtime-config @@ -1264,6 +1273,12 @@ spec: - name: custodian-runtime mountPath: /var/custodian/runtime/prompts readOnly: true + - name: railiance-platform + mountPath: /opt/railiance-platform + readOnly: true + - name: kubeconfigs + mountPath: /kube + readOnly: true volumes: - name: external-activity-definitions configMap: @@ -1281,6 +1296,14 @@ spec: - name: custodian-runtime configMap: name: actcore-custodian-runtime + - name: railiance-platform + hostPath: + path: /home/tegwick/railiance-platform + type: Directory + - name: kubeconfigs + hostPath: + path: /home/tegwick/.kube + type: Directory --- apiVersion: apps/v1 kind: Deployment diff --git a/k8s/railiance/25-worker-backup-mounts.yaml b/k8s/railiance/25-worker-backup-mounts.yaml new file mode 100644 index 0000000..71e8f89 --- /dev/null +++ b/k8s/railiance/25-worker-backup-mounts.yaml @@ -0,0 +1,41 @@ +# Strategic merge patch fragments for actcore-worker backup automation. +# Apply with: kubectl -n activity-core patch deploy actcore-worker --patch-file ... +# +# Adds: +# - hostPath kubeconfigs (/home/tegwick/.kube → /kube) +# - ensure railiance-platform hostPath (already live on railiance01) +# +# RAILIANCE-WP-0016-T04/T05 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: actcore-worker + namespace: activity-core +spec: + template: + spec: + containers: + - name: worker + env: + - name: KUBECONFIG_R01 + value: /kube/config-hosteurope + - name: KUBECONFIG_CORE + value: /kube/config + - name: PATH + value: /opt/railiance-platform/tools/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + volumeMounts: + - name: kubeconfigs + mountPath: /kube + readOnly: true + - name: railiance-platform + mountPath: /opt/railiance-platform + readOnly: true + volumes: + - name: kubeconfigs + hostPath: + path: /home/tegwick/.kube + type: Directory + - name: railiance-platform + hostPath: + path: /home/tegwick/railiance-platform + type: Directory diff --git a/k8s/railiance/26-worker-cnpg-backup-rbac.yaml b/k8s/railiance/26-worker-cnpg-backup-rbac.yaml new file mode 100644 index 0000000..e04770c --- /dev/null +++ b/k8s/railiance/26-worker-cnpg-backup-rbac.yaml @@ -0,0 +1,38 @@ +# Optional in-cluster RBAC if using service-account kubeconfig for railiance01 +# databases dumps (alternative to host kubeconfig admin). Prefer host kubeconfig +# with limited certs for multi-cluster; this Role covers same-cluster exec. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: actcore-cnpg-logical-backup + namespace: databases + labels: + app.kubernetes.io/part-of: activity-core + railiance.apps/backup-lane: option-a +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "create", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: actcore-cnpg-logical-backup + namespace: databases + labels: + app.kubernetes.io/part-of: activity-core + railiance.apps/backup-lane: option-a +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: actcore-cnpg-logical-backup +subjects: + - kind: ServiceAccount + name: default + namespace: activity-core diff --git a/scripts/openbao-eso-token-apply.sh b/scripts/openbao-eso-token-apply.sh index 5fbf6b5..f65956d 100755 --- a/scripts/openbao-eso-token-apply.sh +++ b/scripts/openbao-eso-token-apply.sh @@ -3,14 +3,16 @@ # it on railiance01 for ClusterSecretStore openbao-activity-core / ExternalSecrets: # - actcore-issue-core-runtime (ISSUE_CORE_API_KEY) # - actcore-forgejo-admin (FORGEJO_TOKEN for package prune) +# - actcore-backup-offsite (NC_WEBDAV_* for Option A backups) # # Default policies (already present in OpenBao; files re-written when present): # workload-kv-read-issue-core-runtime # workload-kv-read-forgejo-admin +# workload-kv-read-railiance-backup-offsite-lane set -euo pipefail # Space-separated policy names. Override with OPENBAO_ACTIVITY_CORE_POLICIES. -DEFAULT_POLICIES="workload-kv-read-issue-core-runtime workload-kv-read-forgejo-admin" +DEFAULT_POLICIES="workload-kv-read-issue-core-runtime workload-kv-read-forgejo-admin workload-kv-read-railiance-backup-offsite-lane" POLICIES="${OPENBAO_ACTIVITY_CORE_POLICIES:-$DEFAULT_POLICIES}" POLICY_DIR="${OPENBAO_POLICY_DIR:-$HOME/railiance-platform/openbao/policies}" BAO_ADDR="${BAO_ADDR:-https://bao.coulomb.social}" diff --git a/src/activity_core/context_resolvers/cnpg_backup.py b/src/activity_core/context_resolvers/cnpg_backup.py new file mode 100644 index 0000000..aa87258 --- /dev/null +++ b/src/activity_core/context_resolvers/cnpg_backup.py @@ -0,0 +1,96 @@ +"""CNPG Option A backup shell context query (RAILIANCE-WP-0016).""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + +_DEFAULT_SCRIPT = Path("/opt/railiance-platform/tools/cmd/cnpg-option-a-backup") +_DEFAULT_TIMEOUT_SECONDS = 7200 + + +def cnpg_option_a_backup(params: dict[str, Any]) -> dict[str, Any]: + """Run multi-host Option A backup CLI and return its JSON summary.""" + script = Path(str(params.get("backup_script", _DEFAULT_SCRIPT))).expanduser() + if not script.is_file(): + raise FileNotFoundError(f"cnpg_option_a_backup script not found: {script}") + + dry_run = bool(params.get("dry_run", False)) + timeout = float(params.get("timeout_seconds", _DEFAULT_TIMEOUT_SECONDS)) + targets = params.get("targets") # optional CSV + + env = os.environ.copy() + # Ensure vendor bin (age/kubectl) is visible when hostPath is mounted. + vendor = Path("/opt/railiance-platform/tools/vendor/bin") + if vendor.is_dir(): + env["PATH"] = f"{vendor}:{env.get('PATH', '')}" + + if dry_run: + env["RAILIANCE_BACKUP_DRY_RUN"] = "1" + else: + env["RAILIANCE_BACKUP_DRY_RUN"] = "0" + + # Prefer env already injected via actcore-runtime-secret (ESO). + for src, dst in ( + ("NC_WEBDAV_TOKEN", "RAILIANCE_BACKUP_NC_TOKEN"), + ("NC_WEBDAV_URL", "RAILIANCE_BACKUP_NC_WEBDAV_URL"), + ("AGE_PUBLIC_KEY", "RAILIANCE_BACKUP_AGE_PUBLIC_KEY"), + ): + if env.get(src) and not env.get(dst): + env[dst] = env[src] + + if params.get("kubeconfig_core"): + env["KUBECONFIG_CORE"] = str(params["kubeconfig_core"]) + if params.get("kubeconfig_r01"): + env["KUBECONFIG_R01"] = str(params["kubeconfig_r01"]) + + # Optional hostPath kubeconfigs mounted into the worker. + if not env.get("KUBECONFIG_CORE") and Path("/kube/config").is_file(): + env["KUBECONFIG_CORE"] = "/kube/config" + if not env.get("KUBECONFIG_R01") and Path("/kube/config-hosteurope").is_file(): + env["KUBECONFIG_R01"] = "/kube/config-hosteurope" + if not env.get("KUBECONFIG_R01") and Path("/kube/config").is_file(): + # Single-cluster worker host: treat as railiance01. + env.setdefault("KUBECONFIG_R01", "/kube/config") + + if targets: + env["CNPG_BACKUP_TARGETS"] = str(targets) + + cmd = [str(script)] + completed = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + env=env, + ) + stdout = (completed.stdout or "").strip() + stderr = (completed.stderr or "").strip() + if completed.returncode not in (0, 2) and not stdout: + raise RuntimeError( + f"cnpg_option_a_backup failed (exit {completed.returncode}): {stderr[:500]}" + ) + + try: + summary = json.loads(stdout) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"cnpg_option_a_backup returned invalid JSON: {exc}; stderr={stderr[:300]}" + ) from exc + + if completed.returncode not in (0, 2): + summary.setdefault("errors", []).append( + f"script_exit_code:{completed.returncode}" + ) + summary["kind"] = "cnpg_option_a_backup" + summary["script_exit_code"] = completed.returncode + if stderr: + summary["log_tail"] = stderr[-1500:] + return summary diff --git a/src/activity_core/context_resolvers/reuse_surface.py b/src/activity_core/context_resolvers/reuse_surface.py index 2676f12..e590d27 100644 --- a/src/activity_core/context_resolvers/reuse_surface.py +++ b/src/activity_core/context_resolvers/reuse_surface.py @@ -22,6 +22,7 @@ import yaml from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY, ContextResolver from activity_core.context_resolvers.forgejo_prune import forgejo_package_prune +from activity_core.context_resolvers.cnpg_backup import cnpg_option_a_backup from activity_core.context_resolvers.kaizen import KaizenContextResolver from activity_core.context_resolvers.state_hub import StateHubContextResolver @@ -512,6 +513,8 @@ class ShellContextResolver(ContextResolver): return reuse_surface_report_gaps(params) if query == "forgejo_package_prune": return forgejo_package_prune(params) + if query == "cnpg_option_a_backup": + return cnpg_option_a_backup(params) return KaizenContextResolver().resolve(query, event, params) diff --git a/src/activity_core/ops_console.py b/src/activity_core/ops_console.py index 6aea991..48f5e6b 100644 --- a/src/activity_core/ops_console.py +++ b/src/activity_core/ops_console.py @@ -39,6 +39,7 @@ _AUDIT_BUFFER: deque[dict[str, Any]] = deque(maxlen=100) SIDE_EFFECT_MARKERS = ( "forgejo_package_prune", + "cnpg_option_a_backup", "apply: true", '"apply": true', "'apply': true", diff --git a/src/activity_core/ops_evidence_sinks.py b/src/activity_core/ops_evidence_sinks.py index a6a0f8c..b0bcd8b 100644 --- a/src/activity_core/ops_evidence_sinks.py +++ b/src/activity_core/ops_evidence_sinks.py @@ -139,6 +139,10 @@ def _post_state_hub_progress( compact = probe_result summary = _forgejo_package_prune_summary_text(probe_result) source_type = "shell" + elif probe_result.get("kind") == "cnpg_option_a_backup": + compact = probe_result + summary = _cnpg_option_a_backup_summary_text(probe_result) + source_type = "shell" elif probe_result.get("checks") is not None: compact = probe_result summary = _phase5_summary_text(probe_result) @@ -552,6 +556,18 @@ def _forgejo_package_prune_summary_text(result: dict[str, Any]) -> str: ) +def _cnpg_option_a_backup_summary_text(result: dict[str, Any]) -> str: + overall = result.get("overall", "?") + dumped = result.get("dumped", 0) + uploaded = result.get("uploaded", 0) + failed = result.get("failed", 0) + dry = "dry-run" if result.get("dry_run") else "live" + return ( + f"CNPG Option A backup ({dry}): overall={overall}; " + f"dumped={dumped} uploaded={uploaded} failed={failed}" + ) + + def _legacy_meter_summary_text(result: dict[str, Any]) -> str: candidates = result.get("retirement_candidate_count", 0) calls = result.get("window_legacy_calls", 0) diff --git a/tests/test_cnpg_option_a_backup_resolver.py b/tests/test_cnpg_option_a_backup_resolver.py new file mode 100644 index 0000000..e0d3fd3 --- /dev/null +++ b/tests/test_cnpg_option_a_backup_resolver.py @@ -0,0 +1,62 @@ +"""Unit tests for cnpg_option_a_backup shell resolver.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + +from activity_core.context_resolvers.cnpg_backup import cnpg_option_a_backup + + +def test_missing_script_raises(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + cnpg_option_a_backup({"backup_script": str(tmp_path / "nope")}) + + +def test_happy_path_parses_json(tmp_path: Path) -> None: + script = tmp_path / "cnpg-option-a-backup" + script.write_text("#!/bin/sh\n") + script.chmod(0o755) + payload = { + "kind": "cnpg_option_a_backup", + "overall": "ok", + "dry_run": True, + "dumped": 1, + "uploaded": 0, + "failed": 0, + "targets": [], + } + completed = MagicMock( + returncode=0, + stdout=json.dumps(payload), + stderr="log line", + ) + with patch("activity_core.context_resolvers.cnpg_backup.subprocess.run", return_value=completed): + result = cnpg_option_a_backup( + {"backup_script": str(script), "dry_run": True, "timeout_seconds": 30} + ) + assert result["kind"] == "cnpg_option_a_backup" + assert result["overall"] == "ok" + assert result["script_exit_code"] == 0 + + +def test_degraded_exit_2_still_parses(tmp_path: Path) -> None: + script = tmp_path / "cnpg-option-a-backup" + script.write_text("#!/bin/sh\n") + script.chmod(0o755) + payload = { + "kind": "cnpg_option_a_backup", + "overall": "degraded", + "dumped": 1, + "uploaded": 0, + "failed": 1, + "targets": [], + } + completed = MagicMock(returncode=2, stdout=json.dumps(payload), stderr="") + with patch("activity_core.context_resolvers.cnpg_backup.subprocess.run", return_value=completed): + result = cnpg_option_a_backup({"backup_script": str(script)}) + assert result["overall"] == "degraded" + assert result["script_exit_code"] == 2