feat: daily CNPG Option A backup shell activity (RAILIANCE-WP-0016)

Add cnpg_option_a_backup resolver, disabled ActivityDefinition, ESO
manifest, worker kubeconfig hostPath, databases RBAC, and unit tests.
Enable after ESO token re-mint and host kubeconfig wiring.

activity-definitions/daily-cnpg-option-a-backup.md

@@ -0,0 +1,50 @@
+---
+id: daily-cnpg-option-a-backup
+name: Daily CNPG Option A Backup
+enabled: false
+owner: custodian-agent
+governance: custodian
+status: active
+trigger:
+  type: cron
+  cron_expression: "30 2 * * *"
+  timezone: UTC
+  misfire_policy: skip
+context_sources:
+  - type: shell
+    query: cnpg_option_a_backup
+    required: true
+    params:
+      backup_script: /opt/railiance-platform/tools/cmd/cnpg-option-a-backup
+      dry_run: false
+      timeout_seconds: 7200
+      kubeconfig_r01: /kube/config-hosteurope
+      kubeconfig_core: /kube/config
+      evidence_sinks:
+        - type: state-hub-progress
+          event_type: cnpg_option_a_backup
+          author: activity-core
+    bind_to: context.backup
+---
+
+# Daily CNPG Option A Backup
+
+Runs daily at **02:30 UTC** (RPO 24h). Invokes
+`railiance-platform/tools/cmd/cnpg-option-a-backup` on the railiance01
+activity-core worker to age-encrypt logical dumps and upload to Nextcloud.
+
+**Workplan:** `RAILIANCE-WP-0016` (railiance-apps).
+
+## Enable checklist
+
+1. Vendor tools on railiance01 hostPath tree:
+   `tools/cmd/install-cnpg-backup-vendor-tools` (age + kubectl).
+2. ExternalSecret `actcore-backup-offsite` synced into `actcore-runtime-secret`
+   (`NC_WEBDAV_TOKEN`, `NC_WEBDAV_URL`, `AGE_PUBLIC_KEY`).
+3. Worker mounts: `/opt/railiance-platform`, `/kube` (host kubeconfigs).
+4. CoulombCore kubeconfig present at `/kube/config` for Core clusters; R01 at
+   `/kube/config-hosteurope`.
+5. `enabled: true` + `make sync` / schedule reconcile on railiance01.
+6. Forced run succeeds; `make cnpg-backup-status` stays healthy.
+
+**Starts disabled** until T02–T04 wiring is verified on railiance01.

k8s/railiance/15-externalsecret-backup-offsite.yaml

@@ -0,0 +1,41 @@
+# Sync Option A offsite backup credentials into actcore-runtime-secret.
+#
+# Prereqs on railiance01:
+#   - ClusterSecretStore openbao-activity-core
+#   - ESO token includes policy workload-kv-read-railiance-backup-offsite-lane
+#     (scripts/openbao-eso-token-apply.sh defaults updated in RAILIANCE-WP-0016)
+#
+# OpenBao path: platform/workloads/railiance/backup/offsite-lane
+# Fields synced (never AGE_PRIVATE_KEY):
+#   NC_WEBDAV_TOKEN, NC_WEBDAV_URL
+# AGE_PUBLIC_KEY is not secret material; injected via ConfigMap/env default in CLI.
+#
+# Catalog: warden route show railiance-backup-offsite-lane
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: actcore-backup-offsite
+  namespace: activity-core
+  labels:
+    app.kubernetes.io/name: activity-core
+    app.kubernetes.io/part-of: activity-core
+  annotations:
+    argocd.argoproj.io/sync-wave: "0"
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao-activity-core
+  target:
+    name: actcore-runtime-secret
+    creationPolicy: Merge
+    deletionPolicy: Retain
+  data:
+    - secretKey: NC_WEBDAV_TOKEN
+      remoteRef:
+        key: platform/workloads/railiance/backup/offsite-lane
+        property: NC_WEBDAV_TOKEN
+    - secretKey: NC_WEBDAV_URL
+      remoteRef:
+        key: platform/workloads/railiance/backup/offsite-lane
+        property: NC_WEBDAV_URL

k8s/railiance/20-runtime.yaml

@@ -1244,6 +1244,15 @@ spec:
           ports:
             - name: metrics
               containerPort: 9090
+          env:
+            - name: ISSUE_SINK_TYPE
+              value: state-hub
+            - name: KUBECONFIG_R01
+              value: /kube/config-hosteurope
+            - name: KUBECONFIG_CORE
+              value: /kube/config
+            - name: PATH
+              value: /opt/railiance-platform/tools/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
           envFrom:
             - configMapRef:
                 name: actcore-runtime-config
@@ -1264,6 +1273,12 @@ spec:
             - name: custodian-runtime
               mountPath: /var/custodian/runtime/prompts
               readOnly: true
+            - name: railiance-platform
+              mountPath: /opt/railiance-platform
+              readOnly: true
+            - name: kubeconfigs
+              mountPath: /kube
+              readOnly: true
       volumes:
         - name: external-activity-definitions
           configMap:
@@ -1281,6 +1296,14 @@ spec:
         - name: custodian-runtime
           configMap:
             name: actcore-custodian-runtime
+        - name: railiance-platform
+          hostPath:
+            path: /home/tegwick/railiance-platform
+            type: Directory
+        - name: kubeconfigs
+          hostPath:
+            path: /home/tegwick/.kube
+            type: Directory
 ---
 apiVersion: apps/v1
 kind: Deployment

k8s/railiance/25-worker-backup-mounts.yaml

@@ -0,0 +1,41 @@
+# Strategic merge patch fragments for actcore-worker backup automation.
+# Apply with: kubectl -n activity-core patch deploy actcore-worker --patch-file ...
+#
+# Adds:
+#   - hostPath kubeconfigs (/home/tegwick/.kube → /kube)
+#   - ensure railiance-platform hostPath (already live on railiance01)
+#
+# RAILIANCE-WP-0016-T04/T05
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: actcore-worker
+  namespace: activity-core
+spec:
+  template:
+    spec:
+      containers:
+        - name: worker
+          env:
+            - name: KUBECONFIG_R01
+              value: /kube/config-hosteurope
+            - name: KUBECONFIG_CORE
+              value: /kube/config
+            - name: PATH
+              value: /opt/railiance-platform/tools/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+          volumeMounts:
+            - name: kubeconfigs
+              mountPath: /kube
+              readOnly: true
+            - name: railiance-platform
+              mountPath: /opt/railiance-platform
+              readOnly: true
+      volumes:
+        - name: kubeconfigs
+          hostPath:
+            path: /home/tegwick/.kube
+            type: Directory
+        - name: railiance-platform
+          hostPath:
+            path: /home/tegwick/railiance-platform
+            type: Directory

k8s/railiance/26-worker-cnpg-backup-rbac.yaml

@@ -0,0 +1,38 @@
+# Optional in-cluster RBAC if using service-account kubeconfig for railiance01
+# databases dumps (alternative to host kubeconfig admin). Prefer host kubeconfig
+# with limited certs for multi-cluster; this Role covers same-cluster exec.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: actcore-cnpg-logical-backup
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: activity-core
+    railiance.apps/backup-lane: option-a
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: actcore-cnpg-logical-backup
+  namespace: databases
+  labels:
+    app.kubernetes.io/part-of: activity-core
+    railiance.apps/backup-lane: option-a
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: actcore-cnpg-logical-backup
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: activity-core

scripts/openbao-eso-token-apply.sh

@@ -3,14 +3,16 @@
 # it on railiance01 for ClusterSecretStore openbao-activity-core / ExternalSecrets:
 #   - actcore-issue-core-runtime  (ISSUE_CORE_API_KEY)
 #   - actcore-forgejo-admin       (FORGEJO_TOKEN for package prune)
+#   - actcore-backup-offsite      (NC_WEBDAV_* for Option A backups)
 #
 # Default policies (already present in OpenBao; files re-written when present):
 #   workload-kv-read-issue-core-runtime
 #   workload-kv-read-forgejo-admin
+#   workload-kv-read-railiance-backup-offsite-lane
 set -euo pipefail
 
 # Space-separated policy names. Override with OPENBAO_ACTIVITY_CORE_POLICIES.
-DEFAULT_POLICIES="workload-kv-read-issue-core-runtime workload-kv-read-forgejo-admin"
+DEFAULT_POLICIES="workload-kv-read-issue-core-runtime workload-kv-read-forgejo-admin workload-kv-read-railiance-backup-offsite-lane"
 POLICIES="${OPENBAO_ACTIVITY_CORE_POLICIES:-$DEFAULT_POLICIES}"
 POLICY_DIR="${OPENBAO_POLICY_DIR:-$HOME/railiance-platform/openbao/policies}"
 BAO_ADDR="${BAO_ADDR:-https://bao.coulomb.social}"

src/activity_core/context_resolvers/cnpg_backup.py

@@ -0,0 +1,96 @@
+"""CNPG Option A backup shell context query (RAILIANCE-WP-0016)."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_SCRIPT = Path("/opt/railiance-platform/tools/cmd/cnpg-option-a-backup")
+_DEFAULT_TIMEOUT_SECONDS = 7200
+
+
+def cnpg_option_a_backup(params: dict[str, Any]) -> dict[str, Any]:
+    """Run multi-host Option A backup CLI and return its JSON summary."""
+    script = Path(str(params.get("backup_script", _DEFAULT_SCRIPT))).expanduser()
+    if not script.is_file():
+        raise FileNotFoundError(f"cnpg_option_a_backup script not found: {script}")
+
+    dry_run = bool(params.get("dry_run", False))
+    timeout = float(params.get("timeout_seconds", _DEFAULT_TIMEOUT_SECONDS))
+    targets = params.get("targets")  # optional CSV
+
+    env = os.environ.copy()
+    # Ensure vendor bin (age/kubectl) is visible when hostPath is mounted.
+    vendor = Path("/opt/railiance-platform/tools/vendor/bin")
+    if vendor.is_dir():
+        env["PATH"] = f"{vendor}:{env.get('PATH', '')}"
+
+    if dry_run:
+        env["RAILIANCE_BACKUP_DRY_RUN"] = "1"
+    else:
+        env["RAILIANCE_BACKUP_DRY_RUN"] = "0"
+
+    # Prefer env already injected via actcore-runtime-secret (ESO).
+    for src, dst in (
+        ("NC_WEBDAV_TOKEN", "RAILIANCE_BACKUP_NC_TOKEN"),
+        ("NC_WEBDAV_URL", "RAILIANCE_BACKUP_NC_WEBDAV_URL"),
+        ("AGE_PUBLIC_KEY", "RAILIANCE_BACKUP_AGE_PUBLIC_KEY"),
+    ):
+        if env.get(src) and not env.get(dst):
+            env[dst] = env[src]
+
+    if params.get("kubeconfig_core"):
+        env["KUBECONFIG_CORE"] = str(params["kubeconfig_core"])
+    if params.get("kubeconfig_r01"):
+        env["KUBECONFIG_R01"] = str(params["kubeconfig_r01"])
+
+    # Optional hostPath kubeconfigs mounted into the worker.
+    if not env.get("KUBECONFIG_CORE") and Path("/kube/config").is_file():
+        env["KUBECONFIG_CORE"] = "/kube/config"
+    if not env.get("KUBECONFIG_R01") and Path("/kube/config-hosteurope").is_file():
+        env["KUBECONFIG_R01"] = "/kube/config-hosteurope"
+    if not env.get("KUBECONFIG_R01") and Path("/kube/config").is_file():
+        # Single-cluster worker host: treat as railiance01.
+        env.setdefault("KUBECONFIG_R01", "/kube/config")
+
+    if targets:
+        env["CNPG_BACKUP_TARGETS"] = str(targets)
+
+    cmd = [str(script)]
+    completed = subprocess.run(
+        cmd,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
+        check=False,
+        env=env,
+    )
+    stdout = (completed.stdout or "").strip()
+    stderr = (completed.stderr or "").strip()
+    if completed.returncode not in (0, 2) and not stdout:
+        raise RuntimeError(
+            f"cnpg_option_a_backup failed (exit {completed.returncode}): {stderr[:500]}"
+        )
+
+    try:
+        summary = json.loads(stdout)
+    except json.JSONDecodeError as exc:
+        raise RuntimeError(
+            f"cnpg_option_a_backup returned invalid JSON: {exc}; stderr={stderr[:300]}"
+        ) from exc
+
+    if completed.returncode not in (0, 2):
+        summary.setdefault("errors", []).append(
+            f"script_exit_code:{completed.returncode}"
+        )
+    summary["kind"] = "cnpg_option_a_backup"
+    summary["script_exit_code"] = completed.returncode
+    if stderr:
+        summary["log_tail"] = stderr[-1500:]
+    return summary

src/activity_core/context_resolvers/reuse_surface.py

@@ -22,6 +22,7 @@ import yaml
 
 from activity_core.context_resolvers.base import CONTEXT_RESOLVER_REGISTRY, ContextResolver
 from activity_core.context_resolvers.forgejo_prune import forgejo_package_prune
+from activity_core.context_resolvers.cnpg_backup import cnpg_option_a_backup
 from activity_core.context_resolvers.kaizen import KaizenContextResolver
 from activity_core.context_resolvers.state_hub import StateHubContextResolver
 
@@ -512,6 +513,8 @@ class ShellContextResolver(ContextResolver):
             return reuse_surface_report_gaps(params)
         if query == "forgejo_package_prune":
             return forgejo_package_prune(params)
+        if query == "cnpg_option_a_backup":
+            return cnpg_option_a_backup(params)
         return KaizenContextResolver().resolve(query, event, params)
 
 

src/activity_core/ops_console.py

@@ -39,6 +39,7 @@ _AUDIT_BUFFER: deque[dict[str, Any]] = deque(maxlen=100)
 
 SIDE_EFFECT_MARKERS = (
     "forgejo_package_prune",
+    "cnpg_option_a_backup",
     "apply: true",
     '"apply": true',
     "'apply': true",

src/activity_core/ops_evidence_sinks.py

@@ -139,6 +139,10 @@ def _post_state_hub_progress(
         compact = probe_result
         summary = _forgejo_package_prune_summary_text(probe_result)
         source_type = "shell"
+    elif probe_result.get("kind") == "cnpg_option_a_backup":
+        compact = probe_result
+        summary = _cnpg_option_a_backup_summary_text(probe_result)
+        source_type = "shell"
     elif probe_result.get("checks") is not None:
         compact = probe_result
         summary = _phase5_summary_text(probe_result)
@@ -552,6 +556,18 @@ def _forgejo_package_prune_summary_text(result: dict[str, Any]) -> str:
     )
 
 
+def _cnpg_option_a_backup_summary_text(result: dict[str, Any]) -> str:
+    overall = result.get("overall", "?")
+    dumped = result.get("dumped", 0)
+    uploaded = result.get("uploaded", 0)
+    failed = result.get("failed", 0)
+    dry = "dry-run" if result.get("dry_run") else "live"
+    return (
+        f"CNPG Option A backup ({dry}): overall={overall}; "
+        f"dumped={dumped} uploaded={uploaded} failed={failed}"
+    )
+
+
 def _legacy_meter_summary_text(result: dict[str, Any]) -> str:
     candidates = result.get("retirement_candidate_count", 0)
     calls = result.get("window_legacy_calls", 0)

tests/test_cnpg_option_a_backup_resolver.py

@@ -0,0 +1,62 @@
+"""Unit tests for cnpg_option_a_backup shell resolver."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from activity_core.context_resolvers.cnpg_backup import cnpg_option_a_backup
+
+
+def test_missing_script_raises(tmp_path: Path) -> None:
+    with pytest.raises(FileNotFoundError):
+        cnpg_option_a_backup({"backup_script": str(tmp_path / "nope")})
+
+
+def test_happy_path_parses_json(tmp_path: Path) -> None:
+    script = tmp_path / "cnpg-option-a-backup"
+    script.write_text("#!/bin/sh\n")
+    script.chmod(0o755)
+    payload = {
+        "kind": "cnpg_option_a_backup",
+        "overall": "ok",
+        "dry_run": True,
+        "dumped": 1,
+        "uploaded": 0,
+        "failed": 0,
+        "targets": [],
+    }
+    completed = MagicMock(
+        returncode=0,
+        stdout=json.dumps(payload),
+        stderr="log line",
+    )
+    with patch("activity_core.context_resolvers.cnpg_backup.subprocess.run", return_value=completed):
+        result = cnpg_option_a_backup(
+            {"backup_script": str(script), "dry_run": True, "timeout_seconds": 30}
+        )
+    assert result["kind"] == "cnpg_option_a_backup"
+    assert result["overall"] == "ok"
+    assert result["script_exit_code"] == 0
+
+
+def test_degraded_exit_2_still_parses(tmp_path: Path) -> None:
+    script = tmp_path / "cnpg-option-a-backup"
+    script.write_text("#!/bin/sh\n")
+    script.chmod(0o755)
+    payload = {
+        "kind": "cnpg_option_a_backup",
+        "overall": "degraded",
+        "dumped": 1,
+        "uploaded": 0,
+        "failed": 1,
+        "targets": [],
+    }
+    completed = MagicMock(returncode=2, stdout=json.dumps(payload), stderr="")
+    with patch("activity_core.context_resolvers.cnpg_backup.subprocess.run", return_value=completed):
+        result = cnpg_option_a_backup({"backup_script": str(script)})
+    assert result["overall"] == "degraded"
+    assert result["script_exit_code"] == 2