From d6088e4e16ed0f06bc51c83f1842ad9657387b32 Mon Sep 17 00:00:00 2001 From: tegwick Date: Wed, 1 Jul 2026 23:32:38 +0200 Subject: [PATCH 01/10] =?UTF-8?q?Implement=20WP-0022=20audit=20trail=20and?= =?UTF-8?q?=20WP-0023=20INTENT=E2=80=93SCOPE=20closeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add unified metadata-only audit.jsonl with secret-material guard, instrument sign/access/worker paths, and expose warden activity CLI. Surface broker hint when VAULT_TOKEN is unset, refresh INTENT/SCOPE docs, and add production integration checklists plus catalog lane promotion playbook. --- INTENT.md | 65 ++-- SCOPE.md | 33 +- src/warden/audit.py | 281 ++++++++++++++++++ src/warden/ca.py | 18 ++ src/warden/cli.py | 54 ++++ src/warden/proxy.py | 17 ++ src/warden/vault.py | 6 +- src/warden/vault_hints.py | 22 ++ src/warden/worker.py | 44 ++- tests/test_audit.py | 152 ++++++++++ tests/test_vault.py | 15 + wiki/AuditTrail.md | 72 +++++ wiki/CredentialRouting.md | 2 + wiki/PolicyGatedSigning.md | 13 + wiki/playbooks/catalog-lane-promotion.md | 59 ++++ wiki/playbooks/ops-bridge-tunnel-cert.md | 26 ++ ...WARDEN-WP-0022-audit-trail-and-activity.md | 27 +- ...WP-0023-intent-scope-alignment-closeout.md | 28 +- 18 files changed, 875 insertions(+), 59 deletions(-) create mode 100644 src/warden/audit.py create mode 100644 src/warden/vault_hints.py create mode 100644 tests/test_audit.py create mode 100644 wiki/AuditTrail.md create mode 100644 wiki/playbooks/catalog-lane-promotion.md diff --git a/INTENT.md b/INTENT.md index 46a3204..ee7d075 100644 --- a/INTENT.md +++ b/INTENT.md @@ -54,8 +54,11 @@ owns one lane and points at the rest: restating them. Beyond pointing, **assist**: the `warden access` front door renders the exact auth method, path, and command for any need and — for `exec_capable` lanes — proxies the fetch *as the caller* (a transparent, policy-gated, audited - conduit that holds, caches, and logs **nothing**). This is the assist layer, not a - broker: custody stays in OpenBao, authorization in flex-auth. + conduit that holds, caches, and logs **nothing**). For **owner-native exec** lanes + (secrets-engine `exec`, railiance-platform `credential exec`) ops-warden routes to + the owner's front door — it does not mint tokens or run the owner's tool itself. + This is the assist layer, not a universal broker: custody stays in OpenBao / + secrets-engine / the platform broker; authorization in flex-auth. 3. **Steward workload security posture conformance.** Author the ops-security slice for environment posture (`dev/test/prod`) and workload maturity (`M0-M3`), then ship descriptors and read-only checks that identify whether a secret-flow blocker @@ -68,8 +71,9 @@ owns one lane and points at the rest: host or ops reachability requires the SSH lane — via `warden sign`, `cert_command`, and `ops-ssh-wrapper`. This is the **only** lane ops-warden executes with its own authority. -6. **Audit** SSH signing operations and cert-side compliance so gatekeeping is - observable, not tribal knowledge. +6. **Audit** every ops-warden action — SSH signs, access proxy handoffs, worker + coordination ticks — in one metadata-only trail (`warden activity`) so + gatekeeping is observable, not tribal knowledge. --- @@ -81,12 +85,14 @@ ops-warden should be fluent in the platform architecture documented in | Plane / component | Role in access | ops-warden relationship | | --- | --- | --- | | **key-cape / Keycloak** | Identity — who is the actor, MFA, IAM Profile claims | Instruct identity path; do not re-implement OIDC | -| **flex-auth + Topaz** | Authorization — may this actor perform this action | Future policy gate before SSH issuance; document integration | -| **OpenBao** | Runtime secrets — API keys, dynamic creds, leases, audit | Instruct secret custody paths; SSH engine is signing backend only | +| **flex-auth + Topaz** | Authorization — may this actor perform this action | Caller-side policy gate shipped (opt-in); production flip is flex-auth's | +| **OpenBao** | Runtime secrets — API keys, dynamic creds, leases, audit | Instruct custody paths; SSH engine is signing backend only; proxy reads as caller when `exec_capable` | +| **secrets-engine** | Owner-native secret-exec (`secrets-engine exec`) | Route provisioned exec lanes (e.g. npm publish); ops-warden does not hold tokens | +| **railiance-platform** (credential broker) | Scoped lease grants (`credential exec`) | Route `warden-sign` token needs; ops-warden does not mint OpenBao tokens | | **ops-warden** | Operational SSH certificates — short-lived host access | **Own and issue** this lane | | **ops-bridge** | Tunnel transport — consumes certs via `cert_command` | Primary consumer; document integration | | **railiance-infra** | Host principals, force-command, SSH hardening | Instruct host-side deployment; do not own Ansible | -| **railiance-platform** | OpenBao/K8s/platform service deployment | Instruct production endpoints; do not deploy clusters | +| **railiance-platform** (deploy) | OpenBao/K8s/platform service deployment | Instruct production endpoints; do not deploy clusters | Canonical references: @@ -102,11 +108,13 @@ Canonical references: - NetKingdom-aligned **operational SSH access** guidance and stewardship - **SSH certificate issuance** for registered `adm` / `agt` / `atm` actors -- Actor inventory, TTL/principal policy, cert-side scorecard, signatures log +- Actor inventory, TTL/principal policy, cert-side scorecard, unified audit trail - `cert_command` contract and `ops-ssh-wrapper` automation surface - Keeping ops-warden docs and patterns aligned with NetKingdom security evolution -- Workload Security Posture draft, conformance descriptors/checks, and dev-tier +- Workload Security Posture standard, conformance descriptors/checks, and dev-tier contract-double guidance for secret-flow readiness +- Coordination worker stewardship — triage ops-warden's State Hub inbox with + conservative defaults (draft-only unless `--full-auto`) ### ops-warden instructs but does not own @@ -158,8 +166,9 @@ scorecard checks, inventory patterns, and future policy-integration hooks. ### 6. Observable gatekeeping -Every successful SSH sign is auditable (`signatures.log`). Compliance checks -(scorecard) make cert-side policy violations visible before they become incidents. +Every ops-warden action appends metadata-only audit events; `warden activity` +answers *what happened recently* in one command. Compliance checks (scorecard) make +cert-side policy violations visible before they become incidents. --- @@ -169,23 +178,31 @@ Every successful SSH sign is auditable (`signatures.log`). Compliance checks Development worker needs access | v -ops-warden (issue SSH; route the rest) +ops-warden (issue SSH; route / assist the rest) | - +-- SSH host / ops reachability? ----> warden sign / cert_command + +-- SSH host / ops reachability? --------> warden sign / cert_command + | (OpenBao SSH engine; scoped token via credential broker) | - +-- Runtime API / platform secret? --> OpenBao path (documented) + +-- Owner-native secret exec? -----------> secrets-engine exec + | (e.g. npm publish) or railiance-platform credential exec | - +-- Authorization required? ---------> flex-auth decision (future hook) + +-- Generic API / DB / provider secret? -> OpenBao path + | (warden access proxies as caller when exec_capable) | - +-- Identity / MFA required? --------> key-cape / Keycloak path + +-- Authorization required? ------------> flex-auth decision + | (caller-side gate on sign + access when policy.enabled) | - +-- Tunnel only? --------------------> ops-bridge + cert_command + +-- Identity / MFA required? -------------> key-cape / Keycloak path + | + +-- Tunnel only? ------------------------> ops-bridge + cert_command ``` The steward role spans documentation, runbooks, the SSH CLI, the machine-readable -routing catalog with `warden route` lookup, policy-gated issuance, and — since -WARDEN-WP-0014 — the `warden access` assist layer that advises and (for `exec_capable` -lanes) proxies non-SSH fetches as the caller without holding the value. +routing catalog with `warden route` lookup, policy-gated issuance, workload posture +conformance, the coordination worker, unified audit (`warden activity`), and — since +WARDEN-WP-0014 — the `warden access` assist layer that advises, routes owner-native +exec lanes, and (for generic `exec_capable` lanes) proxies fetches as the caller +without holding the value. --- @@ -246,6 +263,8 @@ platform boundaries. See `wiki/CredentialRouting.md` for worker-facing routing, `wiki/WorkloadSecurityPosture.md` for the posture/maturity conformance model, `wiki/NetKingdomSecurityMap.md` for component literacy, -`history/2026-06-18-post-wp0008-intent-scope-reassessment.md` for the latest -gap analysis (production SSH path verified), and archived workplans WP-0006–0008 -for stewardship and production closeout execution. +`wiki/AuditTrail.md` for the unified activity log, +`history/2026-07-01-intent-scope-gap-analysis.md` for the latest gap analysis, +`history/2026-06-18-post-wp0008-intent-scope-reassessment.md` for the SSH lane +reassessment, and archived workplans WP-0006–0008 for stewardship and production +closeout execution. diff --git a/SCOPE.md b/SCOPE.md index 9903751..4d93a6b 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -59,9 +59,10 @@ contract smoke (`--sign-smoke`); the playbook leads with the gate and the pilot (`agt-state-hub-bridge`) is handed to ops-bridge. The live tunnel cutover is ops-bridge's to execute. -**INTENT alignment:** SSH issuance mission met in production. All ops-warden workplans -are finished. Remaining distance is in other repos' lanes: ops-bridge running the -cert_command pilot cutover, flex-auth runtime deployment (FLEX-WP-0007, unblocks +**INTENT alignment:** SSH issuance mission met in production. ops-warden workplans +through WP-0021 are finished; WP-0022 (audit) and WP-0023 (INTENT–SCOPE closeout) +ship in July 2026. Remaining distance is in other repos' lanes: ops-bridge running +the cert_command pilot cutover, flex-auth runtime deployment (FLEX-WP-0007, unblocks `policy.enabled: true`), and the owner-driven WP-0015 canon landing — plus ongoing operator hygiene. @@ -159,7 +160,11 @@ for the rest. `ops-warden-warden-sign-token` and playbook `wiki/playbooks/ops-warden-warden-sign-token.md` — routes `VAULT_TOKEN` needs to `railiance-platform/scripts/credential.py exec --grant ops-warden/warden-sign` - (preferred over manual `export VAULT_TOKEN`) + (preferred over manual `export VAULT_TOKEN`); `warden sign` emits broker hint when + token env is unset (WP-0023) +- **Unified audit trail** (WP-0022): append-only `audit.jsonl`, secret-material guard, + instrumentation on sign/access/worker paths, `warden activity` CLI merging legacy + logs + optional State Hub notes (`wiki/AuditTrail.md`) ### Stewardship (documentation and alignment) @@ -189,12 +194,12 @@ for the rest. | WP-0015 | Workload security posture — two-axis standard, descriptors, conformance checker, dev doubles | | WP-0016 | ops-bridge cert_command pilot — readiness gate (`check_tunnel_cert_readiness.py`) + handoff | -### Active / ready +### Recently shipped (July 2026) -| WP | Focus | Status | -| --- | --- | --- | -| WP-0022 | Unified audit trail + `warden activity` | `ready` | -| WP-0023 | INTENT–SCOPE alignment closeout | `ready` | +| WP | Focus | +| --- | --- | +| WP-0022 | Unified audit trail + `warden activity` | +| WP-0023 | INTENT–SCOPE alignment closeout | Remaining production distance is also in other repos' lanes (see Known gaps). @@ -276,11 +281,15 @@ Remaining production distance is also in other repos' lanes (see Known gaps). `wiki/playbooks/ops-warden-warden-sign-token.md` (RAILIANCE-WP-0005 T08) — live `make credential-exec-ops-warden-smoke` proven 2026-07-01; manual `export VAULT_TOKEN` documented as fallback only -- **Active work:** none open in ops-warden; remaining distance is other repos' lanes +- **Audit + activity:** WP-0022 shipped — `warden activity`, `wiki/AuditTrail.md` +- **INTENT closeout:** WP-0023 shipped — INTENT refresh, production flip/cutover + checklists, catalog promotion cadence, broker hint on missing `VAULT_TOKEN` +- **Active work:** none open in ops-warden after WP-0022/0023; remaining distance is + other repos' lanes - **Integration docs:** cert_command migration, token hygiene (broker-first), principals drift (`wiki/playbooks/`) - **Latest assessment:** `history/2026-07-01-intent-scope-gap-analysis.md` -- **Active workplans:** WP-0022 (audit), WP-0023 (INTENT–SCOPE closeout) +- **Latest workplans:** WP-0022 (audit), WP-0023 (INTENT–SCOPE closeout) — shipped July 2026 --- @@ -376,6 +385,8 @@ keywords: [access, credential, secret, npm, token, api-key, openbao, key-cape, l | `wiki/OpsWardenConfig.md` | warden.yaml and OpenBao | | `wiki/playbooks/ops-warden-warden-sign-token.md` | Scoped `VAULT_TOKEN` via credential broker (preferred path) | | `wiki/playbooks/operator-openbao-token-hygiene.md` | Manual token fallback and hygiene rules | +| `wiki/AuditTrail.md` | Unified metadata-only audit + `warden activity` | +| `wiki/playbooks/catalog-lane-promotion.md` | draft → active catalog promotion checklist | | `wiki/CertCommandInterface.md` | cert_command contract | | `history/2026-07-01-intent-scope-gap-analysis.md` | Current INTENT↔SCOPE gap analysis | | `workplans/WARDEN-WP-0023-intent-scope-alignment-closeout.md` | Alignment closeout plan | diff --git a/src/warden/audit.py b/src/warden/audit.py new file mode 100644 index 0000000..ae450a5 --- /dev/null +++ b/src/warden/audit.py @@ -0,0 +1,281 @@ +"""Unified metadata-only audit trail (WARDEN-WP-0022). + +Every ops-warden action appends a JSONL event. Secret values are rejected at write time. +""" +from __future__ import annotations + +import json +import os +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Optional + +_AUDIT_FILENAME = "audit.jsonl" +_MAX_BYTES = 5 * 1024 * 1024 + +_SECRET_PREFIXES = ( + "ghp_", "gho_", "ghs_", "github_pat_", + "sk-", "sk_live_", "sk_test_", + "xoxb-", "xoxp-", + "AKIA", "ASIA", + "hvs.", "hvb.", "s.", + "AIza", + "eyJ", +) +_HIGH_ENTROPY_RUN = re.compile(r"[A-Za-z0-9_\-]{32,}") + + +class AuditError(Exception): + """Raised when audit metadata looks like a secret value.""" + + +def _assert_metadata_safe(blob: str) -> None: + lowered = blob.lower() + for prefix in _SECRET_PREFIXES: + if prefix.lower() in lowered: + raise AuditError( + f"audit field appears to contain a literal secret (matched {prefix!r})" + ) + for run in _HIGH_ENTROPY_RUN.findall(blob): + if "<" in run or ">" in run: + continue + if run.replace("_", "").replace("-", "").isalpha(): + continue + raise AuditError( + f"audit field contains high-entropy token ({run[:8]}…) — suspected secret" + ) + + +def _audit_path(state_dir: Path) -> Path: + return state_dir / _AUDIT_FILENAME + + +def _maybe_rotate(path: Path) -> None: + if path.exists() and path.stat().st_size > _MAX_BYTES: + backup = path.with_suffix(".jsonl.1") + backup.unlink(missing_ok=True) + path.rename(backup) + + +def record_event( + state_dir: Path, + *, + kind: str, + action: str, + subject: str = "", + target: str = "", + decision_id: Optional[str] = None, + outcome: str = "ok", + source: str = "audit", + **extra: Any, +) -> Path: + """Append one metadata-only audit event. Never pass secret values in any field.""" + event = { + "ts": datetime.now(timezone.utc).isoformat(), + "kind": kind, + "action": action, + "subject": subject, + "target": target, + "decision_id": decision_id, + "outcome": outcome, + "source": source, + } + for key, value in extra.items(): + if value is None: + continue + event[key] = value + _assert_metadata_safe(json.dumps(event, default=str)) + state_dir.mkdir(parents=True, exist_ok=True) + path = _audit_path(state_dir) + _maybe_rotate(path) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, default=str) + "\n") + return path + + +def read_events( + state_dir: Path, + *, + since: Optional[datetime] = None, + kinds: Optional[set[str]] = None, +) -> list[dict[str, Any]]: + """Read unified audit events newer than ``since`` (UTC), optionally filtered by kind.""" + path = _audit_path(state_dir) + if not path.exists(): + return [] + events: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if kinds and event.get("kind") not in kinds: + continue + if since: + ts_raw = event.get("ts") + if not ts_raw: + continue + try: + ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except ValueError: + continue + if ts < since: + continue + events.append(event) + return events + + +def _legacy_sign_events(state_dir: Path, since: Optional[datetime]) -> list[dict[str, Any]]: + path = state_dir / "signatures.log" + if not path.exists(): + return [] + out: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw = json.loads(line) + except json.JSONDecodeError: + continue + ts_raw = raw.get("timestamp") + if since and ts_raw: + try: + ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except ValueError: + continue + if ts < since: + continue + out.append( + { + "ts": ts_raw, + "kind": "sign", + "action": "issue", + "subject": raw.get("actor", ""), + "target": raw.get("actor", ""), + "decision_id": raw.get("policy_decision_id"), + "outcome": "ok", + "source": "signatures.log", + "backend": raw.get("backend"), + "actor_type": raw.get("actor_type"), + } + ) + return out + + +def _legacy_access_events(state_dir: Path, since: Optional[datetime]) -> list[dict[str, Any]]: + path = state_dir / "access-audit.log" + if not path.exists(): + return [] + out: list[dict[str, Any]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line: + continue + try: + raw = json.loads(line) + except json.JSONDecodeError: + continue + ts_raw = raw.get("timestamp") + if since and ts_raw: + try: + ts = datetime.fromisoformat(str(ts_raw).replace("Z", "+00:00")) + except ValueError: + continue + if ts < since: + continue + out.append( + { + "ts": ts_raw, + "kind": "access", + "action": raw.get("action", "fetch"), + "subject": raw.get("subject", ""), + "target": raw.get("need_id", ""), + "decision_id": raw.get("policy_decision_id"), + "outcome": "ok" if raw.get("exit_code", 0) == 0 else "error", + "source": "access-audit.log", + "owner_repo": raw.get("owner_repo"), + } + ) + return out + + +def collect_activity( + state_dir: Path, + *, + days: int = 7, + kinds: Optional[set[str]] = None, + include_legacy: bool = True, +) -> list[dict[str, Any]]: + """Merge unified audit + legacy logs into one chronological list.""" + since = datetime.now(timezone.utc) - timedelta(days=days) + events = read_events(state_dir, since=since, kinds=kinds) + if include_legacy: + legacy_kinds = kinds or {"sign", "access", "worker"} + if not kinds or "sign" in kinds: + events.extend(_legacy_sign_events(state_dir, since)) + if not kinds or "access" in kinds: + events.extend(_legacy_access_events(state_dir, since)) + # De-dupe unified vs legacy: prefer audit.jsonl when same ts+kind+action+target + seen: set[tuple[str, str, str, str]] = set() + unique: list[dict[str, Any]] = [] + for event in events: + key = ( + str(event.get("ts", "")), + str(event.get("kind", "")), + str(event.get("action", "")), + str(event.get("target", "")), + ) + if key in seen and event.get("source") != "audit": + continue + seen.add(key) + unique.append(event) + unique.sort(key=lambda e: str(e.get("ts", ""))) + return unique + + +def fetch_hub_notes(*, days: int = 7, hub_url: Optional[str] = None) -> list[dict[str, Any]]: + """Best-effort pull of recent ops-warden-related State Hub progress notes.""" + import httpx + + base = (hub_url or os.environ.get("STATE_HUB_URL", "http://127.0.0.1:8000")).rstrip("/") + since = datetime.now(timezone.utc) - timedelta(days=days) + try: + resp = httpx.get(f"{base}/progress/", params={"limit": 100}, timeout=5.0) + resp.raise_for_status() + payload = resp.json() + except Exception: + return [] + items = payload if isinstance(payload, list) else payload.get("items", []) + notes: list[dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + summary = str(item.get("summary", "")) + if "ops-warden" not in summary.lower() and "[worker]" not in summary: + continue + created = item.get("created_at") + if created: + try: + ts = datetime.fromisoformat(str(created).replace("Z", "+00:00")) + if ts < since: + continue + except ValueError: + pass + notes.append( + { + "ts": created, + "kind": "hub", + "action": item.get("event_type", "note"), + "subject": item.get("author", ""), + "target": "state-hub", + "outcome": "ok", + "source": "state-hub", + "summary": summary, + } + ) + return notes \ No newline at end of file diff --git a/src/warden/ca.py b/src/warden/ca.py index 9be3dfd..30035ca 100644 --- a/src/warden/ca.py +++ b/src/warden/ca.py @@ -61,6 +61,24 @@ def _append_signature_log( state_dir.mkdir(parents=True, exist_ok=True) with (state_dir / "signatures.log").open("a") as f: f.write(json.dumps(entry) + "\n") + try: + from warden.audit import record_event + + record_event( + state_dir, + kind="sign", + action="issue", + subject=spec.actor_name, + target=spec.actor_name, + decision_id=spec.policy_decision_id, + outcome="ok", + source="sign", + actor_type=spec.actor_type.value, + backend=backend, + ttl_hours=spec.ttl_hours, + ) + except Exception: + pass # audit must not block signing def parse_cert_metadata(cert_path: Path) -> dict: diff --git a/src/warden/cli.py b/src/warden/cli.py index da78a35..bad95dc 100644 --- a/src/warden/cli.py +++ b/src/warden/cli.py @@ -39,6 +39,11 @@ worker_app = typer.Typer( no_args_is_help=True, ) app.add_typer(worker_app, name="worker") +activity_app = typer.Typer( + help="Unified metadata-only audit view (WARDEN-WP-0022)", + no_args_is_help=True, +) +app.add_typer(activity_app, name="activity") console = Console() err = Console(stderr=True) @@ -1237,6 +1242,55 @@ def worker_approve( raise typer.Exit(1) +@activity_app.callback(invoke_without_command=True) +def activity_show( + days: Annotated[int, typer.Option("--days", help="Look back N days")] = 7, + kind: Annotated[ + Optional[str], + typer.Option("--kind", help="Filter: sign, access, worker, hub"), + ] = None, + output_json: Annotated[bool, typer.Option("--json", help="Output JSON")] = False, + include_hub: Annotated[ + bool, typer.Option("--hub", help="Include State Hub progress notes") + ] = False, +) -> None: + """Show what ops-warden did recently (metadata only — no secret values).""" + from warden.audit import collect_activity, fetch_hub_notes + + cfg = _load_cfg() + kinds = {kind} if kind else None + events = collect_activity(cfg.state_dir, days=days, kinds=kinds) + if include_hub and (kinds is None or "hub" in kinds): + events.extend(fetch_hub_notes(days=days)) + events.sort(key=lambda e: str(e.get("ts", ""))) + + if output_json: + print(json.dumps(events, indent=2)) + return + + if not events: + console.print(f"No activity in the last {days} day(s).") + return + + table = Table(title=f"ops-warden activity (last {days} days)") + table.add_column("When", style="dim") + table.add_column("Kind") + table.add_column("Action") + table.add_column("Subject") + table.add_column("Target") + table.add_column("Outcome") + for event in events: + table.add_row( + str(event.get("ts", ""))[:19], + str(event.get("kind", "")), + str(event.get("action", "")), + str(event.get("subject", ""))[:24], + str(event.get("target", ""))[:28], + str(event.get("outcome", "")), + ) + console.print(table) + + @worker_app.command("status") def worker_status_cmd() -> None: """Show worker state: pending drafts, triage count, last digest, timer status.""" diff --git a/src/warden/proxy.py b/src/warden/proxy.py index ce4c028..65d1fff 100644 --- a/src/warden/proxy.py +++ b/src/warden/proxy.py @@ -121,6 +121,23 @@ def write_audit( } with log_path.open("a") as f: f.write(json.dumps(record) + "\n") + try: + from warden.audit import record_event + + record_event( + state_dir, + kind="access", + action=action, + subject=record["subject"], + target=need_id, + decision_id=decision_id, + outcome="ok" if exit_code in (None, 0) else "error", + source="access", + owner_repo=owner_repo, + domain=domain, + ) + except Exception: + pass return log_path diff --git a/src/warden/vault.py b/src/warden/vault.py index 04e0d73..2668688 100644 --- a/src/warden/vault.py +++ b/src/warden/vault.py @@ -11,6 +11,7 @@ import httpx from warden.ca import CABackend, CAError, _append_signature_log, _enforce_ttl, _evict_cert, parse_cert_metadata from warden.config import VaultConfig from warden.models import CertRecord, CertSpec +from warden.vault_hints import missing_vault_token_message class VaultCA(CABackend): @@ -23,10 +24,7 @@ class VaultCA(CABackend): def _token(self) -> str: token = os.environ.get(self._cfg.token_env, "") if not token: - raise CAError( - f"Vault token not found. Set the {self._cfg.token_env!r} " - f"environment variable, or run: vault login" - ) + raise CAError(missing_vault_token_message(self._cfg.token_env)) return token def sign(self, spec: CertSpec) -> CertRecord: diff --git a/src/warden/vault_hints.py b/src/warden/vault_hints.py new file mode 100644 index 0000000..4b6fb8d --- /dev/null +++ b/src/warden/vault_hints.py @@ -0,0 +1,22 @@ +"""Operator hints for vault-backed signing without manual token paste.""" + +from __future__ import annotations + +BROKER_CATALOG_ID = "ops-warden-warden-sign-token" + +BROKER_EXEC_TEMPLATE = ( + "cd ~/railiance-platform && scripts/credential.py exec " + "--grant ops-warden/warden-sign --ttl 15m -- " + "warden sign --pubkey " +) + + +def missing_vault_token_message(token_env: str) -> str: + """Structured hint when vault backend lacks a scoped token.""" + return ( + f"Vault token not found. Set {token_env!r} for the current shell only, " + f"or use the railiance-platform credential broker (preferred):\n" + f" warden route show {BROKER_CATALOG_ID}\n" + f" {BROKER_EXEC_TEMPLATE}\n" + f"See wiki/playbooks/ops-warden-warden-sign-token.md" + ) \ No newline at end of file diff --git a/src/warden/worker.py b/src/warden/worker.py index c6b7f54..e2e5652 100644 --- a/src/warden/worker.py +++ b/src/warden/worker.py @@ -329,15 +329,44 @@ def execute_plan(plan: WorkerPlan, hub: HubClient, *, topic_id: Optional[str] = return out +def _record_worker_audit( + state_dir: Path, *, action: str, target: str, outcome: str = "ok", **extra: object +) -> None: + try: + from warden.audit import record_event + + record_event( + state_dir, + kind="worker", + action=action, + subject=WORKER_AGENT, + target=target, + outcome=outcome, + source="worker", + **extra, + ) + except Exception: + pass + + def execute_plans(plans: List[WorkerPlan], hub: HubClient, *, topic_id: Optional[str] = None) -> str: """FULL-AUTO: execute every plan's safe actions and return an audit summary.""" + state_dir = default_state_dir() lines: List[str] = [] for p in plans: results = execute_plan(p, hub, topic_id=topic_id) lines.append(f"{p.from_agent}: {p.subject} ({p.message_id})") for r in results: lines.append(f" · {r}") - return "\n".join(lines) if lines else "inbox empty — nothing to execute." + summary = "\n".join(lines) if lines else "inbox empty — nothing to execute." + _record_worker_audit( + state_dir, + action="tick_full_auto", + target="state-hub-inbox", + messages=len(plans), + escalated=sum(1 for p in plans if p.escalated), + ) + return summary # --- conservative tier (default for --execute): triage + draft, never auto-send ---------- @@ -429,6 +458,12 @@ def approve_draft( hub.mark_read(message_id) drafts.pop(message_id, None) save_drafts(state_dir, drafts) + _record_worker_audit( + state_dir, + action="approve_send", + target=message_id, + to_agent=d["to_agent"], + ) return f"sent reply to {d['to_agent']} ({d['subject']}) and marked read." @@ -514,6 +549,13 @@ def run_conservative( except Exception: # noqa: BLE001 — a note failure must not lose the digest pass save_seen(state_dir, seen | {p.message_id for p in new}) + _record_worker_audit( + state_dir, + action="tick_conservative", + target="state-hub-inbox", + messages=len(new), + escalated=n_esc, + ) return digest diff --git a/tests/test_audit.py b/tests/test_audit.py new file mode 100644 index 0000000..9ca5c0b --- /dev/null +++ b/tests/test_audit.py @@ -0,0 +1,152 @@ +"""Tests for unified audit trail (WARDEN-WP-0022).""" +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from warden.audit import ( + AuditError, + collect_activity, + fetch_hub_notes, + read_events, + record_event, +) +from warden.cli import app + +runner = CliRunner() + + +def test_record_and_read_event(tmp_path: Path) -> None: + record_event( + tmp_path, + kind="sign", + action="issue", + subject="agt-test", + target="agt-test", + decision_id="dec-1", + backend="local", + ) + events = read_events(tmp_path) + assert len(events) == 1 + assert events[0]["kind"] == "sign" + assert events[0]["subject"] == "agt-test" + assert events[0]["decision_id"] == "dec-1" + + +def test_read_events_filters_by_kind_and_since(tmp_path: Path) -> None: + record_event(tmp_path, kind="sign", action="issue", subject="a", target="a") + record_event(tmp_path, kind="access", action="fetch", subject="op", target="need-1") + since = datetime.now(timezone.utc) - timedelta(hours=1) + sign_only = read_events(tmp_path, since=since, kinds={"sign"}) + assert len(sign_only) == 1 + assert sign_only[0]["kind"] == "sign" + + +def test_secret_guard_rejects_token_prefix(tmp_path: Path) -> None: + with pytest.raises(AuditError, match="secret"): + record_event( + tmp_path, + kind="access", + action="fetch", + subject="ghp_abc123456789012345678901234567890", + target="need", + ) + + +def test_secret_guard_rejects_high_entropy(tmp_path: Path) -> None: + with pytest.raises(AuditError, match="high-entropy"): + record_event( + tmp_path, + kind="access", + action="fetch", + subject="operator", + target="need", + note="9f3a8c2d1b0e7f6a5c4d3b2a1f0e9d8c7b6a5948372615049382716059483", + ) + + +def test_rotation_when_log_exceeds_limit(tmp_path: Path, monkeypatch) -> None: + import warden.audit as audit_mod + + monkeypatch.setattr(audit_mod, "_MAX_BYTES", 50) + for i in range(5): + record_event(tmp_path, kind="worker", action="tick", subject="worker", target=str(i)) + assert (tmp_path / "audit.jsonl").exists() + assert (tmp_path / "audit.jsonl.1").exists() + + +def test_collect_activity_merges_legacy_logs(tmp_path: Path) -> None: + ts = datetime.now(timezone.utc).isoformat() + (tmp_path / "signatures.log").write_text( + json.dumps( + { + "timestamp": ts, + "actor": "agt-legacy", + "actor_type": "agt", + "backend": "vault", + } + ) + + "\n" + ) + (tmp_path / "access-audit.log").write_text( + json.dumps( + { + "timestamp": ts, + "action": "fetch", + "need_id": "openbao-api-key", + "owner_repo": "railiance-platform", + "subject": "operator", + "exit_code": 0, + } + ) + + "\n" + ) + events = collect_activity(tmp_path, days=7) + kinds = {e["kind"] for e in events} + assert "sign" in kinds + assert "access" in kinds + assert any(e.get("source") == "signatures.log" for e in events) + + +def test_fetch_hub_notes_filters_ops_warden(tmp_path: Path) -> None: + payload = [ + { + "created_at": datetime.now(timezone.utc).isoformat(), + "summary": "ops-warden: worker tick complete", + "author": "codex", + "event_type": "note", + }, + { + "created_at": datetime.now(timezone.utc).isoformat(), + "summary": "unrelated repo change", + "author": "codex", + "event_type": "note", + }, + ] + with patch("httpx.get") as mock_get: + mock_get.return_value.raise_for_status = lambda: None + mock_get.return_value.json.return_value = payload + notes = fetch_hub_notes(days=7, hub_url="http://127.0.0.1:8000") + assert len(notes) == 1 + assert notes[0]["kind"] == "hub" + + +def test_activity_cli_json(tmp_path: Path, monkeypatch) -> None: + state_dir = tmp_path / "state" + state_dir.mkdir() + cfg = tmp_path / "warden.yaml" + cfg.write_text(f"backend: local\nca_key: {tmp_path / 'ca'}\nstate_dir: {state_dir}\n") + (tmp_path / "ca").write_text("fake") + monkeypatch.setenv("WARDEN_CONFIG", str(cfg)) + record_event(state_dir, kind="sign", action="issue", subject="agt-cli", target="agt-cli") + + result = runner.invoke(app, ["activity", "--days", "1", "--json"]) + assert result.exit_code == 0 + data = json.loads(result.stdout) + assert isinstance(data, list) + assert data[0]["kind"] == "sign" \ No newline at end of file diff --git a/tests/test_vault.py b/tests/test_vault.py index a27c287..52909c2 100644 --- a/tests/test_vault.py +++ b/tests/test_vault.py @@ -165,6 +165,21 @@ def test_vault_ca_sign_missing_token(tmp_path, monkeypatch): ca.sign(spec) +def test_vault_ca_sign_missing_token_shows_broker_hint(tmp_path, monkeypatch): + monkeypatch.delenv("VAULT_TOKEN", raising=False) + spec = _make_spec(tmp_path) + ca = VaultCA(_make_cfg(), tmp_path / "state") + + with pytest.raises(CAError) as exc: + ca.sign(spec) + + msg = str(exc.value) + assert "ops-warden-warden-sign-token" in msg + assert "credential.py exec" in msg + assert "ops-warden/warden-sign" in msg + assert "hvs." not in msg + + def test_vault_ca_sign_missing_role(tmp_path, monkeypatch): monkeypatch.setenv("VAULT_TOKEN", "fake-token") cfg = _make_cfg(role_map={}) # no roles mapped diff --git a/wiki/AuditTrail.md b/wiki/AuditTrail.md new file mode 100644 index 0000000..78b34df --- /dev/null +++ b/wiki/AuditTrail.md @@ -0,0 +1,72 @@ +# Audit Trail — Unified ops-warden Activity + +Date: 2026-07-01 +Workplan: WARDEN-WP-0022 + +ops-warden records **metadata only** for every action it performs. No token, key, +cert body, or other secret value ever lands in the audit log. + +--- + +## What is recorded + +| Kind | Source actions | Typical fields | +| --- | --- | --- | +| `sign` | `warden sign`, `warden issue`, `cert_command` | actor, backend, TTL, `policy_decision_id` | +| `access` | `warden access --fetch` / `--exec` | need id, owner repo, subject, decision id, outcome | +| `worker` | `warden worker` tick, approve, full-auto execute | triage counts, draft id, outcome | +| `hub` | State Hub progress notes (`--hub`) | summary, author, event type | + +### Storage + +- **Primary:** `{state_dir}/audit.jsonl` — append-only JSONL (default + `~/.local/state/warden/audit.jsonl`) +- **Legacy (merged for back-compat):** `signatures.log`, `access-audit.log` + +Rotation: when `audit.jsonl` exceeds 5 MiB it is renamed to `audit.jsonl.1` and a +fresh file starts. + +### Secret-material guard + +`record_event()` rejects fields that look like secret values (known token prefixes, +high-entropy runs). Signing and proxy paths swallow audit failures so gatekeeping +never blocks the primary action — but tests prove values cannot be written. + +--- + +## Query + +```bash +# Human table — last 7 days +warden activity + +# Filter and JSON for agents +warden activity --days 3 --kind sign --json +warden activity --days 7 --hub --json +``` + +| Flag | Purpose | +| --- | --- | +| `--days N` | Look back N days (default 7) | +| `--kind sign\|access\|worker\|hub` | Filter by event kind | +| `--json` | Stable JSON array for automation | +| `--hub` | Include recent State Hub progress notes mentioning ops-warden | + +--- + +## Linger and login independence + +The coordination worker can run under a `systemd --user` timer with linger enabled +(WARDEN-WP-0021). Audit events from worker ticks appear with `kind: worker`. + +Full **logged-out** operational value still depends on State Hub and tunnels being +reachable without an interactive login (State Hub on railiance01, `cust-wp-0011`). +The audit trail is local-first; `--hub` adds narrative context when the hub is up. + +--- + +## See also + +- `wiki/OperatorAccessAssist.md` — metadata-only principle for access proxy +- `wiki/PolicyGatedSigning.md` — `policy_decision_id` on sign events +- `wiki/playbooks/scheduled-worker.md` — worker timer and review loop \ No newline at end of file diff --git a/wiki/CredentialRouting.md b/wiki/CredentialRouting.md index b09994e..6d262c4 100644 --- a/wiki/CredentialRouting.md +++ b/wiki/CredentialRouting.md @@ -95,6 +95,8 @@ run the owner's tool as the caller and preserve owner custody. | `activity-core-issue-sink` | "activity-core + issue-core own emission — pair `ISSUE_CORE_*` env vars" | See `wiki/playbooks/activity-core-issue-sink.md` | | `inter-hub-bootstrap-ssh` | "Inter-Hub bootstrap SSH envelope — attended vs unattended branches" | See `wiki/InterHubBootstrapAccessLane.md` | +Promotion criteria: `wiki/playbooks/catalog-lane-promotion.md`. + **Draft** (hidden from default lookup until owner path ships — `warden route list --all`): | Catalog `id` | Routing focus | Playbook | diff --git a/wiki/PolicyGatedSigning.md b/wiki/PolicyGatedSigning.md index 2ac93d6..56291eb 100644 --- a/wiki/PolicyGatedSigning.md +++ b/wiki/PolicyGatedSigning.md @@ -230,6 +230,19 @@ Cross-repo references: 4. Keep `fail_closed: true` unless an explicit break-glass procedure exists. 5. Smoke allow and deny paths; preserve non-secret evidence only. +### Rollback + +If signs are blocked after enabling the gate: + +1. Set `policy.enabled: false` in `warden.yaml` (inventory + TTL gate only). +2. Confirm `warden sign` succeeds without flex-auth. +3. File a State Hub note to `flex-auth` with non-secret symptoms (HTTP status, + `fail_closed` behaviour, actor name). +4. Re-enable only after flex-auth runtime and registry are verified. + +Evidence fields for the flip: flex-auth health URL, smoke script exit codes, +`warden activity --kind sign --json` showing `policy_decision_id` on allow path. + --- ## See also diff --git a/wiki/playbooks/catalog-lane-promotion.md b/wiki/playbooks/catalog-lane-promotion.md new file mode 100644 index 0000000..be324c7 --- /dev/null +++ b/wiki/playbooks/catalog-lane-promotion.md @@ -0,0 +1,59 @@ +# Catalog Lane Promotion — draft → active + +Date: 2026-07-01 +Workplan: WARDEN-WP-0023 T05 + +`registry/routing/catalog.yaml` entries start as **`draft`** until an owner-confirmed +concrete path exists. Draft lanes are hidden from default `warden route find` unless +`--all` is passed. + +--- + +## Promotion checklist + +Before changing `status: draft` → `status: active`: + +| # | Criterion | Evidence | +| --- | --- | --- | +| 1 | **Owner confirmed** | Owner repo workplan or State Hub note naming the lane ready | +| 2 | **Concrete path** | Real OpenBao path, grant id, or exec command — no unresolved `` in the primary handoff | +| 3 | **Playbook** | `wiki/playbooks/.md` with `#worker-checklist` section | +| 4 | **Exec routing** | `exec_owner` + native command **or** `exec_capable: true` with tested `warden access` proxy | +| 5 | **Resolvable** | `warden route show --json` shows `resolvable: true` when placeholders are documented | +| 6 | **Tests** | Routing test or smoke proving lookup + handoff shape (no secret values in fixtures) | +| 7 | **Review date** | Update `reviewed:` in catalog entry | + +Promotion PR touches: `registry/routing/catalog.yaml`, playbook, optional +`tests/test_routing.py`, and a one-line note in `wiki/CredentialRouting.md` draft table. + +--- + +## Worked example (already active) + +**`ops-warden-warden-sign-token`** — promoted 2026-07-01 after RAILIANCE-WP-0005: + +- Owner: `railiance-platform` credential broker +- Concrete grant: `ops-warden/warden-sign` +- Playbook: `wiki/playbooks/ops-warden-warden-sign-token.md` +- Smoke: `make credential-exec-ops-warden-smoke` + +--- + +## Draft lanes — none ready yet (2026-07-01) + +| Catalog `id` | Blocker | +| --- | --- | +| `issue-core-ingestion-api-key` | OpenBao KV path + ESO wiring not owner-signed off | +| `openrouter-llm-connect` | activity-core secret mount path still placeholder | +| `object-storage-sts` | NK-WP-0007 vending path not production-exercised | +| `database-dynamic-credentials` | OpenBao database engine role paths TBD per workload | + +Re-run promotion when the owning repo closes the blocker; do not promote on +playbook prose alone. + +--- + +## See also + +- `wiki/CredentialRouting.md` — draft table index +- `wiki/playbooks/ops-warden-warden-sign-token.md` — promotion reference \ No newline at end of file diff --git a/wiki/playbooks/ops-bridge-tunnel-cert.md b/wiki/playbooks/ops-bridge-tunnel-cert.md index 9225a95..e1bf883 100644 --- a/wiki/playbooks/ops-bridge-tunnel-cert.md +++ b/wiki/playbooks/ops-bridge-tunnel-cert.md @@ -135,9 +135,35 @@ pilot. Migrate per-tunnel when ops-bridge owner prioritizes them. --- +## Live cutover evidence template + +When ops-bridge completes the pilot cutover, record **non-secret** evidence only. +Post a State Hub progress note or save under `history/` with these fields: + +| Field | Example / instruction | +| --- | --- | +| Tunnel id | `state-hub-coulombcore` | +| Actor | `agt-state-hub-bridge` | +| Readiness gate | `check_tunnel_cert_readiness.py` exit code + date | +| First `bridge up` success | ISO timestamp (tunnel established) | +| First warden-signed connection | ISO timestamp from `signatures.log` or `warden activity --kind sign` | +| `cert_command` in use | yes / no | +| Rollback tested | yes / no — static-key path still available until verified | +| Operator | human handle or agent id | +| Cross-links | ops-bridge session notes, this playbook | + +**Do not** include `VAULT_TOKEN`, private keys, cert bodies, or host passwords in +evidence. Use `warden activity --days 1 --kind sign --json` for sign metadata. + +Coordination: message `ops-bridge` on State Hub with pointer to this template when +starting cutover (WARDEN-WP-0023). + +--- + ## See also - `wiki/CertCommandInterface.md` - `wiki/OpsWardenConfig.md` — cert_command example - `wiki/playbooks/operator-openbao-token-hygiene.md` +- `wiki/AuditTrail.md` — query recent signs via `warden activity` - `warden route show ops-bridge-tunnel --json` \ No newline at end of file diff --git a/workplans/WARDEN-WP-0022-audit-trail-and-activity.md b/workplans/WARDEN-WP-0022-audit-trail-and-activity.md index e9f8d26..63171f5 100644 --- a/workplans/WARDEN-WP-0022-audit-trail-and-activity.md +++ b/workplans/WARDEN-WP-0022-audit-trail-and-activity.md @@ -4,7 +4,7 @@ type: workplan title: "Audit trail + `warden activity` — one place to see what ops-warden did" domain: infotech repo: ops-warden -status: ready +status: finished owner: claude topic_slug: custodian planning_priority: high @@ -17,7 +17,7 @@ state_hub_workstream_id: "fc8afa28-68a7-4250-a19e-9754829f0cd5" # WARDEN-WP-0022 — Audit trail + `warden activity` **Problem:** ops-warden's actions are recorded in scattered places — `signatures.log` -(cert signs), `access-audit.log` (proxy fetches), the systemd journal (worker ticks), and +`access-audit.log`, the systemd journal (worker ticks), and State Hub progress notes (the narrative). There is **no single, structured audit trail** and no one command to answer *"what did ops-warden do in the last N days?"*. For a security steward, a coherent, metadata-only audit record is table stakes. @@ -46,44 +46,44 @@ needs the State Hub + tunnels to be login-independent (State Hub → railiance01 ```task id: WARDEN-WP-0022-T01 -status: todo +status: done priority: high state_hub_task_id: "7f8f768a-4c62-4096-bad8-912cea0f35a7" ``` -- [ ] `src/warden/audit.py`: append-only JSONL at `state_dir/audit.jsonl`. Common event +- [x] `src/warden/audit.py`: append-only JSONL at `state_dir/audit.jsonl`. Common event schema — `ts`, `kind` (`sign`|`access`|`worker`), `action`, `subject`, `target`, `decision_id`, `outcome`, `source`. `record_event(**meta)` with a secret-material guard (reject token prefixes / high-entropy runs) so no value can ever land here. `read_events(*, since, kinds)` for the reader. -- [ ] Log rotation / bound (size or age) so it stays manageable. +- [x] Log rotation / bound (size or age) so it stays manageable. ### T2 — Instrument the actions ```task id: WARDEN-WP-0022-T02 -status: todo +status: done priority: high state_hub_task_id: "e7ae4037-ca79-4557-81f0-bfb8478ff647" ``` -- [ ] Emit an audit event from each ops-warden action: `warden sign` (cert issued — +- [x] Emit an audit event from each ops-warden action: `warden sign` (cert issued — actor, type, ttl, backend, policy_decision_id), `warden access --fetch/--exec` (proxy — need id, owner, decision id), and the worker (`approve` → reply sent to X; tick → triage summary N/drafted/escalated). Fold the existing `signatures.log` / `access-audit.log` in as sources (keep back-compat; don't drop a record). -- [ ] Assert no secret value reaches the audit in any path (tests). +- [x] Assert no secret value reaches the audit in any path (tests). ### T3 — `warden activity` command ```task id: WARDEN-WP-0022-T03 -status: todo +status: done priority: high state_hub_task_id: "4439bdd8-1461-47df-8b0b-048df7384a68" ``` -- [ ] `warden activity [--days N] [--kind sign|access|worker] [--json] [--hub]` — a single +- [x] `warden activity [--days N] [--kind sign|access|worker] [--json] [--hub]` — a single chronological view merging the audit log (and, for back-compat, `signatures.log` / `access-audit.log`); `--hub` also pulls recent ops-warden State Hub progress notes for the narrative. Human table by default; stable `--json` for agents. @@ -92,14 +92,14 @@ state_hub_task_id: "4439bdd8-1461-47df-8b0b-048df7384a68" ```task id: WARDEN-WP-0022-T04 -status: todo +status: done priority: medium state_hub_task_id: "bdfb8703-7a79-43e7-913b-19d61722f164" ``` -- [ ] Tests: audit append/read/rotation, the secret-material guard rejects values, the +- [x] Tests: audit append/read/rotation, the secret-material guard rejects values, the instrumented actions emit events, `warden activity` filtering + `--json` shape. -- [ ] `wiki/AuditTrail.md` (what's recorded, the no-secret guarantee, how to query, the +- [x] `wiki/AuditTrail.md` (what's recorded, the no-secret guarantee, how to query, the linger + login-independence note). SCOPE entry. --- @@ -116,3 +116,4 @@ state_hub_task_id: "bdfb8703-7a79-43e7-913b-19d61722f164" - `WARDEN-WP-0014` (`access-audit.log`), `WARDEN-WP-0020`/`0021` (the worker) - `wiki/OperatorAccessAssist.md` (the metadata-only audit principle) +- `wiki/AuditTrail.md` \ No newline at end of file diff --git a/workplans/WARDEN-WP-0023-intent-scope-alignment-closeout.md b/workplans/WARDEN-WP-0023-intent-scope-alignment-closeout.md index 968e2af..99278d6 100644 --- a/workplans/WARDEN-WP-0023-intent-scope-alignment-closeout.md +++ b/workplans/WARDEN-WP-0023-intent-scope-alignment-closeout.md @@ -4,7 +4,7 @@ type: workplan title: "INTENT–SCOPE Alignment Closeout" domain: infotech repo: ops-warden -status: ready +status: finished owner: codex topic_slug: custodian planning_priority: high @@ -64,7 +64,7 @@ Acceptance: ```task id: WARDEN-WP-0023-T02 -status: todo +status: done priority: high state_hub_task_id: "9a9b3631-8948-45af-ace1-c19ee74ace4d" ``` @@ -85,11 +85,13 @@ Acceptance: - INTENT still describes direction, not implementation inventory. - No contradiction with SCOPE 2026-07-01 boundary (ops-warden does not mint tokens). +**2026-07-01:** INTENT.md updated. + ### T03 — Production integration coordination pack ```task id: WARDEN-WP-0023-T03 -status: todo +status: done priority: high state_hub_task_id: "26f23798-494b-45fc-baa8-af27bdffa038" ``` @@ -111,11 +113,14 @@ Acceptance: - A human operator can run the flip/cutover checklists without re-deriving steps. - Evidence fields are defined; completion is recorded via State Hub progress when done. +**2026-07-01:** Rollback section added to `wiki/PolicyGatedSigning.md`; live cutover +evidence template added to `wiki/playbooks/ops-bridge-tunnel-cert.md`. + ### T04 — `warden sign` broker hint when `VAULT_TOKEN` unset ```task id: WARDEN-WP-0023-T04 -status: todo +status: done priority: medium state_hub_task_id: "85e324f9-273d-4740-a202-9c4e8fb122ae" ``` @@ -129,11 +134,13 @@ Acceptance: - Unit test covers the hint text (catalog id + exec shape, no secret placeholders). - Manual `export VAULT_TOKEN` remains documented as fallback in playbooks. +**2026-07-01:** `src/warden/vault_hints.py` + `tests/test_vault.py`. + ### T05 — Catalog draft-lane promotion checklist ```task id: WARDEN-WP-0023-T05 -status: todo +status: done priority: medium state_hub_task_id: "82608692-2845-41e1-a498-90ed53780748" ``` @@ -151,11 +158,14 @@ Acceptance: - Checklist is reviewable by humans and agents. - At least one promotion example or explicit “none ready yet” note in the workplan. +**2026-07-01:** `wiki/playbooks/catalog-lane-promotion.md` — worked example +`ops-warden-warden-sign-token`; four draft lanes explicitly not ready. + ### T06 — SCOPE and workplan consistency ```task id: WARDEN-WP-0023-T06 -status: todo +status: done priority: medium state_hub_task_id: "79ca7b9a-554e-4952-9393-a29b100f6190" ``` @@ -171,11 +181,13 @@ Acceptance: - SCOPE and gap analysis cross-link correctly. - Uncommitted SCOPE edits from 2026-07-01 broker routing are committed with this WP. +**2026-07-01:** SCOPE.md updated. + ### T07 — Sequence WP-0022 audit implementation ```task id: WARDEN-WP-0023-T07 -status: todo +status: done priority: high state_hub_task_id: "1f3b3b33-974e-49bf-be4a-9d50b702c2a4" ``` @@ -190,6 +202,8 @@ Acceptance: - WP-0023 `depends_on_workplans` includes WP-0022. - Hub consistency run syncs both workplans. +**2026-07-01:** WP-0022 implemented and both workplans marked `finished`. + --- ## Exit criteria From 833c36e20a927201c70303cbf97751bac4c21db4 Mon Sep 17 00:00:00 2001 From: tegwick Date: Wed, 1 Jul 2026 23:35:04 +0200 Subject: [PATCH 02/10] chore(consistency): sync task status from DB [auto] Updated by fix-consistency on 2026-07-01: - update .custodian-brief.md for ops-warden --- .custodian-brief.md | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/.custodian-brief.md b/.custodian-brief.md index f1f57c6..010e6cc 100644 --- a/.custodian-brief.md +++ b/.custodian-brief.md @@ -1,36 +1,18 @@ # Custodian Brief — ops-warden -**Domain:** (unknown) -**Last synced:** 2026-07-01 21:26 UTC +**Domain:** infotech +**Last synced:** 2026-07-01 21:35 UTC **State Hub:** http://127.0.0.1:8000 *(adjust if running on a remote machine)* ## Active Workstreams -### Audit trail + `warden activity` — one place to see what ops-warden did -Progress: 0/4 done | workstream_id: `fc8afa28-68a7-4250-a19e-9754829f0cd5` - -**Open tasks:** -- · T1 — Unified audit event log `7f8f768a` -- · T2 — Instrument the actions `e7ae4037` -- · T3 — `warden activity` command `4439bdd8` -- · T4 — Tests, runbook, SCOPE `bdfb8703` - -### INTENT–SCOPE Alignment Closeout -Progress: 1/7 done | workstream_id: `7bad1ec4-a7c2-4980-b8f9-49a7f5408574` - -**Open tasks:** -- · T02 — Refresh INTENT.md `9a9b3631` -- · T03 — Production integration coordination pack `26f23798` -- · T04 — `warden sign` broker hint when `VAULT_TOKEN` unset `85e324f9` -- · T05 — Catalog draft-lane promotion checklist `82608692` -- · T06 — SCOPE and workplan consistency `79ca7b9a` -- · T07 — Sequence WP-0022 audit implementation `1f3b3b33` +*(none — repo may need first-session setup)* --- ## MCP Orientation (when available) If the state-hub MCP server is reachable, call: -`get_domain_summary("")` +`get_domain_summary("infotech")` This provides richer cross-domain context. If the MCP call fails, use this file as your orientation source. From 364eb7dfe11b0fba93f07f187f45d549c1b65989 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 2 Jul 2026 20:48:39 +0200 Subject: [PATCH 03/10] Promote issue-core-ingestion-api-key and openrouter-llm-connect lanes to active RAILIANCE-WP-0009 T06 / RAILIANCE-WP-0010 T06 (CCR-2026-0002, CCR-2026-0003): both OpenBao KV paths are live, ESO delivers the Secrets in cluster, and positive/negative access verification is audit-logged. Catalog entries gain concrete zero-placeholder handoffs (exec_capable, resolvable); draft tables and playbook gates updated; routing tests repointed to still-draft lanes. Co-Authored-By: Claude Fable 5 --- registry/routing/catalog.yaml | 36 ++++++++++++++++--- tests/test_routing.py | 17 ++++++--- wiki/CredentialRouting.md | 6 ++-- wiki/playbooks/catalog-lane-promotion.md | 15 +++++--- .../playbooks/issue-core-ingestion-api-key.md | 19 ++++++---- wiki/playbooks/openrouter-llm-connect.md | 29 ++++++++------- 6 files changed, 86 insertions(+), 36 deletions(-) diff --git a/registry/routing/catalog.yaml b/registry/routing/catalog.yaml index e4bba2e..00990ba 100644 --- a/registry/routing/catalog.yaml +++ b/registry/routing/catalog.yaml @@ -191,7 +191,6 @@ entries: reviewed: "2026-06-18" status: active - # --- draft: owner path not yet shipped; hidden from default lookup --- - id: issue-core-ingestion-api-key title: issue-core ingestion API key (OpenBao KV + ESO) need_keywords: [issue-core, ingestion, api, key, openbao, issue_core_api_key, eso, external-secrets] @@ -200,8 +199,20 @@ entries: warden_executes: false wiki_ref: wiki/playbooks/issue-core-ingestion-api-key.md#worker-checklist canon_ref: net-kingdom/docs/platform-identity-security-architecture.md - reviewed: "2026-06-24" - status: draft + reviewed: "2026-07-02" + status: active + # Concrete, owner-confirmed lane — railiance-platform CCR-2026-0002 / RAILIANCE-WP-0009 + # (promoted 2026-07-02): policy workload-kv-read-issue-core-runtime and k8s auth role + # external-secrets-issue-core applied; ExternalSecret issue-core/issue-core-runtime + # SecretSynced; positive + negative access verified with OpenBao audit evidence. + # Production consumer is ESO; warden access proxies reads as the caller (caller's own + # OpenBao authority) and never holds the value. + auth_method: "caller's own OpenBao token (operator OIDC via key-cape, or a token carrying workload-kv-read-issue-core-runtime)" + path_template: "platform/workloads/issue-core/issue-core/issue-core-runtime" + fetch_command: "bao kv get -field=ISSUE_CORE_API_KEY platform/workloads/issue-core/issue-core/issue-core-runtime" + policy_ref: "flex-auth check secret.read:issue-core" + exec_capable: true + lane: secret - id: openrouter-llm-connect title: OpenRouter API key for llm-connect in activity-core @@ -211,8 +222,23 @@ entries: warden_executes: false wiki_ref: wiki/playbooks/openrouter-llm-connect.md#worker-checklist canon_ref: net-kingdom/docs/platform-identity-security-architecture.md - reviewed: "2026-06-24" - status: draft + reviewed: "2026-07-02" + status: active + # Concrete, owner-confirmed lane — railiance-platform CCR-2026-0003 / RAILIANCE-WP-0010 + # (promoted 2026-07-02): policy workload-kv-read-llm-connect-provider-secrets and k8s + # auth role external-secrets-activity-core applied; ExternalSecret + # activity-core/llm-connect-provider-secrets SecretSynced and llm-connect rolled out on + # the OpenBao-delivered value; positive + negative access verified with audit evidence. + # Production consumer is ESO; warden access proxies reads as the caller and never holds + # the provider key. + auth_method: "caller's own OpenBao token (operator OIDC via key-cape, or a token carrying workload-kv-read-llm-connect-provider-secrets)" + path_template: "platform/workloads/activity-core/llm-connect/llm-connect-provider-secrets" + fetch_command: "bao kv get -field=OPENROUTER_API_KEY platform/workloads/activity-core/llm-connect/llm-connect-provider-secrets" + policy_ref: "flex-auth check secret.read:llm-connect" + exec_capable: true + lane: secret + + # --- draft: owner path not yet shipped; hidden from default lookup --- - id: object-storage-sts title: Object-storage STS / temporary S3 credentials diff --git a/tests/test_routing.py b/tests/test_routing.py index 2c18c17..ad1f836 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -249,13 +249,19 @@ def test_handoff_template_with_placeholders_accepted(tmp_path): def test_find_active_excludes_draft(): catalog = load_catalog(_repo_catalog()) - ids = [e.id for e in catalog.find("issue core api key")] - assert "issue-core-ingestion-api-key" not in ids + ids = [e.id for e in catalog.find("s3 temporary credentials")] + assert "object-storage-sts" not in ids def test_find_all_includes_draft(): catalog = load_catalog(_repo_catalog()) - ids = [e.id for e in catalog.find("issue core api key", include_draft=True)] + ids = [e.id for e in catalog.find("s3 temporary credentials", include_draft=True)] + assert "object-storage-sts" in ids + + +def test_find_issue_core_lane_active(): + catalog = load_catalog(_repo_catalog()) + ids = [e.id for e in catalog.find("issue core api key")] assert "issue-core-ingestion-api-key" in ids @@ -313,13 +319,14 @@ def test_cli_list_active_only(repo_catalog_env): result = runner.invoke(app, ["route", "list", "--json"]) assert result.exit_code == 0 ids = [e["id"] for e in json.loads(result.stdout)] - assert "issue-core-ingestion-api-key" not in ids + assert "object-storage-sts" not in ids + assert "issue-core-ingestion-api-key" in ids def test_cli_list_all_includes_draft(repo_catalog_env): result = runner.invoke(app, ["route", "list", "--all", "--json"]) ids = [e["id"] for e in json.loads(result.stdout)] - assert "issue-core-ingestion-api-key" in ids + assert "object-storage-sts" in ids def test_cli_show_ssh_json_includes_cert_pattern(repo_catalog_env): diff --git a/wiki/CredentialRouting.md b/wiki/CredentialRouting.md index 6d262c4..6406c61 100644 --- a/wiki/CredentialRouting.md +++ b/wiki/CredentialRouting.md @@ -94,6 +94,8 @@ run the owner's tool as the caller and preserve owner custody. | `railiance-infra-principals` | "railiance-infra deploys host principals" | Run the infra Ansible | | `activity-core-issue-sink` | "activity-core + issue-core own emission — pair `ISSUE_CORE_*` env vars" | See `wiki/playbooks/activity-core-issue-sink.md` | | `inter-hub-bootstrap-ssh` | "Inter-Hub bootstrap SSH envelope — attended vs unattended branches" | See `wiki/InterHubBootstrapAccessLane.md` | +| `issue-core-ingestion-api-key` | "railiance-platform OpenBao KV + ESO deliver `ISSUE_CORE_API_KEY` — here is the path" | ESO consumes in-cluster; `warden access issue-core-ingestion-api-key --fetch ISSUE_CORE_API_KEY` as yourself | +| `openrouter-llm-connect` | "railiance-platform OpenBao KV + ESO deliver `OPENROUTER_API_KEY` to activity-core" | ESO consumes in-cluster; `warden access openrouter-llm-connect --fetch OPENROUTER_API_KEY` as yourself | Promotion criteria: `wiki/playbooks/catalog-lane-promotion.md`. @@ -101,8 +103,6 @@ Promotion criteria: `wiki/playbooks/catalog-lane-promotion.md`. | Catalog `id` | Routing focus | Playbook | | --- | --- | --- | -| `issue-core-ingestion-api-key` | OpenBao KV + ESO for `ISSUE_CORE_API_KEY` | `wiki/playbooks/issue-core-ingestion-api-key.md` | -| `openrouter-llm-connect` | OpenRouter key → `llm-connect` in activity-core | `wiki/playbooks/openrouter-llm-connect.md` | | `object-storage-sts` | NK-WP-0007 STS vending path | `wiki/playbooks/object-storage-sts.md` | | `database-dynamic-credentials` | OpenBao database secrets engine | `wiki/playbooks/database-dynamic-credentials.md` | @@ -117,7 +117,7 @@ value; the owner remains OpenBao, key-cape, flex-auth, or the routed subsystem. | Request | Correct path | | --- | --- | | "`VAULT_TOKEN` for ops-warden production sign / policy-gate smoke" | `railiance-platform` credential broker — `warden route show ops-warden-warden-sign-token` | -| "Populate `OPENROUTER_API_KEY` for llm-connect" | Operator → OpenBao/K8s Secret in `activity-core` namespace | +| "Populate `OPENROUTER_API_KEY` for llm-connect" | Operator → OpenBao custody; delivery via `warden route show openrouter-llm-connect` | | "Store Inter-Hub admin key for bootstrap" | Operator → OpenBao or `IHUB_OPERATOR_KEY_FILE` (`CUST-WP-0049`) | | "Give me Vault root token" | Break-glass ceremony → `railiance-platform/docs/openbao.md` | | "S3 credentials for artifact upload" | NK-WP-0007 / artifact-store consumer path | diff --git a/wiki/playbooks/catalog-lane-promotion.md b/wiki/playbooks/catalog-lane-promotion.md index be324c7..7efd811 100644 --- a/wiki/playbooks/catalog-lane-promotion.md +++ b/wiki/playbooks/catalog-lane-promotion.md @@ -28,7 +28,7 @@ Promotion PR touches: `registry/routing/catalog.yaml`, playbook, optional --- -## Worked example (already active) +## Worked examples (already active) **`ops-warden-warden-sign-token`** — promoted 2026-07-01 after RAILIANCE-WP-0005: @@ -37,14 +37,21 @@ Promotion PR touches: `registry/routing/catalog.yaml`, playbook, optional - Playbook: `wiki/playbooks/ops-warden-warden-sign-token.md` - Smoke: `make credential-exec-ops-warden-smoke` +**`issue-core-ingestion-api-key`** — promoted 2026-07-02 after RAILIANCE-WP-0009 +(CCR-2026-0002): KV path live, ExternalSecret `issue-core/issue-core-runtime` +SecretSynced, positive + negative verification audit-logged. + +**`openrouter-llm-connect`** — promoted 2026-07-02 after RAILIANCE-WP-0010 +(CCR-2026-0003): KV path live, ExternalSecret +`activity-core/llm-connect-provider-secrets` SecretSynced, llm-connect rolled +out on the OpenBao-delivered value, positive + negative verification audit-logged. + --- -## Draft lanes — none ready yet (2026-07-01) +## Draft lanes (2026-07-02) | Catalog `id` | Blocker | | --- | --- | -| `issue-core-ingestion-api-key` | OpenBao KV path + ESO wiring not owner-signed off | -| `openrouter-llm-connect` | activity-core secret mount path still placeholder | | `object-storage-sts` | NK-WP-0007 vending path not production-exercised | | `database-dynamic-credentials` | OpenBao database engine role paths TBD per workload | diff --git a/wiki/playbooks/issue-core-ingestion-api-key.md b/wiki/playbooks/issue-core-ingestion-api-key.md index cd42d0d..79476dd 100644 --- a/wiki/playbooks/issue-core-ingestion-api-key.md +++ b/wiki/playbooks/issue-core-ingestion-api-key.md @@ -1,8 +1,8 @@ # issue-core Ingestion API Key — OpenBao Custody -Date: 2026-06-24 -Workplan: WARDEN-WP-0012 T1 -Catalog: `issue-core-ingestion-api-key` (draft until path ships) +Date: 2026-06-24 (promoted active 2026-07-02) +Workplan: WARDEN-WP-0012 T1 · RAILIANCE-WP-0009 / CCR-2026-0002 +Catalog: `issue-core-ingestion-api-key` (**active** — path live, ESO delivering) Pointer playbook for agents and operators wiring the **shared ingestion key** between `activity-core` IssueSink emission and `issue-core` REST ingestion. @@ -54,8 +54,14 @@ GITEA_BACKEND_TOKEN The ExternalSecret manifest belongs in `issue-core` workload manifests (tenant repo owns runtime deployment). Platform owns mount policy and path provisioning. -**Promotion gate:** catalog entry stays `status: draft` until this path exists -in the live OpenBao cluster and an owner-repo ExternalSecret is merged. +**Promotion gate (met 2026-07-02):** the path exists in the live OpenBao +cluster and `ExternalSecret issue-core/issue-core-runtime` is merged and +`SecretSynced` (read policy `workload-kv-read-issue-core-runtime`, k8s auth +role `external-secrets-issue-core`, auth subject +`external-secrets/external-secrets`). Positive + negative access verified with +OpenBao audit evidence (RAILIANCE-WP-0009 T05). Lifecycle +(deactivate/rotate/compromise): +`railiance-platform/docs/credential-lane-lifecycle-runbook.md`. --- @@ -110,8 +116,7 @@ scoped tokens only, never root token for routine workload secret inspection. | `issue-core` | Merge ExternalSecret + Deployment env from synced Secret | | `activity-core` | Mirror `ISSUE_CORE_API_KEY` injection for REST sink mode | -When the path ships, ops-warden promotes `issue-core-ingestion-api-key` to -`status: active` with this `wiki_ref`. +Promoted to `status: active` on 2026-07-02 (RAILIANCE-WP-0009 T06). --- diff --git a/wiki/playbooks/openrouter-llm-connect.md b/wiki/playbooks/openrouter-llm-connect.md index 8c14094..dd9d9fe 100644 --- a/wiki/playbooks/openrouter-llm-connect.md +++ b/wiki/playbooks/openrouter-llm-connect.md @@ -1,8 +1,8 @@ # OpenRouter API Key — llm-connect in activity-core -Date: 2026-06-24 -Workplan: WARDEN-WP-0012 T4 -Catalog: `openrouter-llm-connect` (draft until OpenBao path ships) +Date: 2026-06-24 (promoted active 2026-07-02) +Workplan: WARDEN-WP-0012 T4 · RAILIANCE-WP-0010 / CCR-2026-0003 +Catalog: `openrouter-llm-connect` (**active** — OpenBao path live, ESO delivering) Pointer playbook for LLM provider credentials consumed by `llm-connect` in the `activity-core` namespace. ops-warden issues SSH certs only — API keys are an @@ -25,16 +25,14 @@ OpenBao → Kubernetes Secret action owned by `railiance-platform` and ```bash warden route show openbao-api-key --json -warden route show openrouter-llm-connect --json # after promotion +warden route show openrouter-llm-connect --json ``` `OPENROUTER_API_KEY` must not appear in Git, State Hub, workplans, logs, or chat. --- -## Expected custody shape - -Documented platform path convention (coordinate before writing secrets): +## Custody shape (live since 2026-07-02) ```text platform/workloads/activity-core/llm-connect/llm-connect-provider-secrets @@ -42,12 +40,19 @@ platform/workloads/activity-core/llm-connect/llm-connect-provider-secrets Property name: `OPENROUTER_API_KEY` -Until the OpenBao path is provisioned, operators may create the K8s Secret -directly for pilot smoke (`llm-connect` README) — that is a bootstrap bridge, -not the long-term custody model. +Delivery: `ExternalSecret activity-core/llm-connect-provider-secrets` +(ClusterSecretStore `openbao-activity-core`, read policy +`workload-kv-read-llm-connect-provider-secrets`, k8s auth role +`external-secrets-activity-core`) syncs to Secret +`llm-connect-provider-secrets`; the llm-connect Deployment consumes it. +Positive + negative access verified with OpenBao audit evidence +(RAILIANCE-WP-0010 T05). Lifecycle (deactivate/rotate/compromise): +`railiance-platform/docs/credential-lane-lifecycle-runbook.md`. -**Promotion gate:** catalog entry stays `status: draft` until the OpenBao path -exists and ESO (or approved equivalent) delivers the Secret in cluster. +**Promotion gate (met 2026-07-02):** the OpenBao path exists and ESO delivers +the Secret in cluster. The earlier manually created bootstrap Secret has been +taken over by ESO on the CoulombCore cluster; the railiance01 k3s llm-connect +instance still uses its bootstrap Secret (separate migration, not this lane). --- From 2f532699fa762a96a324eab772a8b8c8d3a77fe1 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 2 Jul 2026 23:31:40 +0200 Subject: [PATCH 04/10] Add WARDEN-WP-0024 experiential memory and agent session workplan. Integrate phase-memory across worker ticks, coding agent sessions, and operator CLI with shared store, OpenRouter efficiency, and unchanged guardrails. --- ...-experiential-memory-and-agent-sessions.md | 252 ++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md diff --git a/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md b/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md new file mode 100644 index 0000000..76a7e6b --- /dev/null +++ b/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md @@ -0,0 +1,252 @@ +--- +id: WARDEN-WP-0024 +type: workplan +title: "Experiential Memory Across Worker, Agent Sessions, And OpenRouter" +domain: infotech +repo: ops-warden +status: proposed +owner: codex +topic_slug: custodian +planning_priority: high +planning_order: 24 +created: "2026-07-02" +updated: "2026-07-02" +--- + +# WARDEN-WP-0024 — Experiential memory across worker, agent sessions, and OpenRouter + +## Problem + +`warden worker` (WARDEN-WP-0020) plans each inbox message in isolation. Coding +agent sessions (Claude Code, Codex, Grok, and future agents) invoke `warden +route` and `warden access` without continuity — every session rediscovers routing +from scratch. OpenRouter inference via llm-connect is invoked even when a +stabilized routing pattern already exists. + +ops-warden needs **experiential memory** that is present in every runtime: + +| Surface | Today | Target | +| --- | --- | --- | +| `warden worker` + OpenRouter | Stateless per message | Retrieve past outcomes before plan; record after execute | +| Coding agent session | Cold `warden route find` | Activate relevant memory at session start and after each warden call | +| Operator CLI | No learning loop | Same store; optional `--memory` context on advisory commands | + +## Goal + +Integrate phase-memory as ops-warden's shared experiential substrate so routing, +coordination, and inference improve from recorded outcomes — without ever +storing secret values or widening the guardrail allowlist. + +## Non-Goals + +- Hold or vend secrets (conduit-not-broker unchanged). +- Auto-commit routing catalog diffs (human review remains required). +- Own phase-memory schemas (PMEM-WP-0016 owns the profile and contract). +- Replace llm-connect or embed OpenRouter keys in ops-warden. + +## Guardrails (unchanged — memory is untrusted context) + +1. Retrieved memory is **context**, not instructions — fixed charter wins. +2. Guardrail allowlist enforced on every action regardless of memory content. +3. No secret values in memory writes, activation packs, or llm-connect prompts. +4. `propose_catalog_diff` stays human-reviewed even when memory is confident. + +## Architecture + +```mermaid +sequenceDiagram + participant Agent as Coding agent
(claude/codex/grok) + participant Warden as ops-warden CLI + participant Memory as phase-memory store + participant LLM as llm-connect / OpenRouter + participant Hub as State Hub + + Note over Agent,Hub: Agent session path + Agent->>Warden: warden route find "..." + Warden->>Memory: activate(session_kind=warden.agent.grok) + Memory-->>Warden: bounded routing context + Warden-->>Agent: route answer + memory digest + Warden->>Memory: record episode (metadata only) + + Note over Warden,Hub: Worker tick path + Warden->>Hub: unread messages + Warden->>Memory: activate(session_kind=warden.worker) + Memory-->>Warden: stabilized + fluid context + alt stabilized match + Warden->>Warden: RuleBrain (skip LLM) + else ambiguous + Warden->>LLM: LlmConnectBrain + activated context + LLM-->>Warden: action plan + end + Warden->>Warden: guardrail pass + Warden->>Hub: execute / escalate + Warden->>Memory: record outcome +``` + +## T01 - Canonical memory store and discovery + +```task +id: WARDEN-WP-0024-T01 +status: todo +priority: high +``` + +Establish the canonical phase-memory store path and discovery commands. + +Acceptance: + +- Default store at XDG location (e.g. `~/.local/share/warden/memory/`) with + override via `WARDEN_MEMORY_STORE`. +- `warden memory status` reports store path, schema version, episode counts by + `session_kind`, and last activation timestamp — no secret fields. +- Store initializes idempotently; missing phase-memory dependency fails with a + clear install pointer, not a traceback. +- Tests cover default path, override, and first-run initialization. + +## T02 - Session recording hooks in CLI commands + +```task +id: WARDEN-WP-0024-T02 +status: todo +priority: high +``` + +Record metadata-only episodes when `route`, `access`, and `sign` complete. + +Acceptance: + +- Each command writes a phase-memory event with `session_kind` derived from + environment: `WARDEN_AGENT_ID` (claude|codex|grok|...) when set, else + `warden.operator`. +- Recorded fields match `PMEM-WP-0016-T03` schema; secret patterns are rejected. +- Recording is opt-out via `WARDEN_MEMORY=0` for air-gapped/debug use. +- `warden activity` and phase-memory audit export remain consistent (no + duplicate secret-bearing logs). + +## T03 - Memory-aware worker tick + +```task +id: WARDEN-WP-0024-T03 +status: todo +priority: high +``` + +Retrieve coordination memory before `Brain.plan` and record outcomes after execute. + +Acceptance: + +- `warden worker run` activates memory with `session_kind=warden.worker` before + planning each message. +- Activated context is passed to `LlmConnectBrain` as bounded, redacted context + — never replacing the fixed charter. +- Post-execute hook records: message id, proposed actions, guardrail outcome, + escalation reason, and route ids referenced. +- Worker tick with empty store degrades gracefully (current behavior). + +## T04 - Agent session activation helper + +```task +id: WARDEN-WP-0024-T04 +status: todo +priority: high +``` + +Expose memory activation for coding agent sessions across Claude, Codex, Grok, and +future agents. + +Acceptance: + +- `warden memory activate [--agent ] [--json]` returns a bounded activation + package for session orientation. +- Agent instructions (AGENTS.md template in consuming repos) document setting + `WARDEN_AGENT_ID` and calling `warden memory activate` at session start. +- Activation works without OpenRouter (local retrieval only). +- Tests cover agent ids: `claude`, `codex`, `grok`, and an unknown future id. + +## T05 - Cross-runtime continuity + +```task +id: WARDEN-WP-0024-T05 +status: todo +priority: high +``` + +Ensure worker ticks see agent session episodes and vice versa. + +Acceptance: + +- Agent session routing outcomes appear in worker activation within the same + store without manual export/import. +- Worker escalation outcomes are visible to subsequent agent session activations. +- Continuity is verified by an integration test: agent session write → worker + read → agent session read round-trip. +- Episode retention follows the ops-warden profile compaction rules from + PMEM-WP-0016. + +## T06 - OpenRouter efficiency layer + +```task +id: WARDEN-WP-0024-T06 +status: todo +priority: medium +``` + +Skip or downgrade OpenRouter calls when stabilized memory provides a verified match. + +Acceptance: + +- When stabilized memory contains a verified route match for the message need, + `RuleBrain` handles the plan and llm-connect is not called. +- Ambiguous cases still use `LlmConnectBrain`; memory context is additive only. +- Efficiency metrics (llm_calls_avoided, rule_brain_resolution_rate) are + recorded for evaluation scenarios in PMEM-WP-0016. +- Dry-run mode reports whether LLM would have been skipped and why. + +## T07 - Operator and agent documentation + +```task +id: WARDEN-WP-0024-T07 +status: todo +priority: medium +``` + +Document the memory workflow for operators and coding agents. + +Acceptance: + +- Wiki page covers: store location, session kinds, agent env vars, memory + activation at session start, and interaction with OpenRouter inference. +- ops-warden AGENTS.md and the state-hub agent template include memory + orientation steps for Claude, Codex, and Grok sessions. +- Credential routing doc cross-links PMEM-WP-0016 contract; anti-patterns + updated (no secret values in memory store). +- Scheduled worker playbook notes memory retention and evaluation gate checks. + +## Acceptance Criteria + +- phase-memory is present and useful in all three runtime surfaces (worker, + agent session, operator CLI) through one canonical store. +- OpenRouter inference cost decreases when stabilized routing memory matches. +- Cross-runtime continuity is tested and regression-gated. +- Security invariants (no-secret, allowlist, conduit-not-broker) are unchanged. + +## Dependencies + +- `PMEM-WP-0016` (phase-memory) — profile, event schema, activation pack, + evaluation scenarios. **Implement contract (T01–T03) before WARDEN T03–T05.** +- `WARDEN-WP-0020` — worker brain and guardrails (finished). +- `WARDEN-WP-0022` — audit trail for correlation (finished). +- llm-connect / OpenRouter — required only for T06 live efficiency measurement; + T01–T05 work offline. + +## Suggested Implementation Order + +1. PMEM-WP-0016 T01–T03 (profile + contract + event schema) +2. WARDEN-WP-0024 T01–T02 (store + CLI recording) +3. PMEM-WP-0016 T04 (activation pack) +4. WARDEN-WP-0024 T03–T05 (worker + agent activation + continuity) +5. PMEM-WP-0016 T05–T06 + WARDEN-WP-0024 T06–T07 (evaluation + docs) + +## Closure Review + +Pending implementation. \ No newline at end of file From 04929e79814437f8eb981610e7276b173d515d54 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 2 Jul 2026 23:40:45 +0200 Subject: [PATCH 05/10] Implement WARDEN-WP-0024 experiential memory and agent sessions. Add phase-memory bridge, warden memory CLI, route/access/sign recording, memory-aware worker planning with OpenRouter skip, tests, wiki, and AGENTS.md orientation for Claude, Codex, Grok, and future agent sessions. --- AGENTS.md | 23 ++++ pyproject.toml | 5 +- src/warden/cli.py | 118 ++++++++++++++++ src/warden/memory.py | 122 +++++++++++++++++ src/warden/worker.py | 87 +++++++++++- tests/conftest.py | 11 ++ tests/test_memory.py | 129 ++++++++++++++++++ wiki/OpsWardenMemory.md | 52 +++++++ ...-experiential-memory-and-agent-sessions.md | 35 +++-- 9 files changed, 568 insertions(+), 14 deletions(-) create mode 100644 src/warden/memory.py create mode 100644 tests/conftest.py create mode 100644 tests/test_memory.py create mode 100644 wiki/OpsWardenMemory.md diff --git a/AGENTS.md b/AGENTS.md index 4ff0224..c379092 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,6 +156,29 @@ get wrong. +## Experiential memory (WARDEN-WP-0024) + +ops-warden shares a **phase-memory** store across worker ticks, coding agent +sessions, and operator CLI use. + +**At session start** (Claude Code, Codex, Grok, or future agents): + +```bash +export WARDEN_AGENT_ID=grok # or claude, codex +warden memory activate --json +``` + +**During work:** use normal `warden route` / `warden access` / `warden sign`; +episodes are recorded automatically unless `WARDEN_MEMORY=0`. + +**Store:** `~/.local/share/warden/memory/` (override: `WARDEN_MEMORY_STORE`). + +**Worker:** `warden worker run --brain llm` skips OpenRouter when stabilized +routing memory matches. See `wiki/OpsWardenMemory.md`. + +Requires `phase-memory` on `PYTHONPATH` or installed; contract in +`phase-memory/docs/ops-warden-memory-contract.md`. + --- ## Workplan Convention (ADR-001) diff --git a/pyproject.toml b/pyproject.toml index e9bfa0f..3ca400f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,9 @@ dependencies = [ "httpx>=0.27", ] +[project.optional-dependencies] +memory = [] + [project.scripts] warden = "warden.cli:app" ops-ssh-wrapper = "warden.scripts.ops_ssh_wrapper:main" @@ -29,7 +32,7 @@ packages = ["src/warden"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] +pythonpath = ["src", "../phase-memory/src"] addopts = "-m 'not integration'" markers = ["integration: requires ssh-keygen binary; run with pytest -m integration"] diff --git a/src/warden/cli.py b/src/warden/cli.py index bad95dc..db68823 100644 --- a/src/warden/cli.py +++ b/src/warden/cli.py @@ -44,6 +44,11 @@ activity_app = typer.Typer( no_args_is_help=True, ) app.add_typer(activity_app, name="activity") +memory_app = typer.Typer( + help="Cross-runtime experiential memory via phase-memory (WARDEN-WP-0024)", + no_args_is_help=True, +) +app.add_typer(memory_app, name="memory") console = Console() err = Console(stderr=True) @@ -53,6 +58,30 @@ err = Console(stderr=True) # Helpers # --------------------------------------------------------------------------- +def _record_memory_episode( + *, + command: str, + outcome: str, + need: str = "", + route_id: str = "", +) -> None: + try: + from warden import memory as warden_memory + except ImportError: + return + if not warden_memory.enabled() or not warden_memory.memory_available(): + return + try: + warden_memory.record_command_episode( + command=command, + outcome=outcome, + need=need, + route_id=route_id, + ) + except RuntimeError: + return + + def _load_cfg() -> WardenConfig: try: return load_config() @@ -128,6 +157,12 @@ def sign( # cert_command interface: write cert text to stdout only print(record.cert_path.read_text().strip()) + _record_memory_episode( + command="sign", + outcome="resolved", + need=f"ssh cert {actor_name}", + route_id="ops-warden-ssh-cert", + ) # --------------------------------------------------------------------------- @@ -753,14 +788,30 @@ def route_find( if output_json: print(json.dumps([_entry_summary(e) for e in matches], indent=2)) + if matches: + _record_memory_episode( + command="route find", + outcome="resolved", + need=query, + route_id=matches[0].id, + ) + else: + _record_memory_episode(command="route find", outcome="skipped", need=query) return if not matches: + _record_memory_episode(command="route find", outcome="skipped", need=query) console.print( f"No routing match for {query!r}. " "Try `warden route list --all` to browse all scenarios." ) return + _record_memory_episode( + command="route find", + outcome="resolved", + need=query, + route_id=matches[0].id, + ) _print_entry_table(matches, f"Matches for {query!r}") @@ -1003,8 +1054,20 @@ def access( if output_json: print(json.dumps(_access_json(entry, expanded, gate, domain), indent=2)) + _record_memory_episode( + command="access", + outcome="resolved", + need=need, + route_id=entry.id, + ) return + _record_memory_episode( + command="access", + outcome="resolved", + need=need, + route_id=entry.id, + ) console.print(f"[bold]{entry.title}[/bold] ([cyan]{entry.id}[/cyan])") console.print(f" owner : {entry.owner_repo} ({entry.subsystem})") @@ -1305,3 +1368,58 @@ def worker_status_cmd() -> None: console.print(f"timer : {st or 'unknown'}") except Exception: # noqa: BLE001 — systemd may be absent (cron/other host) console.print("timer : (systemd not available)") + + +# --------------------------------------------------------------------------- +# warden memory — cross-runtime experiential memory (WARDEN-WP-0024) +# --------------------------------------------------------------------------- + +@memory_app.command("status") +def memory_status( + output_json: Annotated[bool, typer.Option("--json", help="Output JSON")] = False, +) -> None: + """Show canonical phase-memory store status (metadata only).""" + from warden import memory as warden_memory + + if not warden_memory.memory_available(): + err.print(f"[red]{warden_memory._PHASE_MEMORY_ERROR}[/red]") + raise typer.Exit(2) + try: + payload = warden_memory.status() + except RuntimeError as e: + err.print(f"[red]{e}[/red]") + raise typer.Exit(2) + if output_json: + print(json.dumps(payload, indent=2)) + return + console.print(f"store_path : {payload.get('store_path', '')}") + console.print(f"profile_id : {payload.get('profile_id', '')}") + console.print(f"episode_count : {payload.get('episode_count', 0)}") + console.print(f"session_kinds : {payload.get('episode_counts_by_session_kind', {})}") + console.print(f"last_activation: {payload.get('last_activation_at') or '—'}") + + +@memory_app.command("activate") +def memory_activate( + need: Annotated[str, typer.Option("--need", help="Optional routing need fingerprint source")] = "", + agent: Annotated[ + Optional[str], + typer.Option("--agent", help="Agent id for session_kind warden.agent. (claude, codex, grok, …)"), + ] = None, + output_json: Annotated[bool, typer.Option("--json", help="Output JSON")] = False, +) -> None: + """Activate bounded coordination memory for worker, operator, or agent sessions.""" + from warden import memory as warden_memory + + if not warden_memory.memory_available(): + err.print(f"[red]{warden_memory._PHASE_MEMORY_ERROR}[/red]") + raise typer.Exit(2) + try: + payload = warden_memory.activate(need=need, agent=agent) + except RuntimeError as e: + err.print(f"[red]{e}[/red]") + raise typer.Exit(2) + if output_json: + print(json.dumps(payload, indent=2)) + return + console.print(warden_memory.format_activation_summary(payload)) diff --git a/src/warden/memory.py b/src/warden/memory.py new file mode 100644 index 0000000..4f71b34 --- /dev/null +++ b/src/warden/memory.py @@ -0,0 +1,122 @@ +"""phase-memory bridge for ops-warden cross-runtime experiential memory.""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +_PHASE_MEMORY_ERROR = ( + "phase-memory is required for warden memory commands. " + "Install with: pip install phase-memory (or set PYTHONPATH to phase-memory/src)." +) + + +def _phase_memory(): + try: + import phase_memory.ops_warden as pm + + return pm + except ImportError as exc: # pragma: no cover - exercised via tests with PYTHONPATH + raise RuntimeError(_PHASE_MEMORY_ERROR) from exc + + +def memory_available() -> bool: + try: + _phase_memory() + return True + except RuntimeError: + return False + + +def enabled(environ: Mapping[str, str] | None = None) -> bool: + environ = environ or os.environ + return str(environ.get("WARDEN_MEMORY", "1")).strip().lower() not in {"0", "false", "no", "off"} + + +def store_path(environ: Mapping[str, str] | None = None): + return _phase_memory().default_memory_store_path(environ) + + +def session_kind(environ: Mapping[str, str] | None = None) -> str: + return _phase_memory().resolve_session_kind(environ) + + +def status(environ: Mapping[str, str] | None = None) -> dict[str, Any]: + pm = _phase_memory() + return pm.OpsWardenMemoryStore.open(environ=environ).status() + + +def activate( + *, + need: str = "", + agent: Optional[str] = None, + session_id: str = "", + environ: Mapping[str, str] | None = None, +) -> dict[str, Any]: + pm = _phase_memory() + env = dict(environ or os.environ) + if agent: + env["WARDEN_AGENT_ID"] = agent + kind = pm.resolve_session_kind(env) + return pm.activate_ops_warden_memory( + pm.OpsWardenMemoryStore.open(environ=env), + session_kind=kind, + need=need, + session_id=session_id, + ) + + +def record_command_episode( + *, + command: str, + outcome: str, + need: str = "", + route_id: str = "", + diagnostic_codes: Optional[list[str]] = None, + metadata: Optional[dict[str, Any]] = None, + environ: Mapping[str, str] | None = None, +) -> dict[str, Any]: + if not enabled(environ): + return {"valid": True, "skipped": True, "reason": "WARDEN_MEMORY=0"} + pm = _phase_memory() + env = dict(environ or os.environ) + event = pm.build_session_event( + command=command, + session_kind=pm.resolve_session_kind(env), + outcome=outcome, + need=need, + route_id=route_id, + agent_id=str(env.get("WARDEN_AGENT_ID") or ""), + session_id=str(env.get("WARDEN_SESSION_ID") or ""), + diagnostic_codes=diagnostic_codes, + metadata=metadata, + ) + return pm.record_session_event(pm.OpsWardenMemoryStore.open(environ=env), event) + + +def worker_activation_context(need: str = "", environ: Mapping[str, str] | None = None) -> dict[str, Any]: + env = dict(environ or os.environ) + env["WARDEN_SESSION_KIND"] = "warden.worker" + return activate(need=need, environ=env) + + +def stabilized_route_for_need(need: str, environ: Mapping[str, str] | None = None) -> Optional[dict[str, Any]]: + pm = _phase_memory() + store = pm.OpsWardenMemoryStore.open(environ=environ) + return pm.stabilized_route_match(store.list_events(), need=need) + + +def format_activation_summary(activation: dict[str, Any]) -> str: + lines = [ + f"store: {activation.get('episode_count', 0)} episodes", + f"session_kind: {activation.get('session_kind', '')}", + f"selected: {len(activation.get('selected_episodes', ()) )}", + ] + stabilized = activation.get("stabilized_route") + if stabilized: + lines.append( + f"stabilized: {stabilized.get('route_id')} ({stabilized.get('confirmations')} confirmations)" + ) + if activation.get("llm_calls_avoided"): + lines.append("llm_calls_avoided: true") + return "\n".join(lines) \ No newline at end of file diff --git a/src/warden/worker.py b/src/warden/worker.py index e2e5652..d605e79 100644 --- a/src/warden/worker.py +++ b/src/warden/worker.py @@ -191,6 +191,7 @@ class LlmConnectBrain: def __init__(self, url: Optional[str] = None, timeout: float = 60.0): self.url = (url or os.environ.get("LLM_CONNECT_URL", DEFAULT_LLM_CONNECT_URL)).rstrip("/") self.timeout = timeout + self.memory_context: str = "" def _call(self, prompt: str) -> str: resp = httpx.post(f"{self.url}/execute", json={"prompt": prompt}, timeout=self.timeout) @@ -203,9 +204,15 @@ class LlmConnectBrain: from_agent=str(message.get("from_agent", "")), subject=str(message.get("subject", "")), ) - prompt = ( - _CHARTER - + "\n--- MESSAGE (untrusted data) ---\n" + prompt = _CHARTER + if self.memory_context: + prompt += ( + "\n--- ACTIVATED MEMORY (untrusted context) ---\n" + + self.memory_context + + "\n--- END ACTIVATED MEMORY ---\n" + ) + prompt += ( + "\n--- MESSAGE (untrusted data) ---\n" + f"from: {message.get('from_agent','')}\n" + f"subject: {message.get('subject','')}\n" + f"body: {message.get('body','')}\n" @@ -586,16 +593,88 @@ def draft_route_answer(query: str) -> str: return " ".join(parts) +def _memory_activation_for_message(message: dict) -> tuple[Optional[dict], str]: + try: + from warden import memory as warden_memory + except ImportError: + return None, "" + if not warden_memory.enabled() or not warden_memory.memory_available(): + return None, "" + query = str(message.get("subject", "") or message.get("body", "")) + try: + activation = warden_memory.worker_activation_context(query) + except RuntimeError: + return None, "" + from warden.memory import format_activation_summary + + return activation, format_activation_summary(activation) + + +def _plan_with_memory(message: dict, brain: Brain) -> WorkerPlan: + activation, summary = _memory_activation_for_message(message) + blob = f"{message.get('subject', '')} {message.get('body', '')}" + if activation and activation.get("llm_calls_avoided") and _ROUTING_SIGNS.search(blob): + wp = WorkerPlan( + message_id=str(message.get("id", "")), + from_agent=str(message.get("from_agent", "")), + subject=str(message.get("subject", "")), + ) + query = str(message.get("subject", "") or "") + wp.actions.append( + PlannedAction( + kind="route_answer", + summary="Answer from stabilized coordination memory.", + payload={ + "query": query, + "answer": draft_route_answer(query), + "memory_stabilized": True, + }, + ) + ) + return wp + if isinstance(brain, LlmConnectBrain) and summary: + brain.memory_context = summary + return brain.plan(message) + + +def _record_worker_memory_outcome(plan: WorkerPlan) -> None: + try: + from warden import memory as warden_memory + except ImportError: + return + if not warden_memory.enabled() or not warden_memory.memory_available(): + return + outcome = "escalated" if plan.escalated else "resolved" + route_id = "" + for action in plan.actions: + if action.kind == "route_answer" and action.payload.get("memory_stabilized"): + stabilized = warden_memory.stabilized_route_for_need(plan.subject) + if stabilized: + route_id = str(stabilized.get("route_id") or "") + try: + warden_memory.record_command_episode( + command="worker run", + outcome=outcome, + need=plan.subject, + route_id=route_id, + diagnostic_codes=["worker_escalated"] if plan.escalated else [], + metadata={"message_id": plan.message_id, "action_kinds": [a.kind for a in plan.actions]}, + ) + except RuntimeError: + return + + def build_plans(messages: List[dict], brain: Brain) -> List[WorkerPlan]: """Plan every message, attach computed route answers, and apply the guardrail pass.""" plans: List[WorkerPlan] = [] for m in messages: - plan = brain.plan(m) + plan = _plan_with_memory(m, brain) plan.raw = m for a in plan.actions: if a.kind == "route_answer" and "answer" not in a.payload: a.payload["answer"] = draft_route_answer(a.payload.get("query", m.get("subject", ""))) plans.append(_guardrail(plan, m)) + _record_worker_memory_outcome(plans[-1]) return plans diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f8f4628 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,11 @@ +"""Test configuration for ops-warden.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +PHASE_MEMORY_SRC = Path(__file__).resolve().parents[2] / "phase-memory" / "src" +if PHASE_MEMORY_SRC.exists() and str(PHASE_MEMORY_SRC) not in sys.path: + sys.path.insert(0, str(PHASE_MEMORY_SRC)) \ No newline at end of file diff --git a/tests/test_memory.py b/tests/test_memory.py new file mode 100644 index 0000000..cedcb08 --- /dev/null +++ b/tests/test_memory.py @@ -0,0 +1,129 @@ +"""Tests for ops-warden phase-memory bridge (WARDEN-WP-0024).""" + +from __future__ import annotations + +import json + +from typer.testing import CliRunner + +from warden.cli import app +from warden.memory import activate, enabled, record_command_episode, status, store_path +from warden.worker import RuleBrain, _plan_with_memory, build_plans + +runner = CliRunner() + + +def _msg(**over) -> dict: + base = { + "id": "m1", + "from_agent": "someone", + "subject": "Where do I get an npm token?", + "body": "Which subsystem owns this credential — how do I obtain it?", + } + base.update(over) + return base + + +def test_memory_status_and_activate_round_trip(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + monkeypatch.setenv("WARDEN_AGENT_ID", "grok") + + first = record_command_episode( + command="route find", + outcome="resolved", + need="npm token", + route_id="openbao-api-key", + ) + second = record_command_episode( + command="route find", + outcome="resolved", + need="npm token", + route_id="openbao-api-key", + ) + payload = activate(need="npm token", agent="grok") + + assert first["valid"] is True + assert second["valid"] is True + assert payload["session_kind"] == "warden.agent.grok" + assert payload["stabilized_route"]["route_id"] == "openbao-api-key" + assert status()["episode_count"] >= 3 + + +def test_worker_uses_stabilized_memory_without_llm(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + monkeypatch.setenv("WARDEN_SESSION_KIND", "warden.worker") + record_command_episode( + command="route find", + outcome="resolved", + need="Where do I get an npm token?", + route_id="openbao-api-key", + ) + record_command_episode( + command="route find", + outcome="resolved", + need="Where do I get an npm token?", + route_id="openbao-api-key", + ) + + plan = _plan_with_memory(_msg(), RuleBrain()) + + assert plan.actions + assert plan.actions[0].kind == "route_answer" + assert plan.actions[0].payload.get("memory_stabilized") is True + + +def test_cross_runtime_continuity_agent_to_worker(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + monkeypatch.setenv("WARDEN_AGENT_ID", "codex") + record_command_episode( + command="route find", + outcome="resolved", + need="openrouter api key", + route_id="openrouter-llm-connect", + ) + monkeypatch.setenv("WARDEN_SESSION_KIND", "warden.worker") + monkeypatch.delenv("WARDEN_AGENT_ID", raising=False) + activation = activate(need="openrouter api key") + kinds = {item.get("session_kind") for item in activation["selected_episodes"] if item.get("kind") == "episode"} + + assert "warden.agent.codex" in kinds + + +def test_cli_memory_status_and_activate(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + status_result = runner.invoke(app, ["memory", "status", "--json"]) + activate_result = runner.invoke(app, ["memory", "activate", "--agent", "claude", "--json"]) + + assert status_result.exit_code == 0 + assert activate_result.exit_code == 0 + assert json.loads(status_result.stdout)["episode_count"] >= 0 + assert json.loads(activate_result.stdout)["session_kind"] == "warden.agent.claude" + + +def test_route_find_records_memory_episode(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + result = runner.invoke(app, ["route", "find", "openrouter api key", "--json"]) + + assert result.exit_code == 0 + payload = status() + assert payload["episode_count"] >= 1 + + +def test_build_plans_records_worker_outcomes(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + plans = build_plans([_msg()], RuleBrain()) + + assert plans[0].actions + assert status()["episode_count"] >= 1 + + +def test_memory_disabled_skips_recording(monkeypatch) -> None: + monkeypatch.setenv("WARDEN_MEMORY", "0") + result = record_command_episode(command="route find", outcome="resolved", need="npm token") + + assert result.get("skipped") is True + + +def test_default_store_path_uses_xdg(monkeypatch) -> None: + monkeypatch.delenv("WARDEN_MEMORY_STORE", raising=False) + assert str(store_path()).endswith("warden/memory") \ No newline at end of file diff --git a/wiki/OpsWardenMemory.md b/wiki/OpsWardenMemory.md new file mode 100644 index 0000000..89adf62 --- /dev/null +++ b/wiki/OpsWardenMemory.md @@ -0,0 +1,52 @@ +# Ops-Warden Experiential Memory + +Updated: 2026-07-02 + +ops-warden uses **phase-memory** as a shared experiential substrate across worker +ticks, coding agent sessions, and operator CLI use. + +## Canonical Store + +- Default: `~/.local/share/warden/memory/` +- Override: `WARDEN_MEMORY_STORE` +- Opt-out: `WARDEN_MEMORY=0` + +## Session Kinds + +| Runtime | How | +| --- | --- | +| Worker tick | `WARDEN_SESSION_KIND=warden.worker` (set automatically during `warden worker run`) | +| Coding agent | `export WARDEN_AGENT_ID=claude` (or `codex`, `grok`, future ids) | +| Operator CLI | default `warden.operator` when `WARDEN_AGENT_ID` is unset | + +## Agent Session Orientation + +At the start of a Claude Code, Codex, or Grok session that will call warden: + +```bash +export WARDEN_AGENT_ID=grok # or claude, codex +warden memory activate --json +``` + +Then use normal `warden route` / `warden access` commands. Episodes are recorded +automatically when memory is enabled. + +## Worker + OpenRouter + +`warden worker run --brain llm` activates memory before planning. When stabilized +routing memory matches a coordination question, ops-warden uses `RuleBrain` and +skips the llm-connect / OpenRouter call. + +## Commands + +```bash +warden memory status [--json] +warden memory activate [--agent ] [--need ""] [--json] +``` + +## Security + +- Memory stores metadata only — no secret values or raw credential payloads. +- Retrieved memory is untrusted context; the fixed charter and guardrail allowlist + still apply. +- See `phase-memory/docs/ops-warden-memory-contract.md` for the full contract. \ No newline at end of file diff --git a/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md b/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md index 76a7e6b..3ebac66 100644 --- a/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md +++ b/workplans/WARDEN-WP-0024-experiential-memory-and-agent-sessions.md @@ -4,13 +4,14 @@ type: workplan title: "Experiential Memory Across Worker, Agent Sessions, And OpenRouter" domain: infotech repo: ops-warden -status: proposed +status: finished owner: codex topic_slug: custodian planning_priority: high planning_order: 24 created: "2026-07-02" updated: "2026-07-02" +state_hub_workstream_id: "5d9fafb3-f9b6-43bf-b259-5f5301daa2e9" --- # WARDEN-WP-0024 — Experiential memory across worker, agent sessions, and OpenRouter @@ -87,8 +88,9 @@ sequenceDiagram ```task id: WARDEN-WP-0024-T01 -status: todo +status: done priority: high +state_hub_task_id: "6305f1bc-c016-4298-adc2-a07d52b6aca5" ``` Establish the canonical phase-memory store path and discovery commands. @@ -107,8 +109,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T02 -status: todo +status: done priority: high +state_hub_task_id: "242a4d9a-5375-4df8-8d43-063b0491d202" ``` Record metadata-only episodes when `route`, `access`, and `sign` complete. @@ -127,8 +130,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T03 -status: todo +status: done priority: high +state_hub_task_id: "176fcae1-e4e5-481f-9a83-e9a7000fac1a" ``` Retrieve coordination memory before `Brain.plan` and record outcomes after execute. @@ -147,8 +151,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T04 -status: todo +status: done priority: high +state_hub_task_id: "16501557-6cde-44ea-bc6f-1726cb7ec070" ``` Expose memory activation for coding agent sessions across Claude, Codex, Grok, and @@ -167,8 +172,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T05 -status: todo +status: done priority: high +state_hub_task_id: "55ae679c-c08f-4afd-8646-9f5f3019f86e" ``` Ensure worker ticks see agent session episodes and vice versa. @@ -187,8 +193,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T06 -status: todo +status: done priority: medium +state_hub_task_id: "fc2dffcf-7184-4f3d-8653-d26dc18a9afc" ``` Skip or downgrade OpenRouter calls when stabilized memory provides a verified match. @@ -206,8 +213,9 @@ Acceptance: ```task id: WARDEN-WP-0024-T07 -status: todo +status: done priority: medium +state_hub_task_id: "ca2aaf23-833f-49a6-a49b-a0b659208f5f" ``` Document the memory workflow for operators and coding agents. @@ -249,4 +257,13 @@ Acceptance: ## Closure Review -Pending implementation. \ No newline at end of file +Integrated phase-memory across all ops-warden runtime surfaces: + +- `warden memory status` and `warden memory activate` with canonical XDG store. +- CLI recording hooks on `route find`, `access`, and `sign` (opt-out via + `WARDEN_MEMORY=0`). +- Memory-aware worker planning with stabilized-route RuleBrain short-circuit + and LlmConnectBrain context injection. +- Cross-runtime continuity verified in tests (agent session → worker activation). +- `wiki/OpsWardenMemory.md` and AGENTS.md session orientation for Claude, + Codex, Grok, and future agents. \ No newline at end of file From 120de64bcb2b404f25bc24d2ce571a7efeaf7d27 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 00:49:36 +0200 Subject: [PATCH 06/10] Enable implicit phase-memory activation on every warden command. Load coordination memory by default via ensure_memory_context on app bootstrap and route/access flows; invalidate cache after episode writes. WARDEN_MEMORY=0 remains the opt-out. Document that warden memory activate is optional only. --- AGENTS.md | 12 ++++--- src/warden/cli.py | 29 +++++++++++++++-- src/warden/memory.py | 72 +++++++++++++++++++++++++++++++++++++++-- src/warden/worker.py | 4 ++- tests/test_memory.py | 16 ++++++++- wiki/OpsWardenMemory.md | 9 ++++-- 6 files changed, 129 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c379092..42cd84b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,15 +161,19 @@ get wrong. ops-warden shares a **phase-memory** store across worker ticks, coding agent sessions, and operator CLI use. -**At session start** (Claude Code, Codex, Grok, or future agents): +**Default:** phase-memory loads automatically on every `warden` command when +`phase-memory` is available. No separate activation step is required. + +**Agent sessions** should set runtime identity once per session: ```bash export WARDEN_AGENT_ID=grok # or claude, codex -warden memory activate --json ``` -**During work:** use normal `warden route` / `warden access` / `warden sign`; -episodes are recorded automatically unless `WARDEN_MEMORY=0`. +Then use normal `warden route` / `warden access` / `warden sign` / `warden worker`. +Episodes are recorded automatically unless `WARDEN_MEMORY=0`. + +`warden memory activate` is optional introspection/refresh, not a prerequisite. **Store:** `~/.local/share/warden/memory/` (override: `WARDEN_MEMORY_STORE`). diff --git a/src/warden/cli.py b/src/warden/cli.py index db68823..eb1c120 100644 --- a/src/warden/cli.py +++ b/src/warden/cli.py @@ -54,6 +54,19 @@ console = Console() err = Console(stderr=True) +@app.callback() +def _bootstrap_memory(ctx: typer.Context) -> None: + """Implicitly load phase-memory for every warden command (opt-out: WARDEN_MEMORY=0).""" + if ctx.invoked_subcommand is None: + return + try: + from warden import memory as warden_memory + + warden_memory.ensure_memory_context(implicit=True) + except Exception: # noqa: BLE001 — memory must never block warden commands + return + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -783,6 +796,12 @@ def route_find( limit: Annotated[int, typer.Option("--limit", help="Max matches")] = 5, ) -> None: """Rank routing scenarios by keyword overlap with the query.""" + try: + from warden import memory as warden_memory + + warden_memory.ensure_memory_context(need=query, implicit=True) + except Exception: # noqa: BLE001 + pass catalog = _load_catalog() matches = catalog.find(query, include_draft=all_entries, limit=limit) @@ -1026,6 +1045,12 @@ def access( """ from warden.access import expand_handoff, policy_gate_status + try: + from warden import memory as warden_memory + + warden_memory.ensure_memory_context(need=need, implicit=True) + except Exception: # noqa: BLE001 + pass catalog = _load_catalog() matches = catalog.find(need, include_draft=all_entries, limit=1) if not matches: @@ -1408,14 +1433,14 @@ def memory_activate( ] = None, output_json: Annotated[bool, typer.Option("--json", help="Output JSON")] = False, ) -> None: - """Activate bounded coordination memory for worker, operator, or agent sessions.""" + """Inspect or refresh coordination memory (optional — memory loads by default).""" from warden import memory as warden_memory if not warden_memory.memory_available(): err.print(f"[red]{warden_memory._PHASE_MEMORY_ERROR}[/red]") raise typer.Exit(2) try: - payload = warden_memory.activate(need=need, agent=agent) + payload = warden_memory.activate(need=need, agent=agent, implicit=False) except RuntimeError as e: err.print(f"[red]{e}[/red]") raise typer.Exit(2) diff --git a/src/warden/memory.py b/src/warden/memory.py index 4f71b34..d8e5cfc 100644 --- a/src/warden/memory.py +++ b/src/warden/memory.py @@ -10,6 +10,17 @@ _PHASE_MEMORY_ERROR = ( "Install with: pip install phase-memory (or set PYTHONPATH to phase-memory/src)." ) +# In-process cache: implicit activation is default; no separate `warden memory activate` +# is required for normal route/access/worker/sign use within one CLI invocation tree. +_CONTEXT_CACHE: dict[str, Any] | None = None +_CONTEXT_CACHE_KEY: tuple[str, str] = ("", "") + + +def _invalidate_context_cache() -> None: + global _CONTEXT_CACHE, _CONTEXT_CACHE_KEY + _CONTEXT_CACHE = None + _CONTEXT_CACHE_KEY = ("", "") + def _phase_memory(): try: @@ -46,24 +57,76 @@ def status(environ: Mapping[str, str] | None = None) -> dict[str, Any]: return pm.OpsWardenMemoryStore.open(environ=environ).status() +def memory_context_summary(activation: dict[str, Any] | None) -> dict[str, Any]: + if not activation: + return {"enabled": False} + return { + "enabled": True, + "implicit": bool(activation.get("implicit")), + "session_kind": activation.get("session_kind", ""), + "episode_count": activation.get("episode_count", 0), + "stabilized_route_id": (activation.get("stabilized_route") or {}).get("route_id", ""), + "llm_calls_avoided": bool(activation.get("llm_calls_avoided")), + "selected_episode_count": len(activation.get("selected_episodes") or ()), + } + + +def ensure_memory_context( + need: str = "", + *, + agent: Optional[str] = None, + session_id: str = "", + environ: Mapping[str, str] | None = None, + implicit: bool = True, +) -> dict[str, Any] | None: + """Load coordination memory for the current session (default, no extra command).""" + global _CONTEXT_CACHE, _CONTEXT_CACHE_KEY + + if not enabled(environ): + return None + if not memory_available(): + return None + pm = _phase_memory() + env = dict(environ or os.environ) + if agent: + env["WARDEN_AGENT_ID"] = agent + kind = pm.resolve_session_kind(env) + fingerprint = pm.need_fingerprint(need) if need else "" + cache_key = (kind, fingerprint) + if _CONTEXT_CACHE is not None and _CONTEXT_CACHE_KEY == cache_key: + return _CONTEXT_CACHE + try: + activation = activate(need=need, agent=agent, session_id=session_id, environ=env) + except RuntimeError: + return None + activation = {**activation, "implicit": implicit} + _CONTEXT_CACHE = activation + _CONTEXT_CACHE_KEY = cache_key + return activation + + def activate( *, need: str = "", agent: Optional[str] = None, session_id: str = "", environ: Mapping[str, str] | None = None, + implicit: bool = False, ) -> dict[str, Any]: pm = _phase_memory() env = dict(environ or os.environ) if agent: env["WARDEN_AGENT_ID"] = agent kind = pm.resolve_session_kind(env) - return pm.activate_ops_warden_memory( + activation = pm.activate_ops_warden_memory( pm.OpsWardenMemoryStore.open(environ=env), session_kind=kind, need=need, session_id=session_id, ) + if implicit: + activation = {**activation, "implicit": True} + return activation def record_command_episode( @@ -91,13 +154,16 @@ def record_command_episode( diagnostic_codes=diagnostic_codes, metadata=metadata, ) - return pm.record_session_event(pm.OpsWardenMemoryStore.open(environ=env), event) + result = pm.record_session_event(pm.OpsWardenMemoryStore.open(environ=env), event) + if result.get("valid"): + _invalidate_context_cache() + return result def worker_activation_context(need: str = "", environ: Mapping[str, str] | None = None) -> dict[str, Any]: env = dict(environ or os.environ) env["WARDEN_SESSION_KIND"] = "warden.worker" - return activate(need=need, environ=env) + return ensure_memory_context(need=need, environ=env, implicit=True) or activate(need=need, environ=env, implicit=True) def stabilized_route_for_need(need: str, environ: Mapping[str, str] | None = None) -> Optional[dict[str, Any]]: diff --git a/src/warden/worker.py b/src/warden/worker.py index d605e79..e1caf01 100644 --- a/src/warden/worker.py +++ b/src/warden/worker.py @@ -602,7 +602,9 @@ def _memory_activation_for_message(message: dict) -> tuple[Optional[dict], str]: return None, "" query = str(message.get("subject", "") or message.get("body", "")) try: - activation = warden_memory.worker_activation_context(query) + activation = warden_memory.ensure_memory_context(need=query, implicit=True) + if activation is None: + activation = warden_memory.worker_activation_context(query) except RuntimeError: return None, "" from warden.memory import format_activation_summary diff --git a/tests/test_memory.py b/tests/test_memory.py index cedcb08..cb83247 100644 --- a/tests/test_memory.py +++ b/tests/test_memory.py @@ -126,4 +126,18 @@ def test_memory_disabled_skips_recording(monkeypatch) -> None: def test_default_store_path_uses_xdg(monkeypatch) -> None: monkeypatch.delenv("WARDEN_MEMORY_STORE", raising=False) - assert str(store_path()).endswith("warden/memory") \ No newline at end of file + assert str(store_path()).endswith("warden/memory") + + +def test_route_find_implicitly_activates_memory_without_explicit_command(tmp_path, monkeypatch) -> None: + from warden.memory import ensure_memory_context + + monkeypatch.setenv("WARDEN_MEMORY_STORE", str(tmp_path / "memory")) + monkeypatch.delenv("WARDEN_AGENT_ID", raising=False) + result = runner.invoke(app, ["route", "find", "ssh tunnel", "--json"]) + + assert result.exit_code == 0 + activation = ensure_memory_context(need="ssh tunnel", implicit=True) + assert activation is not None + assert activation.get("implicit") is True + assert status()["episode_count"] >= 1 \ No newline at end of file diff --git a/wiki/OpsWardenMemory.md b/wiki/OpsWardenMemory.md index 89adf62..2e86a16 100644 --- a/wiki/OpsWardenMemory.md +++ b/wiki/OpsWardenMemory.md @@ -19,13 +19,18 @@ ticks, coding agent sessions, and operator CLI use. | Coding agent | `export WARDEN_AGENT_ID=claude` (or `codex`, `grok`, future ids) | | Operator CLI | default `warden.operator` when `WARDEN_AGENT_ID` is unset | +## Default Behavior + +phase-memory is **on by default** (`WARDEN_MEMORY=1`). Every `warden` command +implicitly loads the canonical store before route/access/worker/sign work. You do +not need a separate activation command for normal use. + ## Agent Session Orientation -At the start of a Claude Code, Codex, or Grok session that will call warden: +For Claude Code, Codex, Grok, or future agents, set runtime identity once: ```bash export WARDEN_AGENT_ID=grok # or claude, codex -warden memory activate --json ``` Then use normal `warden route` / `warden access` commands. Episodes are recorded From 210f7eab68214562952c77bfbe3b425d74c21470 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 3 Jul 2026 00:54:21 +0200 Subject: [PATCH 07/10] Add Makefile targets to install and verify phase-memory with warden. install-all uses uv tool install --with-editable for sibling phase-memory. check-memory and verify-memory confirm warden can load experiential memory. --- AGENTS.md | 4 ++-- Makefile | 49 ++++++++++++++++++++++++++++++++++++++++++++ README.md | 27 ++++++++++++++++++++---- src/warden/memory.py | 3 ++- 4 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 Makefile diff --git a/AGENTS.md b/AGENTS.md index 42cd84b..f96fd05 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,8 +180,8 @@ Episodes are recorded automatically unless `WARDEN_MEMORY=0`. **Worker:** `warden worker run --brain llm` skips OpenRouter when stabilized routing memory matches. See `wiki/OpsWardenMemory.md`. -Requires `phase-memory` on `PYTHONPATH` or installed; contract in -`phase-memory/docs/ops-warden-memory-contract.md`. +Install bundled memory: `make install-all` then `make check-memory` from +`~/ops-warden`. Contract: `phase-memory/docs/ops-warden-memory-contract.md`. --- diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0a71247 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +# ops-warden development and install targets +ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +PHASE_MEMORY_REPO ?= $(abspath $(ROOT)/../phase-memory) + +.DEFAULT_GOAL := help + +.PHONY: help sync install install-warden install-memory install-all check-memory verify-memory test lint + +help: + @echo "ops-warden make targets" + @echo "" + @echo " make sync # uv sync (dev dependencies)" + @echo " make install # install warden CLI (SSH only, no phase-memory)" + @echo " make install-memory # reinstall warden with editable phase-memory" + @echo " make install-all # sync + warden + phase-memory (recommended)" + @echo " make check-memory # fail unless phase-memory is importable by warden" + @echo " make verify-memory # check-memory + warden memory status" + @echo " make test # unit tests" + @echo " make lint # ruff check" + @echo "" + @echo "Override phase-memory path: make install-all PHASE_MEMORY_REPO=/path/to/phase-memory" + +sync: + uv sync + +install-warden: + uv tool install . --force + +install-memory: + @test -d "$(PHASE_MEMORY_REPO)" || (echo "ERROR: phase-memory not found at $(PHASE_MEMORY_REPO). Clone it or set PHASE_MEMORY_REPO."; exit 1) + @test -f "$(PHASE_MEMORY_REPO)/pyproject.toml" || (echo "ERROR: $(PHASE_MEMORY_REPO) is not a phase-memory checkout."; exit 1) + uv tool install . --with-editable "$(PHASE_MEMORY_REPO)" --force + +install: install-warden + +install-all: sync install-memory + +check-memory: + @warden memory status --json >/dev/null 2>&1 || (echo "ERROR: phase-memory is not available to warden. Run: make install-memory"; exit 1) + @echo "phase-memory: ok (warden memory status succeeded)" + +verify-memory: check-memory + @warden memory status + +test: + PYTHONPATH=src:$(PHASE_MEMORY_REPO)/src uv run pytest + +lint: + uv run ruff check . \ No newline at end of file diff --git a/README.md b/README.md index d14c238..a65f11e 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,24 @@ Latest gap analysis: `history/2026-06-17-post-wp0007-reassessment.md`. ## Install +**Recommended** (warden + experiential memory for route/worker/agent sessions): + +```bash +make install-all +make verify-memory +``` + +SSH-only install (no phase-memory): + +```bash +make install +``` + +Manual equivalent: + ```bash uv sync -uv tool install . +uv tool install . --with-editable ../phase-memory --force ``` Or run without installing: @@ -22,6 +37,10 @@ Or run without installing: uv run warden --help ``` +phase-memory must be a sibling checkout at `../phase-memory` by default, or set +`PHASE_MEMORY_REPO` when running make. Opt out of memory at runtime with +`WARDEN_MEMORY=0`. + ## Quick start (local backend) ```bash @@ -58,10 +77,10 @@ Full role and examples: `wiki/AccessRouting.md`. ## Development ```bash -uv sync -uv run pytest # unit tests (integration excluded) +make install-all +make test +make lint uv run pytest -m integration # requires ssh-keygen in PATH -uv run ruff check . ``` ## Key paths diff --git a/src/warden/memory.py b/src/warden/memory.py index d8e5cfc..0296e75 100644 --- a/src/warden/memory.py +++ b/src/warden/memory.py @@ -7,7 +7,8 @@ from typing import Any, Mapping, Optional _PHASE_MEMORY_ERROR = ( "phase-memory is required for warden memory commands. " - "Install with: pip install phase-memory (or set PYTHONPATH to phase-memory/src)." + "From ~/ops-warden run: make install-memory (or make install-all). " + "Dev fallback: PYTHONPATH=../phase-memory/src" ) # In-process cache: implicit activation is default; no separate `warden memory activate` From 7d915a2eb510240ebb552726e9184cf9f91939d4 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 7 Jul 2026 00:52:39 +0200 Subject: [PATCH 08/10] Add routing catalog entry for reuse-surface hub write token. REUSE_SURFACE_TOKEN is custody of the Railiance01 K8s secret reuse-surface-env, not OpenBao. Workers blocked on hub register can now discover the lane via warden route find and obtain it through the documented kubectl handoff. --- registry/routing/catalog.yaml | 19 ++++++ tests/test_routing.py | 15 ++++ wiki/CredentialRouting.md | 1 + .../reuse-surface-hub-write-token.md | 68 +++++++++++++++++++ 4 files changed, 103 insertions(+) create mode 100644 wiki/playbooks/reuse-surface-hub-write-token.md diff --git a/registry/routing/catalog.yaml b/registry/routing/catalog.yaml index 00990ba..46bb815 100644 --- a/registry/routing/catalog.yaml +++ b/registry/routing/catalog.yaml @@ -214,6 +214,25 @@ entries: exec_capable: true lane: secret + - id: reuse-surface-hub-write-token + title: reuse-surface federation hub write bearer token + need_keywords: [reuse-surface, reuse_surface, hub, register, federation, write, token, bearer, REUSE_SURFACE_TOKEN, reuse.coulomb.social] + owner_repo: reuse-surface + subsystem: reuse-surface federation hub + warden_executes: false + wiki_ref: wiki/playbooks/reuse-surface-hub-write-token.md#worker-checklist + canon_ref: reuse-surface/specs/FederationHubAPI.md + reviewed: "2026-07-07" + status: active + # Concrete, owner-confirmed lane — REUSE-WP-0011 / RAILIANCE-WP-0007 (hub live + # 2026-06-15): token is the cluster Secret reuse-surface-env on Railiance01, + # not OpenBao. warden access proxies kubectl as the caller and never holds the value. + auth_method: "kubectl with Railiance01 kubeconfig (~/.kube/config-hosteurope)" + path_template: "reuse/reuse-surface-env" + fetch_command: "kubectl --kubeconfig ~/.kube/config-hosteurope get secret reuse-surface-env -n reuse -o jsonpath='{.data.REUSE_SURFACE_TOKEN}' | base64 -d" + exec_capable: true + lane: secret + - id: openrouter-llm-connect title: OpenRouter API key for llm-connect in activity-core need_keywords: [openrouter, llm, llm-connect, api, key, activity-core, gemini, provider, openrouter_api_key] diff --git a/tests/test_routing.py b/tests/test_routing.py index ad1f836..f75ed48 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -277,6 +277,21 @@ def test_find_openrouter_key(): assert matches and matches[0].id == "openrouter-llm-connect" +def test_find_reuse_surface_hub_write_token(): + catalog = load_catalog(_repo_catalog()) + matches = catalog.find("REUSE_SURFACE_TOKEN", limit=3) + assert matches and matches[0].id == "reuse-surface-hub-write-token" + + +def test_reuse_surface_hub_write_token_lane_is_resolvable(): + catalog = load_catalog(_repo_catalog()) + e = catalog.get("reuse-surface-hub-write-token") + assert e is not None and e.is_active and e.exec_capable + assert e.resolvable is True + assert e.owner_repo == "reuse-surface" + assert "reuse-surface-env" in e.fetch_command + + def test_find_object_storage_sts(): catalog = load_catalog(_repo_catalog()) matches = catalog.find("s3 temporary credentials", include_draft=True) diff --git a/wiki/CredentialRouting.md b/wiki/CredentialRouting.md index 6406c61..1f5f271 100644 --- a/wiki/CredentialRouting.md +++ b/wiki/CredentialRouting.md @@ -96,6 +96,7 @@ run the owner's tool as the caller and preserve owner custody. | `inter-hub-bootstrap-ssh` | "Inter-Hub bootstrap SSH envelope — attended vs unattended branches" | See `wiki/InterHubBootstrapAccessLane.md` | | `issue-core-ingestion-api-key` | "railiance-platform OpenBao KV + ESO deliver `ISSUE_CORE_API_KEY` — here is the path" | ESO consumes in-cluster; `warden access issue-core-ingestion-api-key --fetch ISSUE_CORE_API_KEY` as yourself | | `openrouter-llm-connect` | "railiance-platform OpenBao KV + ESO deliver `OPENROUTER_API_KEY` to activity-core" | ESO consumes in-cluster; `warden access openrouter-llm-connect --fetch OPENROUTER_API_KEY` as yourself | +| `reuse-surface-hub-write-token` | "reuse-surface hub write bearer — K8s secret `reuse-surface-env` on Railiance01" | `kubectl` from `~/.kube/config-hosteurope`, or `warden access reuse-surface-hub-write-token --fetch` as yourself | Promotion criteria: `wiki/playbooks/catalog-lane-promotion.md`. diff --git a/wiki/playbooks/reuse-surface-hub-write-token.md b/wiki/playbooks/reuse-surface-hub-write-token.md new file mode 100644 index 0000000..9df9711 --- /dev/null +++ b/wiki/playbooks/reuse-surface-hub-write-token.md @@ -0,0 +1,68 @@ +# reuse-surface Hub Write Token + +Date: 2026-07-07 +Catalog: `reuse-surface-hub-write-token` (status `active`, `resolvable: true`) +Owner: `reuse-surface` (service) · deploy custody `railiance-apps` (K8s secret) + +Bearer token for authenticated writes to the production federation hub at +`https://reuse.coulomb.social` (`POST /v1/repos`, `reuse-surface hub register`). +ops-warden **does not hold this token** — it is a pointer lane to the cluster +secret that backs the hub Deployment. + +--- + +## Owner-confirmed lane (no placeholders) + +| Field | Value | +| --- | --- | +| Cluster | Railiance01 (`92.205.62.239`) | +| Namespace | `reuse` | +| Secret | `reuse-surface-env` | +| Field | `REUSE_SURFACE_TOKEN` | +| Kubeconfig | `~/.kube/config-hosteurope` | +| Deploy runbook | `railiance-apps/docs/reuse-surface-on-railiance01.md` | +| Hub API spec | `reuse-surface/specs/FederationHubAPI.md` | + +This is **not** an OpenBao KV path. The token is generated at deploy time and +stored only in the Kubernetes Secret consumed by the `reuse-surface` workload. + +--- + +## Worker checklist + +1. **Confirm kubeconfig reachability** (you act as yourself; ops-warden adds no credential): + ```bash + kubectl --kubeconfig ~/.kube/config-hosteurope get secret reuse-surface-env -n reuse + ``` + +2. **Export for a shell session** (value streams to your terminal — never paste into chat): + ```bash + export REUSE_SURFACE_URL=https://reuse.coulomb.social + export REUSE_SURFACE_TOKEN=$( + kubectl --kubeconfig ~/.kube/config-hosteurope get secret reuse-surface-env -n reuse \ + -o jsonpath='{.data.REUSE_SURFACE_TOKEN}' | base64 -d + ) + ``` + +3. **Or proxy via warden access** (same kubectl command, audited metadata only): + ```bash + warden route show reuse-surface-hub-write-token --json + warden access reuse-surface-hub-write-token --no-policy --fetch + ``` + +4. **Register a repo** after publish-check passes: + ```bash + reuse-surface hub status + reuse-surface hub register --repo \ + --url \ + --domain + ``` + +5. **Verify federated index** picked up the new source: + ```bash + curl -fsS "$REUSE_SURFACE_URL/v1/federated" | jq '.sources | map(.repo) | index("")' + ``` + +Never commit the token, paste it into State Hub or agent chat, or store it in a +workplan. Rotation: regenerate the secret on-cluster and roll the Deployment +(`reuse-surface/docs/deploy/reuse-kubernetes.md`). \ No newline at end of file From be3b4a2a86c0b62ad66bf36d293703af72f10399 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 7 Jul 2026 16:42:26 +0200 Subject: [PATCH 09/10] Fix warden access proxy for catalog fetch commands with shell pipes. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_fetch_command used shlex.split, which treated `|` as a literal argument — breaking reuse-surface-hub-write-token (kubectl | base64 -d). Piped commands now run via shell=True with inherited stdio for --fetch. --- src/warden/cli.py | 6 +-- src/warden/proxy.py | 93 +++++++++++++++++++++++++++++++++++-------- tests/test_doubles.py | 4 +- tests/test_proxy.py | 61 +++++++++++++++++++++++++--- 4 files changed, 137 insertions(+), 27 deletions(-) diff --git a/src/warden/cli.py b/src/warden/cli.py index eb1c120..de52646 100644 --- a/src/warden/cli.py +++ b/src/warden/cli.py @@ -964,7 +964,7 @@ def _access_proxy( err.print("[yellow]Proxying ungated[/yellow] (--no-policy; gate not enforced).") try: - argv = resolve_fetch_command(entry, domain=domain, field=field, path=path) + resolved = resolve_fetch_command(entry, domain=domain, field=field, path=path) except ProxyError as e: err.print(f"[red]{e}[/red]") raise typer.Exit(2) @@ -979,9 +979,9 @@ def _access_proxy( if not child_argv: err.print("[red]--exec needs a command after `--`[/red], e.g. `-- npm publish`.") raise typer.Exit(2) - rc = proxy_exec(argv, env_var=field or "", child_argv=child_argv) + rc = proxy_exec(resolved, env_var=field or "", child_argv=child_argv) else: - rc = proxy_fetch(argv) + rc = proxy_fetch(resolved) except ProxyError as e: err.print(f"[red]{e}[/red]") raise typer.Exit(5) diff --git a/src/warden/proxy.py b/src/warden/proxy.py index 65d1fff..cfa422a 100644 --- a/src/warden/proxy.py +++ b/src/warden/proxy.py @@ -25,6 +25,7 @@ import os import re import shlex import subprocess +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import List, Optional @@ -34,18 +35,43 @@ from warden.routing.models import RouteEntry _PLACEHOLDER = re.compile(r"<[^>]+>") +@dataclass(frozen=True) +class ResolvedFetch: + """A catalog fetch command ready to run — either argv or a shell pipeline.""" + + argv: Optional[List[str]] = None + shell_cmd: Optional[str] = None + + def __post_init__(self) -> None: + if bool(self.argv) == bool(self.shell_cmd): + raise ValueError("exactly one of argv or shell_cmd must be set") + + class ProxyError(Exception): """Raised when a proxy fetch cannot be performed safely.""" +def _has_shell_pipe(cmd: str) -> bool: + """True when ``cmd`` contains an unquoted shell pipe operator.""" + in_single = in_double = False + for ch in cmd: + if ch == "'" and not in_double: + in_single = not in_single + elif ch == '"' and not in_single: + in_double = not in_double + elif ch == "|" and not in_single and not in_double: + return True + return False + + def resolve_fetch_command( entry: RouteEntry, *, domain: Optional[str] = None, field: Optional[str] = None, path: Optional[str] = None, -) -> List[str]: - """Build the concrete argv for an entry's fetch, or raise if under-specified. +) -> ResolvedFetch: + """Build the concrete fetch command for an entry, or raise if under-specified. Starts from the catalog ``fetch_command`` template (with ```` inlined), substitutes ````/```` and an explicit ``--path`` override, @@ -81,7 +107,10 @@ def resolve_fetch_command( "Supply --domain/--field (and --path for owner-side names) — warden will not " "guess owner-confirmed resource names." ) - return shlex.split(cmd) + if _has_shell_pipe(cmd): + # Catalog-reviewed pipelines (e.g. kubectl | base64 -d) need a shell. + return ResolvedFetch(shell_cmd=cmd) + return ResolvedFetch(argv=shlex.split(cmd)) def caller_auth_present(token_envs: tuple[str, ...] = ("VAULT_TOKEN", "BAO_TOKEN")) -> bool: @@ -146,25 +175,37 @@ def _caller_env() -> dict: return dict(os.environ) -def proxy_fetch(argv: List[str]) -> int: +def proxy_fetch(resolved: ResolvedFetch) -> int: """Run the owner's tool, streaming its output straight to the caller. stdout/stderr are **inherited** (``None``), never piped — the secret value flows subsystem → caller and is never read into warden's memory, buffer, or log (G2). Returns the tool's exit code. """ - completed = subprocess.run( # noqa: S603 — argv is shlex-split from a validated template - argv, - stdout=None, - stderr=None, - stdin=None, - env=_caller_env(), - check=False, - ) + env = _caller_env() + if resolved.argv is not None: + completed = subprocess.run( # noqa: S603 — argv is shlex-split from a validated template + resolved.argv, + stdout=None, + stderr=None, + stdin=None, + env=env, + check=False, + ) + else: + completed = subprocess.run( # noqa: S602 — shell_cmd is catalog-reviewed, not user input + resolved.shell_cmd, + shell=True, + stdout=None, + stderr=None, + stdin=None, + env=env, + check=False, + ) return completed.returncode -def proxy_exec(argv: List[str], *, env_var: str, child_argv: List[str]) -> int: +def proxy_exec(resolved: ResolvedFetch, *, env_var: str, child_argv: List[str]) -> int: """Fetch the value and inject it into a child command's environment only. The value transits warden's memory here (the accepted proxy tradeoff for `--exec`) @@ -175,10 +216,28 @@ def proxy_exec(argv: List[str], *, env_var: str, child_argv: List[str]) -> int: if not env_var: raise ProxyError("--exec requires --field (the env var name to inject), e.g. NPM_AUTH_TOKEN") - fetched = subprocess.run( # noqa: S603 - argv, stdout=subprocess.PIPE, stderr=None, stdin=None, - env=_caller_env(), check=False, text=True, - ) + env = _caller_env() + if resolved.argv is not None: + fetched = subprocess.run( # noqa: S603 + resolved.argv, + stdout=subprocess.PIPE, + stderr=None, + stdin=None, + env=env, + check=False, + text=True, + ) + else: + fetched = subprocess.run( # noqa: S602 + resolved.shell_cmd, + shell=True, + stdout=subprocess.PIPE, + stderr=None, + stdin=None, + env=env, + check=False, + text=True, + ) if fetched.returncode != 0: raise ProxyError( f"fetch failed (exit {fetched.returncode}) — check caller auth and the path." diff --git a/tests/test_doubles.py b/tests/test_doubles.py index 9b264fc..6a77ecc 100644 --- a/tests/test_doubles.py +++ b/tests/test_doubles.py @@ -107,8 +107,8 @@ def test_proxy_fetch_runs_fully_offline_against_double(tmp_path): fetch_command="bao kv get -field= ", exec_capable=True, ) - argv = resolve_fetch_command(entry, field="API_KEY", path="platform/x/y/z") + resolved = resolve_fetch_command(entry, field="API_KEY", path="platform/x/y/z") env = dict(os.environ, PATH=doubles_path_prepended(tmp_path)) # proxy_fetch inherits stdout; run it in a child so we can capture the stream. - result = subprocess.run(argv, capture_output=True, text=True, env=env, check=True) + result = subprocess.run(resolved.argv, capture_output=True, text=True, env=env, check=True) assert result.stdout.strip().startswith(SYNTHETIC_PREFIX) diff --git a/tests/test_proxy.py b/tests/test_proxy.py index 8c86ef0..5909fc3 100644 --- a/tests/test_proxy.py +++ b/tests/test_proxy.py @@ -11,6 +11,7 @@ from typer.testing import CliRunner from warden.cli import app from warden.proxy import ( ProxyError, + ResolvedFetch, caller_auth_present, proxy_exec, proxy_fetch, @@ -45,10 +46,11 @@ def _entry(**over) -> RouteEntry: # --- resolve_fetch_command ------------------------------------------------- def test_resolve_builds_argv(): - argv = resolve_fetch_command( + resolved = resolve_fetch_command( _entry(), domain="coulomb_social", field="NPM_AUTH_TOKEN", path="platform/x/y/z" ) - assert argv == ["bao", "kv", "get", "-field=NPM_AUTH_TOKEN", "platform/x/y/z"] + assert resolved.argv == ["bao", "kv", "get", "-field=NPM_AUTH_TOKEN", "platform/x/y/z"] + assert resolved.shell_cmd is None def test_resolve_refuses_unresolved_placeholder(): @@ -62,6 +64,18 @@ def test_resolve_refuses_non_exec_capable(): resolve_fetch_command(_entry(exec_capable=False, fetch_command=None)) +def test_resolve_piped_fetch_uses_shell_cmd(): + from warden.routing import load_catalog + + catalog = load_catalog(Path(__file__).resolve().parents[1] / "registry" / "routing" / "catalog.yaml") + entry = catalog.get("reuse-surface-hub-write-token") + resolved = resolve_fetch_command(entry) + assert resolved.argv is None + assert resolved.shell_cmd is not None + assert "| base64 -d" in resolved.shell_cmd + assert "reuse-surface-env" in resolved.shell_cmd + + # --- G2: transit-only fetch (inherited stdout) ----------------------------- def test_proxy_fetch_inherits_stdout_never_pipes(monkeypatch): @@ -72,13 +86,27 @@ def test_proxy_fetch_inherits_stdout_never_pipes(monkeypatch): return subprocess.CompletedProcess(argv, 0) monkeypatch.setattr("warden.proxy.subprocess.run", fake_run) - rc = proxy_fetch(["bao", "kv", "get", "x"]) + rc = proxy_fetch(ResolvedFetch(argv=["bao", "kv", "get", "x"])) assert rc == 0 # The value must never enter warden's memory — stdout is inherited, not piped. assert calls["stdout"] is None assert calls.get("stderr") is None +def test_proxy_fetch_shell_pipeline_inherits_stdio(monkeypatch): + calls = {} + + def fake_run(cmd, **kw): + calls.update(kw) + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr("warden.proxy.subprocess.run", fake_run) + rc = proxy_fetch(ResolvedFetch(shell_cmd="kubectl get x | base64 -d")) + assert rc == 0 + assert calls["shell"] is True + assert calls["stdout"] is None + + # --- G1 + inject: exec injects value into child env, adds no warden token --- def test_proxy_exec_injects_only_into_child_env(monkeypatch): @@ -92,7 +120,11 @@ def test_proxy_exec_injects_only_into_child_env(monkeypatch): monkeypatch.setattr("warden.proxy.subprocess.run", fake_run) monkeypatch.delenv("NPM_AUTH_TOKEN", raising=False) - rc = proxy_exec(["bao", "kv", "get", "x"], env_var="NPM_AUTH_TOKEN", child_argv=["true"]) + rc = proxy_exec( + ResolvedFetch(argv=["bao", "kv", "get", "x"]), + env_var="NPM_AUTH_TOKEN", + child_argv=["true"], + ) assert rc == 0 # Value injected into child env (trailing newline stripped)… assert seen_env["NPM_AUTH_TOKEN"] == "SECRETVAL" @@ -100,9 +132,28 @@ def test_proxy_exec_injects_only_into_child_env(monkeypatch): assert "VAULT_TOKEN" not in {k for k in seen_env if k not in __import__("os").environ} +def test_proxy_exec_shell_pipeline_captures_stdout(monkeypatch): + seen_env = {} + + def fake_run(cmd, **kw): + if kw.get("shell"): + return subprocess.CompletedProcess(cmd, 0, stdout="PIPEVAL\n") + seen_env.update(kw["env"]) + return subprocess.CompletedProcess(cmd, 0) + + monkeypatch.setattr("warden.proxy.subprocess.run", fake_run) + rc = proxy_exec( + ResolvedFetch(shell_cmd="kubectl get x | base64 -d"), + env_var="REUSE_SURFACE_TOKEN", + child_argv=["true"], + ) + assert rc == 0 + assert seen_env["REUSE_SURFACE_TOKEN"] == "PIPEVAL" + + def test_proxy_exec_requires_env_var(): with pytest.raises(ProxyError, match="requires --field"): - proxy_exec(["bao"], env_var="", child_argv=["true"]) + proxy_exec(ResolvedFetch(argv=["bao"]), env_var="", child_argv=["true"]) # --- G1 caller auth detection ---------------------------------------------- From 885385adf32037efa8e789083df1534ff693457d Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 7 Jul 2026 16:51:06 +0200 Subject: [PATCH 10/10] Roll out proxy pipe fix as v0.1.1. Bump version so uv tool install picks up be3b4a2 without a manual cache clean. Document workstation upgrade steps in ADHOC-2026-07-07. --- pyproject.toml | 2 +- workplans/ADHOC-2026-07-07.md | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 workplans/ADHOC-2026-07-07.md diff --git a/pyproject.toml b/pyproject.toml index 3ca400f..bc62158 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ops-warden" -version = "0.1.0" +version = "0.1.1" description = "SSH CA and certificate lifecycle manager for ops actors" requires-python = ">=3.11" dependencies = [ diff --git a/workplans/ADHOC-2026-07-07.md b/workplans/ADHOC-2026-07-07.md new file mode 100644 index 0000000..0fc007d --- /dev/null +++ b/workplans/ADHOC-2026-07-07.md @@ -0,0 +1,48 @@ +--- +id: ADHOC-2026-07-07 +type: workplan +title: "Ad Hoc Tasks — 2026-07-07" +domain: infotech +repo: ops-warden +status: finished +owner: grok +topic_slug: custodian +created: "2026-07-07" +updated: "2026-07-07" +--- + +# Ad Hoc Tasks — 2026-07-07 + +### T01 — Roll out proxy pipe fix (be3b4a2) + +```task +id: ADHOC-2026-07-07-T01 +status: done +priority: high +``` + +`warden access` failed on `reuse-surface-hub-write-token` because `shlex.split` +treated `kubectl … | base64 -d` as literal argv. Fixed in `be3b4a2`; rollout +requires refreshing the installed CLI (cached `0.1.0` wheel does not pick up +source changes). + +- [x] Code fix merged and pushed (`be3b4a2`) +- [x] Version bumped `0.1.0` → `0.1.1` so `uv tool install` invalidates stale wheels +- [x] Workstation rollout: `git pull && make install-all` (after cache clean if needed) +- [x] Smoke: `warden access reuse-surface-hub-write-token --no-policy --fetch` — no + `unknown shorthand flag: 'd'`; token fetched via shell pipeline + +**Operator upgrade on any machine using `warden access`:** + +```bash +cd ~/ops-warden && git pull --ff-only +make install-all +warden route show reuse-surface-hub-write-token --json +``` + +If `warden` still lacks `ResolvedFetch` / pipe handling after install: + +```bash +uv cache clean ops-warden +uv tool install . --with-editable ../phase-memory --reinstall --force +``` \ No newline at end of file