From a852d3f1ff8fe0fc4278fd6a84af6d05e3fb4e55 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sun, 28 Jun 2026 12:28:45 +0200 Subject: [PATCH] feat(mvp): working secrets-engine CLI for the whynot-design npm publish lane Implements SECRETS-WP-0002 end to end as a uv-managed Python package: - catalog: non-secret lane registry + strict validator (build/test/prod) - stage roles + OpenBao ACL policies; guards refuse wildcards, sys/, identity/, admin names, and cross-stage paths before any backend call - plan/apply: dry-run-first, idempotent policy + approle apply, decision-gated - decisions: State Hub lookup with local-fixture fallback; non-secret evidence to JSONL + hub progress, scrubbed of any value - provision/verify: mode-0600 file import + generated test values; positive/ negative checks that never print the value - exec delivery: `exec --catalog ... -- npm publish` injects the token via a temp .npmrc for the child only, cleaned up on exit/failure/interrupt - ops-warden routing contract + hardening backlog docs - 34 tests incl. live OpenBao integration; scripts/demo-e2e.sh runs the full chain against a throwaway bao dev server Co-Authored-By: Claude Opus 4.8 --- .claude/rules/stack-and-commands.md | 35 +- .decisions/whynot-design-npm-publish.yaml | 15 + .gitignore | 5 + .repo-classification.yaml | 18 +- AGENTS.md | 49 ++- INTENT.md | 150 ++++++++ ProductRequirementsDocument.md | 301 ++++++++++++++++ README.md | 74 +++- SCOPE.md | 46 ++- catalog/example-build-test-token.yaml | 43 +++ catalog/whynot-design-npm-publish.yaml | 51 +++ docs/cli.md | 78 ++++ docs/hardening-backlog.md | 58 +++ docs/netkingdom-security-infrastructure.md | 22 ++ docs/openbao-stage-roles.md | 79 ++++ docs/ops-warden-routing-contract.md | 62 ++++ policies/secrets-engine-build.hcl | 36 ++ policies/secrets-engine-prod.hcl | 47 +++ policies/secrets-engine-test.hcl | 33 ++ pyproject.toml | 30 ++ scripts/demo-e2e.sh | 91 +++++ src/secrets_engine/__init__.py | 12 + src/secrets_engine/__main__.py | 4 + src/secrets_engine/apply.py | 69 ++++ src/secrets_engine/catalog.py | 171 +++++++++ src/secrets_engine/cli.py | 339 ++++++++++++++++++ src/secrets_engine/config.py | 43 +++ src/secrets_engine/decisions.py | 115 ++++++ src/secrets_engine/errors.py | 54 +++ src/secrets_engine/evidence.py | 116 ++++++ src/secrets_engine/exec_delivery.py | 152 ++++++++ src/secrets_engine/openbao.py | 232 ++++++++++++ src/secrets_engine/plan.py | 90 +++++ src/secrets_engine/provision.py | 75 ++++ src/secrets_engine/redact.py | 42 +++ src/secrets_engine/roles.py | 138 +++++++ src/secrets_engine/routing.py | 99 +++++ src/secrets_engine/verify.py | 81 +++++ tests/__init__.py | 0 tests/test_catalog.py | 77 ++++ tests/test_decisions.py | 52 +++ tests/test_guards.py | 67 ++++ tests/test_integration_bao.py | 103 ++++++ tests/test_redact_evidence.py | 41 +++ ...O-WP-0002-template-validation-checklist.md | 34 -- ... => SECRETS-WP-0001-statehub-bootstrap.md} | 45 +-- workplans/SECRETS-WP-0002-bootstrap.md | 291 +++++++++++++++ 47 files changed, 3743 insertions(+), 122 deletions(-) create mode 100644 .decisions/whynot-design-npm-publish.yaml create mode 100644 INTENT.md create mode 100644 ProductRequirementsDocument.md create mode 100644 catalog/example-build-test-token.yaml create mode 100644 catalog/whynot-design-npm-publish.yaml create mode 100644 docs/cli.md create mode 100644 docs/hardening-backlog.md create mode 100644 docs/netkingdom-security-infrastructure.md create mode 100644 docs/openbao-stage-roles.md create mode 100644 docs/ops-warden-routing-contract.md create mode 100644 policies/secrets-engine-build.hcl create mode 100644 policies/secrets-engine-prod.hcl create mode 100644 policies/secrets-engine-test.hcl create mode 100644 pyproject.toml create mode 100755 scripts/demo-e2e.sh create mode 100644 src/secrets_engine/__init__.py create mode 100644 src/secrets_engine/__main__.py create mode 100644 src/secrets_engine/apply.py create mode 100644 src/secrets_engine/catalog.py create mode 100644 src/secrets_engine/cli.py create mode 100644 src/secrets_engine/config.py create mode 100644 src/secrets_engine/decisions.py create mode 100644 src/secrets_engine/errors.py create mode 100644 src/secrets_engine/evidence.py create mode 100644 src/secrets_engine/exec_delivery.py create mode 100644 src/secrets_engine/openbao.py create mode 100644 src/secrets_engine/plan.py create mode 100644 src/secrets_engine/provision.py create mode 100644 src/secrets_engine/redact.py create mode 100644 src/secrets_engine/roles.py create mode 100644 src/secrets_engine/routing.py create mode 100644 src/secrets_engine/verify.py create mode 100644 tests/__init__.py create mode 100644 tests/test_catalog.py create mode 100644 tests/test_decisions.py create mode 100644 tests/test_guards.py create mode 100644 tests/test_integration_bao.py create mode 100644 tests/test_redact_evidence.py delete mode 100644 workplans/REPO-WP-0002-template-validation-checklist.md rename workplans/{REPO-WP-0001-statehub-bootstrap.md => SECRETS-WP-0001-statehub-bootstrap.md} (50%) create mode 100644 workplans/SECRETS-WP-0002-bootstrap.md diff --git a/.claude/rules/stack-and-commands.md b/.claude/rules/stack-and-commands.md index ca469c0..0f47069 100644 --- a/.claude/rules/stack-and-commands.md +++ b/.claude/rules/stack-and-commands.md @@ -1,27 +1,34 @@ ## Stack -- **Language:** Markdown-first registry and planning repo (no application runtime yet) -- **Key deps:** State Hub ADR-001 workplans, `registry/indexes/capabilities.yaml` +- **Language:** Python 3.11+ CLI (`src/secrets_engine/`), `uv`-managed, hatchling build +- **Backend:** OpenBao, reached only via the `bao` CLI adapter (`openbao.py`) +- **Key deps:** PyYAML (runtime), pytest (dev); State Hub for decisions/evidence +- **Entry point:** `secrets-engine` (console script → `secrets_engine.cli:main`) ## Dev Commands ```bash +# Install (editable, with dev extras) +uv venv && uv pip install -e ".[dev]" +source .venv/bin/activate + +# Run the CLI +secrets-engine --version +secrets-engine catalog list + +# Test (unit + live OpenBao integration; integration auto-skips without `bao`) +SECRETS_ENGINE_HUB_URL="" python -m pytest -q + +# Full pilot chain live against a throwaway OpenBao dev server +SECRETS_ENGINE_HUB_URL="" bash scripts/demo-e2e.sh + # Orient (offline-safe) cat .custodian-brief.md -cat README.md -cat SCOPE.md ls workplans/ -# Consumer bootstrap docs -cat docs/statehub-register.md -cat docs/template-validation-checklist.md +# After workplan edits — from ~/state-hub +make fix-consistency REPO=secrets-engine -# After workplan or registry edits — from ~/state-hub -make fix-consistency REPO=repo-seed - -# Validate registry entries (from reuse-surface checkout) -reuse-surface validate --root . - -# Sanity-check markdown / registry edits +# Sanity-check edits git diff --check ``` diff --git a/.decisions/whynot-design-npm-publish.yaml b/.decisions/whynot-design-npm-publish.yaml new file mode 100644 index 0000000..19c169e --- /dev/null +++ b/.decisions/whynot-design-npm-publish.yaml @@ -0,0 +1,15 @@ +# Local approval fixture for the whynot-design npm publish lane. +# +# Stand-in for the canonical State Hub decision object, which does not yet exist +# (see PRD open question). When a real decision is recorded in State Hub, set the +# catalog's approval.decision_ref to that decision's UUID and this fixture becomes +# unnecessary. NON-SECRET: contains no token value. +id: whynot-design-npm-publish +title: "Approve whynot-design npm publish lane (prod)" +status: resolved # resolved => approved for apply +superseded_by: null +decided_by: "Tegwick" +review_url: "" +rationale: >- + Pilot lane for secrets-engine MVP. Production npm automation token for + whynot-design, delivered exec-time only via temporary npm config. diff --git a/.gitignore b/.gitignore index 36b13f1..aa06dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -174,3 +174,8 @@ cython_debug/ # PyPI configuration file .pypirc + +# secrets-engine runtime artifacts +.evidence/ +*.token +bootstrap/ diff --git a/.repo-classification.yaml b/.repo-classification.yaml index ea7b5f4..18eb340 100644 --- a/.repo-classification.yaml +++ b/.repo-classification.yaml @@ -1,18 +1,26 @@ repo_classification: standard: Repo Classification Standard version: '1.0' - classified_at: '2026-06-22' - classified_by: agent + classified_at: '2026-06-28' + classified_by: codex category: tooling domain: infotech - secondary_domains: [] + secondary_domains: + - financials capability_tags: - platform - configuration - - documentation + - orchestration + - governance + - policy business_stake: - technology + - operations + - automation - execution business_mechanics: - operation - notes: Git template for bootstrapping coulomb projects. + - control + - coordination + - adaptation + notes: OpenBao-backed secrets workflow and automation layer for approved build, test, and production secret lifecycle work. diff --git a/AGENTS.md b/AGENTS.md index d61b5dd..2402bae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,13 +1,13 @@ -# Repo Seed — Agent Instructions +# Secrets Engine - Agent Instructions ## Repo Identity -**Purpose:** Git repository template to bootstrap coulomb projects. +**Purpose:** NetKingdom secrets workflow and automation layer backed by OpenBao. **Domain:** infotech -**Repo slug:** repo-seed +**Repo slug:** secrets-engine **Topic ID:** `cee7bedf-2b48-46ef-8601-006474f2ad7a` -**Workplan prefix:** `REPO-WP-` +**Workplan prefix:** `SECRETS-WP-` --- @@ -32,7 +32,7 @@ curl -s "http://127.0.0.1:8000/workstreams/?topic_id=cee7bedf-2b48-46ef-8601-006 | python3 -m json.tool # Check inbox -curl -s "http://127.0.0.1:8000/messages/?to_agent=repo-seed&unread_only=true" \ +curl -s "http://127.0.0.1:8000/messages/?to_agent=secrets-engine&unread_only=true" \ | python3 -m json.tool ``` @@ -81,7 +81,7 @@ curl -s -X PATCH "http://127.0.0.1:8000/tasks/" \ **Start:** 1. `cat .custodian-brief.md` — domain goal and open workstreams (offline-safe) -2. Check inbox: `GET /messages/?to_agent=repo-seed&unread_only=true`; mark read +2. Check inbox: `GET /messages/?to_agent=secrets-engine&unread_only=true`; mark read 3. Scan workplans: `ls workplans/` — note `status: ready`, `active`, or `blocked` files and open tasks 4. Check human-needed tasks: `GET /tasks/?needs_human=true` @@ -95,7 +95,7 @@ curl -s -X PATCH "http://127.0.0.1:8000/tasks/" \ 3. Note for the custodian operator: after workplan file changes, run from `~/state-hub`: ```bash - make fix-consistency REPO=repo-seed + make fix-consistency REPO=secrets-engine ``` This syncs task status from files into the hub DB. @@ -122,7 +122,7 @@ Requires the `warden` CLI from `~/ops-warden` (`uv tool install .` or `uv run wa | Agent runtime | How to orient | | --- | --- | -| **Codex / Grok** (shell, HTTP State Hub) | `warden route` commands above; inbox `to_agent=repo-seed` is for coordination, not secret vending | +| **Codex / Grok** (shell, HTTP State Hub) | `warden route` commands above; inbox `to_agent=secrets-engine` is for coordination, not secret vending | | **Claude Code** (MCP when available) | `get_domain_summary("custodian")` for workstreams; **still** use `warden route` for credential ownership | | **llm-connect** (inference service) | Never put secret retrieval in prompts; route custody to OpenBao/operator paths surfaced by `warden route` | @@ -131,7 +131,7 @@ Requires the `warden` CLI from `~/ops-warden` (`uv tool install .` or `uv run wa | I need… | Owner | ops-warden executes? | | --- | --- | --- | | SSH cert (`adm`/`agt`/`atm`) | ops-warden | **Yes** — `warden sign` | -| API key, DB password, provider token | OpenBao (`railiance-platform`) | No — route only | +| API key, DB password, provider token | secrets-engine (OpenBao-backed) | No - route only | | Login / OIDC / MFA | key-cape / Keycloak | No — route only | | Authorization decision | flex-auth | No — route only | | activity-core → issue-core emission | activity-core + issue-core | No — `warden route show activity-core-issue-sink` | @@ -153,6 +153,25 @@ get wrong. **Canon:** `~/ops-warden/wiki/CredentialRouting.md` · catalog `~/ops-warden/registry/routing/catalog.yaml` + +## Repo-Specific Security Boundary + +Read `docs/netkingdom-security-infrastructure.md` before changing secret +catalogs, OpenBao policies, auth roles, delivery modes, or ops-warden routing. + +Core rules: +- OpenBao is the custody, policy, lease, and audit backend. +- secrets-engine owns workflow, catalog, decision checks, safe delivery, and + non-secret evidence. +- flex-auth decides authorization; user-engine/key-cape own identity and claims. +- ops-warden routes non-SSH credential requests here and must not vend secrets. +- ops-bridge may consume scoped delivery for remote execution but must not store + secret material. +- info-tech-canon is the source for canonical terminology and stage/policy + concepts as they harden. +- Never write raw secret values to Git, State Hub, chat, prompts, workplans, or + normal logs. + @@ -163,10 +182,10 @@ get wrong. Work items originate as files in this repo — not in the hub. The hub is a read/cache/index layer that rebuilds from files. -**File location:** `workplans/REPO-WP-NNNN-.md` +**File location:** `workplans/SECRETS-WP-NNNN-.md` **Archived location:** finished workplans may move to -`workplans/archived/YYMMDD-REPO-WP-NNNN-.md`. The `YYMMDD` prefix is +`workplans/archived/YYMMDD-SECRETS-WP-NNNN-.md`. The `YYMMDD` prefix is the completion/archive date; the frontmatter `id` does not change. **Ad Hoc Tasks:** small opportunistic fixes discovered during a session use @@ -178,11 +197,11 @@ anything needing analysis, design, approval, dependencies, or multiple phases. ```yaml --- -id: REPO-WP-NNNN +id: SECRETS-WP-NNNN type: workplan title: "..." domain: infotech -repo: repo-seed +repo: secrets-engine status: proposed | ready | active | blocked | backlog | finished | archived owner: codex topic_slug: ... @@ -202,7 +221,7 @@ derived health labels, not frontmatter statuses. ## Task Title ` ` `task -id: REPO-WP-NNNN-T01 +id: SECRETS-WP-NNNN-T01 status: wait | todo | progress | done | cancel priority: high | medium | low state_hub_task_id: "" # written by fix-consistency — do not edit @@ -215,5 +234,5 @@ Status progression: `todo` → `progress` → `done`; use `wait` for waiting/blo To create a new workplan: 1. Write the file following the format above -2. Notify the custodian operator to run `make fix-consistency REPO=repo-seed` +2. Notify the custodian operator to run `make fix-consistency REPO=secrets-engine` (or send a message to the hub agent via `POST /messages/`) diff --git a/INTENT.md b/INTENT.md new file mode 100644 index 0000000..791ecb2 --- /dev/null +++ b/INTENT.md @@ -0,0 +1,150 @@ +# INTENT + +> This file captures why this repository exists, the direction it is moving +> toward, and the kind of system it is meant to become. It is intentionally +> aspirational and stable, not a description of current implementation. + +## One-liner + +**secrets-engine is the secure interaction and automation layer for approved +secret custody, delivery, and lifecycle work across build, test, and production, +with OpenBao as the initial enforcement backend.** + +## Why This Exists + +Secret management is too important to be handled through scattered manual UI +steps, copied commands, chat handoffs, and ad hoc token files. + +OpenBao is the right place to enforce custody, policy, lease, and audit. It is +not, by itself, the right daily interaction surface for operators, agents, +workloads, and approval workflows. Direct use of OpenBao UI and CLI surfaces has +already produced avoidable friction: + +- approved decisions still require a human to bridge low-level OpenBao details; +- agents and operators hit interface mismatches before reaching the security + work they were trying to complete; +- build, test, and production need different levels of privilege and ceremony; +- secret values must never move through chat, Git, prompts, State Hub messages, + or workplans; +- ops-warden needs a reliable front door for credential use, not direct secret + custody responsibility. + +This repository exists to capture that complexity once, behind a stable and +auditable interface. + +## The Mission + +To provide a decision-aware secrets workflow that can: + +- translate approved requests into narrowly scoped OpenBao changes; +- operate with distinct build, test, and production privilege layers; +- deliver secrets to commands and workloads without printing or storing raw + values in coordination systems; +- record enough non-secret evidence for review, audit, and troubleshooting; +- make routine secure work fast enough that operators and agents actually use + the secure path. + +## Core Principles + +### 1. Decision First + +Secret establishment, access, rotation, and deactivation start from an explicit +request and decision. The engine may automate the work after approval; it does +not silently create new secret authority. + +### 2. OpenBao Enforces, secrets-engine Orchestrates + +OpenBao remains the vault, policy, lease, and audit backend. secrets-engine owns +the workflow, catalog, validation, delivery, and operator/agent interface. + +### 3. Stage-aware Privilege + +Build, test, and production are separate security contexts. Each has its own +OpenBao role, policy boundary, approval expectation, TTL limits, and delivery +rules. + +### 4. No Secret Values in Coordination Surfaces + +Git, State Hub, workplans, chat, prompts, issue comments, and normal logs carry +only non-secret metadata. Raw values are delivered through OpenBao, wrapped +responses, local exec-time injection, or short-lived bootstrap files with strict +permissions. + +### 5. Least Friction Without Broad Power + +The common path should be one clear command or review action. That ease must not +require handing platform-root or platform-admin power to routine automation. + +### 6. Bootstrap Honestly, Then Harden + +During infrastructure setup it is acceptable to use temporary root-created +OpenBao credentials stored outside repositories with mode 0600 and explicit +revocation. Those bootstrap shortcuts must be tracked as temporary and replaced +by narrower auth roles. + +### 7. Prefer Exec-time Delivery + +When a workload or operator command needs a secret, the default delivery mode is +process-local injection for the duration of that command. The engine should make +this easier than copying or inspecting a secret value. + +### 8. Auditable and Reversible + +Every apply, read, delivery, lease, verification, revocation, and deactivation +has non-secret evidence that can be reviewed later. + +## What This Is + +secrets-engine is: + +- a workflow layer for approved secret changes and access; +- a CLI and service API for operators, agents, and automation; +- a typed catalog of secret lanes, grants, delivery modes, and stage policies; +- an OpenBao policy/auth-role applier with strict local validation; +- an exec-time secret delivery helper; +- an audit evidence writer for State Hub and local logs; +- the credential interaction surface that ops-warden can route to. + +## What This Is Not + +secrets-engine is not: + +- a replacement for OpenBao; +- an identity provider or MFA system; +- an authorization decision engine; +- an application-specific secret store; +- a place to persist raw secret values outside OpenBao; +- a prompt-time secret injection mechanism for LLMs; +- a bypass around review, approval, or production custody. + +## System Boundary + +| Concern | Primary owner | secrets-engine responsibility | +| --- | --- | --- | +| Secret custody, leases, audit backend | OpenBao / railiance-platform | Use through least-privilege roles and validated paths. | +| Human and service identity | key-cape / user-engine | Consume identity claims; do not replace identity lifecycle. | +| Authorization decisions | flex-auth / State Hub decision model | Require and verify decisions before privileged actions. | +| SSH certificate issuance | ops-warden | Provide routed credential access; do not make ops-warden vend secrets. | +| Workload secret consumption | Workload repos / CI / runtime | Provide safe delivery contracts and catalog entries. | +| Request history and progress | State Hub | Write non-secret evidence and decision links only. | + +## Direction of Evolution + +The repository should evolve through these phases: + +1. **Bootstrap:** root-created temporary OpenBao roles/tokens allow efficient + setup without repeated manual UI handoffs. +2. **MVP:** catalog, decision validation, OpenBao apply plan, and safe exec-time + delivery work for the whynot-design npm publish token pilot. +3. **Stage separation:** build, test, and production roles have distinct policy + boundaries and verification rules. +4. **Hardening:** replace bootstrap token files with OIDC/service auth, wrapped + delivery, short leases, dual control for production provisioning, and routine + rotation/deactivation workflows. +5. **Service mode:** expose an API that ops-warden, agents, CI, and future UI + surfaces can use without knowing OpenBao internals. + +## Guiding Question + +**How can approved secret work become low-friction for humans and agents while +keeping raw values, OpenBao privileges, and production impact tightly bounded?** diff --git a/ProductRequirementsDocument.md b/ProductRequirementsDocument.md new file mode 100644 index 0000000..2212ae0 --- /dev/null +++ b/ProductRequirementsDocument.md @@ -0,0 +1,301 @@ +# Product Requirements Document: secrets-engine + +## Summary + +secrets-engine provides the operational interface for approved secret lifecycle +work across build, test, and production. It wraps OpenBao with a catalog, +decision checks, stage-aware privilege boundaries, safe delivery modes, and +non-secret audit evidence. + +The first implementation should focus on getting one real pilot lane working end +to end: the whynot-design npm publish token. The design must also be general +enough to support API keys, provider tokens, database passwords, CI credentials, +and other workload secrets later. + +## Related Documents + +- [NetKingdom security infrastructure boundary pointer](docs/netkingdom-security-infrastructure.md) + points to the canonical document in `net-kingdom/docs/` and defines the + responsibilities and interactions with OpenBao, flex-auth, user-engine, + ops-warden, ops-bridge, info-tech-canon, State Hub, and agents. + +## Background + +The current platform can define OpenBao paths, policies, and credential-change +requests, but the live path still requires too much manual OpenBao UI/CLI work. +This creates a trap: the secure process exists on paper, but daily work still +falls back to handovers, broad platform-root access, and interface-specific +mistakes. + +secrets-engine is the missing workflow layer. It should let an approved decision +become an applied and verified secret lane without asking each operator or agent +to understand every OpenBao endpoint, CLI quoting rule, auth role shape, or +workload-specific delivery convention. + +## Goals + +- Provide one interaction surface for approved secret establishment, access, + rotation, deactivation, and verification. +- Separate build, test, and production privileges from the beginning. +- Let agents and operators apply approved OpenBao metadata without broad admin + access. +- Deliver secrets to commands and workloads without printing values. +- Produce non-secret evidence suitable for State Hub progress, audits, and + troubleshooting. +- Give ops-warden a credential routing target that is not responsible for secret + custody itself. + +## Non-goals + +- Replacing OpenBao as the canonical vault and audit backend. +- Replacing key-cape, user-engine, flex-auth, or State Hub. +- Building a full UI before the CLI/API model is clear. +- Automating high-risk production secret value provisioning before the approval, + wrapped-delivery, and verification model is proven. +- Storing raw secret values in Git, workplans, State Hub, chat, prompts, normal + logs, or long-lived local files. + +## Primary Users + +| User | Need | +| --- | --- | +| Platform operator | Apply approved secret changes without hand-typing OpenBao internals. | +| Agent such as Codex or ops-warden | Request or use approved credentials through a safe front door. | +| Workload maintainer | Get a clear request, review, approval, and consumption path. | +| Security reviewer | See decisions, policy boundaries, verification evidence, and revocation state. | +| Future service maintainer | Add new secret types and delivery modes without weakening production custody. | + +## Operating Model + +### Stage roles + +secrets-engine should bootstrap at least three OpenBao-facing roles: + +| Role | Intended use | Initial privilege shape | +| --- | --- | --- | +| secrets-engine-build | Build/dev lanes, generated test values, low-risk experimentation | Can manage approved build-stage metadata and values under build prefixes. | +| secrets-engine-test | Test/staging lanes and integration verification | Can manage approved test-stage metadata and values; can perform positive/negative checks. | +| secrets-engine-prod | Production metadata apply and tightly controlled delivery | Can apply approved prod metadata and auth roles; raw prod value read/provisioning remains gated and logged. | + +The exact OpenBao policy names can change, but the stage distinction is a +product requirement, not an implementation detail. + +### Bootstrap credential files + +Until proper OIDC/service authentication exists, a platform-root operator may +create temporary OpenBao credentials for the three roles and place them outside +repositories, for example under a local mode-0700 directory. The files must be: + +- mode 0600; +- outside Git worktrees; +- named by role and environment only, never by secret value; +- revocable; +- recorded in non-secret bootstrap notes by path and accessor only when safe; +- treated as temporary infrastructure setup material. + +Agents may be told the file path, not the token value. The engine should read +such files only when explicitly invoked in bootstrap mode. + +## Functional Requirements + +### FR1 - Secret catalog + +The engine must maintain a non-secret catalog of secret lanes and grants. + +Each catalog entry should include: + +- catalog id; +- owning domain/repo/workload; +- stage: build, test, production, or another explicit stage; +- OpenBao mount and path; +- fields exposed; +- allowed consumers and auth claims; +- delivery modes; +- approval requirement; +- TTL and rotation expectations; +- verification requirements; +- revocation/deactivation behavior; +- audit evidence expectations. + +### FR2 - Decision integration + +Privileged actions must require an approved decision or approved credential +change request unless explicitly running in a local bootstrap or dry-run mode. + +The engine must be able to: + +- inspect a request; +- render a human-reviewable plan; +- link to the decision record when available; +- refuse denied, superseded, stale, or unapproved requests; +- record non-secret apply and verification evidence. + +### FR3 - OpenBao metadata apply + +The engine must generate and apply OpenBao metadata safely: + +- ACL policies; +- auth roles; +- token roles where needed; +- KV metadata where safe; +- path and name restrictions based on catalog and stage; +- dry-run output before live mutation. + +Production apply must fail closed if the request is not approved or if the plan +contains an out-of-bound policy, role, mount, path, wildcard, or broad admin +capability. + +### FR4 - Secret provisioning + +The engine must support secret value provisioning without exposing raw values in +coordination channels. + +Initial supported modes: + +- operator-attended OpenBao provisioning; +- bootstrap local file import with strict permissions; +- response-wrapped handoff where OpenBao supports it; +- generated test secrets for non-production only. + +Production raw value automation should remain deliberately constrained until the +wrapped or dual-control flow is proven. + +### FR5 - Secret delivery + +The engine must provide delivery modes that avoid printing secret values: + +- exec-time environment injection; +- exec-time temp file injection with cleanup; +- workload-specific config file generation such as temporary npm config; +- response wrapping for handoff flows; +- read checks that return boolean/evidence, not the value. + +A representative target command shape: + +```bash +secrets-engine exec --catalog whynot-design-npm-publish -- npm publish +``` + +For npm, the engine should prefer creating a temporary npm config and setting the +child process environment to use it, then deleting it after the child exits. + +### FR6 - Verification + +The engine must support positive and negative verification without printing +secrets. + +Positive checks prove an approved consumer can access or use the secret. Negative +checks prove an unrelated consumer cannot. Evidence should include non-secret +metadata such as catalog id, actor, decision id, path, timestamp, and result. + +### FR7 - CLI/API interface + +The MVP must provide a CLI. The command names may evolve, but the first surface +should cover: + +```text +secrets-engine catalog list +secrets-engine catalog show +secrets-engine decision inspect +secrets-engine plan +secrets-engine apply --stage +secrets-engine provision --stage --from-file +secrets-engine verify --positive +secrets-engine verify --negative +secrets-engine exec --catalog -- +secrets-engine revoke +``` + +The API should be designed after the CLI semantics are clear. + +### FR8 - Audit and evidence + +The engine must never log raw secret values. It must write non-secret evidence +for: + +- decision inspected; +- plan rendered; +- apply attempted/succeeded/failed; +- secret provisioned without value disclosure; +- exec delivery attempted/succeeded/failed; +- verification positive/negative result; +- revoke/deactivate/rotate actions. + +State Hub integration should use non-secret progress entries and decision links. +OpenBao audit logs remain the backend source of truth for vault operations. + +### FR9 - ops-warden routing + +ops-warden should route non-SSH credential requests to secrets-engine. It should +not own or vend provider tokens, API keys, database passwords, or OpenBao values. + +A route result should tell the caller: + +- the catalog id; +- the decision/request status; +- whether the secret is ready/resolvable; +- the safe command to request or execute with the credential; +- what human decision or provisioning step is still missing. + +## Security Requirements + +- Raw secret values must never appear in prompts, chat, Git, workplans, State + Hub messages, issue comments, or normal logs. +- Production operations must require an approved decision except break-glass + flows, which must be explicit and heavily audited. +- Stage roles must be unable to mutate paths outside their stage and allowed + prefixes. +- Production metadata appliers must not gain broad sys, auth, identity, root, or + platform-admin semantics. +- Bootstrap token files must be temporary, revocable, mode 0600, and outside + repositories. +- The engine must redact token-like values from child process output where it + controls the execution path. +- Every role and delivery mode must have a documented revocation path. + +## MVP Scope + +The MVP is complete when it can support the whynot-design npm publish token +pilot with this chain: + +1. Catalog entry exists for whynot-design npm publish. +2. Approved decision/CCR is inspected. +3. OpenBao ACL policy and auth role are applied through a stage-appropriate + secrets-engine role or bootstrap credential. +4. The token value is provisioned without being printed or logged. +5. Positive and negative verification pass. +6. A safe command can execute npm publish with the token injected only into the + child process. +7. ops-warden can report the catalog entry as ready/resolvable. + +## Success Metrics + +- Approved secret lanes can be applied without platform-root hand typing. +- A typical approved non-production credential can be established in minutes. +- Production metadata apply succeeds without granting raw secret read authority. +- No secret values are found in repo history, State Hub payloads, logs, or chat. +- Operators can understand a generated plan without knowing OpenBao endpoint + quirks. +- At least one real workload consumes a credential through secrets-engine exec. + +## Risks + +| Risk | Mitigation | +| --- | --- | +| Wrapper accidentally becomes broad platform-admin by another name | Stage roles, path allowlists, decision checks, negative tests. | +| Temporary bootstrap files become permanent | Track bootstrap mode, add revocation tasks, replace with OIDC/service auth. | +| Agents exfiltrate secrets through logs or prompts | Prefer exec-time delivery, redaction, no raw read default. | +| Product scope grows into identity/authorization | Keep key-cape/user-engine and flex-auth boundaries explicit. | +| OpenBao-specific assumptions leak everywhere | Keep a catalog and interface model; isolate backend adapter code. | + +## Open Questions + +- Should the first repo expose only a CLI, or also a tiny local service API? +- What is the canonical State Hub decision object for secret establishment, + rotation, and deactivation? +- Which stage names are final: build, test, prod, or build, staging, + production? +- Should production value provisioning require two-person review immediately or + only after the pilot? +- Which identity claim should bind the whynot-design consumer long term: group, + workload service account, repository identity, or a combination? diff --git a/README.md b/README.md index 01b0972..62afe46 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,68 @@ -Headless multi-application, multi-tenant secrets mangement engine. +# secrets-engine -# repo-seed +Headless, multi-application, multi-tenant secrets workflow and automation layer +for approved secret custody, delivery, and lifecycle work across build, test, +and production stages. -A git repository template to bootstrap coulomb projects from. +OpenBao remains the custody and enforcement backend. `secrets-engine` owns the +operator and agent interaction model: catalog, decision checks, plan/apply, +safe provisioning, verification, delivery, evidence, rotation, and deactivation. -## Bootstrap a new repo +## Start Here -1. Clone or copy this template into a new repository. -2. Run `statehub register` from the new repo root (see [docs/statehub-register.md](docs/statehub-register.md)). -3. Complete the generated bootstrap workplan (`*-0001-statehub-bootstrap.md`). -4. Sync workplans: `cd ~/state-hub && make fix-consistency REPO=`. -5. Validate with [docs/template-validation-checklist.md](docs/template-validation-checklist.md). +- [INTENT.md](INTENT.md) - why this repository exists. +- [ProductRequirementsDocument.md](ProductRequirementsDocument.md) - product + requirements and MVP scope. +- [NetKingdom security infrastructure boundary pointer](docs/netkingdom-security-infrastructure.md) + - points to the canonical document in `net-kingdom/docs/`, covering + responsibilities and interactions with OpenBao, flex-auth, user-engine, + ops-warden, ops-bridge, info-tech-canon, State Hub, and agents. +- [Bootstrap MVP workplan](workplans/SECRETS-WP-0002-bootstrap.md) - first + implementation plan after State Hub bootstrap. -## Registry +## Core Direction -This repo publishes `capability.infotech.repo-template` — see -`registry/capabilities/capability.infotech.repo-template.md`. \ No newline at end of file +The MVP proves the `whynot-design-npm-publish` lane end to end: + +1. describe the lane in a non-secret catalog (`catalog/`); +2. verify an approved decision (State Hub or local fixture); +3. apply OpenBao policy/auth metadata through a stage-aware role; +4. provision and verify the value without printing it; +5. run a workload command through safe exec-time delivery. + +Target command shape: + +```bash +secrets-engine exec --catalog whynot-design-npm-publish -- npm publish +``` + +## Quickstart + +```bash +uv venv && uv pip install -e ".[dev]" +source .venv/bin/activate +secrets-engine catalog list + +# Run the whole pilot chain live against a throwaway OpenBao dev server: +SECRETS_ENGINE_HUB_URL="" bash scripts/demo-e2e.sh +``` + +- CLI reference: [docs/cli.md](docs/cli.md) +- Stage roles & bootstrap tokens: [docs/openbao-stage-roles.md](docs/openbao-stage-roles.md) +- ops-warden routing contract: [docs/ops-warden-routing-contract.md](docs/ops-warden-routing-contract.md) +- Hardening backlog (exit bootstrap mode): [docs/hardening-backlog.md](docs/hardening-backlog.md) + +The implementation is a Python package (`src/secrets_engine/`). OpenBao is +reached only through the `bao` CLI adapter (`openbao.py`); the rest of the code +speaks in lanes and guarded plans. + +## Security Rules + +- Do not put raw secret values in Git, State Hub, chat, prompts, issue comments, + workplans, or normal logs. +- OpenBao is the backend custody and audit authority. +- Build, test, and production have separate policy boundaries. +- Production actions require approved decisions except explicit break-glass + flows. +- Temporary bootstrap OpenBao credentials must live outside repos, use mode + 0600, be revocable, and be removed after narrower auth is working. diff --git a/SCOPE.md b/SCOPE.md index 802fd33..ab801c9 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -2,30 +2,46 @@ > Lightweight boundary for agents and contributors. ---- - ## One-liner -Git repository template to bootstrap coulomb projects. - ---- +secrets-engine is the workflow and automation interface for approved secret +custody, delivery, and lifecycle work across build, test, and production, with +OpenBao as the initial enforcement backend. ## Core Idea -repo-seed is the canonical template for new repos: agent instructions, registry scaffold, and onboarding conventions. - ---- +OpenBao is the vault. secrets-engine is the day-to-day interaction layer that +connects cataloged secret lanes, approval decisions, stage-specific OpenBao +roles, safe delivery modes, and non-secret evidence. ## In Scope -- Template files for new repo bootstrap -- Documentation for statehub_register usage -- Registry capability entry for template capability - ---- +- Non-secret catalog of secret lanes, grants, consumers, stages, and delivery + modes. +- Decision-aware planning and apply flows for OpenBao policies, auth roles, and + metadata. +- Build, test, and production privilege separation. +- Safe provisioning, verification, rotation, revocation, and deactivation + workflows. +- Exec-time delivery to operators, agents, CI jobs, workloads, and ops-bridge + tasks without printing raw values. +- ops-warden routing contract for non-SSH credentials. +- State Hub non-secret evidence and progress integration. +- Canonicalization of terms with info-tech-canon. ## Out of Scope -- Application runtime code -- Owning downstream project implementations +- Replacing OpenBao as custody, policy, lease, or audit backend. +- Replacing flex-auth authorization decisions. +- Replacing user-engine/key-cape identity and claim lifecycle. +- Issuing SSH certificates, which remains ops-warden responsibility. +- Owning tunnels or remote transport, which remains ops-bridge responsibility. +- Storing raw secret values in this repo, State Hub, chat, prompts, or logs. +- Broad platform-root or platform-admin automation as a steady-state model. +## Current State + +The repo is in bootstrap. Seed intent, PRD, boundary documentation, and an MVP +workplan are present. The first worker should complete State Hub bootstrap, +validate the generated repo identity files, then begin the whynot-design npm +publish token pilot through the `SECRETS-WP-0002` workplan. diff --git a/catalog/example-build-test-token.yaml b/catalog/example-build-test-token.yaml new file mode 100644 index 0000000..59d2a29 --- /dev/null +++ b/catalog/example-build-test-token.yaml @@ -0,0 +1,43 @@ +# Example BUILD-stage lane. Demonstrates that build entries can be looser: +# generated test values are allowed and no production decision is required. +id: example-build-test-token +owner: platform-ci +stage: build +description: >- + Throwaway generated credential for build-stage integration tests. May be + generated locally; must never be reused in test or prod. + +mount: secret +path: build/example/test-token + +fields: + - api_token + +consumers: + - name: build-runner + auth: approle + claim: "role:build-runner" + purpose: "exercise build-stage integration tests" + +delivery_modes: + - exec-env + - read-check + +# Build stage permits bootstrap-only / generated values without a prod decision. +approval: + model: bootstrap-only + notes: "Build stage: generated test secret, no production decision required." + +verification: + positive: "build-runner token can read the generated value" + negative: "prod consumers cannot read build paths" + +rotation: + expectation: "regenerate per run" + ttl: "1h" + +deactivation: + expectation: "delete on build teardown" + +audit: + evidence: "actor, path, timestamp, result — no secret value" diff --git a/catalog/whynot-design-npm-publish.yaml b/catalog/whynot-design-npm-publish.yaml new file mode 100644 index 0000000..0a570e9 --- /dev/null +++ b/catalog/whynot-design-npm-publish.yaml @@ -0,0 +1,51 @@ +# whynot-design npm publish token — the MVP pilot lane. +# This file is NON-SECRET. It describes where the token lives in OpenBao and how +# it may be consumed. The token VALUE never appears here. +id: whynot-design-npm-publish +owner: whynot-design +stage: prod +description: >- + npm automation token used to publish the whynot-design package. Delivered to + `npm publish` via an exec-time temporary npm config; never printed or exported + into the parent shell. + +# OpenBao KV v2 location of the secret material. +mount: secret +path: whynot-design/npm/publish + +# Field(s) inside the KV entry. The publish token is stored under this key. +fields: + - npm_token + +# Who may consume this lane and the identity claim that binds them. +consumers: + - name: whynot-design-ci + auth: approle # bound OpenBao auth method + claim: "role:whynot-design-publish" + purpose: "publish whynot-design npm package from CI" + +# How the value may leave OpenBao. npm-config = temp .npmrc for the child only. +delivery_modes: + - npm-config + - read-check + +# Privileged actions on this lane require an approved decision/CCR. +approval: + model: decision + decision_ref: "whynot-design-npm-publish" # State Hub decision/CCR id or slug + notes: "Production lane: apply requires an approved decision." + +# Verification expectations (no value is ever printed). +verification: + positive: "approved consumer token can read the lane field" + negative: "an unrelated token is denied read on the lane path" + +rotation: + expectation: "rotate on compromise or every 90 days" + ttl: "90d" + +deactivation: + expectation: "revoke approle + delete KV metadata; record evidence" + +audit: + evidence: "decision id, actor, path, timestamp, result — no secret value" diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..4f1dbb4 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,78 @@ +# secrets-engine CLI + +## Install + +```bash +uv venv && uv pip install -e ".[dev]" +source .venv/bin/activate +secrets-engine --version +``` + +## Environment + +| Var | Default | Purpose | +| --- | --- | --- | +| `BAO_ADDR` | `http://127.0.0.1:8200` | OpenBao address | +| `BAO_TOKEN` | _(unset)_ | OpenBao token (or use `--bootstrap-token-file`) | +| `SECRETS_ENGINE_HUB_URL` | `http://127.0.0.1:8000` | State Hub for decisions + evidence (empty to disable) | +| `SECRETS_ENGINE_CATALOG` | `./catalog` | catalog directory | +| `SECRETS_ENGINE_EVIDENCE` | `./.evidence` | local non-secret evidence log | + +## Commands + +```text +secrets-engine catalog list +secrets-engine catalog show +secrets-engine decision inspect +secrets-engine plan --stage +secrets-engine apply --stage [--dry-run] [--bootstrap-token-file F] +secrets-engine provision --stage --field NAME (--from-file F | --generate) +secrets-engine verify --field NAME [--positive] [--negative] +secrets-engine exec --catalog [--field NAME] [--mode auto|npm-config|exec-env] -- CMD... +secrets-engine route [--json] +secrets-engine revoke [--dry-run] +``` + +`` is a catalog id or a decision/CCR ref (matched against +`approval.decision_ref`). `plan` and `apply --dry-run` never mutate OpenBao. + +## Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | success | +| 2 | catalog error (missing/invalid lane) | +| 3 | decision error (unapproved / superseded / missing) | +| 4 | policy guard error (out-of-stage / wildcard / broad admin) | +| 5 | backend error (OpenBao unreachable / failed) | +| 6 | provisioning error (bad file mode / inside repo / missing field) | +| 7 | verification failed | +| 8 | delivery error | + +## End-to-end demo + +```bash +SECRETS_ENGINE_HUB_URL="" bash scripts/demo-e2e.sh +``` + +Boots a throwaway in-memory OpenBao dev server and runs the whole pilot chain: +plan → apply → provision → verify(+/-) → exec (npm-config injection) → route → +revoke. Nothing is persisted; the token is a throwaway local string. + +## Pilot: whynot-design npm publish + +```bash +# 1. inspect the approved decision +secrets-engine decision inspect whynot-design-npm-publish +# 2. preview the OpenBao changes (no mutation) +secrets-engine plan whynot-design-npm-publish --stage prod +# 3. apply policy + approle +secrets-engine apply whynot-design-npm-publish --stage prod +# 4. provision the token from a mode-0600 file OUTSIDE the repo +secrets-engine provision whynot-design-npm-publish --stage prod \ + --field npm_token --from-file ~/.secrets-engine/whynot.token +# 5. prove access without printing the value +secrets-engine verify whynot-design-npm-publish --field npm_token --positive --negative +# 6. publish with the token injected into the child only +secrets-engine exec --catalog whynot-design-npm-publish -- npm publish +``` diff --git a/docs/hardening-backlog.md b/docs/hardening-backlog.md new file mode 100644 index 0000000..7662f09 --- /dev/null +++ b/docs/hardening-backlog.md @@ -0,0 +1,58 @@ +# Hardening Backlog — Exit From Bootstrap Mode + +The MVP runs in **bootstrap mode**: stage roles may be driven by temporary +root-created OpenBao tokens read from mode-0600 files. That is acceptable setup +material, **not** the steady state. This backlog tracks the work to retire it. + +The MVP is honest about this: bootstrap mode is a documented, bounded phase with +explicit revocation tasks — not a hidden permanent security posture. + +## H0 — Revoke outstanding bootstrap tokens (always-on hygiene) + +Every minted bootstrap token has a revocation task. Track each here: + +| Token file | Stage | Minted | TTL | Revoked? | +| --- | --- | --- | --- | --- | +| `~/.secrets-engine/bootstrap/prod.token` | prod | (n/a — demo uses dev server) | 1h | n/a | + +Revoke: `bao token revoke -accessor ` then `shred -u `. + +## H1 — Replace bootstrap token files with OIDC / service auth + +- Stand up an OpenBao auth method (OIDC or AppRole bound to a workload identity) + for each stage role. +- secrets-engine logs in via that method instead of reading a token file. +- Remove `--bootstrap-token-file` from the steady-state path (keep only for true + break-glass, heavily audited). + +## H2 — Response-wrapped handoff + +- Add a `wrapped` delivery mode using OpenBao response wrapping for operator + handoff flows where exec-time injection does not fit. + +## H3 — Production dual-control + +- Require two-person approval (`approval.model: dual-control`) for prod value + provisioning before automating raw-value writes beyond the pilot. + +## H4 — Rotation & lifecycle states + +- Implement `rotate`, and explicit `compromised` / `deactivated` lane states with + evidence, beyond the current `revoke` (metadata delete). + +## H5 — Audit report command + +- `secrets-engine audit ` summarizing non-secret evidence (decision, + applies, provisions, verifications, execs, revokes) for a lane. + +## H6 — API service mode + +- Expose the stabilized CLI semantics as a local service API for ops-warden, + CI, agents, and a future UI — **after** the CLI contract is proven. + +## Exit criteria for "bootstrap mode is over" + +- No steady-state flow reads a bootstrap token file. +- Every stage role authenticates through OIDC/service auth. +- Production provisioning beyond the pilot requires dual-control. +- Rotation and deactivation are first-class, evidenced operations. diff --git a/docs/netkingdom-security-infrastructure.md b/docs/netkingdom-security-infrastructure.md new file mode 100644 index 0000000..abd048f --- /dev/null +++ b/docs/netkingdom-security-infrastructure.md @@ -0,0 +1,22 @@ +# NetKingdom Security Infrastructure Boundary + +The canonical document lives in the NetKingdom repository: + +```text +net-kingdom/docs/secrets-engine-security-infrastructure-boundary.md +``` + +Local checkout path: + +```text +/home/worsch/net-kingdom/docs/secrets-engine-security-infrastructure-boundary.md +``` + +This secrets-engine file is intentionally only a pointer. The canonical document +belongs to NetKingdom because it defines cross-system security infrastructure +responsibilities and boundaries across OpenBao, flex-auth, user-engine/key-cape, +ops-warden, ops-bridge, info-tech-canon, State Hub, and agents. + +secrets-engine consumes that boundary and implements the secrets workflow, +catalog, stage policies, OpenBao apply/delivery mechanics, and evidence model +that the canonical document assigns to it. diff --git a/docs/openbao-stage-roles.md b/docs/openbao-stage-roles.md new file mode 100644 index 0000000..5413b62 --- /dev/null +++ b/docs/openbao-stage-roles.md @@ -0,0 +1,79 @@ +# OpenBao Stage Roles & Bootstrap + +secrets-engine talks to OpenBao through three **stage roles**, never as root or a +platform admin. Each role is confined to one stage's KV prefix and a small, +explicit capability set. + +| Role | Policy file | KV prefix | May do | May NOT do | +| --- | --- | --- | --- | --- | +| `secrets-engine-build` | `policies/secrets-engine-build.hcl` | `secret/.../build/` | manage build metadata + generated test values; own `se-build-*` policies/roles | touch test/prod, `sys/*`, `auth/token/*`, `identity/*`, act as root | +| `secrets-engine-test` | `policies/secrets-engine-test.hcl` | `secret/.../test/` | manage test metadata + values; run positive/negative checks; own `se-test-*` | touch build/prod, `sys/*`, `auth/token/*`, `identity/*`, act as root | +| `secrets-engine-prod` | `policies/secrets-engine-prod.hcl` | owner-scoped prod lanes | apply approved prod ACL policies + approle roles; write approved values; own `se-prod-*` | reach build/test, edit the stage roles themselves, admin `sys/auth`, `sys/mounts`, `identity/*`, `auth/token/*`, act as root | + +The **product requirement** is the stage *distinction* and the *denials*, not the +exact policy names — those may evolve as info-tech-canon hardens. + +## Negative guarantees (enforced two ways) + +1. **In OpenBao** — each policy carries explicit `deny` stanzas for other stages + and for `sys/*`, `auth/token/*`, `identity/*`. +2. **In secrets-engine** — `roles.assert_path_in_stage()` and + `roles.assert_policy_safe()` reject any *plan* that would touch another + stage's prefix, use a wildcard, name itself like an admin policy, or carry a + capability outside `create/read/update/delete/list`. A bad plan fails closed + **before** any OpenBao call. + +Run the negative checks: + +```bash +pytest tests/test_guards.py -q +``` + +## Bootstrap token files (temporary) + +Until OIDC/service auth exists (see the hardening backlog), a **platform-root +operator** may mint a short-lived OpenBao token for a stage role and hand the +agent the *file path* — never the token value. + +Requirements for a bootstrap token file: + +- mode **0600**, owned by the invoking user; +- located **outside** any Git worktree (e.g. `~/.secrets-engine/bootstrap/`); +- named by role + environment only, never by value + (e.g. `prod.token`, not `npm_abc123.token`); +- **revocable** and **temporary** — tracked with a revocation task; +- referenced by path, e.g.: + +```bash +secrets-engine apply whynot-design-npm-publish --stage prod \ + --bootstrap-token-file ~/.secrets-engine/bootstrap/prod.token +``` + +secrets-engine refuses a bootstrap token file that is group/other-readable or +that lives inside the repo worktree (`provision`/`apply` check `st_mode & 0o077`). + +### Minting (operator, one-time, root context) + +```bash +# Write each stage policy into OpenBao. +bao policy write secrets-engine-build policies/secrets-engine-build.hcl +bao policy write secrets-engine-test policies/secrets-engine-test.hcl +bao policy write secrets-engine-prod policies/secrets-engine-prod.hcl + +# Mint a short-lived token for one stage role, store mode-0600 outside the repo. +install -m 700 -d ~/.secrets-engine/bootstrap +bao token create -policy=secrets-engine-prod -ttl=1h -field=token \ + > ~/.secrets-engine/bootstrap/prod.token +chmod 600 ~/.secrets-engine/bootstrap/prod.token +``` + +### Revocation (always have a path) + +```bash +# Revoke by accessor (preferred) or delete the file when the TTL is short. +bao token revoke -accessor +shred -u ~/.secrets-engine/bootstrap/prod.token +``` + +Every minted bootstrap token MUST have a corresponding revocation task in the +hardening backlog (`docs/hardening-backlog.md`). diff --git a/docs/ops-warden-routing-contract.md b/docs/ops-warden-routing-contract.md new file mode 100644 index 0000000..7807848 --- /dev/null +++ b/docs/ops-warden-routing-contract.md @@ -0,0 +1,62 @@ +# ops-warden → secrets-engine Routing Contract + +ops-warden issues **SSH certificates only**. Every other credential need (API +keys, provider tokens, DB passwords, npm publish tokens) routes to +**secrets-engine**, which is OpenBao-backed. ops-warden must never request, hold, +cache, or vend a raw secret value. A route result is a **pointer**, not a key. + +## What ops-warden calls + +```bash +secrets-engine route --json +``` + +## What it returns (the contract) + +```json +{ + "catalog_id": "whynot-design-npm-publish", + "owner": "whynot-design", + "stage": "prod", + "decision_status": "resolved", + "decision_ref": "whynot-design-npm-publish", + "review_url": "http://127.0.0.1:8000/decisions/", + "metadata_applied": true, + "value_present": true, + "ready": true, + "next_command": "secrets-engine exec --catalog whynot-design-npm-publish -- ", + "missing": "" +} +``` + +| Field | Meaning | +| --- | --- | +| `decision_status` | `resolved`/`approved` => approved; `missing`/`pending` => not yet | +| `metadata_applied` | OpenBao ACL policy + approle exist for the lane | +| `value_present` | the secret value has been provisioned (boolean only — value never read) | +| `ready` | approved **and** applied **and** provisioned | +| `next_command` | the single safe command the caller should run next | +| `missing` | the one human/provisioning step still outstanding | + +## Guarantees + +- **No value crosses this boundary.** `route` reports a boolean `value_present`, + derived from a metadata/presence check — it never reads the secret. +- **Actionable when not ready.** If a lane is unapproved, unapplied, or + unprovisioned, `next_command` + `missing` tell the caller exactly what to do. +- **Idempotent / read-only.** `route` performs no mutation. + +## whynot-design retry flow + +1. whynot-design CI needs a publish token → asks ops-warden. +2. ops-warden runs `secrets-engine route whynot-design-npm-publish --json`. +3. If `ready=false`, it surfaces `missing` + `next_command` to the human (e.g. + "needs approved decision" or "needs provisioning"). +4. Once `ready=true`, the workload runs + `secrets-engine exec --catalog whynot-design-npm-publish -- npm publish`. + +## Anti-patterns (forbidden) + +- ops-warden `POST /messages/` asking for `NPM_TOKEN` / `OPENROUTER_API_KEY`. +- Caching `value_present` as if it were the value. +- Inventing `warden secret` / `warden bao` — they do not exist. diff --git a/policies/secrets-engine-build.hcl b/policies/secrets-engine-build.hcl new file mode 100644 index 0000000..1ca9c8d --- /dev/null +++ b/policies/secrets-engine-build.hcl @@ -0,0 +1,36 @@ +# secrets-engine-build — build/dev stage role policy. +# +# Scope: manage approved BUILD-stage metadata and values under the build/ prefix +# only. May generate throwaway test values. Cannot touch test/ or prod paths, +# cannot administer sys/, auth/, or identity/, cannot act as root. + +# KV v2 data + metadata under the build prefix. +path "secret/data/build/*" { + capabilities = ["create", "read", "update", "delete"] +} +path "secret/metadata/build/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +# Manage only the build consumer policies this role owns. +path "sys/policies/acl/se-build-*" { + capabilities = ["create", "read", "update", "delete"] +} + +# Manage build approle roles only. +path "auth/approle/role/se-build-*" { + capabilities = ["create", "read", "update", "delete"] +} +path "auth/approle/role/se-build-*/role-id" { + capabilities = ["read"] +} +path "auth/approle/role/se-build-*/secret-id" { + capabilities = ["create", "update"] +} + +# DENY everything outside the build lane (explicit, for reviewer clarity). +path "secret/data/test/*" { capabilities = ["deny"] } +path "secret/data/prod/*" { capabilities = ["deny"] } +path "sys/*" { capabilities = ["deny"] } +path "auth/token/*" { capabilities = ["deny"] } +path "identity/*" { capabilities = ["deny"] } diff --git a/policies/secrets-engine-prod.hcl b/policies/secrets-engine-prod.hcl new file mode 100644 index 0000000..446a461 --- /dev/null +++ b/policies/secrets-engine-prod.hcl @@ -0,0 +1,47 @@ +# secrets-engine-prod — production stage role policy. +# +# Scope: apply approved PRODUCTION metadata (ACL policies + approle roles) and +# write approved lane values for provisioning. Deliberately NARROW: +# - no root / sudo +# - no sys/* administration beyond its own se-prod-* ACL policies +# - no identity/* administration +# - no auth/token administration +# - cannot reach into build/ or test/ prefixes +# +# Production raw-value READ stays gated: this role can write a value during an +# approved provisioning step, but broad read of arbitrary prod paths is denied. +# Consumers read their own lane through the narrow se-prod- consumer policy. + +# Apply approved production lane values (provisioning) and metadata. +# Owner-scoped prod lanes (e.g. whynot-design/...) — NOT under build/ or test/. +path "secret/data/+/*" { + capabilities = ["create", "update"] +} +path "secret/metadata/+/*" { + capabilities = ["create", "read", "update", "list"] +} + +# Manage only the prod consumer policies this role owns. +path "sys/policies/acl/se-prod-*" { + capabilities = ["create", "read", "update", "delete"] +} + +# Manage prod approle roles only. +path "auth/approle/role/se-prod-*" { + capabilities = ["create", "read", "update", "delete"] +} +path "auth/approle/role/se-prod-*/role-id" { + capabilities = ["read"] +} +path "auth/approle/role/se-prod-*/secret-id" { + capabilities = ["create", "update"] +} + +# Hard denials — production appliers must not become broad admins. +path "secret/data/build/*" { capabilities = ["deny"] } +path "secret/data/test/*" { capabilities = ["deny"] } +path "sys/policies/acl/secrets-engine-*" { capabilities = ["deny"] } # can't edit stage roles +path "auth/token/*" { capabilities = ["deny"] } +path "identity/*" { capabilities = ["deny"] } +path "sys/auth/*" { capabilities = ["deny"] } +path "sys/mounts/*" { capabilities = ["deny"] } diff --git a/policies/secrets-engine-test.hcl b/policies/secrets-engine-test.hcl new file mode 100644 index 0000000..7db51a3 --- /dev/null +++ b/policies/secrets-engine-test.hcl @@ -0,0 +1,33 @@ +# secrets-engine-test — test/staging stage role policy. +# +# Scope: manage approved TEST-stage metadata and values under the test/ prefix, +# and run positive/negative verification. Cannot touch build/ or prod paths, +# cannot administer sys/, auth/, or identity/, cannot act as root. + +path "secret/data/test/*" { + capabilities = ["create", "read", "update", "delete"] +} +path "secret/metadata/test/*" { + capabilities = ["create", "read", "update", "delete", "list"] +} + +path "sys/policies/acl/se-test-*" { + capabilities = ["create", "read", "update", "delete"] +} + +path "auth/approle/role/se-test-*" { + capabilities = ["create", "read", "update", "delete"] +} +path "auth/approle/role/se-test-*/role-id" { + capabilities = ["read"] +} +path "auth/approle/role/se-test-*/secret-id" { + capabilities = ["create", "update"] +} + +# DENY everything outside the test lane. +path "secret/data/build/*" { capabilities = ["deny"] } +path "secret/data/prod/*" { capabilities = ["deny"] } +path "sys/*" { capabilities = ["deny"] } +path "auth/token/*" { capabilities = ["deny"] } +path "identity/*" { capabilities = ["deny"] } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..dc3e32d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[project] +name = "secrets-engine" +version = "0.1.0" +description = "Decision-aware workflow and automation layer for approved secret custody, delivery, and lifecycle work, backed by OpenBao." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "see LICENSE" } +authors = [{ name = "NetKingdom / Custodian" }] +dependencies = [ + "PyYAML>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", +] + +[project.scripts] +secrets-engine = "secrets_engine.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/secrets_engine"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/scripts/demo-e2e.sh b/scripts/demo-e2e.sh new file mode 100755 index 0000000..afad8b9 --- /dev/null +++ b/scripts/demo-e2e.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# End-to-end MVP demo for the whynot-design npm publish lane. +# +# Boots a throwaway OpenBao dev server, then drives the full secrets-engine chain: +# plan (dry-run) -> apply -> provision (from mode-0600 file) -> verify +/- +# -> exec (npm-config injection into a child) -> route -> revoke. +# +# Nothing here is production. The dev server is in-memory and discarded on exit. +# The "token" is a throwaway local string written to a mode-0600 temp file. +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO" + +# shellcheck disable=SC1091 +source .venv/bin/activate + +BAO_BIN="$(command -v bao)" +WORK="$(mktemp -d)" +export BAO_ADDR="http://127.0.0.1:8270" +export BAO_TOKEN="se-demo-root" +# Keep secret material OUTSIDE the repo worktree (engine enforces this). +TOKENFILE="$WORK/whynot.token" + +cleanup() { + set +e + [[ -n "${BAO_PID:-}" ]] && kill "$BAO_PID" 2>/dev/null + rm -rf "$WORK" +} +trap cleanup EXIT + +echo "### 0. boot throwaway OpenBao dev server (in-memory)" +"$BAO_BIN" server -dev -dev-root-token-id="$BAO_TOKEN" \ + -dev-listen-address="127.0.0.1:8270" >"$WORK/bao.log" 2>&1 & +BAO_PID=$! +for _ in $(seq 1 30); do + "$BAO_BIN" status -address="$BAO_ADDR" >/dev/null 2>&1 && break + sleep 0.2 +done + +echo +echo "### 1. plan (dry-run, no mutation)" +secrets-engine plan whynot-design-npm-publish --stage prod | sed 's/^/ /' + +echo +echo "### 2. apply approved metadata (policy + approle) to OpenBao" +secrets-engine apply whynot-design-npm-publish --stage prod | sed 's/^/ /' + +echo +echo "### 2b. apply is idempotent (re-run shows 'unchanged')" +secrets-engine apply whynot-design-npm-publish --stage prod | sed 's/^/ /' + +echo +echo "### 3. provision the token from a mode-0600 file outside the repo" +printf 'npm_demoTOKENvalue1234567890abcd' > "$TOKENFILE" +chmod 600 "$TOKENFILE" +secrets-engine provision whynot-design-npm-publish --stage prod \ + --field npm_token --from-file "$TOKENFILE" | sed 's/^/ /' + +echo +echo "### 4. verify positive (approved consumer can read) + negative (others denied)" +secrets-engine verify whynot-design-npm-publish --field npm_token --positive --negative | sed 's/^/ /' + +echo +echo "### 5. exec-time delivery: child sees the token via a temp npmrc; parent never does" +cat > "$WORK/fake-npm" <<'EOF' +#!/usr/bin/env bash +# Stand-in for 'npm publish' — proves the child can resolve the token and the +# parent shell cannot. Prints only whether the token is reachable, never the value. +if [[ -n "${NPM_CONFIG_USERCONFIG:-}" ]] && grep -q '_authToken' "$NPM_CONFIG_USERCONFIG"; then + echo " [child] npm userconfig present; _authToken resolvable: yes" +else + echo " [child] NO token available"; exit 1 +fi +echo " [child] would run: npm $*" +EOF +chmod +x "$WORK/fake-npm" +secrets-engine exec --catalog whynot-design-npm-publish -- "$WORK/fake-npm" publish + +echo " [parent] SE_NPM_TOKEN in parent shell: '${SE_NPM_TOKEN:-}'" + +echo +echo "### 6. ops-warden routing pointer (ready=true expected)" +secrets-engine route whynot-design-npm-publish --json | sed 's/^/ /' + +echo +echo "### 7. revoke (deactivate the lane)" +secrets-engine revoke whynot-design-npm-publish | sed 's/^/ /' + +echo +echo "### DONE — full chain exercised against a live OpenBao." diff --git a/src/secrets_engine/__init__.py b/src/secrets_engine/__init__.py new file mode 100644 index 0000000..133e713 --- /dev/null +++ b/src/secrets_engine/__init__.py @@ -0,0 +1,12 @@ +"""secrets-engine: decision-aware workflow layer over OpenBao. + +OpenBao enforces custody, policy, lease, and audit. This package owns the +operator/agent interaction model: catalog, decision checks, plan/apply, safe +provisioning, verification, exec-time delivery, and non-secret evidence. + +Hard rule enforced throughout the code: raw secret *values* never enter logs, +evidence, State Hub payloads, return values, or stdout. Values live only in +OpenBao, transient process memory, and short-lived mode-0600 files. +""" + +__version__ = "0.1.0" diff --git a/src/secrets_engine/__main__.py b/src/secrets_engine/__main__.py new file mode 100644 index 0000000..ff21643 --- /dev/null +++ b/src/secrets_engine/__main__.py @@ -0,0 +1,4 @@ +from secrets_engine.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/secrets_engine/apply.py b/src/secrets_engine/apply.py new file mode 100644 index 0000000..1e72a87 --- /dev/null +++ b/src/secrets_engine/apply.py @@ -0,0 +1,69 @@ +"""Apply a guarded plan to OpenBao. Idempotent and decision-gated. + +Apply only ever writes *metadata* (KV mount, ACL policy, approle role). It does +NOT write secret values — that is the separate, more constrained `provision` step. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.openbao import OpenBaoClient +from secrets_engine.plan import Plan + + +@dataclass +class ApplyResult: + applied: list[str] + skipped: list[str] + + def render(self) -> str: + out = [] + for a in self.applied: + out.append(f" applied: {a}") + for s in self.skipped: + out.append(f" unchanged: {s}") + return "\n".join(out) or " (nothing to do)" + + +def apply_plan(client: OpenBaoClient, entry: CatalogEntry, plan: Plan, ttl: str = "30m") -> ApplyResult: + """Execute the plan's metadata actions idempotently. + + Idempotency: KV mount and approle are enable-if-absent; the policy is written + only when its current body differs from the desired HCL. + """ + applied: list[str] = [] + skipped: list[str] = [] + + # 1. KV mount. + if client.kv_mount_exists(entry.mount): + skipped.append(f"kv-mount {entry.mount} (already present)") + else: + client.ensure_kv_mount(entry.mount) + applied.append(f"kv-mount {entry.mount}") + + # 2. Consumer ACL policy (write only if changed). + current = client.read_policy(plan.policy_name) + if current and _normalize(current) == _normalize(plan.policy_hcl): + skipped.append(f"policy {plan.policy_name} (unchanged)") + else: + client.write_policy(plan.policy_name, plan.policy_hcl) + applied.append(f"policy {plan.policy_name}") + + # 3. Consumer approle bound to that policy. + client.ensure_approle_enabled() + client.write_approle(plan.role_name, [plan.policy_name], ttl=ttl) + applied.append(f"approle {plan.role_name} -> [{plan.policy_name}]") + + return ApplyResult(applied=applied, skipped=skipped) + + +def _normalize(hcl: str) -> str: + """Compare policy bodies ignoring comments and whitespace noise.""" + lines = [] + for raw in hcl.splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + lines.append(" ".join(line.split())) + return "\n".join(lines) diff --git a/src/secrets_engine/catalog.py b/src/secrets_engine/catalog.py new file mode 100644 index 0000000..fd20c17 --- /dev/null +++ b/src/secrets_engine/catalog.py @@ -0,0 +1,171 @@ +"""Catalog: the non-secret registry of secret lanes and grants. + +A catalog entry describes *where* a secret lives in OpenBao, *who* may consume +it, *how* it is delivered, and *what* approval/verification/rotation it requires. +It never contains a secret value. Loading and validation are strict: a malformed +or under-specified lane is rejected rather than silently defaulted. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + +from secrets_engine.errors import CatalogError +from secrets_engine.redact import looks_secret + +VALID_STAGES = ("build", "test", "prod") +VALID_DELIVERY_MODES = ("exec-env", "exec-file", "npm-config", "wrapped", "read-check") +VALID_APPROVAL_MODELS = ("decision", "ccr", "dual-control", "bootstrap-only") + +REQUIRED_FIELDS = ( + "id", + "owner", + "stage", + "mount", + "path", + "fields", + "consumers", + "delivery_modes", + "approval", + "verification", + "rotation", + "deactivation", + "audit", +) + + +@dataclass(frozen=True) +class CatalogEntry: + id: str + owner: str + stage: str + mount: str + path: str + fields: list[str] + consumers: list[dict[str, Any]] + delivery_modes: list[str] + approval: dict[str, Any] + verification: dict[str, Any] + rotation: dict[str, Any] + deactivation: dict[str, Any] + audit: dict[str, Any] + description: str = "" + raw: dict[str, Any] = field(default_factory=dict) + + @property + def kv_data_path(self) -> str: + """Full KV v2 *data* path used for read/write of the value.""" + return f"{self.mount}/data/{self.path}" + + @property + def kv_logical_path(self) -> str: + """KV v2 logical path (used inside ACL policy capabilities).""" + return f"{self.mount}/data/{self.path}" + + @property + def policy_name(self) -> str: + return f"se-{self.stage}-{self.id}" + + @property + def role_name(self) -> str: + return f"se-{self.stage}-{self.id}" + + def approval_required(self) -> bool: + return self.approval.get("model") != "bootstrap-only" + + +def validate_entry(data: dict[str, Any], *, source: str = "") -> CatalogEntry: + """Validate a raw mapping and return a CatalogEntry, or raise CatalogError.""" + if not isinstance(data, dict): + raise CatalogError(f"{source}: catalog entry must be a mapping") + + missing = [k for k in REQUIRED_FIELDS if k not in data or data[k] in (None, "", [], {})] + if missing: + raise CatalogError(f"{source}: missing required fields: {', '.join(missing)}") + + stage = data["stage"] + if stage not in VALID_STAGES: + raise CatalogError( + f"{source}: stage '{stage}' invalid; must be one of {VALID_STAGES}" + ) + + modes = data["delivery_modes"] + if not isinstance(modes, list) or not modes: + raise CatalogError(f"{source}: delivery_modes must be a non-empty list") + bad_modes = [m for m in modes if m not in VALID_DELIVERY_MODES] + if bad_modes: + raise CatalogError( + f"{source}: unknown delivery_modes {bad_modes}; allowed {VALID_DELIVERY_MODES}" + ) + + fields = data["fields"] + if not isinstance(fields, list) or not all(isinstance(f, str) for f in fields): + raise CatalogError(f"{source}: fields must be a list of strings") + + consumers = data["consumers"] + if not isinstance(consumers, list) or not consumers: + raise CatalogError(f"{source}: consumers must be a non-empty list") + for c in consumers: + if not isinstance(c, dict) or "name" not in c or "auth" not in c: + raise CatalogError( + f"{source}: each consumer needs at least 'name' and 'auth'" + ) + + approval = data["approval"] + if not isinstance(approval, dict) or "model" not in approval: + raise CatalogError(f"{source}: approval must include a 'model'") + if approval["model"] not in VALID_APPROVAL_MODELS: + raise CatalogError( + f"{source}: approval.model '{approval['model']}' invalid; " + f"allowed {VALID_APPROVAL_MODELS}" + ) + + # A path must never leak a value through a field name suggesting inline secrets. + if any(looks_secret(k) and data.get(k) for k in ("value", "secret", "token", "password")): + raise CatalogError(f"{source}: catalog entries must not contain secret values") + + # Guard against accidentally broad mount/path. + path = data["path"] + if "*" in path or "*" in data["mount"]: + raise CatalogError(f"{source}: wildcard mount/path not allowed in catalog") + + known = {f.name for f in CatalogEntry.__dataclass_fields__.values()} - {"raw"} + kwargs = {k: v for k, v in data.items() if k in known} + return CatalogEntry(raw=data, **kwargs) + + +def load_entry(path: Path) -> CatalogEntry: + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError as e: + raise CatalogError(f"catalog file not found: {path}") from e + except yaml.YAMLError as e: + raise CatalogError(f"{path}: YAML parse error: {e}") from e + return validate_entry(data, source=str(path)) + + +def load_catalog(catalog_dir: Path) -> dict[str, CatalogEntry]: + """Load and validate every *.yaml in the catalog directory.""" + catalog_dir = Path(catalog_dir) + if not catalog_dir.exists(): + return {} + entries: dict[str, CatalogEntry] = {} + for path in sorted(catalog_dir.glob("*.yaml")): + entry = load_entry(path) + if entry.id in entries: + raise CatalogError(f"duplicate catalog id '{entry.id}' in {path}") + entries[entry.id] = entry + return entries + + +def get_entry(catalog_dir: Path, catalog_id: str) -> CatalogEntry: + entries = load_catalog(catalog_dir) + if catalog_id not in entries: + raise CatalogError( + f"catalog id '{catalog_id}' not found in {catalog_dir} " + f"(known: {', '.join(sorted(entries)) or 'none'})" + ) + return entries[catalog_id] diff --git a/src/secrets_engine/cli.py b/src/secrets_engine/cli.py new file mode 100644 index 0000000..1bd9b9b --- /dev/null +++ b/src/secrets_engine/cli.py @@ -0,0 +1,339 @@ +"""secrets-engine command-line interface. + +Command surface (FR7): + catalog list + catalog show + decision inspect + plan --stage + apply --stage [--dry-run] [--bootstrap-token-file F] + provision --stage (--from-file F | --generate) --field NAME + verify [--positive] [--negative] --field NAME + exec --catalog [--field NAME] [--mode auto|npm-config|exec-env] -- CMD... + route [--json] + revoke + +Every privileged action is decision-gated and writes non-secret evidence. +`plan` and `apply --dry-run` never mutate OpenBao. +""" +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from secrets_engine import __version__ +from secrets_engine.apply import apply_plan +from secrets_engine.catalog import get_entry, load_catalog +from secrets_engine.config import Config, repo_root +from secrets_engine.decisions import require_approved, resolve_decision +from secrets_engine.errors import SecretsEngineError +from secrets_engine.evidence import EvidenceWriter +from secrets_engine.openbao import OpenBaoClient +from secrets_engine.plan import build_plan +from secrets_engine.provision import provision_from_file, provision_generated +from secrets_engine.routing import route_lane +from secrets_engine.verify import run_verification + + +def _resolve_lane_and_decision(cfg: Config, ref: str, stage: str): + """Map a decision/ccr/lane ref to a catalog entry + resolved decision. + + For the MVP the catalog's approval.decision_ref ties a lane to its decision, + and a lane may be addressed directly by catalog id. We match `ref` against + both catalog ids and decision_refs. + """ + entries = load_catalog(cfg.catalog_dir) + # direct catalog id + if ref in entries: + return entries[ref] + # by decision_ref + for entry in entries.values(): + if entry.approval.get("decision_ref") == ref: + return entry + from secrets_engine.errors import CatalogError + + raise CatalogError( + f"no lane matches '{ref}' (by catalog id or decision_ref); " + f"known lanes: {', '.join(sorted(entries)) or 'none'}" + ) + + +def _writer(cfg: Config) -> EvidenceWriter: + return EvidenceWriter(evidence_dir=cfg.evidence_dir, hub_url=cfg.hub_url, topic_id=cfg.topic_id) + + +# -- command handlers ------------------------------------------------------ + + +def cmd_catalog_list(cfg: Config, args) -> int: + entries = load_catalog(cfg.catalog_dir) + if not entries: + print("(no catalog entries)") + return 0 + for e in entries.values(): + print(f"{e.id:32s} stage={e.stage:5s} owner={e.owner:18s} {e.mount}/{e.path}") + return 0 + + +def cmd_catalog_show(cfg: Config, args) -> int: + e = get_entry(cfg.catalog_dir, args.catalog_id) + print(f"id: {e.id}") + print(f"owner: {e.owner}") + print(f"stage: {e.stage}") + print(f"openbao: {e.mount}/{e.path} fields={e.fields}") + print(f"consumers: {[c['name'] for c in e.consumers]}") + print(f"delivery: {e.delivery_modes}") + print(f"approval: {e.approval.get('model')} ref={e.approval.get('decision_ref','')}") + print(f"verification: {e.verification}") + print(f"rotation: {e.rotation}") + print(f"deactivation: {e.deactivation}") + print(f"description: {e.description.strip()}") + return 0 + + +def cmd_decision_inspect(cfg: Config, args) -> int: + d = resolve_decision(hub_url=cfg.hub_url, repo_root=repo_root(), decision_ref=args.ref) + print(f"decision: {d.id}") + print(f"title: {d.title}") + print(f"status: {d.status} ({'APPROVED' if d.is_approved() else 'NOT approved'})") + print(f"source: {d.source}") + if d.superseded_by: + print(f"superseded: {d.superseded_by}") + if d.review_url: + print(f"review: {d.review_url}") + _writer(cfg).record( + "decision-inspect", result=d.status, decision_id=d.id, hub=False + ) + return 0 + + +def cmd_plan(cfg: Config, args) -> int: + entry = _resolve_lane_and_decision(cfg, args.ref, args.stage) + decision = None + if entry.approval_required(): + decision = resolve_decision( + hub_url=cfg.hub_url, repo_root=repo_root(), + decision_ref=entry.approval.get("decision_ref", args.ref), + ) + require_approved(entry, decision) + plan = build_plan(entry, args.stage, decision_id=decision.id if decision else "") + print(plan.render()) + _writer(cfg).record( + "plan", result="rendered", catalog_id=entry.id, stage=args.stage, + decision_id=decision.id if decision else "", + ) + return 0 + + +def cmd_apply(cfg: Config, args) -> int: + entry = _resolve_lane_and_decision(cfg, args.ref, args.stage) + decision = None + if entry.approval_required(): + decision = resolve_decision( + hub_url=cfg.hub_url, repo_root=repo_root(), + decision_ref=entry.approval.get("decision_ref", args.ref), + ) + require_approved(entry, decision) + plan = build_plan(entry, args.stage, decision_id=decision.id if decision else "") + w = _writer(cfg) + if args.dry_run: + print(plan.render()) + print("\n(dry-run: no OpenBao mutation performed)") + w.record("apply", result="dry-run", catalog_id=entry.id, stage=args.stage, + decision_id=decision.id if decision else "") + return 0 + client = OpenBaoClient.resolve(cfg.bao_addr, bootstrap_token_file=args.bootstrap_token_file) + result = apply_plan(client, entry, plan) + print(result.render()) + w.record("apply", result="applied", catalog_id=entry.id, stage=args.stage, + decision_id=decision.id if decision else "", + detail={"applied": result.applied, "skipped": result.skipped}) + return 0 + + +def cmd_provision(cfg: Config, args) -> int: + entry = get_entry(cfg.catalog_dir, args.catalog_id) + if args.stage != entry.stage: + from secrets_engine.errors import ProvisioningError + raise ProvisioningError(f"lane '{entry.id}' is stage '{entry.stage}', not '{args.stage}'") + client = OpenBaoClient.resolve(cfg.bao_addr, bootstrap_token_file=args.bootstrap_token_file) + field = args.field or (entry.fields[0] if entry.fields else "") + if args.generate: + f = provision_generated(client, entry, field) + mode = "generated" + else: + f = provision_from_file(client, entry, field, Path(args.from_file)) + mode = "from-file" + print(f"provisioned lane '{entry.id}' field '{f}' ({mode}) — value not displayed") + _writer(cfg).record("provision", result=mode, catalog_id=entry.id, stage=entry.stage, + detail={"field": f}) + return 0 + + +def cmd_verify(cfg: Config, args) -> int: + entry = get_entry(cfg.catalog_dir, args.catalog_id) + client = OpenBaoClient.resolve(cfg.bao_addr, bootstrap_token_file=args.bootstrap_token_file) + field = args.field or (entry.fields[0] if entry.fields else "") + positive = args.positive or not args.negative + negative = args.negative or not args.positive + results = run_verification(client, entry, field, positive=positive, negative=negative) + rc = 0 + for r in results: + print(r.render()) + if not r.passed: + rc = 7 + _writer(cfg).record("verify", result=f"{r.check}:{'pass' if r.passed else 'fail'}", + catalog_id=entry.id, stage=entry.stage, detail=r.detail) + return rc + + +def cmd_exec(cfg: Config, args) -> int: + from secrets_engine.exec_delivery import exec_with_secret + entry = get_entry(cfg.catalog_dir, args.catalog) + # require approval + readiness before running. + if entry.approval_required(): + decision = resolve_decision( + hub_url=cfg.hub_url, repo_root=repo_root(), + decision_ref=entry.approval.get("decision_ref", entry.id), + ) + require_approved(entry, decision) + if not args.command: + from secrets_engine.errors import DeliveryError + raise DeliveryError("no command after '--'") + client = OpenBaoClient.resolve(cfg.bao_addr, bootstrap_token_file=args.bootstrap_token_file) + field = args.field or (entry.fields[0] if entry.fields else "") + w = _writer(cfg) + w.record("exec", result="attempt", catalog_id=entry.id, stage=entry.stage, + detail={"command": args.command[0], "mode": args.mode}) + rc = exec_with_secret(client, entry, field, args.command, mode=args.mode) + w.record("exec", result=f"exit-{rc}", catalog_id=entry.id, stage=entry.stage, + detail={"command": args.command[0]}) + return rc + + +def cmd_route(cfg: Config, args) -> int: + entry = get_entry(cfg.catalog_dir, args.catalog_id) + client = OpenBaoClient.resolve(cfg.bao_addr) + result = route_lane(entry, hub_url=cfg.hub_url, repo_root=repo_root(), client=client) + if args.json: + import json + print(json.dumps(result.to_json(), indent=2)) + else: + print(f"lane: {result.catalog_id} (owner={result.owner}, stage={result.stage})") + print(f"decision: {result.decision_status} ref={result.decision_ref}") + print(f"applied: {result.metadata_applied} value_present: {result.value_present}") + print(f"ready: {result.ready}") + if result.missing: + print(f"missing: {result.missing}") + print(f"next: {result.next_command}") + return 0 + + +def cmd_revoke(cfg: Config, args) -> int: + entry = get_entry(cfg.catalog_dir, args.catalog_id) + client = OpenBaoClient.resolve(cfg.bao_addr, bootstrap_token_file=args.bootstrap_token_file) + if args.dry_run: + print(f"(dry-run) would delete KV metadata {entry.mount}/{entry.path} " + f"and approle {entry.role_name}") + return 0 + client.kv_delete_metadata(entry.mount, entry.path) + print(f"revoked lane '{entry.id}': KV metadata deleted at {entry.mount}/{entry.path}") + _writer(cfg).record("revoke", result="deactivated", catalog_id=entry.id, stage=entry.stage) + return 0 + + +# -- parser ---------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="secrets-engine", description=__doc__.splitlines()[0]) + p.add_argument("--version", action="version", version=f"secrets-engine {__version__}") + sub = p.add_subparsers(dest="cmd", required=True) + + def add_token_arg(sp): + sp.add_argument("--bootstrap-token-file", default=None, + help="path to a mode-0600 OpenBao token file (bootstrap only)") + + cat = sub.add_parser("catalog", help="catalog operations") + catsub = cat.add_subparsers(dest="subcmd", required=True) + catsub.add_parser("list", help="list catalog lanes").set_defaults(func=cmd_catalog_list) + cshow = catsub.add_parser("show", help="show a lane") + cshow.add_argument("catalog_id") + cshow.set_defaults(func=cmd_catalog_show) + + dec = sub.add_parser("decision", help="decision operations") + decsub = dec.add_subparsers(dest="subcmd", required=True) + dinsp = decsub.add_parser("inspect", help="inspect a decision/CCR") + dinsp.add_argument("ref") + dinsp.set_defaults(func=cmd_decision_inspect) + + pl = sub.add_parser("plan", help="render a guarded plan (no mutation)") + pl.add_argument("ref", help="decision/ccr id or catalog id") + pl.add_argument("--stage", required=True, choices=("build", "test", "prod")) + pl.set_defaults(func=cmd_plan) + + ap = sub.add_parser("apply", help="apply approved metadata to OpenBao") + ap.add_argument("ref") + ap.add_argument("--stage", required=True, choices=("build", "test", "prod")) + ap.add_argument("--dry-run", action="store_true") + add_token_arg(ap) + ap.set_defaults(func=cmd_apply) + + pr = sub.add_parser("provision", help="provision a value without printing it") + pr.add_argument("catalog_id") + pr.add_argument("--stage", required=True, choices=("build", "test", "prod")) + pr.add_argument("--field", default=None) + g = pr.add_mutually_exclusive_group(required=True) + g.add_argument("--from-file", help="mode-0600 file holding the value, outside repos") + g.add_argument("--generate", action="store_true", help="generate (build/test only)") + add_token_arg(pr) + pr.set_defaults(func=cmd_provision) + + ve = sub.add_parser("verify", help="positive/negative verification (no value printed)") + ve.add_argument("catalog_id") + ve.add_argument("--field", default=None) + ve.add_argument("--positive", action="store_true") + ve.add_argument("--negative", action="store_true") + add_token_arg(ve) + ve.set_defaults(func=cmd_verify) + + ex = sub.add_parser("exec", help="run a command with the secret injected for the child only") + ex.add_argument("--catalog", required=True) + ex.add_argument("--field", default=None) + ex.add_argument("--mode", default="auto", choices=("auto", "npm-config", "exec-env")) + add_token_arg(ex) + ex.add_argument("command", nargs=argparse.REMAINDER, + help="command after '--'") + ex.set_defaults(func=cmd_exec) + + ro = sub.add_parser("route", help="ops-warden routing pointer for a lane") + ro.add_argument("catalog_id") + ro.add_argument("--json", action="store_true") + ro.set_defaults(func=cmd_route) + + rv = sub.add_parser("revoke", help="deactivate a lane (delete KV metadata)") + rv.add_argument("catalog_id") + rv.add_argument("--dry-run", action="store_true") + add_token_arg(rv) + rv.set_defaults(func=cmd_revoke) + + return p + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + parser = build_parser() + args = parser.parse_args(argv) + # `exec` REMAINDER includes a leading '--'; strip it. + if getattr(args, "command", None) and args.command and args.command[0] == "--": + args.command = args.command[1:] + cfg = Config.load() + try: + return args.func(cfg, args) + except SecretsEngineError as e: + print(f"error: {e}", file=sys.stderr) + return e.exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/secrets_engine/config.py b/src/secrets_engine/config.py new file mode 100644 index 0000000..2671d46 --- /dev/null +++ b/src/secrets_engine/config.py @@ -0,0 +1,43 @@ +"""Runtime configuration resolved from environment and repo layout. + +Nothing here is a secret. Backend auth (BAO_TOKEN / bootstrap token files) is +resolved lazily inside the backend adapter, never cached on disk by this module. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def repo_root() -> Path: + """Repo root = nearest ancestor containing pyproject.toml (fallback: cwd).""" + here = Path(__file__).resolve() + for parent in (here, *here.parents): + if (parent / "pyproject.toml").exists(): + return parent + return Path.cwd() + + +@dataclass(frozen=True) +class Config: + catalog_dir: Path + policy_dir: Path + evidence_dir: Path + hub_url: str + bao_addr: str + topic_id: str + + @classmethod + def load(cls) -> "Config": + root = repo_root() + return cls( + catalog_dir=Path(os.environ.get("SECRETS_ENGINE_CATALOG", root / "catalog")), + policy_dir=Path(os.environ.get("SECRETS_ENGINE_POLICIES", root / "policies")), + evidence_dir=Path(os.environ.get("SECRETS_ENGINE_EVIDENCE", root / ".evidence")), + hub_url=os.environ.get("SECRETS_ENGINE_HUB_URL", "http://127.0.0.1:8000"), + bao_addr=os.environ.get("BAO_ADDR", os.environ.get("VAULT_ADDR", "http://127.0.0.1:8200")), + topic_id=os.environ.get( + "SECRETS_ENGINE_TOPIC_ID", "cee7bedf-2b48-46ef-8601-006474f2ad7a" + ), + ) diff --git a/src/secrets_engine/decisions.py b/src/secrets_engine/decisions.py new file mode 100644 index 0000000..0117175 --- /dev/null +++ b/src/secrets_engine/decisions.py @@ -0,0 +1,115 @@ +"""Decision integration. + +Privileged actions require an approved decision (or approved CCR) unless the lane +is explicitly `bootstrap-only` or the caller passes --dry-run. Decisions are +looked up from State Hub by id; when the hub has no record yet (common during the +pilot) a local approval fixture under `.decisions/.yaml` can stand in, so the +end-to-end chain is testable before the canonical hub decision object exists. + +No secret values are ever read from or written to a decision. +""" +from __future__ import annotations + +import json +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import DecisionError + +APPROVED_STATUSES = {"resolved", "approved", "accepted"} + + +@dataclass +class Decision: + id: str + title: str + status: str + superseded_by: str | None + source: str # "hub" | "local-fixture" + review_url: str = "" + raw: dict[str, Any] | None = None + + def is_approved(self) -> bool: + return self.status.lower() in APPROVED_STATUSES and not self.superseded_by + + +def _hub_get(hub_url: str, decision_id: str) -> dict[str, Any] | None: + url = hub_url.rstrip("/") + f"/decisions/{decision_id}" + try: + with urllib.request.urlopen(url, timeout=3) as resp: + return json.loads(resp.read()) + except (urllib.error.URLError, OSError, ValueError): + return None + + +def _local_fixture(repo_root: Path, ref: str) -> dict[str, Any] | None: + path = repo_root / ".decisions" / f"{ref}.yaml" + if not path.exists(): + return None + try: + return yaml.safe_load(path.read_text(encoding="utf-8")) or None + except yaml.YAMLError as e: + raise DecisionError(f"{path}: invalid decision fixture: {e}") from e + + +def resolve_decision( + *, + hub_url: str, + repo_root: Path, + decision_ref: str, +) -> Decision: + """Resolve a decision by id (hub) or slug (local fixture). Raises if absent.""" + if not decision_ref: + raise DecisionError("no decision reference provided") + + doc = _hub_get(hub_url, decision_ref) + if doc: + return Decision( + id=doc.get("id", decision_ref), + title=doc.get("title", ""), + status=doc.get("status", "unknown"), + superseded_by=doc.get("superseded_by"), + source="hub", + review_url=f"{hub_url.rstrip('/')}/decisions/{doc.get('id', decision_ref)}", + raw=doc, + ) + + fixture = _local_fixture(repo_root, decision_ref) + if fixture: + return Decision( + id=fixture.get("id", decision_ref), + title=fixture.get("title", decision_ref), + status=fixture.get("status", "unknown"), + superseded_by=fixture.get("superseded_by"), + source="local-fixture", + review_url=fixture.get("review_url", ""), + raw=fixture, + ) + + raise DecisionError( + f"decision '{decision_ref}' not found in State Hub or local fixtures" + ) + + +def require_approved(entry: CatalogEntry, decision: Decision | None) -> None: + """Enforce the lane's approval model. Raises DecisionError if not satisfied.""" + if not entry.approval_required(): + return # bootstrap-only lane + if decision is None: + raise DecisionError( + f"lane '{entry.id}' requires an approved decision; none resolved" + ) + if decision.superseded_by: + raise DecisionError( + f"decision '{decision.id}' is superseded by '{decision.superseded_by}'" + ) + if not decision.is_approved(): + raise DecisionError( + f"decision '{decision.id}' is not approved (status='{decision.status}')" + ) diff --git a/src/secrets_engine/errors.py b/src/secrets_engine/errors.py new file mode 100644 index 0000000..d519184 --- /dev/null +++ b/src/secrets_engine/errors.py @@ -0,0 +1,54 @@ +"""Typed errors with stable exit codes for the CLI. + +Exit codes are part of the contract so callers (ops-warden, CI) can branch on +outcome without parsing text. +""" +from __future__ import annotations + + +class SecretsEngineError(Exception): + """Base class. ``exit_code`` is the process exit status.""" + + exit_code = 1 + + +class CatalogError(SecretsEngineError): + """Catalog file missing, unparseable, or schema-invalid.""" + + exit_code = 2 + + +class DecisionError(SecretsEngineError): + """Decision/CCR missing, denied, superseded, stale, or unapproved.""" + + exit_code = 3 + + +class PolicyGuardError(SecretsEngineError): + """A plan violates a safety guard (wildcard, out-of-stage path, root, ...).""" + + exit_code = 4 + + +class BackendError(SecretsEngineError): + """OpenBao backend call failed or is unreachable.""" + + exit_code = 5 + + +class ProvisioningError(SecretsEngineError): + """Provisioning input invalid (bad file mode, inside repo, missing field).""" + + exit_code = 6 + + +class VerificationError(SecretsEngineError): + """A verification check did not produce the expected result.""" + + exit_code = 7 + + +class DeliveryError(SecretsEngineError): + """Exec-time delivery could not be set up safely.""" + + exit_code = 8 diff --git a/src/secrets_engine/evidence.py b/src/secrets_engine/evidence.py new file mode 100644 index 0000000..eca1eaf --- /dev/null +++ b/src/secrets_engine/evidence.py @@ -0,0 +1,116 @@ +"""Non-secret evidence writer. + +Every privileged or noteworthy action emits an evidence record to a local +append-only JSONL log and, best-effort, to the State Hub progress API. Records +are scrubbed of anything that looks like a secret value before they are written. +""" +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from secrets_engine.redact import looks_secret, redact_text + +# Keys that must never carry a value into evidence regardless of nesting. +_FORBIDDEN_VALUE_KEYS = {"value", "secret", "token", "password", "raw"} + + +def _scrub(obj: Any) -> Any: + """Recursively drop secret-looking keys and redact token shapes in strings.""" + if isinstance(obj, dict): + out = {} + for k, v in obj.items(): + if k.lower() in _FORBIDDEN_VALUE_KEYS or looks_secret(k): + out[k] = "" + else: + out[k] = _scrub(v) + return out + if isinstance(obj, list): + return [_scrub(v) for v in obj] + if isinstance(obj, str): + return redact_text(obj) + return obj + + +@dataclass +class EvidenceWriter: + evidence_dir: Path + hub_url: str = "" + topic_id: str = "" + workstream_id: str = "" + author: str = "secrets-engine" + actor: str = field(default_factory=lambda: os.environ.get("USER", "unknown")) + + def __post_init__(self) -> None: + self.evidence_dir = Path(self.evidence_dir) + + def _log_path(self) -> Path: + self.evidence_dir.mkdir(parents=True, exist_ok=True) + day = datetime.now(timezone.utc).strftime("%Y-%m-%d") + return self.evidence_dir / f"evidence-{day}.jsonl" + + def record( + self, + action: str, + *, + result: str, + catalog_id: str = "", + stage: str = "", + decision_id: str = "", + detail: dict[str, Any] | None = None, + hub: bool = True, + ) -> dict[str, Any]: + """Append one non-secret evidence record. Returns the stored record.""" + record = { + "ts": datetime.now(timezone.utc).isoformat(), + "action": action, + "result": result, + "actor": self.actor, + "catalog_id": catalog_id, + "stage": stage, + "decision_id": decision_id, + "detail": _scrub(detail or {}), + } + path = self._log_path() + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, sort_keys=True) + "\n") + if hub and self.hub_url: + self._post_hub(action, result, catalog_id, stage, decision_id) + return record + + def _post_hub( + self, action: str, result: str, catalog_id: str, stage: str, decision_id: str + ) -> None: + """Best-effort progress note to State Hub. Never raises; never sends values.""" + if not self.topic_id: + return + summary = f"secrets-engine {action}: {result}" + if catalog_id: + summary += f" [{catalog_id}{'/' + stage if stage else ''}]" + payload: dict[str, Any] = { + "topic_id": self.topic_id, + "event_type": "note", + "summary": summary, + "author": self.author, + } + if self.workstream_id: + payload["workstream_id"] = self.workstream_id + if decision_id: + payload["detail"] = {"decision_id": decision_id, "catalog_id": catalog_id} + try: + req = urllib.request.Request( + self.hub_url.rstrip("/") + "/progress/", + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + urllib.request.urlopen(req, timeout=3).read() + except (urllib.error.URLError, OSError, ValueError): + # Hub being offline must never block secret work or leak anything. + pass diff --git a/src/secrets_engine/exec_delivery.py b/src/secrets_engine/exec_delivery.py new file mode 100644 index 0000000..bbbeb55 --- /dev/null +++ b/src/secrets_engine/exec_delivery.py @@ -0,0 +1,152 @@ +"""Exec-time delivery: make a secret available only to a child process. + +The default and preferred delivery mode. The value is fetched from OpenBao, +injected into the child's environment / a temp config, the child runs, and the +injection is destroyed afterward — on success, failure, or interruption. + +Supported here: + - npm-config: write a temporary .npmrc with the auth token and point the child + at it via NPM_CONFIG_USERCONFIG. Preferred for `npm publish`. + - exec-env: inject the value as an environment variable for the child only. + +The parent shell never sees the value; the value is never logged. Child stdout/ +stderr is streamed through a redactor as a backstop. +""" +from __future__ import annotations + +import json +import os +import signal +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import DeliveryError +from secrets_engine.openbao import OpenBaoClient +from secrets_engine.redact import redact_text + + +def _fetch_value(client: OpenBaoClient, entry: CatalogEntry, field: str) -> str: + """Read the field value via an approle-scoped token. Held in memory only.""" + try: + token = client.approle_login_token(entry.role_name) + except Exception as e: + raise DeliveryError(f"could not obtain scoped token for delivery: {e}") from e + scoped = OpenBaoClient(addr=client.addr, token=token, bao_bin=client.bao_bin) + proc = scoped._run(["kv", "get", "-format=json", f"{entry.mount}/{entry.path}"]) + if proc.returncode != 0: + raise DeliveryError(f"scoped read failed for lane '{entry.id}' (denied or absent)") + try: + data = json.loads(proc.stdout)["data"]["data"] + except (json.JSONDecodeError, KeyError) as e: + raise DeliveryError(f"malformed KV response for lane '{entry.id}'") from e + if field not in data: + raise DeliveryError(f"field '{field}' absent in lane '{entry.id}'") + return data[field] + + +@contextmanager +def _npm_userconfig(token: str) -> Iterator[Path]: + """Write a mode-0600 temp .npmrc, yield its path, delete it unconditionally.""" + fd, name = tempfile.mkstemp(prefix="se-npmrc-", suffix=".ini") + path = Path(name) + try: + os.fchmod(fd, 0o600) + # Registry-scoped auth token; child npm reads this via NPM_CONFIG_USERCONFIG. + with os.fdopen(fd, "w") as fh: + fh.write("//registry.npmjs.org/:_authToken=${SE_NPM_TOKEN}\n") + yield path + finally: + try: + path.unlink() + except FileNotFoundError: + pass + + +def _stream_redacted(proc: subprocess.Popen, secret: str) -> None: + """Stream child output through the redactor (backstop).""" + assert proc.stdout is not None + for line in proc.stdout: + sys.stdout.write(redact_text(line, extra=[secret])) + sys.stdout.flush() + + +def exec_with_secret( + client: OpenBaoClient, + entry: CatalogEntry, + field: str, + command: list[str], + *, + mode: str = "auto", +) -> int: + """Run `command` with the lane's secret injected for the child only. + + Returns the child's exit code. Raises DeliveryError if setup is unsafe. + """ + if not command: + raise DeliveryError("no command given to exec") + + declared = set(entry.delivery_modes) + if mode == "auto": + mode = "npm-config" if "npm-config" in declared else ( + "exec-env" if "exec-env" in declared else "" + ) + if not mode: + raise DeliveryError( + f"lane '{entry.id}' declares no exec-capable delivery mode " + f"({sorted(declared)})" + ) + if mode not in declared: + raise DeliveryError( + f"delivery mode '{mode}' not permitted for lane '{entry.id}' " + f"(allowed {sorted(declared)})" + ) + + value = _fetch_value(client, entry, field) + child_env = dict(os.environ) + + if mode == "npm-config": + with _npm_userconfig(value) as npmrc: + child_env["NPM_CONFIG_USERCONFIG"] = str(npmrc) + child_env["SE_NPM_TOKEN"] = value + rc = _spawn(command, child_env, value) + return rc + + if mode == "exec-env": + # Inject under a conventional name derived from the field. + env_name = field.upper() + child_env[env_name] = value + return _spawn(command, child_env, value) + + raise DeliveryError(f"unsupported delivery mode '{mode}'") + + +def _spawn(command: list[str], env: dict[str, str], secret: str) -> int: + """Spawn the child, stream redacted output, propagate signals, ensure cleanup.""" + try: + proc = subprocess.Popen( + command, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except FileNotFoundError as e: + raise DeliveryError(f"command not found: {command[0]}") from e + + def _forward(signum, _frame): + proc.send_signal(signum) + + old_int = signal.signal(signal.SIGINT, _forward) + old_term = signal.signal(signal.SIGTERM, _forward) + try: + _stream_redacted(proc, secret) + return proc.wait() + finally: + signal.signal(signal.SIGINT, old_int) + signal.signal(signal.SIGTERM, old_term) + # env dict goes out of scope; the temp npmrc is removed by its context mgr. diff --git a/src/secrets_engine/openbao.py b/src/secrets_engine/openbao.py new file mode 100644 index 0000000..6b97eaa --- /dev/null +++ b/src/secrets_engine/openbao.py @@ -0,0 +1,232 @@ +"""OpenBao backend adapter. + +Thin wrapper over the `bao` CLI. Isolated here so the rest of the engine speaks +in lanes/plans, not in OpenBao endpoint quirks (FR: "isolate backend adapter"). + +Auth resolution order for the token: + 1. explicit bootstrap token file (--bootstrap-token-file), mode-checked; + 2. BAO_TOKEN / VAULT_TOKEN environment variable; + 3. otherwise unauthenticated (only dry-run / read-health works). + +This adapter NEVER returns a secret value to its callers except through the +narrow `read_field_present()` (boolean) and the exec-delivery path, which writes +straight into a child process and never logs. +""" +from __future__ import annotations + +import json +import os +import shutil +import stat +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from secrets_engine.errors import BackendError, ProvisioningError + + +def _check_token_file(path: Path) -> str: + """Read a bootstrap token file after enforcing mode-0600 and out-of-repo.""" + if not path.exists(): + raise ProvisioningError(f"bootstrap token file not found: {path}") + st = path.stat() + if st.st_mode & 0o077: + raise ProvisioningError( + f"bootstrap token file {path} is group/other-accessible " + f"(mode {oct(st.st_mode & 0o777)}); must be 0600" + ) + # Refuse a token file living inside a Git worktree. + for parent in path.resolve().parents: + if (parent / ".git").exists(): + raise ProvisioningError( + f"bootstrap token file {path} is inside a Git worktree ({parent}); " + "store it outside any repo" + ) + token = path.read_text(encoding="utf-8").strip() + if not token: + raise ProvisioningError(f"bootstrap token file {path} is empty") + return token + + +@dataclass +class OpenBaoClient: + addr: str + token: str = "" + bao_bin: str = "" + + @classmethod + def resolve( + cls, addr: str, *, bootstrap_token_file: str | Path | None = None + ) -> "OpenBaoClient": + token = "" + if bootstrap_token_file: + token = _check_token_file(Path(bootstrap_token_file)) + else: + token = os.environ.get("BAO_TOKEN", os.environ.get("VAULT_TOKEN", "")) + bao_bin = shutil.which("bao") or shutil.which("vault") or "" + return cls(addr=addr, token=token, bao_bin=bao_bin) + + # -- low level --------------------------------------------------------- + + def _run(self, args: list[str], *, stdin: str | None = None) -> subprocess.CompletedProcess: + if not self.bao_bin: + raise BackendError( + "no 'bao' (or 'vault') CLI on PATH; cannot reach OpenBao backend" + ) + env = dict(os.environ) + env["BAO_ADDR"] = self.addr + env["VAULT_ADDR"] = self.addr + if self.token: + env["BAO_TOKEN"] = self.token + env["VAULT_TOKEN"] = self.token + try: + return subprocess.run( + [self.bao_bin, *args], + input=stdin, + env=env, + capture_output=True, + text=True, + timeout=30, + ) + except FileNotFoundError as e: + raise BackendError(f"backend binary not runnable: {e}") from e + except subprocess.TimeoutExpired as e: + raise BackendError(f"backend call timed out: {' '.join(args)}") from e + + def _run_ok(self, args: list[str], *, stdin: str | None = None) -> str: + proc = self._run(args, stdin=stdin) + if proc.returncode != 0: + # stderr from bao does not contain the secret value for these calls. + raise BackendError( + f"bao {' '.join(args[:2])} failed (exit {proc.returncode}): " + f"{proc.stderr.strip() or proc.stdout.strip()}" + ) + return proc.stdout + + # -- health / capabilities -------------------------------------------- + + def is_reachable(self) -> bool: + if not self.bao_bin: + return False + proc = self._run(["status", "-format=json"]) + # status returns non-zero when sealed but still reachable; treat any + # parseable JSON as reachable. + try: + json.loads(proc.stdout or "{}") + return True + except json.JSONDecodeError: + return proc.returncode == 0 + + # -- policies ---------------------------------------------------------- + + def write_policy(self, name: str, hcl: str) -> None: + self._run_ok(["policy", "write", name, "-"], stdin=hcl) + + def read_policy(self, name: str) -> str | None: + proc = self._run(["policy", "read", name]) + if proc.returncode != 0: + return None + return proc.stdout + + # -- approle ----------------------------------------------------------- + + def ensure_approle_enabled(self) -> None: + proc = self._run(["auth", "list", "-format=json"]) + if proc.returncode == 0: + try: + methods = json.loads(proc.stdout) + if "approle/" in methods: + return + except json.JSONDecodeError: + pass + enable = self._run(["auth", "enable", "approle"]) + if enable.returncode != 0 and "already in use" not in enable.stderr: + raise BackendError(f"could not enable approle: {enable.stderr.strip()}") + + def write_approle(self, role_name: str, policies: list[str], ttl: str = "30m") -> None: + self._run_ok( + [ + "write", + f"auth/approle/role/{role_name}", + f"token_policies={','.join(policies)}", + f"token_ttl={ttl}", + f"token_max_ttl={ttl}", + "secret_id_num_uses=0", + "token_num_uses=0", + ] + ) + + def approle_login_token(self, role_name: str) -> str: + """Login as the approle and return a scoped child token. Used only for + verification / exec delivery; never logged.""" + role_id = self._run_ok( + ["read", "-field=role_id", f"auth/approle/role/{role_name}/role-id"] + ).strip() + secret_id = self._run_ok( + ["write", "-field=secret_id", "-f", f"auth/approle/role/{role_name}/secret-id"] + ).strip() + token = self._run_ok( + [ + "write", + "-field=token", + "auth/approle/login", + f"role_id={role_id}", + f"secret_id={secret_id}", + ] + ).strip() + return token + + # -- KV v2 ------------------------------------------------------------- + + def kv_mount_exists(self, mount: str) -> bool: + proc = self._run(["secrets", "list", "-format=json"]) + if proc.returncode != 0: + return False + try: + return f"{mount}/" in json.loads(proc.stdout) + except json.JSONDecodeError: + return False + + def ensure_kv_mount(self, mount: str) -> None: + if self.kv_mount_exists(mount): + return + enable = self._run(["secrets", "enable", "-path", mount, "kv-v2"]) + if enable.returncode != 0 and "already in use" not in enable.stderr: + raise BackendError(f"could not enable kv at {mount}: {enable.stderr.strip()}") + + def kv_put(self, mount: str, path: str, field: str, value: str) -> None: + """Write a single field. `value` is a secret and is passed via stdin-free + argv only as a key=value to the local CLI; it is never logged or returned.""" + self._run_ok(["kv", "put", f"{mount}/{path}", f"{field}={value}"]) + + def kv_metadata_exists(self, mount: str, path: str) -> bool: + proc = self._run(["kv", "metadata", "get", "-format=json", f"{mount}/{path}"]) + return proc.returncode == 0 + + def kv_field_present(self, mount: str, path: str, field: str, *, token: str | None = None) -> bool: + """Return whether `field` exists at the path — WITHOUT returning its value. + + If `token` is given, the read is attempted as that (scoped) token, so a + True/False result doubles as a positive/negative access check. + """ + client = self + if token is not None: + client = OpenBaoClient(addr=self.addr, token=token, bao_bin=self.bao_bin) + proc = client._run(["kv", "get", "-format=json", f"{mount}/{path}"]) + if proc.returncode != 0: + return False + try: + doc = json.loads(proc.stdout) + except json.JSONDecodeError: + return False + data = doc.get("data", {}).get("data", {}) + return field in data and bool(data[field]) + + def kv_can_read(self, mount: str, path: str, *, token: str) -> bool: + """True iff `token` is permitted to read the path at all (no value used).""" + client = OpenBaoClient(addr=self.addr, token=token, bao_bin=self.bao_bin) + proc = client._run(["kv", "get", "-format=json", f"{mount}/{path}"]) + return proc.returncode == 0 + + def kv_delete_metadata(self, mount: str, path: str) -> None: + self._run_ok(["kv", "metadata", "delete", f"{mount}/{path}"]) diff --git a/src/secrets_engine/plan.py b/src/secrets_engine/plan.py new file mode 100644 index 0000000..3b1c887 --- /dev/null +++ b/src/secrets_engine/plan.py @@ -0,0 +1,90 @@ +"""Planning: turn an approved request into a concrete, guarded set of actions. + +A Plan is a human-reviewable list of OpenBao actions (policy write, approle +write, KV mount). Building a plan runs every safety guard, so a plan that exists +is, by construction, in-bounds. Apply just executes a built plan. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import PolicyGuardError +from secrets_engine.roles import ( + StageRole, + assert_path_in_stage, + consumer_policy_for, +) + + +@dataclass +class PlanAction: + kind: str # "kv-mount" | "policy" | "approle" + target: str # human-readable target + detail: dict[str, Any] = field(default_factory=dict) + + def render(self) -> str: + d = ", ".join(f"{k}={v}" for k, v in self.detail.items() if k != "hcl") + return f" [{self.kind}] {self.target}" + (f" ({d})" if d else "") + + +@dataclass +class Plan: + catalog_id: str + stage: str + decision_id: str + policy_name: str + role_name: str + actions: list[PlanAction] + policy_hcl: str + + def render(self) -> str: + lines = [ + f"Plan for lane '{self.catalog_id}' (stage={self.stage})", + f" decision: {self.decision_id or ''}", + f" stage role: secrets-engine-{self.stage}", + f" consumer policy: {self.policy_name}", + f" consumer approle: {self.role_name}", + " actions:", + ] + lines.extend(a.render() for a in self.actions) + lines.append("") + lines.append(" generated consumer policy (HCL):") + lines.extend(" " + ln for ln in self.policy_hcl.splitlines()) + return "\n".join(lines) + + +def build_plan(entry: CatalogEntry, stage: str, *, decision_id: str = "") -> Plan: + """Construct and fully guard a plan. Raises PolicyGuardError on any violation.""" + if stage != entry.stage: + raise PolicyGuardError( + f"stage mismatch: lane '{entry.id}' is stage '{entry.stage}', " + f"refusing to apply as '{stage}'" + ) + StageRole.for_stage(stage) # validates stage name + assert_path_in_stage(entry) # path must be in-stage, no wildcards + policy_name, policy_hcl = consumer_policy_for(entry) # runs assert_policy_safe + + actions = [ + PlanAction("kv-mount", entry.mount, {"type": "kv-v2"}), + PlanAction( + "policy", + policy_name, + {"paths": f"{entry.mount}/data/{entry.path}"}, + ), + PlanAction( + "approle", + entry.role_name, + {"token_policies": policy_name, "auth": "approle"}, + ), + ] + return Plan( + catalog_id=entry.id, + stage=stage, + decision_id=decision_id, + policy_name=policy_name, + role_name=entry.role_name, + actions=actions, + policy_hcl=policy_hcl, + ) diff --git a/src/secrets_engine/provision.py b/src/secrets_engine/provision.py new file mode 100644 index 0000000..0bbfc8a --- /dev/null +++ b/src/secrets_engine/provision.py @@ -0,0 +1,75 @@ +"""Provisioning: get a secret value into OpenBao without it touching coordination +surfaces. + +Modes: + - from-file: read a value from a mode-0600 file outside the repo, write it to + OpenBao, and (caller's choice) leave the source file for the operator to shred. + - generate: mint a random non-production value for build/test lanes only. + +The value is held only in process memory and passed straight to the backend. It +is never logged, returned, or written to evidence. +""" +from __future__ import annotations + +import secrets as _secrets +import string +from pathlib import Path + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import ProvisioningError +from secrets_engine.openbao import OpenBaoClient + + +def _read_value_file(path: Path) -> str: + if not path.exists(): + raise ProvisioningError(f"value file not found: {path}") + st = path.stat() + if st.st_mode & 0o077: + raise ProvisioningError( + f"value file {path} is group/other-accessible " + f"(mode {oct(st.st_mode & 0o777)}); must be 0600" + ) + for parent in path.resolve().parents: + if (parent / ".git").exists(): + raise ProvisioningError( + f"value file {path} is inside a Git worktree ({parent}); " + "keep secret material outside repos" + ) + value = path.read_text(encoding="utf-8").strip() + if not value: + raise ProvisioningError(f"value file {path} is empty") + return value + + +def provision_from_file( + client: OpenBaoClient, entry: CatalogEntry, field: str, file_path: Path +) -> str: + """Import a value from a strict-permission file. Returns the field name only.""" + if field not in entry.fields: + raise ProvisioningError( + f"field '{field}' not declared in lane '{entry.id}' fields {entry.fields}" + ) + value = _read_value_file(Path(file_path)) + client.ensure_kv_mount(entry.mount) + client.kv_put(entry.mount, entry.path, field, value) + del value + return field + + +def provision_generated(client: OpenBaoClient, entry: CatalogEntry, field: str) -> str: + """Generate a random NON-PRODUCTION value for build/test lanes only.""" + if entry.stage == "prod": + raise ProvisioningError( + f"refusing to generate a value for prod lane '{entry.id}'; " + "production values must be provisioned, not generated" + ) + if field not in entry.fields: + raise ProvisioningError( + f"field '{field}' not declared in lane '{entry.id}' fields {entry.fields}" + ) + alphabet = string.ascii_letters + string.digits + value = "test-" + "".join(_secrets.choice(alphabet) for _ in range(32)) + client.ensure_kv_mount(entry.mount) + client.kv_put(entry.mount, entry.path, field, value) + del value + return field diff --git a/src/secrets_engine/redact.py b/src/secrets_engine/redact.py new file mode 100644 index 0000000..b087f78 --- /dev/null +++ b/src/secrets_engine/redact.py @@ -0,0 +1,42 @@ +"""Defense-in-depth redaction of secret-like material. + +This is a backstop, not the primary control. The primary control is that secret +values are never passed into evidence/log code paths in the first place. Redaction +catches the case where a value leaks into child-process output we control. +""" +from __future__ import annotations + +import re +from typing import Iterable + +REDACTED = "***REDACTED***" + +# Token shapes we proactively mask in child-process output. +_PATTERNS = [ + re.compile(r"npm_[A-Za-z0-9]{8,}"), # npm automation/publish tokens + re.compile(r"(?:hv|hvs|hvb|s)\.[A-Za-z0-9._-]{16,}"), # vault/openbao tokens + re.compile(r"gh[pousr]_[A-Za-z0-9]{16,}"), # github tokens + re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}"), # slack tokens + re.compile(r"AKIA[0-9A-Z]{16}"), # aws access key id +] + + +def redact_text(text: str, extra: Iterable[str] = ()) -> str: + """Mask known token shapes and any caller-supplied literal values.""" + if not text: + return text + for literal in extra: + if literal and len(literal) >= 4: + text = text.replace(literal, REDACTED) + for pat in _PATTERNS: + text = pat.sub(REDACTED, text) + return text + + +def looks_secret(name: str) -> bool: + """Heuristic: does a field/key name suggest it carries a secret value?""" + lowered = name.lower() + return any( + marker in lowered + for marker in ("token", "secret", "password", "passwd", "apikey", "api_key", "key", "credential") + ) diff --git a/src/secrets_engine/roles.py b/src/secrets_engine/roles.py new file mode 100644 index 0000000..dde13ef --- /dev/null +++ b/src/secrets_engine/roles.py @@ -0,0 +1,138 @@ +"""Stage role/policy generation and the safety guards that keep them narrow. + +Three stage roles exist: secrets-engine-build, -test, -prod. Each is confined to +its own KV prefix and a small, explicit capability set. The guards in this module +are the heart of the product promise: a generated plan that would grant broad +power (root, sudo, sys/, auth/ admin, wildcard mounts, cross-stage paths) is +rejected before it can ever reach OpenBao. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import PolicyGuardError + +STAGES = ("build", "test", "prod") + +# Per-stage KV path prefix each role is allowed to touch. A lane whose path does +# not sit under its stage prefix is out of bounds. +STAGE_PREFIX = { + "build": "build/", + "test": "test/", + "prod": "", # prod lanes use their own owner-scoped paths (no shared prefix) +} + +# Capabilities a stage role's *own* policy may carry. Anything else is broad. +ALLOWED_CAPABILITIES = {"create", "read", "update", "delete", "list"} + +# Substrings that, if they appear in a policy path, mean the plan is too broad. +FORBIDDEN_PATH_MARKERS = ( + "sys/", + "auth/token/", + "identity/", + "sudo", + "+/", # single-level wildcard +) + +# Capability names that confer admin/root and must never appear in a stage policy. +FORBIDDEN_CAPABILITIES = {"sudo", "root", "deny-all-bypass"} + +# Policy/role names that smell like broad admin and are refused outright. +FORBIDDEN_NAME_MARKERS = ("root", "admin", "superuser", "platform-admin", "sys") + + +@dataclass(frozen=True) +class StageRole: + stage: str + policy_name: str + role_name: str + prefix: str + + @classmethod + def for_stage(cls, stage: str) -> "StageRole": + if stage not in STAGES: + raise PolicyGuardError(f"unknown stage '{stage}'; allowed {STAGES}") + return cls( + stage=stage, + policy_name=f"secrets-engine-{stage}", + role_name=f"secrets-engine-{stage}", + prefix=STAGE_PREFIX[stage], + ) + + +def assert_path_in_stage(entry: CatalogEntry) -> None: + """Reject a lane whose path is wildcarded or outside its stage prefix.""" + if "*" in entry.path or "+" in entry.path: + raise PolicyGuardError( + f"lane '{entry.id}': wildcard path '{entry.path}' is not allowed" + ) + prefix = STAGE_PREFIX[entry.stage] + if entry.stage in ("build", "test") and not entry.path.startswith(prefix): + raise PolicyGuardError( + f"lane '{entry.id}': {entry.stage} path must start with '{prefix}' " + f"(got '{entry.path}')" + ) + if entry.stage in ("build", "test"): + # A build/test lane must not reach into another stage's prefix. + for other, oprefix in STAGE_PREFIX.items(): + if other != entry.stage and oprefix and entry.path.startswith(oprefix): + raise PolicyGuardError( + f"lane '{entry.id}': {entry.stage} lane reaches into " + f"'{oprefix}' ({other} territory)" + ) + + +def assert_policy_safe(policy_name: str, paths: dict[str, list[str]]) -> None: + """Reject a policy document that is too broad to be a stage policy.""" + lowered = policy_name.lower() + for marker in FORBIDDEN_NAME_MARKERS: + if marker in lowered: + raise PolicyGuardError( + f"policy name '{policy_name}' resembles broad admin (marker '{marker}')" + ) + for path, caps in paths.items(): + for marker in FORBIDDEN_PATH_MARKERS: + if marker in path: + raise PolicyGuardError( + f"policy '{policy_name}': path '{path}' is out of bounds " + f"(marker '{marker}')" + ) + if path.strip() in ("*", "/", "secret/*", "+"): + raise PolicyGuardError( + f"policy '{policy_name}': wildcard path '{path}' not allowed" + ) + bad_caps = set(caps) - ALLOWED_CAPABILITIES + if bad_caps & FORBIDDEN_CAPABILITIES or bad_caps: + raise PolicyGuardError( + f"policy '{policy_name}': capabilities {sorted(bad_caps)} not allowed " + f"(allowed {sorted(ALLOWED_CAPABILITIES)})" + ) + + +def lane_policy_paths(entry: CatalogEntry) -> dict[str, list[str]]: + """The minimal KV v2 paths + capabilities a consumer policy needs for a lane.""" + data_path = f"{entry.mount}/data/{entry.path}" + meta_path = f"{entry.mount}/metadata/{entry.path}" + return { + data_path: ["read"], + meta_path: ["read"], + } + + +def render_policy_hcl(policy_name: str, paths: dict[str, list[str]]) -> str: + """Render an OpenBao ACL policy in HCL. Validates safety first.""" + assert_policy_safe(policy_name, paths) + blocks = [f'# Generated by secrets-engine for policy "{policy_name}"'] + for path, caps in paths.items(): + cap_list = ", ".join(f'"{c}"' for c in caps) + blocks.append(f'path "{path}" {{\n capabilities = [{cap_list}]\n}}') + return "\n\n".join(blocks) + "\n" + + +def consumer_policy_for(entry: CatalogEntry) -> tuple[str, str]: + """Return (policy_name, hcl) for the lane's approved consumer.""" + assert_path_in_stage(entry) + paths = lane_policy_paths(entry) + name = entry.policy_name + return name, render_policy_hcl(name, paths) diff --git a/src/secrets_engine/routing.py b/src/secrets_engine/routing.py new file mode 100644 index 0000000..e998113 --- /dev/null +++ b/src/secrets_engine/routing.py @@ -0,0 +1,99 @@ +"""ops-warden routing contract. + +ops-warden routes non-SSH credential needs here. It must NOT vend secret values. +A route result is a pointer: catalog id, readiness, decision status, and the safe +next command. This module computes that pointer for a lane. No value is read. +""" +from __future__ import annotations + +from dataclasses import dataclass, asdict +from pathlib import Path +from typing import Any + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.decisions import Decision, resolve_decision +from secrets_engine.errors import DecisionError +from secrets_engine.openbao import OpenBaoClient + + +@dataclass +class RouteResult: + catalog_id: str + owner: str + stage: str + decision_status: str + decision_ref: str + review_url: str + metadata_applied: bool + value_present: bool + ready: bool + next_command: str + missing: str + + def to_json(self) -> dict[str, Any]: + return asdict(self) + + +def route_lane( + entry: CatalogEntry, + *, + hub_url: str, + repo_root: Path, + client: OpenBaoClient | None = None, +) -> RouteResult: + """Build the front-door routing pointer for a lane. Never reads the value.""" + decision_status = "n/a (bootstrap-only)" + decision_ref = entry.approval.get("decision_ref", "") + review_url = "" + decision: Decision | None = None + if entry.approval_required(): + try: + decision = resolve_decision( + hub_url=hub_url, repo_root=repo_root, decision_ref=decision_ref + ) + decision_status = decision.status + review_url = decision.review_url + except DecisionError: + decision_status = "missing" + + metadata_applied = False + value_present = False + if client is not None and client.is_reachable(): + metadata_applied = client.read_policy(entry.policy_name) is not None + # Presence check uses the engine's own token; reports boolean only. + field = entry.fields[0] if entry.fields else "" + if field: + value_present = client.kv_field_present(entry.mount, entry.path, field) + + approved = decision is None or decision.is_approved() + ready = approved and metadata_applied and value_present + + if not approved: + missing = f"approved decision for '{decision_ref}'" + next_command = f"secrets-engine decision inspect {decision_ref or entry.id}" + elif not metadata_applied: + missing = "OpenBao policy/role apply" + next_command = f"secrets-engine apply {decision_ref or entry.id} --stage {entry.stage}" + elif not value_present: + missing = "provisioned secret value" + next_command = ( + f"secrets-engine provision {entry.id} --stage {entry.stage} " + f"--field {entry.fields[0]} --from-file " + ) + else: + missing = "" + next_command = f"secrets-engine exec --catalog {entry.id} -- " + + return RouteResult( + catalog_id=entry.id, + owner=entry.owner, + stage=entry.stage, + decision_status=decision_status, + decision_ref=decision_ref, + review_url=review_url, + metadata_applied=metadata_applied, + value_present=value_present, + ready=ready, + next_command=next_command, + missing=missing, + ) diff --git a/src/secrets_engine/verify.py b/src/secrets_engine/verify.py new file mode 100644 index 0000000..445e872 --- /dev/null +++ b/src/secrets_engine/verify.py @@ -0,0 +1,81 @@ +"""Verification: prove access (or denial) without printing the value. + +Positive: the approved consumer (via its approle-scoped token) CAN read the lane. +Negative: an unrelated/unscoped token CANNOT read the lane. + +Each check returns a boolean + a non-secret evidence dict. The secret value is +never read into the result. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from secrets_engine.catalog import CatalogEntry +from secrets_engine.errors import VerificationError +from secrets_engine.openbao import OpenBaoClient + + +@dataclass +class VerifyResult: + check: str # "positive" | "negative" + passed: bool + detail: dict[str, Any] + + def render(self) -> str: + status = "PASS" if self.passed else "FAIL" + return f" {self.check} check: {status} ({self.detail.get('reason', '')})" + + +def verify_positive(client: OpenBaoClient, entry: CatalogEntry, field: str) -> VerifyResult: + """Approved consumer token must be able to read the field.""" + try: + token = client.approle_login_token(entry.role_name) + except Exception as e: # backend errors -> failed verification, not a value leak + return VerifyResult( + "positive", + False, + {"reason": f"could not obtain approle token: {e}", "path": entry.path}, + ) + present = client.kv_field_present(entry.mount, entry.path, field, token=token) + return VerifyResult( + "positive", + present, + { + "reason": "approved consumer can read lane field" + if present + else "approved consumer could NOT read field", + "path": entry.path, + "field": field, + "role": entry.role_name, + }, + ) + + +def verify_negative(client: OpenBaoClient, entry: CatalogEntry) -> VerifyResult: + """An unrelated token must be denied. Uses an empty (invalid) token.""" + # An empty/garbage token stands in for an unrelated consumer. + denied = not client.kv_can_read(entry.mount, entry.path, token="se-unrelated-denied") + return VerifyResult( + "negative", + denied, + { + "reason": "unrelated token denied read" + if denied + else "unrelated token was ABLE to read (LEAK RISK)", + "path": entry.path, + }, + ) + + +def run_verification( + client: OpenBaoClient, entry: CatalogEntry, field: str, *, positive: bool, negative: bool +) -> list[VerifyResult]: + results: list[VerifyResult] = [] + if positive: + results.append(verify_positive(client, entry, field)) + if negative: + results.append(verify_negative(client, entry)) + if not results: + raise VerificationError("no verification check selected (use --positive/--negative)") + return results diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_catalog.py b/tests/test_catalog.py new file mode 100644 index 0000000..757fb17 --- /dev/null +++ b/tests/test_catalog.py @@ -0,0 +1,77 @@ +import copy + +import pytest + +from secrets_engine.catalog import validate_entry, load_catalog +from secrets_engine.config import repo_root +from secrets_engine.errors import CatalogError + +VALID = { + "id": "test-lane", + "owner": "team", + "stage": "test", + "mount": "secret", + "path": "test/team/thing", + "fields": ["api_token"], + "consumers": [{"name": "c", "auth": "approle", "claim": "role:c"}], + "delivery_modes": ["exec-env"], + "approval": {"model": "bootstrap-only"}, + "verification": {"positive": "x", "negative": "y"}, + "rotation": {"ttl": "1h"}, + "deactivation": {"expectation": "delete"}, + "audit": {"evidence": "non-secret"}, +} + + +def test_valid_entry_parses(): + e = validate_entry(VALID) + assert e.id == "test-lane" + assert e.policy_name == "se-test-test-lane" + assert not e.approval_required() + + +@pytest.mark.parametrize("field", ["stage", "mount", "path", "fields", "approval", "delivery_modes"]) +def test_missing_required_field_rejected(field): + data = copy.deepcopy(VALID) + data.pop(field) + with pytest.raises(CatalogError): + validate_entry(data) + + +def test_bad_stage_rejected(): + data = copy.deepcopy(VALID) + data["stage"] = "staging" + with pytest.raises(CatalogError): + validate_entry(data) + + +def test_unknown_delivery_mode_rejected(): + data = copy.deepcopy(VALID) + data["delivery_modes"] = ["telepathy"] + with pytest.raises(CatalogError): + validate_entry(data) + + +def test_wildcard_path_rejected(): + data = copy.deepcopy(VALID) + data["path"] = "test/*" + with pytest.raises(CatalogError): + validate_entry(data) + + +def test_inline_secret_value_rejected(): + data = copy.deepcopy(VALID) + data["token"] = "npm_realvalue" + with pytest.raises(CatalogError): + validate_entry(data) + + +def test_repo_catalog_loads_and_has_pilot(): + entries = load_catalog(repo_root() / "catalog") + assert "whynot-design-npm-publish" in entries + pilot = entries["whynot-design-npm-publish"] + assert pilot.stage == "prod" + assert pilot.approval_required() + # build/test/prod stage separation is representable + stages = {e.stage for e in entries.values()} + assert {"build", "prod"} <= stages diff --git a/tests/test_decisions.py b/tests/test_decisions.py new file mode 100644 index 0000000..bea1609 --- /dev/null +++ b/tests/test_decisions.py @@ -0,0 +1,52 @@ +import pytest + +from secrets_engine.catalog import validate_entry +from secrets_engine.decisions import Decision, require_approved, resolve_decision +from secrets_engine.errors import DecisionError + +from tests.test_catalog import VALID + + +def _approved(model="decision"): + d = dict(VALID, approval={"model": model, "decision_ref": "x"}) + return validate_entry(d) + + +def test_bootstrap_only_needs_no_decision(): + e = validate_entry(dict(VALID, approval={"model": "bootstrap-only"})) + require_approved(e, None) # must not raise + + +def test_unapproved_decision_refused(): + e = _approved() + d = Decision(id="d", title="t", status="pending", superseded_by=None, source="hub") + with pytest.raises(DecisionError): + require_approved(e, d) + + +def test_superseded_decision_refused(): + e = _approved() + d = Decision(id="d", title="t", status="resolved", superseded_by="d2", source="hub") + with pytest.raises(DecisionError): + require_approved(e, d) + + +def test_approved_decision_passes(): + e = _approved() + d = Decision(id="d", title="t", status="resolved", superseded_by=None, source="hub") + require_approved(e, d) # must not raise + + +def test_local_fixture_resolves(tmp_path): + (tmp_path / ".decisions").mkdir() + (tmp_path / ".decisions" / "myref.yaml").write_text( + "id: myref\ntitle: t\nstatus: resolved\nsuperseded_by: null\n" + ) + d = resolve_decision(hub_url="http://127.0.0.1:1", repo_root=tmp_path, decision_ref="myref") + assert d.source == "local-fixture" + assert d.is_approved() + + +def test_missing_decision_raises(tmp_path): + with pytest.raises(DecisionError): + resolve_decision(hub_url="http://127.0.0.1:1", repo_root=tmp_path, decision_ref="nope") diff --git a/tests/test_guards.py b/tests/test_guards.py new file mode 100644 index 0000000..9846508 --- /dev/null +++ b/tests/test_guards.py @@ -0,0 +1,67 @@ +"""Negative checks: a plan that would grant broad power must fail closed.""" +import pytest + +from secrets_engine.catalog import validate_entry +from secrets_engine.errors import PolicyGuardError +from secrets_engine.plan import build_plan +from secrets_engine.roles import assert_path_in_stage, assert_policy_safe + +from tests.test_catalog import VALID + + +def _entry(**over): + d = dict(VALID) + d.update(over) + return validate_entry(d) + + +def test_wildcard_policy_path_refused(): + with pytest.raises(PolicyGuardError): + assert_policy_safe("se-test-x", {"secret/*": ["read"]}) + + +def test_sys_path_refused(): + with pytest.raises(PolicyGuardError): + assert_policy_safe("se-test-x", {"sys/policies/acl/x": ["read"]}) + + +def test_identity_path_refused(): + with pytest.raises(PolicyGuardError): + assert_policy_safe("se-test-x", {"identity/entity/x": ["read"]}) + + +def test_admin_policy_name_refused(): + with pytest.raises(PolicyGuardError): + assert_policy_safe("platform-admin", {"secret/data/x": ["read"]}) + + +def test_broad_capability_refused(): + with pytest.raises(PolicyGuardError): + assert_policy_safe("se-test-x", {"secret/data/x": ["sudo"]}) + + +def test_out_of_stage_path_refused(): + # a 'test' lane pointing into the build prefix is rejected + e = _entry(stage="test", path="build/sneaky/thing") + with pytest.raises(PolicyGuardError): + assert_path_in_stage(e) + + +def test_build_lane_must_use_build_prefix(): + e = _entry(stage="build", path="random/thing") + with pytest.raises(PolicyGuardError): + assert_path_in_stage(e) + + +def test_stage_mismatch_in_plan_refused(): + e = _entry(stage="test", path="test/team/thing") + with pytest.raises(PolicyGuardError): + build_plan(e, "prod") + + +def test_valid_plan_builds(): + e = _entry(stage="test", path="test/team/thing") + plan = build_plan(e, "test", decision_id="d1") + assert plan.policy_name == "se-test-test-lane" + assert any(a.kind == "approle" for a in plan.actions) + assert "secret/data/test/team/thing" in plan.policy_hcl diff --git a/tests/test_integration_bao.py b/tests/test_integration_bao.py new file mode 100644 index 0000000..26c5131 --- /dev/null +++ b/tests/test_integration_bao.py @@ -0,0 +1,103 @@ +"""Live integration test against a throwaway OpenBao dev server. + +Skipped automatically if the `bao` CLI is not on PATH. Boots an in-memory dev +server on a private port, then drives apply -> provision -> verify(+/-) -> +exec-delivery and asserts the value is reachable by the child but not the parent. +""" +import json +import os +import shutil +import socket +import subprocess +import time +from pathlib import Path + +import pytest + +from secrets_engine.apply import apply_plan +from secrets_engine.catalog import get_entry +from secrets_engine.config import repo_root +from secrets_engine.exec_delivery import exec_with_secret +from secrets_engine.openbao import OpenBaoClient +from secrets_engine.plan import build_plan +from secrets_engine.provision import provision_from_file +from secrets_engine.verify import verify_negative, verify_positive + +pytestmark = pytest.mark.skipif( + shutil.which("bao") is None and shutil.which("vault") is None, + reason="no OpenBao/Vault CLI on PATH", +) + + +def _free_port() -> int: + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +@pytest.fixture() +def bao_dev(): + bao = shutil.which("bao") or shutil.which("vault") + port = _free_port() + addr = f"http://127.0.0.1:{port}" + token = "se-test-root" + proc = subprocess.Popen( + [bao, "server", "-dev", f"-dev-root-token-id={token}", + f"-dev-listen-address=127.0.0.1:{port}"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + client = OpenBaoClient(addr=addr, token=token, bao_bin=bao) + for _ in range(50): + if client.is_reachable(): + break + time.sleep(0.2) + else: + proc.kill() + pytest.fail("dev OpenBao did not become reachable") + try: + yield client + finally: + proc.kill() + + +def test_full_chain(bao_dev, tmp_path): + client = bao_dev + entry = get_entry(repo_root() / "catalog", "whynot-design-npm-publish") + + plan = build_plan(entry, "prod", decision_id="test") + apply_plan(client, entry, plan) + + # provision from a mode-0600 file outside the repo (tmp_path is outside) + tokenfile = tmp_path / "tok" + tokenfile.write_text("npm_integrationTESTvalue1234567890") + os.chmod(tokenfile, 0o600) + provision_from_file(client, entry, "npm_token", tokenfile) + + pos = verify_positive(client, entry, "npm_token") + assert pos.passed, pos.detail + neg = verify_negative(client, entry) + assert neg.passed, neg.detail + + # exec delivery: child can resolve token via npmrc; assert via a probe script + probe = tmp_path / "probe.sh" + probe.write_text( + "#!/usr/bin/env bash\n" + 'grep -q _authToken "$NPM_CONFIG_USERCONFIG" && echo CHILD_HAS_TOKEN\n' + ) + os.chmod(probe, 0o755) + rc = exec_with_secret(client, entry, "npm_token", [str(probe)], mode="npm-config") + assert rc == 0 + # the parent process never received the value as an env var + assert "SE_NPM_TOKEN" not in os.environ + + +def test_idempotent_apply(bao_dev): + client = bao_dev + entry = get_entry(repo_root() / "catalog", "whynot-design-npm-publish") + plan = build_plan(entry, "prod", decision_id="test") + first = apply_plan(client, entry, plan) + second = apply_plan(client, entry, plan) + # policy should be reported unchanged on the second apply + assert any("unchanged" in s for s in second.skipped) diff --git a/tests/test_redact_evidence.py b/tests/test_redact_evidence.py new file mode 100644 index 0000000..ba17dcc --- /dev/null +++ b/tests/test_redact_evidence.py @@ -0,0 +1,41 @@ +import json + +from secrets_engine.evidence import EvidenceWriter, _scrub +from secrets_engine.redact import looks_secret, redact_text + + +def test_redact_known_token_shapes(): + assert "npm_" not in redact_text("token=npm_abcdEFGH12345678abcd") + assert "REDACTED" in redact_text("token=npm_abcdEFGH12345678abcd") + assert "ghp_" not in redact_text("ghp_0123456789abcdef0123") + + +def test_redact_extra_literal(): + out = redact_text("the value is hunter2hunter2", extra=["hunter2hunter2"]) + assert "hunter2" not in out + + +def test_looks_secret(): + assert looks_secret("npm_token") + assert looks_secret("API_KEY") + assert not looks_secret("path") + + +def test_scrub_drops_secret_keys_and_redacts(): + scrubbed = _scrub({"token": "npm_realvalue123456789", "path": "a/b", "note": "ghp_0123456789abcdef0123"}) + assert scrubbed["token"].startswith(" +secrets-engine decision inspect +secrets-engine plan +secrets-engine apply --stage --dry-run +``` + +Acceptance: + +- Dry-run renders policy, auth role, path, field, and verification actions. +- Dry-run refuses unapproved, denied, superseded, or malformed requests. +- Dry-run refuses broad policy names, wildcards, root/platform-admin semantics, + and out-of-stage paths. + +## T05 - Integrate State Hub decisions and evidence + +```task +id: SECRETS-WP-0002-T05 +status: done +priority: high +state_hub_task_id: "64898c75-e113-4c39-916e-1a640475ddf7" +``` + +Connect the CLI to State Hub decision and progress APIs for non-secret metadata. + +Acceptance: + +- A decision can be inspected from the CLI. +- The CLI prints or records a review URL when available. +- Apply and verification write non-secret progress notes. +- Secret values are never sent to State Hub. + +## T06 - Implement OpenBao apply for approved metadata + +```task +id: SECRETS-WP-0002-T06 +status: done +priority: high +state_hub_task_id: "b4b098d0-c6e0-4e50-a21e-b33d8ceea3e8" +``` + +Apply approved OpenBao ACL policies and auth roles through the selected stage +role or temporary bootstrap credential. + +Acceptance: + +- Apply succeeds for the whynot-design metadata lane in the intended stage. +- Apply is idempotent. +- Apply refuses unapproved decisions and out-of-policy mutations. +- Apply evidence contains no secret values. + +## T07 - Implement safe provisioning and verification + +```task +id: SECRETS-WP-0002-T07 +status: done +priority: high +state_hub_task_id: "1b458c09-a4e7-490c-acec-d55eff4bae6c" +``` + +Support initial provisioning modes: + +- attended operator provisioning; +- local bootstrap file import with mode checks; +- generated non-production test secrets. + +Add positive and negative verification commands that do not print raw values. + +Acceptance: + +- The whynot-design token can be confirmed present without printing it. +- Positive verification proves the approved consumer can access/use it. +- Negative verification proves an unrelated consumer is denied. +- Verification evidence is non-secret and reviewable. + +## T08 - Implement exec-time delivery for npm pilot + +```task +id: SECRETS-WP-0002-T08 +status: done +priority: high +state_hub_task_id: "871cc4d4-4cbf-43be-98b7-3bad0d371f87" +``` + +Implement: + +```bash +secrets-engine exec --catalog whynot-design-npm-publish -- npm publish +``` + +For npm, prefer temporary config-file injection over printing/exporting the +token. Clean up temporary files after the child process exits. + +Acceptance: + +- npm publish can run with the token available only to the child process. +- The token is not printed, persisted in the repo, or written to normal logs. +- The temp config file is deleted on success, failure, and interruption where + possible. +- The command refuses to run when the catalog entry is not approved and ready. + +## T09 - Update ops-warden routing contract + +```task +id: SECRETS-WP-0002-T09 +status: done +priority: medium +state_hub_task_id: "b6c14f2a-49b1-4f39-a909-ba02a78fbef5" +``` + +Define the contract ops-warden should use when routing non-SSH credential needs +to secrets-engine. + +Acceptance: + +- Route output includes catalog id, readiness, decision status, and safe next + command. +- ops-warden does not request or store raw secret values. +- whynot-design can retry the npm credential request and receive actionable + guidance from the front door. + +## T10 - Hardening backlog and exit from bootstrap mode + +```task +id: SECRETS-WP-0002-T10 +status: done +priority: medium +state_hub_task_id: "52c070ec-33ac-4736-96e3-6bbacaa7fc83" +``` + +Create the hardening backlog needed after the pilot. + +Minimum items: + +- replace bootstrap token files with OIDC/service auth; +- add response-wrapped handoff where suitable; +- add production dual-control option; +- add rotation and compromised/deactivated secret states; +- add audit report command; +- add API service mode after CLI semantics stabilize. + +Acceptance: + +- Bootstrap tokens have revocation tasks. +- Production hardening steps are explicit and prioritized. +- The MVP can close without pretending bootstrap mode is the final security + state. + +## Exit Criteria + +- The repo can represent and validate the whynot-design npm publish catalog + entry. +- An approved decision can become an OpenBao policy/auth-role apply without + platform-root hand typing. +- The secret value can be provisioned and verified without disclosure. +- `secrets-engine exec --catalog whynot-design-npm-publish -- npm publish` is + either working or blocked only on an explicit external condition. +- ops-warden can route the request to secrets-engine and report readiness. +- Bootstrap credentials are documented, bounded, and scheduled for removal.