diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a019ff7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.venv +**/__pycache__ +*.pyc +.pytest_cache +.mypy_cache +.ruff_cache +tests +workplans +docs +evidence +deploy +scripts +*.egg-info diff --git a/Containerfile b/Containerfile index 2d4a10e..a4eeef5 100644 --- a/Containerfile +++ b/Containerfile @@ -1,9 +1,18 @@ -FROM python:3.12-slim +# Base pinned by digest, not a mutable tag (AUDIT-WP-0005-T03). +# Same python:3.12-slim digest as the deployed user-engine image. +FROM python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 + +ARG GIT_COMMIT=unknown +LABEL org.opencontainers.image.title="audit-core" \ + org.opencontainers.image.source="https://forgejo.coulomb.social/coulomb/audit-core" \ + org.opencontainers.image.revision="${GIT_COMMIT}" + RUN useradd --system --uid 10001 --create-home audit-core WORKDIR /app COPY pyproject.toml README.md LICENSE ./ COPY audit_core ./audit_core RUN pip install --no-cache-dir ".[serve,postgres]" -USER 10001 +USER 10001:10001 +ENV PYTHONUNBUFFERED=1 EXPOSE 8080 CMD ["audit-core-ingest"] diff --git a/Makefile b/Makefile index 3f6fdea..c6e844e 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,35 @@ help: ## Show this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} \ /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-24s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) -.PHONY: test test-pg pg-test-up pg-test-down failure-matrix mock-audit-smoke mock-audit-cleanup help +.PHONY: test test-pg pg-test-up pg-test-down failure-matrix mock-audit-smoke mock-audit-cleanup \ + image-build image-publish deploy-dry-run help + +IMAGE_REGISTRY ?= forgejo.coulomb.social/coulomb/audit-core +GIT_COMMIT := $(shell git rev-parse HEAD) +GIT_COMMIT_SHORT := $(shell git rev-parse --short HEAD) +# railiance01 k3s API is forwarded by ops-bridge tunnel k3s-api-railiance01. +KUBECONFIG_RAILIANCE ?= $(HOME)/.kube/config-hosteurope + +image-build: ## Build the immutable image, labelled with the current commit + docker build -f Containerfile \ + --build-arg GIT_COMMIT=$(GIT_COMMIT) \ + -t $(IMAGE_REGISTRY):$(GIT_COMMIT_SHORT) \ + -t $(IMAGE_REGISTRY):$(GIT_COMMIT) \ + . + +image-publish: image-build ## Push the commit-tagged image to Forgejo + docker push $(IMAGE_REGISTRY):$(GIT_COMMIT_SHORT) + docker push $(IMAGE_REGISTRY):$(GIT_COMMIT) + @echo "Pin the printed digest in deploy/audit-core.yaml and deploy/migrate-job.yaml" + +deploy-dry-run: ## Server-side validate the railiance01 manifests + KUBECONFIG=$(KUBECONFIG_RAILIANCE) kubectl apply --dry-run=server --validate=strict \ + -f deploy/audit-core.yaml \ + -f deploy/networkpolicies.yaml \ + -f deploy/clustersecretstore.yaml \ + -f deploy/vaultdynamicsecrets.yaml \ + -f deploy/externalsecrets.yaml \ + -f deploy/migrate-job.yaml PG_TEST_CONTAINER ?= ac-pg-test PG_TEST_PORT ?= 55445 diff --git a/README.md b/README.md index 17a21e5..53c5a06 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ Reliable multi-tenant auto setup audit capability +Production on railiance01 (AUDIT-WP-0005): PostgreSQL custody, digest-pinned +image, operator procedures in +[`docs/operator-runbook.md`](docs/operator-runbook.md). Manifests live in +`deploy/`. + ## Backend contract The pluggable backend interface, event schema (`audit-core.event.v1alpha1`), diff --git a/audit_core/cli.py b/audit_core/cli.py index 4bb63e3..e729907 100644 --- a/audit_core/cli.py +++ b/audit_core/cli.py @@ -69,6 +69,29 @@ def build_parser() -> argparse.ArgumentParser: cleanup_parser.add_argument("--retention-days", type=int, default=None) cleanup_parser.set_defaults(func=cleanup) + schema_parser = sub.add_parser( + "migrate", + help="Apply pending PostgreSQL schema migrations and exit.", + ) + schema_parser.add_argument( + "--to-postgres", dest="destination", default=None, + help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", + ) + schema_parser.add_argument("--schema", default="audit_core") + schema_parser.set_defaults(func=migrate_schema) + + replay_parser = sub.add_parser( + "replay", + help="Re-submit a stored event through accept(); must reconcile as duplicate.", + ) + replay_parser.add_argument("--event-id", required=True) + replay_parser.add_argument( + "--to-postgres", dest="destination", default=None, + help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.", + ) + replay_parser.add_argument("--schema", default="audit_core") + replay_parser.set_defaults(func=replay_event) + migrate_parser = sub.add_parser( "migrate-store", help="Move audit records from a SQLite store into PostgreSQL." ) @@ -87,6 +110,54 @@ def build_parser() -> argparse.ArgumentParser: return parser +def _postgres_backend(dsn: str | None, schema: str, *, migrate: bool): + import os + + from audit_core.postgres_backend import PostgresAuditBackend + + return PostgresAuditBackend( + dsn if dsn is not None else os.environ.get("AUDIT_CORE_DATABASE_URL") or "", + schema=schema, + credential_dir=os.environ.get("AUDIT_CORE_CREDENTIAL_DIR"), + migrate=migrate, + ) + + +def migrate_schema(args: argparse.Namespace) -> int: + """Apply pending schema migrations using the current connection. + + Production runs this as a Job with the migration lease. The runtime + Deployment sets AUDIT_CORE_AUTO_MIGRATE=0 so the app role never needs + CREATE TABLE. + """ + backend = _postgres_backend(args.destination, args.schema, migrate=False) + try: + applied = backend.migrate() + finally: + backend.close() + print(json.dumps({"ok": True, "schema": args.schema, "applied": applied}, sort_keys=True)) + return 0 + + +def replay_event(args: argparse.Namespace) -> int: + """Reconcile a stored event. A first-acceptance result is a custody defect.""" + backend = _postgres_backend(args.destination, args.schema, migrate=False) + try: + result = backend.replay(args.event_id) + except KeyError: + print(json.dumps({"ok": False, "error": "not_found", "event_id": args.event_id})) + return 1 + finally: + backend.close() + print(json.dumps({ + "ok": True, + "event_id": args.event_id, + "duplicate": result.duplicate, + "reference": result.reference, + }, sort_keys=True)) + return 0 if result.duplicate else 2 + + def migrate_store(args: argparse.Namespace) -> int: import os diff --git a/audit_core/ingestion.py b/audit_core/ingestion.py index 0e1a178..1429734 100644 --- a/audit_core/ingestion.py +++ b/audit_core/ingestion.py @@ -499,6 +499,10 @@ def build_backend() -> IdempotentAuditBackend: statement_timeout_ms=int( os.environ.get("AUDIT_CORE_DB_STATEMENT_TIMEOUT_MS", "30000") ), + # Production mounts the runtime role, which cannot CREATE TABLE. + # Schema changes run as a Job with the migration lease + # (AUDIT-WP-0005-T02). Local and brokered use keep the default. + migrate=_env_flag("AUDIT_CORE_AUTO_MIGRATE", default=True), ) path = os.environ.get("AUDIT_CORE_DATABASE_PATH", "/data/audit-core.db") log.warning( @@ -508,6 +512,14 @@ def build_backend() -> IdempotentAuditBackend: return SQLiteAuditBackend(path) +def _env_flag(name: str, *, default: bool) -> bool: + """Parse a boolean environment flag. Unset or empty keeps ``default``.""" + raw = os.environ.get(name) + if raw is None or not raw.strip(): + return default + return raw.strip().lower() not in {"0", "false", "no", "off"} + + def main() -> None: logging.basicConfig( level=os.environ.get("AUDIT_CORE_LOG_LEVEL", "INFO"), diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..a859689 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,19 @@ +# railiance01 package + +Target: railiance01 only. The workstation kubeconfig that talks to that API +is the `k3s-api-railiance01` tunnel (local port **16444**). +`~/.kube/config-hosteurope` currently points at 16443 (coulombcore); rewrite +the server port or export a copy before applying. + +Apply order is documented in `docs/operator-runbook.md`. Do not apply the +Deployment until: + +1. The image digest is pinned (not `REPLACE_AT_RELEASE`). +2. Secrets `audit-core-database`, `audit-core-database-migrate`, and + `audit-core-senders` exist. +3. Job `audit-core-migrate` has completed. + +```bash +make image-build +make deploy-dry-run +``` diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml index da74dc8..8d6be24 100644 --- a/deploy/audit-core.yaml +++ b/deploy/audit-core.yaml @@ -93,6 +93,10 @@ spec: # to the development store. - name: AUDIT_CORE_REQUIRE_CUSTODY_CLASS value: archive + # Runtime role cannot CREATE TABLE. Schema changes are a Job + # with the migration lease (deploy/migrate-job.yaml). + - name: AUDIT_CORE_AUTO_MIGRATE + value: "0" - name: AUDIT_CORE_DATABASE_SCHEMA value: audit_core - name: AUDIT_CORE_THREADS diff --git a/deploy/clustersecretstore.yaml b/deploy/clustersecretstore.yaml new file mode 100644 index 0000000..ce7efe5 --- /dev/null +++ b/deploy/clustersecretstore.yaml @@ -0,0 +1,31 @@ +# Template of the railiance-platform add-on store. Prefer applying from: +# ~/railiance-platform/argocd/platform-addons/openbao-secretstore/openbao-audit-core.clustersecretstore.yaml +# +# Do not apply until Secret external-secrets/openbao-audit-core-eso-token exists +# (scripts/openbao-eso-token-apply.sh). This store is KV-only: sender registry +# lives at platform/workloads/audit-core/senders. Database leases are dynamic +# and come from VaultDynamicSecret, not this store. +--- +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: openbao-audit-core + labels: + app.kubernetes.io/part-of: railiance-gitops + railiance-platform/component: external-secrets + app.kubernetes.io/name: audit-core +spec: + provider: + vault: + # In-cluster OpenBao on railiance01, not the public bao.coulomb.social UI. + server: http://openbao.openbao.svc:8200 + path: platform + version: v2 + auth: + tokenSecretRef: + name: openbao-audit-core-eso-token + namespace: external-secrets + key: token + conditions: + - namespaces: + - audit-core diff --git a/deploy/externalsecrets.yaml b/deploy/externalsecrets.yaml index 2fc0f91..c40b871 100644 --- a/deploy/externalsecrets.yaml +++ b/deploy/externalsecrets.yaml @@ -1,14 +1,12 @@ # Credential delivery for audit-core (AUDIT-WP-0005-T02). # -# Follows the ClusterSecretStore -> ExternalSecret -> Secret pattern already in -# use by activity-core and rapp-qonto. audit-core never holds a credential in -# its own configuration; it reads whatever is currently mounted. +# Two sources, because they are two kinds of secret: +# - database leases: VaultDynamicSecret -> ExternalSecret (this file) +# - sender registry: ClusterSecretStore openbao-audit-core -> ExternalSecret # -# PREREQUISITE (rapp-postgres / railiance-platform, not this repo): -# - a ClusterSecretStore named openbao-audit-core, scoped to this namespace -# - the OpenBao database role rapp-postgres/audit-core-runtime issuing leases -# against the audit_core_app group role -# Apply order: ClusterSecretStore, then this, then the Deployment. +# audit-core never holds a credential in its own configuration; it reads +# whatever is currently mounted. Apply order: namespace, ESO token, +# ClusterSecretStore, VaultDynamicSecret, this, migrate Job, Deployment. --- apiVersion: external-secrets.io/v1 kind: ExternalSecret @@ -19,9 +17,6 @@ spec: # Shorter than the platform default of 1h: these are dynamic leases, and the # refresh interval bounds how long a revoked lease can remain mounted. refreshInterval: 15m - secretStoreRef: - kind: ClusterSecretStore - name: openbao-audit-core target: name: audit-core-database creationPolicy: Owner @@ -37,15 +32,38 @@ spec: host: platform-pg-rw.databases.svc.cluster.local port: "5432" dbname: audit_core - data: - - secretKey: username - remoteRef: - key: platform/workloads/audit-core/database/audit-core-runtime - property: username - - secretKey: password - remoteRef: - key: platform/workloads/audit-core/database/audit-core-runtime - property: password + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: VaultDynamicSecret + name: audit-core-runtime +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: audit-core-database-migrate + namespace: audit-core +spec: + refreshInterval: 15m + target: + name: audit-core-database-migrate + creationPolicy: Owner + deletionPolicy: Retain + template: + engineVersion: v2 + data: + username: "{{ .username }}" + password: "{{ .password }}" + host: platform-pg-rw.databases.svc.cluster.local + port: "5432" + dbname: audit_core + dataFrom: + - sourceRef: + generatorRef: + apiVersion: generators.external-secrets.io/v1alpha1 + kind: VaultDynamicSecret + name: audit-core-migration --- apiVersion: external-secrets.io/v1 kind: ExternalSecret @@ -71,5 +89,6 @@ spec: # between, so there is no delivery gap. - secretKey: senders.json remoteRef: - key: platform/workloads/audit-core/senders + # CSS already mounts KV path `platform`; do not repeat the prefix. + key: workloads/audit-core/senders property: senders.json diff --git a/deploy/migrate-job.yaml b/deploy/migrate-job.yaml new file mode 100644 index 0000000..c9392d0 --- /dev/null +++ b/deploy/migrate-job.yaml @@ -0,0 +1,66 @@ +# One-shot schema apply (AUDIT-WP-0005-T02). +# +# Uses the migration lease, not the runtime lease. The runtime Deployment +# sets AUDIT_CORE_AUTO_MIGRATE=0 so a leaked app credential cannot change +# the schema. Re-apply after deleting the previous Job when a new migration +# ships (Jobs are immutable). +# +# Image digest must match deploy/audit-core.yaml. +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: audit-core-migrate + namespace: audit-core + labels: + app.kubernetes.io/name: audit-core + app.kubernetes.io/component: migrate +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + app.kubernetes.io/name: audit-core + app.kubernetes.io/component: migrate + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - name: migrate + image: forgejo.coulomb.social/coulomb/audit-core@sha256:REPLACE_AT_RELEASE + imagePullPolicy: IfNotPresent + command: ["python", "-m", "audit_core", "migrate"] + env: + - name: AUDIT_CORE_CREDENTIAL_DIR + value: /etc/audit-core/db + - name: AUDIT_CORE_DATABASE_SCHEMA + value: audit_core + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumeMounts: + - name: tmp + mountPath: /tmp + - name: database-credential + mountPath: /etc/audit-core/db + readOnly: true + volumes: + - name: tmp + emptyDir: {} + - name: database-credential + secret: + secretName: audit-core-database-migrate + defaultMode: 0400 diff --git a/deploy/vaultdynamicsecrets.yaml b/deploy/vaultdynamicsecrets.yaml new file mode 100644 index 0000000..040e4e7 --- /dev/null +++ b/deploy/vaultdynamicsecrets.yaml @@ -0,0 +1,41 @@ +# Dynamic PostgreSQL leases (AUDIT-WP-0005-T02). +# +# These are not KV secrets. Copying a lease into platform/workloads/... would +# freeze a rotating credential and recreate the delivery-gap problem the +# mounted directory exists to avoid. ESO's VaultDynamicSecret generator reads +# database/creds/* on each refresh and writes username/password files that +# audit-core re-reads on the next connection. +# +# Prerequisite: Secret external-secrets/openbao-audit-core-eso-token. +--- +apiVersion: generators.external-secrets.io/v1alpha1 +kind: VaultDynamicSecret +metadata: + name: audit-core-runtime + namespace: audit-core +spec: + path: database/creds/audit-core-runtime + method: GET + provider: + server: http://openbao.openbao.svc:8200 + auth: + tokenSecretRef: + name: openbao-audit-core-eso-token + namespace: external-secrets + key: token +--- +apiVersion: generators.external-secrets.io/v1alpha1 +kind: VaultDynamicSecret +metadata: + name: audit-core-migration + namespace: audit-core +spec: + path: database/creds/audit-core-migration + method: GET + provider: + server: http://openbao.openbao.svc:8200 + auth: + tokenSecretRef: + name: openbao-audit-core-eso-token + namespace: external-secrets + key: token diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md new file mode 100644 index 0000000..64a00aa --- /dev/null +++ b/docs/operator-runbook.md @@ -0,0 +1,192 @@ +# audit-core operator runbook + +Production receiver on railiance01. Custody is PostgreSQL (`platform-pg`, +database `audit_core`). This runbook is the T06 handover: lookup, dead +letters, replay, credential rotation, alerts, and restore. + +In-cluster URL: + +```text +http://audit-core.audit-core.svc.cluster.local:8080 +``` + +Reachability is part of the threat model. Ingress is allowed only from the +`user-engine` namespace (write) and from namespaces labelled +`railiance.io/audit-core-reader=true` (operator read). From a workstation, +`kubectl -n audit-core port-forward svc/audit-core 8080:8080` is the usual +path and does not require that label. + +Do not paste sender tokens, database passwords, or OpenBao tokens into Git, +State Hub, workplans, logs, or chat. Route first: + +```bash +warden route find "database credential" --json +warden route show database-dynamic-credentials --json +``` + +## Readiness and health + +| Check | Meaning | +| --- | --- | +| `GET /healthz` | Process is up. Liveness uses this. A database outage must **not** restart the pod. | +| `GET /readyz` | Custody is reachable and `custody_class=archive`. Readiness uses this; the pod leaves the Service rather than accept events it cannot store. | +| `GET /v1/stats` | In-process counters since start (`accepted`, `duplicate`, `conflict`, `rejected`, `unauthorized`, `forbidden`, `unavailable`, `error`). Resets on restart. Requires `may_read`. | + +A missing `AUDIT_CORE_DATABASE_URL` / credential directory is a startup +failure (`AUDIT_CORE_REQUIRE_CUSTODY_CLASS=archive`), not a silent downgrade +to SQLite. + +## Lookup + +All read routes require a sender identity with `may_read: true`. A write +credential must not be able to read the trail back. + +```bash +# One event +curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \ + http://127.0.0.1:8080/v1/events/ + +# Every event sharing a correlation id +curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \ + 'http://127.0.0.1:8080/v1/events?correlation_id=&limit=100' +``` + +Stored records keep the original `event_id`, `payload_hash`, and +`accepted_at`. Redacted fields appear in `details.redaction.paths`; the +payload itself is stored already masked. + +## Dead letters and replay + +```bash +curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \ + 'http://127.0.0.1:8080/v1/dead-letters?limit=100' +``` + +A dead letter is a **rejected** submission (400). Retrying it unchanged will +not help. Typical reasons: `invalid_event`, `source_not_allowed`, +`tenant_not_allowed`, `secret_shaped_field`. Payloads rejected for secret +shape are withheld (`payload_withheld: true`). + +To replay after the sender is fixed, POST the corrected event with the same +`id` and `Idempotency-Key`. The store is idempotent: an already-accepted +identical payload returns 200 `duplicate` and does not create a second +record. A different payload under the same id returns 409. + +Reconciling an **already stored** event (the WP-0005-T05 S13 case): + +```bash +# From a shell that already has a migration or runtime lease. +python3 -m audit_core replay --event-id +``` + +That command must report `"duplicate": true`. A first-acceptance result is a +custody defect: stop and investigate. + +## Sender credential rotation + +The registry is OpenBao KV `platform/workloads/audit-core/senders`, field +`senders.json`, refreshed by ExternalSecret `audit-core-senders` every hour +and injected as `AUDIT_CORE_SENDERS`. Rotation is overlap-first: + +1. Add the replacement token to the sender's `tokens` list. Both work. +2. Move the sender to the new token. +3. Drop the predecessor from the list. +4. Write the updated document back (`bao kv put` from a mode-0600 file). +5. Either wait for the 1h refresh or annotate the ExternalSecret to force a + sync, then restart the pod so it re-reads `AUDIT_CORE_SENDERS`. + +The write token is bound to `source=user-engine` and the tenants that +identity may claim. The operator token is a separate identity with +`may_read: true`. Do not reuse one token for both. + +A shape (values are placeholders) is in `docs/senders.example.json`. + +## Database credential rotation + +Runtime leases come from `database/creds/audit-core-runtime`. ESO refreshes +every 15 minutes into Secret `audit-core-database`, mounted at +`/etc/audit-core/db`. The process re-reads those files on every new +connection. No restart is required; a restart would be a delivery gap. + +Rotation is logged by password fingerprint, never by value: + +```text +database credential rotated (fingerprint <12 hex chars>) +``` + +A revoked lease surfaces as `/readyz` 503 and request 503 `unavailable`. +The sender contract retries 503/500 and treats 400/401/403/409 as terminal. + +Schema changes are a Job (`deploy/migrate-job.yaml`) using +`database/creds/audit-core-migration`. The runtime Deployment sets +`AUDIT_CORE_AUTO_MIGRATE=0`. After a new migration ships, delete the old Job +and re-apply. + +## Alert conditions + +There is no Prometheus scrape path on railiance01 today (`/v1/stats` is JSON +behind the read privilege). Watch: + +| Signal | Why | +| --- | --- | +| `/readyz` not 200 | Custody is down; senders will retry. Do not bounce the pod. | +| `counts.unavailable` or `counts.error` climbing | Database or unexpected fault. Check `platform-pg` and the mounted lease. | +| `counts.unauthorized` climbing | Sender using a dropped token, or registry not refreshed. | +| `counts.conflict` non-zero | Same `event_id`, different payload. Sender bug or id reuse. | +| `GET /v1/secret-findings` | Secret-shaped fields by path. Fix the sending service. | +| Pod not Ready after rotate | Stale or revoked lease; check ExternalSecret status, not the password. | + +`platform-pg` operations, including what to scrape on port 9187, live in +`rapp-postgres/docs/operations.md`. + +## Restore + +audit-core declares `custody_class=archive` and `immutable=True` (the +append-only trigger). It does **not** declare a retention window shorter +than the platform's, because it currently declares none (`retention_days` +unset = keep). That is only honest if the platform's backup retention is +also unbounded or is an explicit accepted loss. + +As of 2026-08-13, `RAPP-POSTGRES-WP-0002-T06` is `wait`. Production backup +is fail-closed until an S3-compatible Barman target exists. The governed +Nextcloud logical-dump lane cannot provide WAL archiving or PITR. Do not +claim an RPO/RTO for audit-core until those drills have been recorded in +rapp-postgres. + +When the platform restore path is open, the audit-core walk is: + +1. Restore `platform-pg` to a scratch cluster (`platform-pg-restore-full` or + PITR). Never recover in place. +2. Physical restore is instance-wide. A single-consumer restore is a logical + dump of `audit_core` from the scratch cluster, then a controlled import. +3. Verify: event counts, a sample of `event_id`/`payload_hash`/`accepted_at` + triples, and that the append-only trigger is still installed + (`SELECT tgname FROM pg_trigger WHERE tgname = 'events_append_only'`). +4. Record elapsed time. That number is what this service can promise. + +Until that walk exists, a workstation-side logical dump of an empty +`audit_core` is not evidence of recovery. + +## Rollback + +The Deployment annotation `audit-core.railiance.io/rollback-note` is the +source of truth. Migrations 0001-0004 are additive; `kubectl -n audit-core +rollout undo deploy/audit-core` returns to the previous digest and an older +image runs against the newer schema. A future migration that drops or +narrows a column must replace that note before release. + +## Deploy order + +1. Image published, digest pinned in `deploy/audit-core.yaml` and + `deploy/migrate-job.yaml`. +2. Attended `scripts/openbao-eso-token-apply.sh`. +3. `bao kv put platform/workloads/audit-core/senders senders.json=@file` + from a mode-0600 file; shred the file. +4. Apply namespace (carries `railiance.io/postgres-client: platform-pg`), + NetworkPolicies, ClusterSecretStore, VaultDynamicSecrets, ExternalSecrets. +5. Wait until Secrets `audit-core-database`, `audit-core-database-migrate`, + and `audit-core-senders` exist. +6. Apply and wait for Job `audit-core-migrate`. +7. Apply the Deployment. Confirm `/readyz` reports `custody_class=archive`. +8. Run `MODE=remote BASE_URL=… make failure-matrix` and hand the JSON to + NK-WP-0024. diff --git a/docs/senders.example.json b/docs/senders.example.json new file mode 100644 index 0000000..c905baf --- /dev/null +++ b/docs/senders.example.json @@ -0,0 +1,20 @@ +[ + { + "name": "user-engine", + "tokens": ["replace-with-write-token"], + "sources": ["user-engine"], + "tenants": ["*"], + "may_write": true, + "may_read": false, + "secret_policy": "redact" + }, + { + "name": "operator", + "tokens": ["replace-with-read-token"], + "sources": ["user-engine"], + "tenants": ["*"], + "may_write": true, + "may_read": true, + "secret_policy": "redact" + } +] diff --git a/scripts/openbao-eso-token-apply.sh b/scripts/openbao-eso-token-apply.sh new file mode 100755 index 0000000..701aa24 --- /dev/null +++ b/scripts/openbao-eso-token-apply.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Mint a read-limited OpenBao token and store it on railiance01 for +# ClusterSecretStore openbao-audit-core and the VaultDynamicSecret generators. +# +# Policy: railiance-platform/openbao/policies/external-secrets-audit-core.hcl +# +# Does not print secret values. Requires an attended operator OpenBao token +# that can write policies and create child tokens. +set -euo pipefail + +DEFAULT_POLICIES="external-secrets-audit-core" +POLICIES="${OPENBAO_AUDIT_CORE_POLICIES:-$DEFAULT_POLICIES}" +POLICY_DIR="${OPENBAO_POLICY_DIR:-$HOME/railiance-platform/openbao/policies}" +BAO_ADDR="${BAO_ADDR:-https://bao.coulomb.social}" +RAILIANCE01_KUBECONFIG="${RAILIANCE01_KUBECONFIG:-$HOME/.kube/config-hosteurope}" +SECRET_NAME="${OPENBAO_AUDIT_CORE_ESO_SECRET:-openbao-audit-core-eso-token}" +SECRET_NS="${OPENBAO_AUDIT_CORE_ESO_NAMESPACE:-external-secrets}" +TTL="${OPENBAO_AUDIT_CORE_ESO_TTL:-768h}" + +if ! command -v bao >/dev/null 2>&1; then + echo "ERROR: bao CLI not found" >&2 + exit 1 +fi +if ! command -v kubectl >/dev/null 2>&1; then + echo "ERROR: kubectl not found" >&2 + exit 1 +fi + +echo "OpenBao addr: $BAO_ADDR" +echo "Policies: $POLICIES" +echo "K8s secret: $SECRET_NS/$SECRET_NAME (railiance01)" + +if [[ -n "${BAO_TOKEN:-}" ]]; then + : +elif [[ -n "${OPENBAO_TOKEN_FILE:-}" && -f "${OPENBAO_TOKEN_FILE}" ]]; then + BAO_TOKEN="$(head -n 1 "${OPENBAO_TOKEN_FILE}")" +else + read -r -s -p "OpenBao operator token: " BAO_TOKEN + echo >&2 +fi +if [[ -z "${BAO_TOKEN:-}" ]]; then + echo "ERROR: empty OpenBao token" >&2 + exit 1 +fi + +export BAO_ADDR BAO_TOKEN + +health="$(curl -fsS "$BAO_ADDR/v1/sys/health")" +if echo "$health" | grep -q '"sealed":true'; then + echo "ERROR: OpenBao at $BAO_ADDR reports sealed" >&2 + exit 1 +fi + +for policy in $POLICIES; do + policy_file="$POLICY_DIR/${policy}.hcl" + if [[ -f "$policy_file" ]]; then + bao policy write "$policy" "$policy_file" + echo "policy written: $policy" + else + echo "WARN: policy file missing ($policy_file); using existing OpenBao policy '$policy'" >&2 + fi +done + +# Child token: renewable, orphan so operator logout does not revoke delivery. +# shellcheck disable=SC2086 +token_json="$(bao token create -policy="$(echo $POLICIES | tr ' ' ',')" -ttl="$TTL" -renewable=true -orphan -format=json)" +child_token="$(printf '%s' "$token_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["auth"]["client_token"])')" +if [[ -z "$child_token" || ${#child_token} -lt 8 ]]; then + echo "ERROR: failed to mint child token" >&2 + exit 1 +fi +echo "minted child token length=${#child_token} (value not printed)" + +export KUBECONFIG="$RAILIANCE01_KUBECONFIG" +kubectl -n "$SECRET_NS" create secret generic "$SECRET_NAME" \ + --from-literal=token="$child_token" \ + --dry-run=client -o yaml | kubectl apply -f - + +unset child_token BAO_TOKEN +echo "Secret $SECRET_NS/$SECRET_NAME applied on railiance01." +echo "Next: apply ClusterSecretStore openbao-audit-core, then deploy/." diff --git a/tests/test_backend_conformance.py b/tests/test_backend_conformance.py index d9a43b4..c903b10 100644 --- a/tests/test_backend_conformance.py +++ b/tests/test_backend_conformance.py @@ -312,7 +312,10 @@ def test_connects_from_a_brokered_libpq_environment(monkeypatch): def test_missing_connection_information_is_a_clear_error(monkeypatch): for var in ("AUDIT_CORE_DATABASE_URL", "PGHOST", "PGUSER"): monkeypatch.delenv(var, raising=False) - from audit_core.postgres_backend import PostgresAuditBackend + try: + from audit_core.postgres_backend import PostgresAuditBackend + except ImportError: + pytest.skip("psycopg is not installed") with pytest.raises(ValueError, match="no connection information"): PostgresAuditBackend() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..684d9c4 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,120 @@ +import json +import sys +import types + +from audit_core.cli import build_parser +from audit_core.ingestion import build_backend +from audit_core.interface import AcceptResult, RetentionPolicy + + +def _install_fake_postgres(monkeypatch, factory): + """Avoid importing psycopg just to stub the constructor.""" + existing = sys.modules.get("audit_core.postgres_backend") + if existing is not None and getattr(existing, "PostgresAuditBackend", None): + monkeypatch.setattr(existing, "PostgresAuditBackend", factory) + return + mod = types.ModuleType("audit_core.postgres_backend") + mod.PostgresAuditBackend = factory + monkeypatch.setitem(sys.modules, "audit_core.postgres_backend", mod) + + +class _FakePostgres: + def __init__(self, dsn, **kwargs): + self.dsn = dsn + self.kwargs = kwargs + self.closed = False + self.applied = ["0001-events"] + + def migrate(self): + return list(self.applied) + + def replay(self, event_id): + if event_id == "missing": + raise KeyError(event_id) + return AcceptResult(duplicate=True, reference=f"audit:{event_id}") + + def close(self): + self.closed = True + + @property + def retention_policy(self): + return RetentionPolicy( + custody_class="archive", + retention_days=None, + immutable=True, + tamper_evidence=False, + durable=True, + ) + + +def test_build_backend_disables_auto_migrate_when_asked(monkeypatch, tmp_path): + captured = {} + + def fake(dsn, **kwargs): + captured["dsn"] = dsn + captured.update(kwargs) + return _FakePostgres(dsn, **kwargs) + + monkeypatch.setenv("AUDIT_CORE_AUTO_MIGRATE", "0") + monkeypatch.setenv("AUDIT_CORE_CREDENTIAL_DIR", str(tmp_path)) + _install_fake_postgres(monkeypatch, fake) + backend = build_backend() + assert captured["migrate"] is False + assert captured["credential_dir"] == str(tmp_path) + assert isinstance(backend, _FakePostgres) + + +def test_build_backend_auto_migrates_by_default(monkeypatch, tmp_path): + captured = {} + + def fake(dsn, **kwargs): + captured.update(kwargs) + return _FakePostgres(dsn, **kwargs) + + monkeypatch.delenv("AUDIT_CORE_AUTO_MIGRATE", raising=False) + monkeypatch.setenv("AUDIT_CORE_CREDENTIAL_DIR", str(tmp_path)) + _install_fake_postgres(monkeypatch, fake) + build_backend() + assert captured["migrate"] is True + + +def test_migrate_command_applies_pending_schema(monkeypatch, capsys): + seen = {} + + def fake(dsn, **kwargs): + backend = _FakePostgres(dsn, **kwargs) + seen["backend"] = backend + return backend + + monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core") + _install_fake_postgres(monkeypatch, fake) + args = build_parser().parse_args(["migrate", "--schema", "audit_core"]) + assert args.func(args) == 0 + body = json.loads(capsys.readouterr().out) + assert body == {"applied": ["0001-events"], "ok": True, "schema": "audit_core"} + assert seen["backend"].kwargs["migrate"] is False + assert seen["backend"].closed is True + + +def test_replay_command_reconciles(monkeypatch, capsys): + def fake(dsn, **kwargs): + return _FakePostgres(dsn, **kwargs) + + monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core") + _install_fake_postgres(monkeypatch, fake) + args = build_parser().parse_args(["replay", "--event-id", "evt-1"]) + assert args.func(args) == 0 + body = json.loads(capsys.readouterr().out) + assert body["duplicate"] is True + assert body["reference"] == "audit:evt-1" + + +def test_replay_command_missing_event(monkeypatch, capsys): + def fake(dsn, **kwargs): + return _FakePostgres(dsn, **kwargs) + + monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core") + _install_fake_postgres(monkeypatch, fake) + args = build_parser().parse_args(["replay", "--event-id", "missing"]) + assert args.func(args) == 1 + assert json.loads(capsys.readouterr().out)["error"] == "not_found" diff --git a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md index 88b5ac6..a8cbe3c 100644 --- a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md +++ b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: netkingdom created: "2026-08-10" -updated: "2026-08-10" +updated: "2026-08-13" depends_on: - AUDIT-WP-0004 - RAPP-POSTGRES-WP-0002 @@ -179,10 +179,37 @@ after confirming no active sessions; the group roles are untouched and remain NOLOGIN. The harness now uses a per-run random password, `VALID UNTIL` one hour, and an exit trap that drops the roles in every mode including on failure. -Remaining before done: the `openbao-audit-core` ClusterSecretStore and the -`rapp-postgres/audit-core-runtime` OpenBao role are prerequisites owned by -rapp-postgres/railiance-platform, not this repo. Once they exist, apply and -verify a live lease and a live rotation. +Progress 2026-08-13: the in-cluster delivery path is now honest about what +kind of secret it is pulling. + +The previous ExternalSecret pointed at KV +`platform/workloads/audit-core/database/audit-core-runtime`. That path does +not exist, and copying a dynamic lease into KV would freeze it — the same +delivery-gap problem the mounted directory exists to avoid. Database +credentials now come from ESO `VaultDynamicSecret` against +`database/creds/audit-core-runtime` and `database/creds/audit-core-migration`, +refreshed every 15m into one-file-per-field Secrets. The sender registry +stays on ClusterSecretStore `openbao-audit-core` (KV), which is the right +tool for a document. + +A second finding: `PostgresAuditBackend` applied schema on every process +start. The runtime role cannot `CREATE TABLE` (isolation-test, ADR-0001). +Production now sets `AUDIT_CORE_AUTO_MIGRATE=0`; `python -m audit_core +migrate` runs as Job `audit-core-migrate` with the migration lease. + +Package side in this repo: `deploy/clustersecretstore.yaml`, +`deploy/vaultdynamicsecrets.yaml`, `deploy/migrate-job.yaml`, +`scripts/openbao-eso-token-apply.sh`. Platform side drafted in +`railiance-platform` (`openbao/policies/external-secrets-audit-core.hcl` and +the ClusterSecretStore add-on). The OpenBao database roles themselves were +already delivered by RAPP-POSTGRES-WP-0002-T04. + +Remaining before done: attended `platform-admin` OpenBao login to write the +ESO policy, mint `external-secrets/openbao-audit-core-eso-token`, and +`bao kv put` the sender registry. Then apply CSS + generators + ExternalSecrets +and verify a live lease plus a live rotation against the mounted directory. +That step is flagged `needs_human` — this session has no token that can write +policies. ## T03 - Deploy the receiver @@ -261,8 +288,25 @@ other scrape target, so an exposition endpoint would be built for a scrape path that does not exist. This is usable with curl today and a small step from `/metrics` when a metrics stack lands. -Remaining before done: build and publish the image to get a real digest, apply, -and verify restart/rescheduling and the rollback path on the cluster. +Progress 2026-08-13: Containerfile base is digest-pinned +(`python:3.12-slim@sha256:d764629ce0…`, same digest as the deployed +user-engine image) and labelled with `org.opencontainers.image.revision`. +`make image-build` / `make image-publish` exist. Namespace `audit-core` +(with `railiance.io/postgres-client: platform-pg`) and the four +NetworkPolicies are applied on railiance01. Remaining manifests validate +`--dry-run=server --validate=strict`. Deployment is not applied: it still +carries `sha256:REPLACE_AT_RELEASE` until the ESO secrets exist, and +applying it now would ImagePullBackOff and then CrashLoop on a missing +sender registry. + +An image was published to `forgejo.coulomb.social/coulomb/audit-core` during +this session; rebuild and pin after the commit that contains these files so +the label matches the content. + +Remaining before done: pin the post-commit digest in +`deploy/audit-core.yaml` and `deploy/migrate-job.yaml`, apply migrate Job +then Deployment once T02 secrets exist, and verify restart, reschedule, and +`kubectl rollout undo`. ## T04 - Migrate existing SQLite records @@ -389,7 +433,7 @@ OpenBao-leased credentials. ```task id: AUDIT-WP-0005-T06 -status: todo +status: progress priority: medium state_hub_task_id: "0856c80d-abe1-4bff-ba8d-87295cf76819" ``` @@ -404,3 +448,17 @@ provides — the retention window audit-core declares must not exceed the retention the platform guarantees. Done when the runbook exists and the restore path has been walked once. + +Progress 2026-08-13: `docs/operator-runbook.md` covers lookup by correlation +id, dead-letter inspection, accepted-event replay (`python -m audit_core +replay`), overlap-first sender rotation, restart-free database lease +rotation, `/readyz` vs `/healthz` vs `/v1/stats`, and rollback. +`python -m audit_core replay` is the operator surface that WP-0004 +deliberately left off HTTP. + +Restore is written down and is **not** walked. rapp-postgres T06 is `wait` +on an S3 Barman target; the Nextcloud logical-dump lane cannot provide WAL +or PITR; physical restore is instance-wide. audit-core currently declares +no finite `retention_days`, so it does not outrun a platform window that +does not yet exist — but it also cannot promise an RPO until the platform +drills land. The walk listed in the runbook is what closes this task.