Prepare railiance01 delivery: dynamic leases, migrate Job, operator runbook

VaultDynamicSecret pulls database/creds/* so a rotating lease is not frozen
into KV. Runtime sets AUDIT_CORE_AUTO_MIGRATE=0; schema is a Job with the
migration lease. Image base is digest-pinned. Namespace and NetworkPolicies
are on the cluster; Deployment waits for the attended OpenBao ESO token.

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.venv
+**/__pycache__
+*.pyc
+.pytest_cache
+.mypy_cache
+.ruff_cache
+tests
+workplans
+docs
+evidence
+deploy
+scripts
+*.egg-info

Containerfile

@@ -1,9 +1,18 @@
-FROM python:3.12-slim
+# Base pinned by digest, not a mutable tag (AUDIT-WP-0005-T03).
+# Same python:3.12-slim digest as the deployed user-engine image.
+FROM python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9
+
+ARG GIT_COMMIT=unknown
+LABEL org.opencontainers.image.title="audit-core" \
+      org.opencontainers.image.source="https://forgejo.coulomb.social/coulomb/audit-core" \
+      org.opencontainers.image.revision="${GIT_COMMIT}"
+
 RUN useradd --system --uid 10001 --create-home audit-core
 WORKDIR /app
 COPY pyproject.toml README.md LICENSE ./
 COPY audit_core ./audit_core
 RUN pip install --no-cache-dir ".[serve,postgres]"
-USER 10001
+USER 10001:10001
+ENV PYTHONUNBUFFERED=1
 EXPOSE 8080
 CMD ["audit-core-ingest"]

Makefile

@@ -21,7 +21,35 @@ help: ## Show this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} \
 	  /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-24s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST)
 
-.PHONY: test test-pg pg-test-up pg-test-down failure-matrix mock-audit-smoke mock-audit-cleanup help
+.PHONY: test test-pg pg-test-up pg-test-down failure-matrix mock-audit-smoke mock-audit-cleanup \
+	image-build image-publish deploy-dry-run help
+
+IMAGE_REGISTRY ?= forgejo.coulomb.social/coulomb/audit-core
+GIT_COMMIT := $(shell git rev-parse HEAD)
+GIT_COMMIT_SHORT := $(shell git rev-parse --short HEAD)
+# railiance01 k3s API is forwarded by ops-bridge tunnel k3s-api-railiance01.
+KUBECONFIG_RAILIANCE ?= $(HOME)/.kube/config-hosteurope
+
+image-build: ## Build the immutable image, labelled with the current commit
+	docker build -f Containerfile \
+	  --build-arg GIT_COMMIT=$(GIT_COMMIT) \
+	  -t $(IMAGE_REGISTRY):$(GIT_COMMIT_SHORT) \
+	  -t $(IMAGE_REGISTRY):$(GIT_COMMIT) \
+	  .
+
+image-publish: image-build ## Push the commit-tagged image to Forgejo
+	docker push $(IMAGE_REGISTRY):$(GIT_COMMIT_SHORT)
+	docker push $(IMAGE_REGISTRY):$(GIT_COMMIT)
+	@echo "Pin the printed digest in deploy/audit-core.yaml and deploy/migrate-job.yaml"
+
+deploy-dry-run: ## Server-side validate the railiance01 manifests
+	KUBECONFIG=$(KUBECONFIG_RAILIANCE) kubectl apply --dry-run=server --validate=strict \
+	  -f deploy/audit-core.yaml \
+	  -f deploy/networkpolicies.yaml \
+	  -f deploy/clustersecretstore.yaml \
+	  -f deploy/vaultdynamicsecrets.yaml \
+	  -f deploy/externalsecrets.yaml \
+	  -f deploy/migrate-job.yaml
 
 PG_TEST_CONTAINER ?= ac-pg-test
 PG_TEST_PORT ?= 55445

README.md

@@ -1,5 +1,10 @@
 Reliable multi-tenant auto setup audit capability
 
+Production on railiance01 (AUDIT-WP-0005): PostgreSQL custody, digest-pinned
+image, operator procedures in
+[`docs/operator-runbook.md`](docs/operator-runbook.md). Manifests live in
+`deploy/`.
+
 ## Backend contract
 
 The pluggable backend interface, event schema (`audit-core.event.v1alpha1`),

audit_core/cli.py

@@ -69,6 +69,29 @@ def build_parser() -> argparse.ArgumentParser:
     cleanup_parser.add_argument("--retention-days", type=int, default=None)
     cleanup_parser.set_defaults(func=cleanup)
 
+    schema_parser = sub.add_parser(
+        "migrate",
+        help="Apply pending PostgreSQL schema migrations and exit.",
+    )
+    schema_parser.add_argument(
+        "--to-postgres", dest="destination", default=None,
+        help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.",
+    )
+    schema_parser.add_argument("--schema", default="audit_core")
+    schema_parser.set_defaults(func=migrate_schema)
+
+    replay_parser = sub.add_parser(
+        "replay",
+        help="Re-submit a stored event through accept(); must reconcile as duplicate.",
+    )
+    replay_parser.add_argument("--event-id", required=True)
+    replay_parser.add_argument(
+        "--to-postgres", dest="destination", default=None,
+        help="DSN; defaults to AUDIT_CORE_DATABASE_URL or a mounted credential dir.",
+    )
+    replay_parser.add_argument("--schema", default="audit_core")
+    replay_parser.set_defaults(func=replay_event)
+
     migrate_parser = sub.add_parser(
         "migrate-store", help="Move audit records from a SQLite store into PostgreSQL."
     )
@@ -87,6 +110,54 @@ def build_parser() -> argparse.ArgumentParser:
     return parser
 
 
+def _postgres_backend(dsn: str | None, schema: str, *, migrate: bool):
+    import os
+
+    from audit_core.postgres_backend import PostgresAuditBackend
+
+    return PostgresAuditBackend(
+        dsn if dsn is not None else os.environ.get("AUDIT_CORE_DATABASE_URL") or "",
+        schema=schema,
+        credential_dir=os.environ.get("AUDIT_CORE_CREDENTIAL_DIR"),
+        migrate=migrate,
+    )
+
+
+def migrate_schema(args: argparse.Namespace) -> int:
+    """Apply pending schema migrations using the current connection.
+
+    Production runs this as a Job with the migration lease. The runtime
+    Deployment sets AUDIT_CORE_AUTO_MIGRATE=0 so the app role never needs
+    CREATE TABLE.
+    """
+    backend = _postgres_backend(args.destination, args.schema, migrate=False)
+    try:
+        applied = backend.migrate()
+    finally:
+        backend.close()
+    print(json.dumps({"ok": True, "schema": args.schema, "applied": applied}, sort_keys=True))
+    return 0
+
+
+def replay_event(args: argparse.Namespace) -> int:
+    """Reconcile a stored event. A first-acceptance result is a custody defect."""
+    backend = _postgres_backend(args.destination, args.schema, migrate=False)
+    try:
+        result = backend.replay(args.event_id)
+    except KeyError:
+        print(json.dumps({"ok": False, "error": "not_found", "event_id": args.event_id}))
+        return 1
+    finally:
+        backend.close()
+    print(json.dumps({
+        "ok": True,
+        "event_id": args.event_id,
+        "duplicate": result.duplicate,
+        "reference": result.reference,
+    }, sort_keys=True))
+    return 0 if result.duplicate else 2
+
+
 def migrate_store(args: argparse.Namespace) -> int:
     import os
 

audit_core/ingestion.py

@@ -499,6 +499,10 @@ def build_backend() -> IdempotentAuditBackend:
             statement_timeout_ms=int(
                 os.environ.get("AUDIT_CORE_DB_STATEMENT_TIMEOUT_MS", "30000")
             ),
+            # Production mounts the runtime role, which cannot CREATE TABLE.
+            # Schema changes run as a Job with the migration lease
+            # (AUDIT-WP-0005-T02). Local and brokered use keep the default.
+            migrate=_env_flag("AUDIT_CORE_AUTO_MIGRATE", default=True),
         )
     path = os.environ.get("AUDIT_CORE_DATABASE_PATH", "/data/audit-core.db")
     log.warning(
@@ -508,6 +512,14 @@ def build_backend() -> IdempotentAuditBackend:
     return SQLiteAuditBackend(path)
 
 
+def _env_flag(name: str, *, default: bool) -> bool:
+    """Parse a boolean environment flag. Unset or empty keeps ``default``."""
+    raw = os.environ.get(name)
+    if raw is None or not raw.strip():
+        return default
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
 def main() -> None:
     logging.basicConfig(
         level=os.environ.get("AUDIT_CORE_LOG_LEVEL", "INFO"),

deploy/README.md

@@ -0,0 +1,19 @@
+# railiance01 package
+
+Target: railiance01 only. The workstation kubeconfig that talks to that API
+is the `k3s-api-railiance01` tunnel (local port **16444**).
+`~/.kube/config-hosteurope` currently points at 16443 (coulombcore); rewrite
+the server port or export a copy before applying.
+
+Apply order is documented in `docs/operator-runbook.md`. Do not apply the
+Deployment until:
+
+1. The image digest is pinned (not `REPLACE_AT_RELEASE`).
+2. Secrets `audit-core-database`, `audit-core-database-migrate`, and
+   `audit-core-senders` exist.
+3. Job `audit-core-migrate` has completed.
+
+```bash
+make image-build
+make deploy-dry-run
+```

deploy/audit-core.yaml

@@ -93,6 +93,10 @@ spec:
             # to the development store.
             - name: AUDIT_CORE_REQUIRE_CUSTODY_CLASS
               value: archive
+            # Runtime role cannot CREATE TABLE. Schema changes are a Job
+            # with the migration lease (deploy/migrate-job.yaml).
+            - name: AUDIT_CORE_AUTO_MIGRATE
+              value: "0"
             - name: AUDIT_CORE_DATABASE_SCHEMA
               value: audit_core
             - name: AUDIT_CORE_THREADS

deploy/clustersecretstore.yaml

@@ -0,0 +1,31 @@
+# Template of the railiance-platform add-on store. Prefer applying from:
+#   ~/railiance-platform/argocd/platform-addons/openbao-secretstore/openbao-audit-core.clustersecretstore.yaml
+#
+# Do not apply until Secret external-secrets/openbao-audit-core-eso-token exists
+# (scripts/openbao-eso-token-apply.sh). This store is KV-only: sender registry
+# lives at platform/workloads/audit-core/senders. Database leases are dynamic
+# and come from VaultDynamicSecret, not this store.
+---
+apiVersion: external-secrets.io/v1
+kind: ClusterSecretStore
+metadata:
+  name: openbao-audit-core
+  labels:
+    app.kubernetes.io/part-of: railiance-gitops
+    railiance-platform/component: external-secrets
+    app.kubernetes.io/name: audit-core
+spec:
+  provider:
+    vault:
+      # In-cluster OpenBao on railiance01, not the public bao.coulomb.social UI.
+      server: http://openbao.openbao.svc:8200
+      path: platform
+      version: v2
+      auth:
+        tokenSecretRef:
+          name: openbao-audit-core-eso-token
+          namespace: external-secrets
+          key: token
+  conditions:
+    - namespaces:
+        - audit-core

deploy/externalsecrets.yaml

@@ -1,14 +1,12 @@
 # Credential delivery for audit-core (AUDIT-WP-0005-T02).
 #
-# Follows the ClusterSecretStore -> ExternalSecret -> Secret pattern already in
+# Two sources, because they are two kinds of secret:
-# use by activity-core and rapp-qonto. audit-core never holds a credential in
+#   - database leases: VaultDynamicSecret -> ExternalSecret (this file)
-# its own configuration; it reads whatever is currently mounted.
+#   - sender registry: ClusterSecretStore openbao-audit-core -> ExternalSecret
 #
-# PREREQUISITE (rapp-postgres / railiance-platform, not this repo):
+# audit-core never holds a credential in its own configuration; it reads
-#   - a ClusterSecretStore named openbao-audit-core, scoped to this namespace
+# whatever is currently mounted. Apply order: namespace, ESO token,
-#   - the OpenBao database role rapp-postgres/audit-core-runtime issuing leases
+# ClusterSecretStore, VaultDynamicSecret, this, migrate Job, Deployment.
-#     against the audit_core_app group role
-# Apply order: ClusterSecretStore, then this, then the Deployment.
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
@@ -19,9 +17,6 @@ spec:
   # Shorter than the platform default of 1h: these are dynamic leases, and the
   # refresh interval bounds how long a revoked lease can remain mounted.
   refreshInterval: 15m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao-audit-core
   target:
     name: audit-core-database
     creationPolicy: Owner
@@ -37,15 +32,38 @@ spec:
         host: platform-pg-rw.databases.svc.cluster.local
         port: "5432"
         dbname: audit_core
-  data:
+  dataFrom:
-    - secretKey: username
+    - sourceRef:
-      remoteRef:
+        generatorRef:
-        key: platform/workloads/audit-core/database/audit-core-runtime
+          apiVersion: generators.external-secrets.io/v1alpha1
-        property: username
+          kind: VaultDynamicSecret
-    - secretKey: password
+          name: audit-core-runtime
-      remoteRef:
+---
-        key: platform/workloads/audit-core/database/audit-core-runtime
+apiVersion: external-secrets.io/v1
-        property: password
+kind: ExternalSecret
+metadata:
+  name: audit-core-database-migrate
+  namespace: audit-core
+spec:
+  refreshInterval: 15m
+  target:
+    name: audit-core-database-migrate
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    template:
+      engineVersion: v2
+      data:
+        username: "{{ .username }}"
+        password: "{{ .password }}"
+        host: platform-pg-rw.databases.svc.cluster.local
+        port: "5432"
+        dbname: audit_core
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: VaultDynamicSecret
+          name: audit-core-migration
 ---
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
@@ -71,5 +89,6 @@ spec:
     # between, so there is no delivery gap.
     - secretKey: senders.json
       remoteRef:
-        key: platform/workloads/audit-core/senders
+        # CSS already mounts KV path `platform`; do not repeat the prefix.
+        key: workloads/audit-core/senders
         property: senders.json

deploy/migrate-job.yaml

@@ -0,0 +1,66 @@
+# One-shot schema apply (AUDIT-WP-0005-T02).
+#
+# Uses the migration lease, not the runtime lease. The runtime Deployment
+# sets AUDIT_CORE_AUTO_MIGRATE=0 so a leaked app credential cannot change
+# the schema. Re-apply after deleting the previous Job when a new migration
+# ships (Jobs are immutable).
+#
+# Image digest must match deploy/audit-core.yaml.
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: audit-core-migrate
+  namespace: audit-core
+  labels:
+    app.kubernetes.io/name: audit-core
+    app.kubernetes.io/component: migrate
+spec:
+  backoffLimit: 6
+  ttlSecondsAfterFinished: 86400
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: audit-core
+        app.kubernetes.io/component: migrate
+    spec:
+      restartPolicy: OnFailure
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: migrate
+          image: forgejo.coulomb.social/coulomb/audit-core@sha256:REPLACE_AT_RELEASE
+          imagePullPolicy: IfNotPresent
+          command: ["python", "-m", "audit_core", "migrate"]
+          env:
+            - name: AUDIT_CORE_CREDENTIAL_DIR
+              value: /etc/audit-core/db
+            - name: AUDIT_CORE_DATABASE_SCHEMA
+              value: audit_core
+          resources:
+            requests:
+              cpu: 25m
+              memory: 64Mi
+            limits:
+              cpu: 250m
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: database-credential
+              mountPath: /etc/audit-core/db
+              readOnly: true
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: database-credential
+          secret:
+            secretName: audit-core-database-migrate
+            defaultMode: 0400

deploy/vaultdynamicsecrets.yaml

@@ -0,0 +1,41 @@
+# Dynamic PostgreSQL leases (AUDIT-WP-0005-T02).
+#
+# These are not KV secrets. Copying a lease into platform/workloads/... would
+# freeze a rotating credential and recreate the delivery-gap problem the
+# mounted directory exists to avoid. ESO's VaultDynamicSecret generator reads
+# database/creds/* on each refresh and writes username/password files that
+# audit-core re-reads on the next connection.
+#
+# Prerequisite: Secret external-secrets/openbao-audit-core-eso-token.
+---
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: VaultDynamicSecret
+metadata:
+  name: audit-core-runtime
+  namespace: audit-core
+spec:
+  path: database/creds/audit-core-runtime
+  method: GET
+  provider:
+    server: http://openbao.openbao.svc:8200
+    auth:
+      tokenSecretRef:
+        name: openbao-audit-core-eso-token
+        namespace: external-secrets
+        key: token
+---
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: VaultDynamicSecret
+metadata:
+  name: audit-core-migration
+  namespace: audit-core
+spec:
+  path: database/creds/audit-core-migration
+  method: GET
+  provider:
+    server: http://openbao.openbao.svc:8200
+    auth:
+      tokenSecretRef:
+        name: openbao-audit-core-eso-token
+        namespace: external-secrets
+        key: token

docs/operator-runbook.md

@@ -0,0 +1,192 @@
+# audit-core operator runbook
+
+Production receiver on railiance01. Custody is PostgreSQL (`platform-pg`,
+database `audit_core`). This runbook is the T06 handover: lookup, dead
+letters, replay, credential rotation, alerts, and restore.
+
+In-cluster URL:
+
+```text
+http://audit-core.audit-core.svc.cluster.local:8080
+```
+
+Reachability is part of the threat model. Ingress is allowed only from the
+`user-engine` namespace (write) and from namespaces labelled
+`railiance.io/audit-core-reader=true` (operator read). From a workstation,
+`kubectl -n audit-core port-forward svc/audit-core 8080:8080` is the usual
+path and does not require that label.
+
+Do not paste sender tokens, database passwords, or OpenBao tokens into Git,
+State Hub, workplans, logs, or chat. Route first:
+
+```bash
+warden route find "database credential" --json
+warden route show database-dynamic-credentials --json
+```
+
+## Readiness and health
+
+| Check | Meaning |
+| --- | --- |
+| `GET /healthz` | Process is up. Liveness uses this. A database outage must **not** restart the pod. |
+| `GET /readyz` | Custody is reachable and `custody_class=archive`. Readiness uses this; the pod leaves the Service rather than accept events it cannot store. |
+| `GET /v1/stats` | In-process counters since start (`accepted`, `duplicate`, `conflict`, `rejected`, `unauthorized`, `forbidden`, `unavailable`, `error`). Resets on restart. Requires `may_read`. |
+
+A missing `AUDIT_CORE_DATABASE_URL` / credential directory is a startup
+failure (`AUDIT_CORE_REQUIRE_CUSTODY_CLASS=archive`), not a silent downgrade
+to SQLite.
+
+## Lookup
+
+All read routes require a sender identity with `may_read: true`. A write
+credential must not be able to read the trail back.
+
+```bash
+# One event
+curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \
+  http://127.0.0.1:8080/v1/events/<event-id>
+
+# Every event sharing a correlation id
+curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \
+  'http://127.0.0.1:8080/v1/events?correlation_id=<id>&limit=100'
+```
+
+Stored records keep the original `event_id`, `payload_hash`, and
+`accepted_at`. Redacted fields appear in `details.redaction.paths`; the
+payload itself is stored already masked.
+
+## Dead letters and replay
+
+```bash
+curl -sS -H "Authorization: Bearer $OPERATOR_TOKEN" \
+  'http://127.0.0.1:8080/v1/dead-letters?limit=100'
+```
+
+A dead letter is a **rejected** submission (400). Retrying it unchanged will
+not help. Typical reasons: `invalid_event`, `source_not_allowed`,
+`tenant_not_allowed`, `secret_shaped_field`. Payloads rejected for secret
+shape are withheld (`payload_withheld: true`).
+
+To replay after the sender is fixed, POST the corrected event with the same
+`id` and `Idempotency-Key`. The store is idempotent: an already-accepted
+identical payload returns 200 `duplicate` and does not create a second
+record. A different payload under the same id returns 409.
+
+Reconciling an **already stored** event (the WP-0005-T05 S13 case):
+
+```bash
+# From a shell that already has a migration or runtime lease.
+python3 -m audit_core replay --event-id <event-id>
+```
+
+That command must report `"duplicate": true`. A first-acceptance result is a
+custody defect: stop and investigate.
+
+## Sender credential rotation
+
+The registry is OpenBao KV `platform/workloads/audit-core/senders`, field
+`senders.json`, refreshed by ExternalSecret `audit-core-senders` every hour
+and injected as `AUDIT_CORE_SENDERS`. Rotation is overlap-first:
+
+1. Add the replacement token to the sender's `tokens` list. Both work.
+2. Move the sender to the new token.
+3. Drop the predecessor from the list.
+4. Write the updated document back (`bao kv put` from a mode-0600 file).
+5. Either wait for the 1h refresh or annotate the ExternalSecret to force a
+   sync, then restart the pod so it re-reads `AUDIT_CORE_SENDERS`.
+
+The write token is bound to `source=user-engine` and the tenants that
+identity may claim. The operator token is a separate identity with
+`may_read: true`. Do not reuse one token for both.
+
+A shape (values are placeholders) is in `docs/senders.example.json`.
+
+## Database credential rotation
+
+Runtime leases come from `database/creds/audit-core-runtime`. ESO refreshes
+every 15 minutes into Secret `audit-core-database`, mounted at
+`/etc/audit-core/db`. The process re-reads those files on every new
+connection. No restart is required; a restart would be a delivery gap.
+
+Rotation is logged by password fingerprint, never by value:
+
+```text
+database credential rotated (fingerprint <12 hex chars>)
+```
+
+A revoked lease surfaces as `/readyz` 503 and request 503 `unavailable`.
+The sender contract retries 503/500 and treats 400/401/403/409 as terminal.
+
+Schema changes are a Job (`deploy/migrate-job.yaml`) using
+`database/creds/audit-core-migration`. The runtime Deployment sets
+`AUDIT_CORE_AUTO_MIGRATE=0`. After a new migration ships, delete the old Job
+and re-apply.
+
+## Alert conditions
+
+There is no Prometheus scrape path on railiance01 today (`/v1/stats` is JSON
+behind the read privilege). Watch:
+
+| Signal | Why |
+| --- | --- |
+| `/readyz` not 200 | Custody is down; senders will retry. Do not bounce the pod. |
+| `counts.unavailable` or `counts.error` climbing | Database or unexpected fault. Check `platform-pg` and the mounted lease. |
+| `counts.unauthorized` climbing | Sender using a dropped token, or registry not refreshed. |
+| `counts.conflict` non-zero | Same `event_id`, different payload. Sender bug or id reuse. |
+| `GET /v1/secret-findings` | Secret-shaped fields by path. Fix the sending service. |
+| Pod not Ready after rotate | Stale or revoked lease; check ExternalSecret status, not the password. |
+
+`platform-pg` operations, including what to scrape on port 9187, live in
+`rapp-postgres/docs/operations.md`.
+
+## Restore
+
+audit-core declares `custody_class=archive` and `immutable=True` (the
+append-only trigger). It does **not** declare a retention window shorter
+than the platform's, because it currently declares none (`retention_days`
+unset = keep). That is only honest if the platform's backup retention is
+also unbounded or is an explicit accepted loss.
+
+As of 2026-08-13, `RAPP-POSTGRES-WP-0002-T06` is `wait`. Production backup
+is fail-closed until an S3-compatible Barman target exists. The governed
+Nextcloud logical-dump lane cannot provide WAL archiving or PITR. Do not
+claim an RPO/RTO for audit-core until those drills have been recorded in
+rapp-postgres.
+
+When the platform restore path is open, the audit-core walk is:
+
+1. Restore `platform-pg` to a scratch cluster (`platform-pg-restore-full` or
+   PITR). Never recover in place.
+2. Physical restore is instance-wide. A single-consumer restore is a logical
+   dump of `audit_core` from the scratch cluster, then a controlled import.
+3. Verify: event counts, a sample of `event_id`/`payload_hash`/`accepted_at`
+   triples, and that the append-only trigger is still installed
+   (`SELECT tgname FROM pg_trigger WHERE tgname = 'events_append_only'`).
+4. Record elapsed time. That number is what this service can promise.
+
+Until that walk exists, a workstation-side logical dump of an empty
+`audit_core` is not evidence of recovery.
+
+## Rollback
+
+The Deployment annotation `audit-core.railiance.io/rollback-note` is the
+source of truth. Migrations 0001-0004 are additive; `kubectl -n audit-core
+rollout undo deploy/audit-core` returns to the previous digest and an older
+image runs against the newer schema. A future migration that drops or
+narrows a column must replace that note before release.
+
+## Deploy order
+
+1. Image published, digest pinned in `deploy/audit-core.yaml` and
+   `deploy/migrate-job.yaml`.
+2. Attended `scripts/openbao-eso-token-apply.sh`.
+3. `bao kv put platform/workloads/audit-core/senders senders.json=@file`
+   from a mode-0600 file; shred the file.
+4. Apply namespace (carries `railiance.io/postgres-client: platform-pg`),
+   NetworkPolicies, ClusterSecretStore, VaultDynamicSecrets, ExternalSecrets.
+5. Wait until Secrets `audit-core-database`, `audit-core-database-migrate`,
+   and `audit-core-senders` exist.
+6. Apply and wait for Job `audit-core-migrate`.
+7. Apply the Deployment. Confirm `/readyz` reports `custody_class=archive`.
+8. Run `MODE=remote BASE_URL=… make failure-matrix` and hand the JSON to
+   NK-WP-0024.

docs/senders.example.json

@@ -0,0 +1,20 @@
+[
+  {
+    "name": "user-engine",
+    "tokens": ["replace-with-write-token"],
+    "sources": ["user-engine"],
+    "tenants": ["*"],
+    "may_write": true,
+    "may_read": false,
+    "secret_policy": "redact"
+  },
+  {
+    "name": "operator",
+    "tokens": ["replace-with-read-token"],
+    "sources": ["user-engine"],
+    "tenants": ["*"],
+    "may_write": true,
+    "may_read": true,
+    "secret_policy": "redact"
+  }
+]

scripts/openbao-eso-token-apply.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Mint a read-limited OpenBao token and store it on railiance01 for
+# ClusterSecretStore openbao-audit-core and the VaultDynamicSecret generators.
+#
+# Policy: railiance-platform/openbao/policies/external-secrets-audit-core.hcl
+#
+# Does not print secret values. Requires an attended operator OpenBao token
+# that can write policies and create child tokens.
+set -euo pipefail
+
+DEFAULT_POLICIES="external-secrets-audit-core"
+POLICIES="${OPENBAO_AUDIT_CORE_POLICIES:-$DEFAULT_POLICIES}"
+POLICY_DIR="${OPENBAO_POLICY_DIR:-$HOME/railiance-platform/openbao/policies}"
+BAO_ADDR="${BAO_ADDR:-https://bao.coulomb.social}"
+RAILIANCE01_KUBECONFIG="${RAILIANCE01_KUBECONFIG:-$HOME/.kube/config-hosteurope}"
+SECRET_NAME="${OPENBAO_AUDIT_CORE_ESO_SECRET:-openbao-audit-core-eso-token}"
+SECRET_NS="${OPENBAO_AUDIT_CORE_ESO_NAMESPACE:-external-secrets}"
+TTL="${OPENBAO_AUDIT_CORE_ESO_TTL:-768h}"
+
+if ! command -v bao >/dev/null 2>&1; then
+  echo "ERROR: bao CLI not found" >&2
+  exit 1
+fi
+if ! command -v kubectl >/dev/null 2>&1; then
+  echo "ERROR: kubectl not found" >&2
+  exit 1
+fi
+
+echo "OpenBao addr: $BAO_ADDR"
+echo "Policies: $POLICIES"
+echo "K8s secret: $SECRET_NS/$SECRET_NAME (railiance01)"
+
+if [[ -n "${BAO_TOKEN:-}" ]]; then
+  :
+elif [[ -n "${OPENBAO_TOKEN_FILE:-}" && -f "${OPENBAO_TOKEN_FILE}" ]]; then
+  BAO_TOKEN="$(head -n 1 "${OPENBAO_TOKEN_FILE}")"
+else
+  read -r -s -p "OpenBao operator token: " BAO_TOKEN
+  echo >&2
+fi
+if [[ -z "${BAO_TOKEN:-}" ]]; then
+  echo "ERROR: empty OpenBao token" >&2
+  exit 1
+fi
+
+export BAO_ADDR BAO_TOKEN
+
+health="$(curl -fsS "$BAO_ADDR/v1/sys/health")"
+if echo "$health" | grep -q '"sealed":true'; then
+  echo "ERROR: OpenBao at $BAO_ADDR reports sealed" >&2
+  exit 1
+fi
+
+for policy in $POLICIES; do
+  policy_file="$POLICY_DIR/${policy}.hcl"
+  if [[ -f "$policy_file" ]]; then
+    bao policy write "$policy" "$policy_file"
+    echo "policy written: $policy"
+  else
+    echo "WARN: policy file missing ($policy_file); using existing OpenBao policy '$policy'" >&2
+  fi
+done
+
+# Child token: renewable, orphan so operator logout does not revoke delivery.
+# shellcheck disable=SC2086
+token_json="$(bao token create -policy="$(echo $POLICIES | tr ' ' ',')" -ttl="$TTL" -renewable=true -orphan -format=json)"
+child_token="$(printf '%s' "$token_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["auth"]["client_token"])')"
+if [[ -z "$child_token" || ${#child_token} -lt 8 ]]; then
+  echo "ERROR: failed to mint child token" >&2
+  exit 1
+fi
+echo "minted child token length=${#child_token} (value not printed)"
+
+export KUBECONFIG="$RAILIANCE01_KUBECONFIG"
+kubectl -n "$SECRET_NS" create secret generic "$SECRET_NAME" \
+  --from-literal=token="$child_token" \
+  --dry-run=client -o yaml | kubectl apply -f -
+
+unset child_token BAO_TOKEN
+echo "Secret $SECRET_NS/$SECRET_NAME applied on railiance01."
+echo "Next: apply ClusterSecretStore openbao-audit-core, then deploy/."

tests/test_backend_conformance.py

@@ -312,7 +312,10 @@ def test_connects_from_a_brokered_libpq_environment(monkeypatch):
 def test_missing_connection_information_is_a_clear_error(monkeypatch):
     for var in ("AUDIT_CORE_DATABASE_URL", "PGHOST", "PGUSER"):
         monkeypatch.delenv(var, raising=False)
-    from audit_core.postgres_backend import PostgresAuditBackend
+    try:
+        from audit_core.postgres_backend import PostgresAuditBackend
+    except ImportError:
+        pytest.skip("psycopg is not installed")
 
     with pytest.raises(ValueError, match="no connection information"):
         PostgresAuditBackend()

tests/test_cli.py

@@ -0,0 +1,120 @@
+import json
+import sys
+import types
+
+from audit_core.cli import build_parser
+from audit_core.ingestion import build_backend
+from audit_core.interface import AcceptResult, RetentionPolicy
+
+
+def _install_fake_postgres(monkeypatch, factory):
+    """Avoid importing psycopg just to stub the constructor."""
+    existing = sys.modules.get("audit_core.postgres_backend")
+    if existing is not None and getattr(existing, "PostgresAuditBackend", None):
+        monkeypatch.setattr(existing, "PostgresAuditBackend", factory)
+        return
+    mod = types.ModuleType("audit_core.postgres_backend")
+    mod.PostgresAuditBackend = factory
+    monkeypatch.setitem(sys.modules, "audit_core.postgres_backend", mod)
+
+
+class _FakePostgres:
+    def __init__(self, dsn, **kwargs):
+        self.dsn = dsn
+        self.kwargs = kwargs
+        self.closed = False
+        self.applied = ["0001-events"]
+
+    def migrate(self):
+        return list(self.applied)
+
+    def replay(self, event_id):
+        if event_id == "missing":
+            raise KeyError(event_id)
+        return AcceptResult(duplicate=True, reference=f"audit:{event_id}")
+
+    def close(self):
+        self.closed = True
+
+    @property
+    def retention_policy(self):
+        return RetentionPolicy(
+            custody_class="archive",
+            retention_days=None,
+            immutable=True,
+            tamper_evidence=False,
+            durable=True,
+        )
+
+
+def test_build_backend_disables_auto_migrate_when_asked(monkeypatch, tmp_path):
+    captured = {}
+
+    def fake(dsn, **kwargs):
+        captured["dsn"] = dsn
+        captured.update(kwargs)
+        return _FakePostgres(dsn, **kwargs)
+
+    monkeypatch.setenv("AUDIT_CORE_AUTO_MIGRATE", "0")
+    monkeypatch.setenv("AUDIT_CORE_CREDENTIAL_DIR", str(tmp_path))
+    _install_fake_postgres(monkeypatch, fake)
+    backend = build_backend()
+    assert captured["migrate"] is False
+    assert captured["credential_dir"] == str(tmp_path)
+    assert isinstance(backend, _FakePostgres)
+
+
+def test_build_backend_auto_migrates_by_default(monkeypatch, tmp_path):
+    captured = {}
+
+    def fake(dsn, **kwargs):
+        captured.update(kwargs)
+        return _FakePostgres(dsn, **kwargs)
+
+    monkeypatch.delenv("AUDIT_CORE_AUTO_MIGRATE", raising=False)
+    monkeypatch.setenv("AUDIT_CORE_CREDENTIAL_DIR", str(tmp_path))
+    _install_fake_postgres(monkeypatch, fake)
+    build_backend()
+    assert captured["migrate"] is True
+
+
+def test_migrate_command_applies_pending_schema(monkeypatch, capsys):
+    seen = {}
+
+    def fake(dsn, **kwargs):
+        backend = _FakePostgres(dsn, **kwargs)
+        seen["backend"] = backend
+        return backend
+
+    monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core")
+    _install_fake_postgres(monkeypatch, fake)
+    args = build_parser().parse_args(["migrate", "--schema", "audit_core"])
+    assert args.func(args) == 0
+    body = json.loads(capsys.readouterr().out)
+    assert body == {"applied": ["0001-events"], "ok": True, "schema": "audit_core"}
+    assert seen["backend"].kwargs["migrate"] is False
+    assert seen["backend"].closed is True
+
+
+def test_replay_command_reconciles(monkeypatch, capsys):
+    def fake(dsn, **kwargs):
+        return _FakePostgres(dsn, **kwargs)
+
+    monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core")
+    _install_fake_postgres(monkeypatch, fake)
+    args = build_parser().parse_args(["replay", "--event-id", "evt-1"])
+    assert args.func(args) == 0
+    body = json.loads(capsys.readouterr().out)
+    assert body["duplicate"] is True
+    assert body["reference"] == "audit:evt-1"
+
+
+def test_replay_command_missing_event(monkeypatch, capsys):
+    def fake(dsn, **kwargs):
+        return _FakePostgres(dsn, **kwargs)
+
+    monkeypatch.setenv("AUDIT_CORE_DATABASE_URL", "postgresql://example/audit_core")
+    _install_fake_postgres(monkeypatch, fake)
+    args = build_parser().parse_args(["replay", "--event-id", "missing"])
+    assert args.func(args) == 1
+    assert json.loads(capsys.readouterr().out)["error"] == "not_found"

workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md

@@ -8,7 +8,7 @@ status: active
 owner: codex
 topic_slug: netkingdom
 created: "2026-08-10"
-updated: "2026-08-10"
+updated: "2026-08-13"
 depends_on:
   - AUDIT-WP-0004
   - RAPP-POSTGRES-WP-0002
@@ -179,10 +179,37 @@ after confirming no active sessions; the group roles are untouched and remain
 NOLOGIN. The harness now uses a per-run random password, `VALID UNTIL` one
 hour, and an exit trap that drops the roles in every mode including on failure.
 
-Remaining before done: the `openbao-audit-core` ClusterSecretStore and the
+Progress 2026-08-13: the in-cluster delivery path is now honest about what
-`rapp-postgres/audit-core-runtime` OpenBao role are prerequisites owned by
+kind of secret it is pulling.
-rapp-postgres/railiance-platform, not this repo. Once they exist, apply and
+
-verify a live lease and a live rotation.
+The previous ExternalSecret pointed at KV
+`platform/workloads/audit-core/database/audit-core-runtime`. That path does
+not exist, and copying a dynamic lease into KV would freeze it — the same
+delivery-gap problem the mounted directory exists to avoid. Database
+credentials now come from ESO `VaultDynamicSecret` against
+`database/creds/audit-core-runtime` and `database/creds/audit-core-migration`,
+refreshed every 15m into one-file-per-field Secrets. The sender registry
+stays on ClusterSecretStore `openbao-audit-core` (KV), which is the right
+tool for a document.
+
+A second finding: `PostgresAuditBackend` applied schema on every process
+start. The runtime role cannot `CREATE TABLE` (isolation-test, ADR-0001).
+Production now sets `AUDIT_CORE_AUTO_MIGRATE=0`; `python -m audit_core
+migrate` runs as Job `audit-core-migrate` with the migration lease.
+
+Package side in this repo: `deploy/clustersecretstore.yaml`,
+`deploy/vaultdynamicsecrets.yaml`, `deploy/migrate-job.yaml`,
+`scripts/openbao-eso-token-apply.sh`. Platform side drafted in
+`railiance-platform` (`openbao/policies/external-secrets-audit-core.hcl` and
+the ClusterSecretStore add-on). The OpenBao database roles themselves were
+already delivered by RAPP-POSTGRES-WP-0002-T04.
+
+Remaining before done: attended `platform-admin` OpenBao login to write the
+ESO policy, mint `external-secrets/openbao-audit-core-eso-token`, and
+`bao kv put` the sender registry. Then apply CSS + generators + ExternalSecrets
+and verify a live lease plus a live rotation against the mounted directory.
+That step is flagged `needs_human` — this session has no token that can write
+policies.
 
 ## T03 - Deploy the receiver
 
@@ -261,8 +288,25 @@ other scrape target, so an exposition endpoint would be built for a scrape path
 that does not exist. This is usable with curl today and a small step from
 `/metrics` when a metrics stack lands.
 
-Remaining before done: build and publish the image to get a real digest, apply,
+Progress 2026-08-13: Containerfile base is digest-pinned
-and verify restart/rescheduling and the rollback path on the cluster.
+(`python:3.12-slim@sha256:d764629ce0…`, same digest as the deployed
+user-engine image) and labelled with `org.opencontainers.image.revision`.
+`make image-build` / `make image-publish` exist. Namespace `audit-core`
+(with `railiance.io/postgres-client: platform-pg`) and the four
+NetworkPolicies are applied on railiance01. Remaining manifests validate
+`--dry-run=server --validate=strict`. Deployment is not applied: it still
+carries `sha256:REPLACE_AT_RELEASE` until the ESO secrets exist, and
+applying it now would ImagePullBackOff and then CrashLoop on a missing
+sender registry.
+
+An image was published to `forgejo.coulomb.social/coulomb/audit-core` during
+this session; rebuild and pin after the commit that contains these files so
+the label matches the content.
+
+Remaining before done: pin the post-commit digest in
+`deploy/audit-core.yaml` and `deploy/migrate-job.yaml`, apply migrate Job
+then Deployment once T02 secrets exist, and verify restart, reschedule, and
+`kubectl rollout undo`.
 
 ## T04 - Migrate existing SQLite records
 
@@ -389,7 +433,7 @@ OpenBao-leased credentials.
 
 ```task
 id: AUDIT-WP-0005-T06
-status: todo
+status: progress
 priority: medium
 state_hub_task_id: "0856c80d-abe1-4bff-ba8d-87295cf76819"
 ```
@@ -404,3 +448,17 @@ provides — the retention window audit-core declares must not exceed the
 retention the platform guarantees.
 
 Done when the runbook exists and the restore path has been walked once.
+
+Progress 2026-08-13: `docs/operator-runbook.md` covers lookup by correlation
+id, dead-letter inspection, accepted-event replay (`python -m audit_core
+replay`), overlap-first sender rotation, restart-free database lease
+rotation, `/readyz` vs `/healthz` vs `/v1/stats`, and rollback.
+`python -m audit_core replay` is the operator surface that WP-0004
+deliberately left off HTTP.
+
+Restore is written down and is **not** walked. rapp-postgres T06 is `wait`
+on an S3 Barman target; the Nextcloud logical-dump lane cannot provide WAL
+or PITR; physical restore is instance-wide. audit-core currently declares
+no finite `retention_days`, so it does not outrun a platform window that
+does not yet exist — but it also cannot promise an RPO until the platform
+drills land. The walk listed in the runbook is what closes this task.