From c4dbb92a8afb487bcce30e8662d9e31b571dc848 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 13 Aug 2026 14:14:22 +0200 Subject: [PATCH] Finish AUDIT-WP-0005: walk restore and close T06 Logical dump of live audit_core restored locally in 4.1s; counts and sample triples match the rapp-postgres drill. Retention is platform backup, not an audit-core deletion window. Workplan finished. --- deploy/audit-core.yaml | 10 +-- .../restore-walk-20260813T121200Z.json | 36 ++++++++++ docs/operator-runbook.md | 66 +++++++++---------- ...ostgres-store-and-production-deployment.md | 21 +++--- 4 files changed, 85 insertions(+), 48 deletions(-) create mode 100644 docs/evidence/restore-walk-20260813T121200Z.json diff --git a/deploy/audit-core.yaml b/deploy/audit-core.yaml index 3dbc588..1ab18fb 100644 --- a/deploy/audit-core.yaml +++ b/deploy/audit-core.yaml @@ -46,11 +46,11 @@ metadata: # Rollback position. Update both together; `kubectl rollout undo` returns to # the previous digest, and the schema note records whether that is safe. audit-core.railiance.io/rollback-note: >- - Migrations 0001-0004 are additive (CREATE TABLE/INDEX/TRIGGER IF NOT - EXISTS) and are not reversed by a rollback. An older image runs against - the newer schema without harm. A future migration that drops or narrows a - column breaks that property and must state its own rollback position - before it is released. + Migrations 0001-0005 are additive (CREATE TABLE/INDEX/TRIGGER IF NOT + EXISTS, plus GRANTs) and are not reversed by a rollback. An older image + runs against the newer schema without harm. A future migration that drops + or narrows a column breaks that property and must state its own rollback + position before it is released. spec: replicas: 1 revisionHistoryLimit: 5 diff --git a/docs/evidence/restore-walk-20260813T121200Z.json b/docs/evidence/restore-walk-20260813T121200Z.json new file mode 100644 index 0000000..5346ecb --- /dev/null +++ b/docs/evidence/restore-walk-20260813T121200Z.json @@ -0,0 +1,36 @@ +{ + "workplan_task": "AUDIT-WP-0005-T06", + "recorded_at": "2026-08-13T12:12:00Z", + "path": "logical single-consumer dump of live audit_core, restore into local PostgreSQL 16", + "source": "platform-pg-1 / audit_core on railiance01", + "destination": "throwaway postgres:16-alpine (127.0.0.1:55446), deleted after verify", + "elapsed_seconds": 4.072, + "counts": { + "events": 21, + "dead_letters": 3, + "secret_findings": 1, + "migrations": 4 + }, + "trigger": "events_append_only", + "sample_triples_matched": 5, + "sample_event_ids": [ + "ok-1a39fe39", + "rot2-45d720f4", + "rot2-66c15638", + "fm-deliver-a25fd813", + "fm-redact-fccd1dd8" + ], + "matches_platform_drill": true, + "platform_evidence": "rapp-postgres/docs/evidence/backup-restore-20260813T111651Z-remote.json", + "platform_full_restore_seconds": 51.044, + "platform_pitr_seconds": 52.455, + "platform_logical_seconds": 3.468, + "retention": { + "audit_core_retention_days": null, + "audit_core_meaning": "does not expire or delete events", + "platform_planned_window_days": 30, + "production_barman": "fail-closed; no off-host copy yet", + "recovery_bound": "platform backup retention, not audit-core deletion" + }, + "verified": true +} diff --git a/docs/operator-runbook.md b/docs/operator-runbook.md index fbd4ae8..b9af8f6 100644 --- a/docs/operator-runbook.md +++ b/docs/operator-runbook.md @@ -142,52 +142,48 @@ behind the read privilege). Watch: ## Restore -audit-core declares `custody_class=archive` and `immutable=True` (the -append-only trigger). It does **not** declare a retention window shorter -than the platform's, because it currently declares none (`retention_days` -unset = keep). That is only honest if the platform's backup retention is -also unbounded or is an explicit accepted loss. +audit-core does **not** expire events (`retention_days` unset). Recovery is +bounded by what rapp-postgres can restore, not by an audit-core deletion +window. The platform's planned Barman window is 30 days. Production Barman +is still fail-closed (no governed off-host target). Do not promise an RPO +until that target exists. Local WAL on the node is not an off-host copy. -As of 2026-08-13, `RAPP-POSTGRES-WP-0002-T06` is `wait`. Production backup -is fail-closed until an S3-compatible Barman target exists. The governed -Nextcloud logical-dump lane cannot provide WAL archiving or PITR. Do not -claim an RPO/RTO for audit-core until those drills have been recorded in -rapp-postgres. +Physical restore is instance-wide. A consumer-only restore is a logical +export of `audit_core` from a scratch physical restore, then a controlled +import. Never recover in place. Procedure: `rapp-postgres/docs/restore.md`. -When the platform restore path is open, the audit-core walk is: +Walked 2026-08-13: -1. Restore `platform-pg` to a scratch cluster (`platform-pg-restore-full` or - PITR). Never recover in place. -2. Physical restore is instance-wide. A single-consumer restore is a logical - dump of `audit_core` from the scratch cluster, then a controlled import. -3. Verify: event counts, a sample of `event_id`/`payload_hash`/`accepted_at` - triples, and that the append-only trigger is still installed - (`SELECT tgname FROM pg_trigger WHERE tgname = 'events_append_only'`). -4. Record elapsed time. That number is what this service can promise. +| Path | Evidence | Elapsed | Result | +| --- | --- | --- | --- | +| Platform full restore + PITR of live `audit_core` (21 events) | `rapp-postgres/docs/evidence/backup-restore-20260813T111651Z-remote.json` | 51.0s / 52.5s | triples + `events_append_only` | +| Platform logical single-consumer export/import | same | 3.5s | 21 events | +| audit-core logical dump of live `audit_core` → local PG 16 | `evidence/restore-walk-20260813T121200Z.json` | 4.1s | 21/3/1 counts, 5 triples, trigger present | -Until that walk exists, a workstation-side logical dump of an empty -`audit_core` is not evidence of recovery. +Verify after any restore: + +```sql +SELECT count(*) FROM audit_core.events; +SELECT event_id, payload_hash, accepted_at FROM audit_core.events + WHERE event_id IN (/* sample from pre-restore */); +SELECT tgname FROM pg_trigger WHERE tgname = 'events_append_only'; +``` ## Rollback The Deployment annotation `audit-core.railiance.io/rollback-note` is the -source of truth. Migrations 0001-0004 are additive; `kubectl -n audit-core +source of truth. Migrations 0001-0005 are additive; `kubectl -n audit-core rollout undo deploy/audit-core` returns to the previous digest and an older image runs against the newer schema. A future migration that drops or narrows a column must replace that note before release. ## Deploy order -1. Image published, digest pinned in `deploy/audit-core.yaml` and - `deploy/migrate-job.yaml`. -2. Attended `scripts/openbao-eso-token-apply.sh`. -3. `bao kv put platform/workloads/audit-core/senders senders.json=@file` - from a mode-0600 file; shred the file. -4. Apply namespace (carries `railiance.io/postgres-client: platform-pg`), - NetworkPolicies, ClusterSecretStore, VaultDynamicSecrets, ExternalSecrets. -5. Wait until Secrets `audit-core-database`, `audit-core-database-migrate`, - and `audit-core-senders` exist. -6. Apply and wait for Job `audit-core-migrate`. -7. Apply the Deployment. Confirm `/readyz` reports `custody_class=archive`. -8. Run `MODE=remote BASE_URL=… make failure-matrix` and hand the JSON to - NK-WP-0024. +1. Image published and digest-pinned. +2. Namespace (label `railiance.io/postgres-client: platform-pg`), + NetworkPolicies, ClusterSecretStores, ExternalSecrets. +3. Sender registry is Secret `audit-core-senders` (in-cluster mint). + Database leases come from `openbao-audit-core-database`. +4. Job `audit-core-migrate` with `AUDIT_CORE_MIGRATE_ROLE=audit_core_migrate`. +5. Deployment. `/readyz` must report `custody_class=archive`. +6. In-pod `MODE=remote DISRUPT=0` failure matrix. Evidence goes to NK-WP-0024. diff --git a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md index 3c8d8b6..163d36c 100644 --- a/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md +++ b/workplans/AUDIT-WP-0005-postgres-store-and-production-deployment.md @@ -4,7 +4,7 @@ type: workplan title: "Deploy audit-core on Railiance with durable Postgres custody" domain: infotech repo: audit-core -status: active +status: finished owner: codex topic_slug: netkingdom created: "2026-08-10" @@ -469,7 +469,7 @@ locked in migration 0005. That JSON is the NK-WP-0024 hand-back. ```task id: AUDIT-WP-0005-T06 -status: progress +status: done priority: medium state_hub_task_id: "0856c80d-abe1-4bff-ba8d-87295cf76819" ``` @@ -492,9 +492,14 @@ rotation, `/readyz` vs `/healthz` vs `/v1/stats`, and rollback. `python -m audit_core replay` is the operator surface that WP-0004 deliberately left off HTTP. -Restore is written down and is **not** walked. rapp-postgres T06 is `wait` -on an S3 Barman target; the Nextcloud logical-dump lane cannot provide WAL -or PITR; physical restore is instance-wide. audit-core currently declares -no finite `retention_days`, so it does not outrun a platform window that -does not yet exist — but it also cannot promise an RPO until the platform -drills land. The walk listed in the runbook is what closes this task. +Closed 2026-08-13. rapp-postgres T06 is done. Platform remote drill +recovered live `audit_core` (21 events, matching triples, append-only +trigger; full 51s, PITR 52s, logical 3.5s). audit-core walked the +single-consumer path: `pg_dump -Fc` of live `audit_core` restored into +local PostgreSQL 16 in **4.1s**; counts 21/3/1, five triples match the +platform sample, trigger present. +`evidence/restore-walk-20260813T121200Z.json`. + +Retention check: audit-core does not delete events. Recoverable history +is the platform backup window (planned 30 days once Barman is on). +Production Barman is still fail-closed — no off-host RPO is claimed.