From 195a51a62c690c8f54f7b8458537e0c2d32c85a9 Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 5 Sep 2026 22:35:20 +0200 Subject: [PATCH] Record rejected historical backup, verified fresh capture and worker activation Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883 --- .../RPF-WP-0029-fresh-archive-2026-09-05.json | 15 +++++ ...PF-WP-0029-rejected-backup-2026-09-05.json | 24 +++++++ .../RPF-WP-0029-worker-bundle-2026-09-05.json | 8 +++ .../RPF-WP-0029-worker-source-2026-09-05.json | 4 ++ docs/forgejo-backup.md | 29 +++++++-- ...2026-09-05-wp0029-real-offsite-recovery.md | 62 +++++++++++++++++++ ...-0029-backup-credential-default-removal.md | 31 +++++++++- 7 files changed, 166 insertions(+), 7 deletions(-) create mode 100644 docs/evidence/RPF-WP-0029-fresh-archive-2026-09-05.json create mode 100644 docs/evidence/RPF-WP-0029-rejected-backup-2026-09-05.json create mode 100644 docs/evidence/RPF-WP-0029-worker-bundle-2026-09-05.json create mode 100644 docs/evidence/RPF-WP-0029-worker-source-2026-09-05.json create mode 100644 history/2026-09-05-wp0029-real-offsite-recovery.md diff --git a/docs/evidence/RPF-WP-0029-fresh-archive-2026-09-05.json b/docs/evidence/RPF-WP-0029-fresh-archive-2026-09-05.json new file mode 100644 index 0000000..ee43002 --- /dev/null +++ b/docs/evidence/RPF-WP-0029-fresh-archive-2026-09-05.json @@ -0,0 +1,15 @@ +{ + "schema": "platform.forgejo-capture-integrity.v1", + "status": "verified", + "archive_bytes": 5351717533, + "producer_exit_zero": true, + "transfer_hash_matches": true, + "zip_crc_passed": true, + "production_temporary_files_removed": true, + "members": 89197, + "uncompressed_bytes": 5432944912, + "repository_heads": 142, + "archive_sha256": "fe53d156461387ea487cddc5d6a01f0b5e3debda6fa494c8b87afc8bc62b55b2", + "encrypted_bytes": 5353024293, + "encrypted_stage": "/tmp/rpf-wp0029-restore/forgejo-dump-20260905-verified.zip.age" +} diff --git a/docs/evidence/RPF-WP-0029-rejected-backup-2026-09-05.json b/docs/evidence/RPF-WP-0029-rejected-backup-2026-09-05.json new file mode 100644 index 0000000..c972aaa --- /dev/null +++ b/docs/evidence/RPF-WP-0029-rejected-backup-2026-09-05.json @@ -0,0 +1,24 @@ +{ + "schema": "platform.real-offsite-recovery.v1", + "status": "restore_rejected", + "quota_before": { + "quota_bytes": 10737418240, + "used_bytes": 2505506, + "available_bytes": 10734912734 + }, + "artifact": "wp0029-recovery-forgejo-dump-20260904T001507Z.zip.age", + "source_backup": "forgejo-dump-20260904T001507Z.zip.age", + "kv_version": 3, + "upload_http_status": 201, + "download_http_status": 200, + "ciphertext_bytes": 147171923, + "ciphertext_sha256": "475687367d7e1736b169a22732293c534d7a04d4f6315d235f0d649218f7d4cb", + "decrypted": true, + "quota_after": { + "quota_bytes": 10737418240, + "used_bytes": 2505506, + "available_bytes": 10734912734 + }, + "archive_validation": "failed_truncated_zip", + "application_restore": false +} diff --git a/docs/evidence/RPF-WP-0029-worker-bundle-2026-09-05.json b/docs/evidence/RPF-WP-0029-worker-bundle-2026-09-05.json new file mode 100644 index 0000000..a4268cf --- /dev/null +++ b/docs/evidence/RPF-WP-0029-worker-bundle-2026-09-05.json @@ -0,0 +1,8 @@ +{ + "status": "verified", + "configmap": "backup-verified-0220ca56520c", + "worker_ready": true, + "bundle_files_match": true, + "host_checkout_edited": false, + "rollback": "Restore deployment from its prior ReplicaSet with kubectl rollout undo; retain immutable ConfigMap until no revision references it." +} diff --git a/docs/evidence/RPF-WP-0029-worker-source-2026-09-05.json b/docs/evidence/RPF-WP-0029-worker-source-2026-09-05.json new file mode 100644 index 0000000..dabdc2e --- /dev/null +++ b/docs/evidence/RPF-WP-0029-worker-source-2026-09-05.json @@ -0,0 +1,4 @@ +{ + "worker_backup_entrypoint_matches": false, + "worker_capture_helper_present": false +} diff --git a/docs/forgejo-backup.md b/docs/forgejo-backup.md index f5dae34..93b4947 100644 --- a/docs/forgejo-backup.md +++ b/docs/forgejo-backup.md @@ -7,7 +7,7 @@ Workplan: `RAIL-HO-WP-0005` T04/T09 · Decision: Option A (Nextcloud + age) | Artifact | Source | Format | | --- | --- | --- | | Blob state | `forgejo dump` in production pod | zip → age | -| PostgreSQL | `pg_dump -Fc` from CNPG `forgejo-db` | custom dump → age | +| PostgreSQL | `pg_dump --no-owner --no-acl` from CNPG `forgejo-db` | SQL → age | Covers git repos, packages (OCI/npm/generic), attachments, LFS, avatars, and DB metadata. @@ -20,11 +20,12 @@ make forgejo-backup # encrypt + upload to Nextcloud forgejo/ make forgejo-backup-status # last success + 7-day gate hint ``` -Requires: `kubectl`, `age`, `curl`, `KUBECONFIG=~/.kube/config-hosteurope`. +Requires: `kubectl`, `age`, Python 3, and curl or the Python upload backend. +Set `KUBECONFIG` to the currently verified railiance01 access path. -Preflight: `forgejo-db` pod must be Ready (`make forgejo-db-status`). A full dump -with 13 org repos is ~670MiB — allow 10–20 minutes for stream + age on a typical -workstation link. +Preflight: `forgejo-db` pod must be Ready (`make forgejo-db-status`). The validated 2026-09-05 full archive is 5,351,717,533 bytes before age encryption. +Allow for substantial transfer time and check the account quota before upload. +This size supersedes the historical ~670 MiB estimate. Uploads require the governed `CCR-2026-0004` OpenBao offsite lane or explicitly provided `RAILIANCE_BACKUP_NC_TOKEN` / `RAILIANCE_BACKUP_NC_WEBDAV_URL` inputs. @@ -79,3 +80,21 @@ See `railiance-infra`: Download and decrypt a dump from Nextcloud, set `BACKUP_LOCAL` to the zip path, then run the drill script. + +## Archive-integrity incident — 2026-09-05 + +The September 4 encrypted cache sample decrypted but failed ZIP validation. +The backup command now waits for the exact dump process to succeed, checks +transfer size/hash and validates ZIP CRCs before encryption/upload. A fresh +5.35 GB archive passed these checks. This does not certify the other historical +backups or establish a completed offsite application restore. See WP-0029 and +its dated evidence; never report a nonempty file as a successful backup. + + +Activity-core currently receives the validated entry point through the immutable +`backup-verified-0220ca56520c` bundle, applied by +`scripts/deploy_verified_backup_bundle.py`. The wrapper mounts over the existing +backup entry point; helpers/libraries live at `/opt/railiance-backup-verified`. +Reapply the source-owned delivery helper after reviewed backup changes and verify +its hash/rollout receipt. Roll back through the previous worker ReplicaSet; do +not edit another machine's host checkout or remove referenced ConfigMaps. diff --git a/history/2026-09-05-wp0029-real-offsite-recovery.md b/history/2026-09-05-wp0029-real-offsite-recovery.md new file mode 100644 index 0000000..77c3919 --- /dev/null +++ b/history/2026-09-05-wp0029-real-offsite-recovery.md @@ -0,0 +1,62 @@ +# WP-0029 real offsite recovery — 2026-09-05 + +User authorized executing WP-0029. A real September 4 encrypted Forgejo backup +was uploaded through Backup's create-only share (201), independently downloaded +as owner (200), compared byte-for-byte and decrypted. ZIP validation then rejected +it: it has a ZIP header but lacks a complete archive directory. It cannot support +application recovery. Evidence is `docs/evidence/RPF-WP-0029-rejected-backup-2026-09-05.json`. +No conclusion about every other historical backup is inferred from this sample. + +The previous capture code accepted nonempty archive files after its polling +limit, did not require the dump's successful exit, and did not validate the full +transfer or ZIP integrity. Replaced it with an exact-process completion marker, +per-chunk length checks, producer/receiver SHA-256 equality and ZIP CRC validation +before encryption. A new live dump completed at 5,351,717,533 bytes, passed producer/transfer hash +and ZIP CRC validation, and contains 142 repository HEAD entries. It is encrypted +in private staging; drill-created local plaintext and producer temporary files +were removed. Its size materially constrains +the dedicated account's 10 GiB retention capacity. + +Prepared isolated local recovery with pinned Forgejo 11.0.3/PostgreSQL 16 images, +internal-only Docker network, no host ports, disposable database credentials, +independent application configuration, and required fetched-artifact hash receipt. +The old infra drill's missing-file production-dump fallback and copied production +credentials are not used. Database import, application startup, repository clone +and package-blob integrity remain live acceptance gates until their receipt passes. + +Old-share invalidation requires Bernd's owner confirmation or protected owner +custody coordinates; requested without asking for a password in chat. Neither +the existing age-key exposure record nor retained backups may be silently cleared. + +Implementation validation: 200 repository tests passed. Live completion and +cleanup receipts will be appended when available; this entry is not closure. + + +## Resume boundary and runtime delivery + +A fresh OpenBao login for the predecessor check failed before command handoff; +revocation could not be confirmed. No probe/provider mutation ran in that attempt. +It is terminal and cannot be reused. The validated backup still needs a fresh +attended upload/download/decrypt session followed by isolated restore. + +Encrypted staging: `/tmp/rpf-wp0029-restore/forgejo-dump-20260905-verified.zip.age`. +Use `verify_nextcloud_offsite_restore.py` with this source, a new private directory +and receipt; then `restore_forgejo_offsite_locally.py --archive /fetched.zip +--transfer-receipt --receipt `. Both scripts are +committed. The latter requires the hash-bound downloaded artifact. Retain the +encrypted staging until acceptance; remove fetched plaintext after the drill. +The rejected candidate's newly created offsite object is +`forgejo/wp0029-recovery-forgejo-dump-20260904T001507Z.zip.age`; remove only this +failed test copy through owner authority, preserving original historical backups. + +The worker source check found the old capture script still mounted and the new +helper absent. Delivered immutable ConfigMap `backup-verified-0220ca56520c`, with +an entry-point wrapper and the complete reviewed backup helper/library bundle. +The worker rolled successfully and all four delivered source hashes match. +No host checkout was edited. This compatibility delivery is source-owned in +`scripts/deploy_verified_backup_bundle.py`; broader package ownership remains T06. +Evidence: `docs/evidence/RPF-WP-0029-worker-bundle-2026-09-05.json`. + +T02 remains waiting for attended access, actual offsite application restore and +old-share owner invalidation. No full recovery, old-token rejection or age-key +exposure closure is claimed. Fresh archive and rejection receipts are persisted. diff --git a/workplans/RPF-WP-0029-backup-credential-default-removal.md b/workplans/RPF-WP-0029-backup-credential-default-removal.md index d57907a..38120be 100644 --- a/workplans/RPF-WP-0029-backup-credential-default-removal.md +++ b/workplans/RPF-WP-0029-backup-credential-default-removal.md @@ -4,7 +4,7 @@ type: workplan title: "Remove backup credential default and verify governed replacement" domain: financials repo: railiance-platform -status: active +status: blocked owner: codex created: "2026-09-05" updated: "2026-09-05" @@ -36,7 +36,7 @@ redirects/non-success status. Added transport containment and failure tests. ```task id: RPF-WP-0029-T02 -status: progress +status: wait priority: high state_hub_task_id: "b3f3402f-890b-5781-9b3e-1c9c0d28cea8" ``` @@ -86,3 +86,30 @@ Workload KV CAS 2→3 preserved age escrow. ExternalSecret delivery and all thre activity-core deployments verified ready with new credentials loaded. All test fixtures removed. Account quota is enforced by Nextcloud; no automatic pruning. Evidence: `docs/evidence/RPF-WP-0029-backup-account-2026-09-05.json`. + +## Real recovery attempt — 2026-09-05 + +The September 4 encrypted backup passed new-account upload/download/decryption +but failed ZIP integrity, so no application restore was claimed. Fixed the +capture path to require exact producer success, transferred-byte identity and +ZIP CRC validation before encryption. A fresh 5,351,717,533-byte production +archive passed all three gates; it is encrypted in private local staging. +200 tests passed. The isolated restore implementation now requires a hash-bound +receipt for fetched bytes and uses disposable local credentials/network. + +A later attended OpenBao login failed before command handoff, with revocation +not confirmed; the read-only predecessor probe did not execute. Do not reuse +that attempt or infer provider invalidation. T02 waits for a fresh attended +login to upload/fetch the validated archive and execute isolated recovery, +plus the old Bernd-owned share's invalidation receipt or protected owner path. +The failed candidate's newly created offsite copy also needs exact owner cleanup. +History: `history/2026-09-05-wp0029-real-offsite-recovery.md`. + + +Worker follow-through: its mounted source was still old. Deployed the immutable +`backup-verified-0220ca56520c` ConfigMap via the scoped platform helper, rolled +actcore-worker and verified all four source hashes. Future worker entry-point +invocations use the integrity checks without editing the host checkout. +Fresh encrypted archive: 5,353,024,293 bytes; 142 repository HEAD entries. +Local drill plaintext and producer temporary files were removed. Owner login +and the two remaining acceptance results above are still required.