diff --git a/SCOPE.md b/SCOPE.md index 6c7ac3c..08e377a 100644 --- a/SCOPE.md +++ b/SCOPE.md @@ -131,3 +131,20 @@ and a create-only workload share. Encrypted fixture recovery and consumer refresh are verified; full application restore and predecessor invalidation remain RPF-WP-0029-T02. See the [latest blocked-workplan review](history/2026-09-05-blocked-workplan-closure-review.md). + + +## Backup authority correction — 2026-09-06 + +Scaleway Standard Multi-AZ (`nl-ams`) is the primary backup destination under +RESOURCE-WP-0002. Nextcloud is the independent secondary-copy lane. The live +Scaleway paths cover apps-pg, platform-pg and platform-pg-2. An isolated apps-pg +restore from that primary succeeded in 42.64 seconds; production remained ready +and scratch resources were removed. See +[primary recovery evidence](docs/evidence/scaleway-primary-restore-2026-09-06.json). + +Coverage is per asset: live forgejo-db, net-kingdom-pg and state-hub-db have no +native Barman destination. The Forgejo full-archive helper still targets +Nextcloud; no Forgejo blob/archive destination on Scaleway was evidenced. +The account cutover and archive-integrity fix do not establish that coverage. +WP-0029 retains secondary credential invalidation/recovery; its completion +must not be presented as full primary-backup assurance. diff --git a/assurance/ownership-handoffs.json b/assurance/ownership-handoffs.json index 615cb54..c4f9d77 100644 --- a/assurance/ownership-handoffs.json +++ b/assurance/ownership-handoffs.json @@ -12,7 +12,7 @@ "source_files": [ { "path": "tools/cmd/forgejo-backup", - "sha256": "a20f0aebb22f0978c0f45f74e4ac55a927b080910844296acabec10f45110cb6" + "sha256": "448921b702b5251e1b8d97ec16842bf3b31a2701170b110bba51fc94beab26e8" }, { "path": "tools/cmd/forgejo-package-prune", @@ -32,7 +32,7 @@ }, { "path": "docs/forgejo-backup.md", - "sha256": "af99987e900c8d2322da11c361bac4f1b946a577eed4a93d995cefa31a8e35b5" + "sha256": "2888028db46e8afdce7a78bf56772b307a5e1fa7e4ba7afdbadd23881e4fed64" }, { "path": "docs/forgejo-package-prune.md", diff --git a/assurance/service-records.json b/assurance/service-records.json index 2135e04..1876129 100644 --- a/assurance/service-records.json +++ b/assurance/service-records.json @@ -1,6 +1,6 @@ { "schema": "railiance-platform.service-records.v1", - "reviewed": "2026-09-05", + "reviewed": "2026-09-06", "review_owner": "railiance-platform", "review_scope": "S3 disclosure of unsupported guarantees; not external package approval", "services": [ @@ -30,7 +30,7 @@ "decision_owner": "railiance-platform + railiance-platform" }, "retention": "30 days", - "existing_evidence": "docs/evidence/RPF-WP-0019-backup-restore-2026-08-20.md", + "existing_evidence": "docs/evidence/scaleway-primary-restore-2026-09-06.json", "recovery_custody": "OpenBao: 2-of-3 operator quorum plus independent encrypted snapshot custody; database/offsite: governed backup lane and separately available restore key. Availability not verified in this task.", "maintenance_abort": "docs/railiance01-coordinated-reboot.md; stop before mutation when freshness, quorum, consumer readiness or named abort operator is absent", "freshness_policy": "assurance/service-contract.json; diagnostic thresholds only, no installed cadence approval", @@ -156,12 +156,12 @@ "target_seconds": null, "decision_owner": "railiance-platform + railiance-forge" }, - "retention": "14 daily + 4 weekly target; local 7/type", + "retention": "Nextcloud secondary account: 10 GiB total; 14 daily + 4 weekly is an unfulfilled target at the measured 5.35 GB/archive size", "existing_evidence": "docs/forgejo-backup.md", "recovery_custody": "OpenBao: 2-of-3 operator quorum plus independent encrypted snapshot custody; database/offsite: governed backup lane and separately available restore key. Availability not verified in this task.", "maintenance_abort": "docs/railiance01-coordinated-reboot.md; stop before mutation when freshness, quorum, consumer readiness or named abort operator is absent", "freshness_policy": "assurance/service-contract.json; diagnostic thresholds only, no installed cadence approval", - "requirement_assessment": "No accepted numeric consumer availability/RPO/RTO requirement found in the reviewed contracts. Service classes inform placement, not guarantees. Refuse any request for guaranteed HA/node-loss recovery until matched to supported substrate and package proof." + "requirement_assessment": "No accepted numeric consumer availability/RPO/RTO requirement found in the reviewed contracts. Service classes inform placement, not guarantees. Refuse any request for guaranteed HA/node-loss recovery until matched to supported substrate and package proof. Scaleway is the selected primary provider, but no Forgejo primary archive or native database destination is evidenced; coverage remains incomplete." }, { "service": "cnpg-option-a", diff --git a/docs/backup-credential-recovery.md b/docs/backup-credential-recovery.md index dd9e299..ed1ac94 100644 --- a/docs/backup-credential-recovery.md +++ b/docs/backup-credential-recovery.md @@ -1,5 +1,8 @@ # WP-0029 provider recovery procedure +Primary platform backup is Scaleway (RESOURCE-WP-0002). This procedure covers +the independent Nextcloud secondary-copy lane. + Scope: invalidate the exposed Nextcloud upload predecessor and prove replacement encrypted upload and offsite recovery under CCR-2026-0004. The route is a Nextcloud file-drop share, not a platform-admin OpenBao credential. OpenBao diff --git a/docs/backup-provider-coverage.md b/docs/backup-provider-coverage.md new file mode 100644 index 0000000..0e29a66 --- /dev/null +++ b/docs/backup-provider-coverage.md @@ -0,0 +1,47 @@ +# Backup provider coverage — 2026-09-06 + +Primary: Scaleway Standard Multi-AZ, nl-ams, per RESOURCE-WP-0002 and the +operator's confirmation. Independent secondary: governed Nextcloud account +Backup, 10 GiB quota. Provider selection does not establish asset coverage. + +| Asset | Verified primary configuration | Secondary / remaining gap | +| --- | --- | --- | +| apps-pg | Scaleway Barman base backups + WAL, `platform-pg/apps-pg/` | Fresh isolated physical restore passed in 42.64 seconds; Nextcloud logical copy is separate | +| platform-pg | Scaleway Barman base backups + WAL, `platform-pg/` | Earlier package restore evidence; independent logical Nextcloud copy | +| platform-pg-2 | Scaleway Barman base backups + WAL, `platform-pg/platform-pg-2/` | Earlier package restore evidence; independent logical Nextcloud copy | +| forgejo-db | No native Barman destination observed | Logical SQL/full archive helper targets Nextcloud; primary coverage needs implementation | +| Forgejo repositories/packages/blobs | No reviewed Scaleway archive destination found | Corrected full-archive capture; 5.35 GB verified encrypted artifact staged; secondary download/application restore still pending | +| net-kingdom-pg / state-hub-db | No native Barman destination observed | Do not infer protection from the shared cells' healthy backup status | +| OpenBao / S1 host configuration | Not evaluated by this database restore | Their own encrypted snapshot/host backup and recovery contracts still apply | + +All three configured cells reported successful 2026-09-05 02:15 UTC backups. +The fresh apps-pg restore consumed the existing Scaleway base backup and WAL in +a unique scratch namespace, imported only the existing S3 credential fields in +captured memory, preserved expected databases and connection limits, left +production Ready and removed the namespace. This proves physical database +recovery; it does not prove application workflows, PITR targets, or Forgejo +recovery. Evidence: `docs/evidence/scaleway-primary-restore-2026-09-06.json`. + +## Forgejo primary extension requirements + +The existing bucket policy permits the runtime identity only under +`platform-pg/*`. Do not put unrelated archive objects in a Barman server directory +or assume a top-level `forgejo/` prefix is permitted. Before extending coverage: + +1. Accept an exact independent archive prefix and storage/retention contract + with reef-storage/resource-control; distinguish it from native database WAL. +2. Use the scoped backup runtime identity, never the Scaleway bootstrap key. + Review its delivery to the scheduled archive uploader. The current approved + ExternalSecret destination is in `databases`; activity-core must not inherit + write credentials through an undocumented namespace expansion. +3. Use a streaming multipart S3 uploader for growing archives, with abort/cleanup + and immutable object naming. Verify completion and content, then recover by + GET from Scaleway into the isolated Forgejo procedure. +4. Set native forgejo-db Barman coverage through its owning package/source, + with a separate tested recovery and no production in-place restore. +5. Record provider-native retention and primary failure reporting separately + from the 10 GiB secondary budget. No retained backup deletion is implicit. + +WP-0029 remains the secondary credential incident: old Bernd-share invalidation +and replacement recovery. The full primary coverage gap belongs to S3 assurance +(RPF-WP-0036-T03), with forge requirements and package/storage-owner inputs. diff --git a/docs/evidence/scaleway-primary-restore-2026-09-06.json b/docs/evidence/scaleway-primary-restore-2026-09-06.json new file mode 100644 index 0000000..1921798 --- /dev/null +++ b/docs/evidence/scaleway-primary-restore-2026-09-06.json @@ -0,0 +1,26 @@ +{ + "schema": "platform.scaleway-primary-restore.v1", + "status": "verified", + "namespace": "apps-pg-recovery-78fd92b6", + "primary_destination": "s3://railiance-platform-pg-backup/platform-pg/apps-pg/", + "last_successful_backup": "2026-09-05T02:15:07Z", + "source": "Scaleway Barman base backup and WAL", + "started_at": "2026-09-05T22:29:37.062455+00:00", + "stage": "database_acceptance", + "restore_seconds": 42.64, + "databases": [ + "app", + "apps_meta", + "coulomb_social_db", + "postgres", + "vergabe_db" + ], + "public_table_counts": { + "coulomb_social_db": 13, + "vergabe_db": 0 + }, + "consumer_connection_limits_preserved": true, + "production_ready": true, + "cleanup": true, + "finished_at": "2026-09-05T22:30:45.208512+00:00" +} diff --git a/docs/forgejo-backup.md b/docs/forgejo-backup.md index 93b4947..f8572d9 100644 --- a/docs/forgejo-backup.md +++ b/docs/forgejo-backup.md @@ -1,6 +1,12 @@ # Forgejo backup (railiance01) -Workplan: `RAIL-HO-WP-0005` T04/T09 · Decision: Option A (Nextcloud + age) +Workplan: `RAIL-HO-WP-0005` T04/T09 · Secondary copy: Nextcloud + age + +Scaleway is the platform primary backup provider. This helper currently writes +Forgejo archives only to Nextcloud. Live inspection on 2026-09-06 found no +Barman destination on forgejo-db and no reviewed Scaleway blob/archive path. +Treat Forgejo primary coverage as a gap; primary service selection alone does +not prove each asset has migrated. ## What is backed up diff --git a/history/2026-09-06-primary-backup-correction.md b/history/2026-09-06-primary-backup-correction.md new file mode 100644 index 0000000..ae9da21 --- /dev/null +++ b/history/2026-09-06-primary-backup-correction.md @@ -0,0 +1,34 @@ +# Primary backup correction and verified Scaleway recovery + +The operator reaffirmed that primary backup moved to Scaleway. Corrected the +platform scope, Forgejo/credential runbooks and service records: Nextcloud is an +independent secondary, not the primary proof for the platform. + +Live inspection found Scaleway Barman configuration on apps-pg, platform-pg and +platform-pg-2, with successful September 5 backups. forgejo-db, net-kingdom-pg and +state-hub-db have no native destination. The current Forgejo full archive helper +still uploads only to Nextcloud. Prior account migration did not cover that gap. + +Executed the bounded apps-pg restore from Scaleway into a separate namespace. +Ready in 42.64 seconds, expected consumer databases present, 13 public tables in +coulomb_social_db, and both connection limits remained 20. Production stayed +Ready. The scratch namespace and its namespaced resources were removed. +No credential value or application rows were printed or recorded. The exact +existing S3 fields were copied only within the protected apply stream. + +Evidence: `docs/evidence/scaleway-primary-restore-2026-09-06.json`. +Implementation: `scripts/verify_scaleway_primary_restore.py`. +The date is the operator's Europe/Berlin date; evidence retains exact UTC times. + +Prepared exact Forgejo primary extension requirements in +`docs/backup-provider-coverage.md`. It needs an independent archive destination, +reviewed runtime delivery and full artifact recovery, not just a provider-name +change. Existing database restore success does not close the Forgejo or +Nextcloud-secondary acceptance gates in WP-0029. + + +A fresh attended login for the validated Nextcloud secondary archive failed +before command handoff; revocation could not be confirmed. No new secondary +transfer ran. The encrypted staging from September 5 remains available and no +old-share revocation is asserted. A fresh attended login and old-share owner +confirmation/custody remain necessary. All 200 repository tests passed. diff --git a/workplans/RPF-WP-0029-backup-credential-default-removal.md b/workplans/RPF-WP-0029-backup-credential-default-removal.md index 38120be..822d92c 100644 --- a/workplans/RPF-WP-0029-backup-credential-default-removal.md +++ b/workplans/RPF-WP-0029-backup-credential-default-removal.md @@ -113,3 +113,18 @@ invocations use the integrity checks without editing the host checkout. Fresh encrypted archive: 5,353,024,293 bytes; 142 repository HEAD entries. Local drill plaintext and producer temporary files were removed. Owner login and the two remaining acceptance results above are still required. + + +## Primary/secondary boundary correction — 2026-09-06 + +User reaffirmed Scaleway as the primary backup provider. Nextcloud remains the +independent secondary lane. An actual isolated apps-pg recovery from Scaleway +passed in 42.64 seconds, including expected databases and consumer connection +limits; production stayed Ready and scratch resources were deleted. This is +primary database recovery evidence, not Forgejo or Nextcloud recovery proof. + +The live primary covers apps-pg/platform-pg/platform-pg-2; forgejo-db has no +Barman destination and the Forgejo full-archive uploader still targets Nextcloud. +Do not conflate this coverage gap with the old-share incident or silently move +archives into a database-owned prefix. WP-0029's secondary acceptance gates +remain explicit. Source/platform assurance records now name the correct primary. diff --git a/workplans/RPF-WP-0036-platform-service-assurance.md b/workplans/RPF-WP-0036-platform-service-assurance.md index 0fd16f0..5785d2e 100644 --- a/workplans/RPF-WP-0036-platform-service-assurance.md +++ b/workplans/RPF-WP-0036-platform-service-assurance.md @@ -242,3 +242,18 @@ open. The installed generator would reproduce them; exact UUID mapping and remaining owner requirements are persisted in `history/2026-09-05-blocked-workplan-closure-review.md`. No duplicate recovery, monitoring or owner-transfer workplan was created. + +## Primary backup coverage — 2026-09-06 + +Scaleway is the selected primary; Nextcloud is the independent secondary. +Fresh apps-pg recovery from Scaleway passed in 42.64 seconds with expected +consumer databases and limits, production Ready and scratch cleanup complete. +Evidence: `docs/evidence/scaleway-primary-restore-2026-09-06.json`. +T03 now has this fresh physical recovery receipt but still lacks recurring +cadence, the other recovery surfaces and validated evidence adapters. + +The source/live coverage inventory `docs/backup-provider-coverage.md` exposes +missing native primary configuration on forgejo-db/net-kingdom-pg/state-hub-db +and no reviewed Scaleway Forgejo archive destination. Track primary coverage +here with forge/package/storage owners; do not silently claim the Nextcloud +account cutover filled it or weaken WP-0029's separate incident closure.