From d011073e16fa8d160de0b8d3ba44ea38ee144270 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 7 Jul 2026 20:04:54 +0200 Subject: [PATCH] Start DISCTL-WP-0002: policies, registers, runbooks, playbooks Expand AssetRegister and DataRegister with T0/T4 stack assets. Add ResiliencePolicy, PanicSheet, full-server-loss runbook, and five incident playbooks. Update README with mirror status. Mark T02-T05, T07-T08 done; T01 and T06 in progress. --- AssetRegister.yaml | 246 +++++++++++++++++- DataRegister.yaml | 82 +++++- INTENT.md | 5 +- IncidentPlaybooks/accidental-delete.md | 23 ++ IncidentPlaybooks/credential-leak.md | 18 ++ IncidentPlaybooks/loss-of-admin-identity.md | 25 ++ IncidentPlaybooks/provider-outage.md | 19 ++ IncidentPlaybooks/ransomware.md | 27 ++ PanicSheet.md | 96 +++++++ README.md | 58 ++++- ResiliencePolicy.md | 96 +++++++ RestoreRunbooks/01-full-server-loss.md | 126 +++++++++ .../DISCTL-WP-0002-resilience-foundation.md | 22 +- 13 files changed, 814 insertions(+), 29 deletions(-) create mode 100644 IncidentPlaybooks/accidental-delete.md create mode 100644 IncidentPlaybooks/credential-leak.md create mode 100644 IncidentPlaybooks/loss-of-admin-identity.md create mode 100644 IncidentPlaybooks/provider-outage.md create mode 100644 IncidentPlaybooks/ransomware.md create mode 100644 PanicSheet.md create mode 100644 ResiliencePolicy.md create mode 100644 RestoreRunbooks/01-full-server-loss.md diff --git a/AssetRegister.yaml b/AssetRegister.yaml index 034a0d5..11337a5 100644 --- a/AssetRegister.yaml +++ b/AssetRegister.yaml @@ -1,10 +1,10 @@ # Resilience Control Plane — asset register (initial seed) # DISCTL-WP-0003-T01 — railiance-backup tool family and backup lanes -# Broader T0/T4 inventory (domains, Stripe, Bubble, etc.) follows in DISCTL-WP-0002-T02. +# Backup lanes: DISCTL-WP-0003-T01. Stack assets (T0/T4): DISCTL-WP-0002-T02. -schema_version: "0.1" +schema_version: "0.2" updated: "2026-07-07" -register_scope: backup_lanes +register_scope: backup_lanes_and_stack tier_definitions: T0: "Identity / trust / money — recoverable even after breach" @@ -216,6 +216,246 @@ backup_streams: - history/2026-07-04-forgejo-backup-strategy-assessment.md - specs/ResilienceControlSetup.md +assets: + # ── T0 Identity / trust / money ────────────────────────────────────────────── + + - id: asset-dns-coulomb-social + name: coulomb.social DNS zone + tier: T0 + owner: Bernd + provider: IONOS + hosts: + - gitea.coulomb.social + - forgejo.coulomb.social + - hub.coulomb.social + - reuse.coulomb.social + production_ips: + coulombcore_legacy: "92.205.130.254" + railiance01: "92.205.62.239" + recovery_path: + method: IONOS control panel + zone record export + backup_admin: unknown + mfa_recovery: unknown + break_glass: unknown + backup: manual export after changes + status: at_risk + gaps: + - "Registrar lock status not inventoried" + - "MFA recovery codes not recorded here" + - "No automated zone export" + + - id: asset-registrar-ionos + name: Domain registrar (IONOS) + tier: T0 + owner: Bernd + recovery_path: + method: IONOS account login + backup_admin: unknown + mfa_recovery: unknown + break_glass: unknown + status: unknown + gaps: + - "Backup admin contact not recorded" + - "Account lock + MFA recovery not evidenced" + + - id: asset-email-ionos + name: coulomb.social email (IONOS SMTP) + tier: T0 + owner: Bernd + mailboxes: + - forgejo@coulomb.social + recovery_path: + method: IONOS mail control panel + backup_admin: unknown + mfa_recovery: unknown + status: operational + gaps: + - "Recovery path not fully documented" + + - id: asset-stripe-admin + name: Stripe account (payments / subscriptions) + tier: T0 + owner: Bernd + recovery_path: + method: Stripe dashboard login + backup_admin: unknown + mfa_recovery: unknown + break_glass: unknown + backup: scheduled CSV export (not established) + status: at_risk + gaps: + - "No scheduled export evidenced" + - "Recovery contacts not inventoried" + + - id: asset-openbao-recovery + name: OpenBao sealed recovery material + tier: T0 + owner: Bernd + host: railiance01 + recovery_path: + method: OpenBao unseal keys + root token recovery procedure + backup_admin: Bernd + break_glass: ops-warden / railiance-platform runbooks + backup: SOPS/age sealed vault (not fully evidenced for cold disaster) + status: at_risk + gaps: + - "Cold disaster recovery of unseal material not drill-tested" + references: + - railiance-platform/docs/openbao-*.md + + - id: asset-age-backup-keys + name: age encryption keys (backup artifacts) + tier: T0 + owner: Bernd + recovery_path: + method: Password manager recovery copy + operational: ~/.config/age/railiance-backup.key, ~/.config/sops/age/keys.txt + backup: password manager (mandatory) + status: operational + gaps: + - "Password manager copy not verified in register drill" + + - id: asset-backup-offsite-lane + name: Nextcloud backup file drop + tier: T0 + owner: Bernd + recovery_path: + method: Nextcloud admin read access for restore; upload via filesdrop token + credentials: OpenBao platform/workloads/railiance/backup/offsite-lane + backup: N/A (this is the backup destination) + status: operational + references: + - BackupPolicy.md §4 + - railiance-cluster/docs/backup-restore.md + + - id: asset-ssh-warden + name: SSH certificate access (ops-warden) + tier: T0 + owner: Bernd + recovery_path: + method: warden sign / cert_command + break_glass: ops-warden wiki + status: operational + references: + - AGENTS.md credential routing + + # ── T1 Irreplaceable source of truth ───────────────────────────────────────── + + - id: asset-gitea-canonical + name: Gitea instance (canonical forge until cutover) + tier: T1 + owner: Bernd + host: coulombcore + url: https://gitea.coulomb.social + backup_stream: backup-gitea-canonical + recovery_path: + method: gitea dump restore; git mirror + status: at_risk + + - id: asset-forgejo-production + name: Forgejo instance (production forge) + tier: T1 + owner: Bernd + host: railiance01 + url: https://forgejo.coulomb.social + backup_stream: backup-forgejo-production + recovery_path: + method: forgejo-restore-drill.sh + status: not_wired + + - id: asset-git-repos + name: Git repositories (all coulomb/* repos) + tier: T1 + owner: Bernd + source_of_truth: Gitea (migrating to Forgejo) + backup: git push to remote; forgejo/gitea dump for full fidelity + recovery_path: + method: git clone from mirror or dump restore + status: operational + gaps: + - "disaster-control not mirrored off primary Gitea (DISCTL-WP-0002-T01)" + + - id: asset-state-hub + name: Custodian State Hub + tier: T1 + owner: Bernd + host: bnt-lap001 (dev workstation) + backup_stream: backup-dev-workstation + recovery_path: + method: PostgreSQL restore from age-decrypted dump + status: operational + + - id: asset-disaster-control + name: disaster-control repo (this control plane) + tier: T1 + owner: Bernd + host: gitea.coulomb.social + backup: git push only + recovery_path: + method: git clone from mirror (not yet configured) + status: at_risk + gaps: + - "No off-Gitea mirror (DISCTL-WP-0002-T01)" + + # ── T2 Runtime platforms ───────────────────────────────────────────────────── + + - id: asset-coulombcore + name: CoulombCore server + tier: T2 + owner: Bernd + host: coulombcore + ip: "92.205.130.254" + recovery_path: + method: Rebuild from Ansible + restore data; do not resurrect compromised host + backup_stream: backup-gitea-canonical + status: operational + + - id: asset-railiance01 + name: railiance01 k3s cluster + tier: T2 + owner: Bernd + host: railiance01 + ip: "92.205.62.239" + recovery_path: + method: Ansible baseline + Helm redeploy + data restore from backups + backup_streams: + - backup-s1-os-config + - backup-s2-k8s-runtime + status: operational + gaps: + - "S1/S2 backups on-node only" + + # ── T4 External dependencies ───────────────────────────────────────────────── + + - id: asset-bubble-io + name: Bubble.io legacy application + tier: T4 + owner: Bernd + recovery_path: + method: Bubble database export + documented restore workflow + limits: Platform-controlled; export cadence not established + backup: none automated + rpo: 24h + rto: 24h + status: at_risk + gaps: + - "No scheduled export evidenced" + - "Member data on platform we do not control" + + - id: asset-openrouter + name: OpenRouter API (LLM inference) + tier: T4 + owner: Bernd + recovery_path: + method: Alternate LLM provider; API key rotation via OpenBao + limits: No data RPO unless storing prompts/results + backup: API keys in OpenBao + rpo: N/A + rto: 1-4h + status: operational + gaps: + - "Alternate provider path not documented in runbook" + related_streams: # Operational on cluster but outside T01 minimum table — tracked for completeness. - id: backup-adjacent-sso diff --git a/DataRegister.yaml b/DataRegister.yaml index e789f69..b61abae 100644 --- a/DataRegister.yaml +++ b/DataRegister.yaml @@ -1,10 +1,10 @@ # Resilience Control Plane — data register (initial seed) # DISCTL-WP-0003-T01 — data classes covered by railiance-backup lanes -# Full data inventory (Bubble, Stripe exports, member data, etc.) follows in DISCTL-WP-0002-T02. +# Stack data classes (T0/T4): DISCTL-WP-0002-T02. Backup lanes: DISCTL-WP-0003-T01. -schema_version: "0.1" +schema_version: "0.2" updated: "2026-07-07" -register_scope: backup_lane_data_classes +register_scope: backup_lanes_and_stack data_classes: - id: data-custodian-state-hub @@ -201,6 +201,82 @@ data_classes: gaps: - "Not inventoried in AssetRegister T0 section yet — DISCTL-WP-0002-T02 follow-up" + - id: data-dns-zone-records + name: DNS zone records (*.coulomb.social) + tier: T0 + source_of_truth: IONOS DNS panel + backup_stream: null + backup: manual export after changes + recovery_method: Re-enter records from export; cutover per PanicSheet.md + rpo: manual + rto: 2h + owner: Bernd + status: at_risk + gaps: + - "No standing zone export on file" + + - id: data-stripe-customers + name: Stripe customers / subscriptions / payment history + tier: T0 + source_of_truth: Stripe (authoritative) + backup_stream: null + backup: none automated + recovery_method: Stripe dashboard; scheduled CSV/Data Pipeline export (not established) + rpo: 24h-7d + rto: 24h + owner: Bernd + status: at_risk + + - id: data-bubble-members + name: Bubble.io member / application database + tier: T4 + source_of_truth: Bubble.io platform + backup_stream: null + backup: none automated + recovery_method: Bubble database export per vendor docs + rpo: 24h + rto: 24h + owner: Bernd + status: at_risk + gaps: + - "Export cadence not established" + + - id: data-openbao-secrets + name: OpenBao secret material (runtime) + tier: T0 + source_of_truth: OpenBao on railiance01 + backup_stream: null + backup: sealed recovery vault (partial) + recovery_method: Unseal + restore; rotate after breach — never blind restore + rpo: immediate rotation on incident + rto: 2-4h + owner: Bernd + status: at_risk + + - id: data-openrouter-usage + name: OpenRouter API keys and usage metadata + tier: T4 + source_of_truth: OpenRouter account + OpenBao + backup_stream: null + backup: OpenBao credential paths + recovery_method: Issue new key; switch to alternate provider + rpo: N/A + rto: 1-4h + owner: Bernd + status: operational + + - id: data-disaster-control-git + name: disaster-control repository content + tier: T1 + source_of_truth: gitea.coulomb.social/coulomb/disaster-control + backup_stream: backup-gitea-canonical + backup: git push only; no mirror yet + recovery_method: git clone from off-Gitea mirror (T01) or gitea dump + rpo: 24h + rto: 4h + owner: Bernd + status: at_risk + status_legend: operational: "Backup mechanism exists and has run successfully" manual_only: "Restore path exists but backup is operator-triggered only" diff --git a/INTENT.md b/INTENT.md index 8782e94..3650027 100644 --- a/INTENT.md +++ b/INTENT.md @@ -26,4 +26,7 @@ This repo documents, plans, and evidences resilience for the Coulomb stack. It d ## Current state -Early exploration: `specs/ResilienceControlSetup.md` outlines the full design (asset tiers, RTO/RPO targets, 3-2-1 + immutable backup architecture, playbooks, drill cadence). The registers, policies, runbooks, and evidence loop are not yet written. +In progress (`DISCTL-WP-0002`): registers, `ResiliencePolicy.md`, `BackupPolicy.md`, +`RTO-RPO-Matrix.md`, panic sheet, full-server-loss runbook, and incident playbooks +exist. Off-Gitea mirror (`DISCTL-WP-0002-T01`) and next restore drill (`T06` exit +criterion) remain open. diff --git a/IncidentPlaybooks/accidental-delete.md b/IncidentPlaybooks/accidental-delete.md new file mode 100644 index 0000000..317d35d --- /dev/null +++ b/IncidentPlaybooks/accidental-delete.md @@ -0,0 +1,23 @@ +# Playbook: Accidental Delete / Data Corruption + +**Triggers:** deleted repo, dropped table, bad migration, destructive script, bad Ansible run. + +## Steps + +1. **Stop writes** — scale deployment to zero, enable maintenance mode, or revoke DB write access. +2. **Assess scope** — what was deleted/corrupted, when, last known good state. +3. **Clone backup to staging** — never restore directly over production first. +4. **Identify last good point** — backup timestamp, git ref, DB snapshot. +5. **Restore selectively** — one repo, one table, or one PVC — not full blast restore unless required. +6. **Diff** — compare staging restore against current production; list deltas. +7. **Apply to production** — merge recovered data; re-run migrations if needed. +8. **Resume writes** — remove maintenance mode; monitor. +9. **Document** — `TestEvidence/` note with RPO achieved; update registers if backup gap contributed. + +## Fast rollback + +If local snapshot or recent git push exists, prefer that over full restore. + +## References + +`RTO-RPO-Matrix.md` · `BackupPolicy.md` · `AssetRegister.yaml` \ No newline at end of file diff --git a/IncidentPlaybooks/credential-leak.md b/IncidentPlaybooks/credential-leak.md new file mode 100644 index 0000000..4ebae46 --- /dev/null +++ b/IncidentPlaybooks/credential-leak.md @@ -0,0 +1,18 @@ +# Playbook: Credential Leak + +**Triggers:** token in Git/logs/chat, stolen laptop, exposed `.env`, compromised CI secret. + +## Steps + +1. **Identify scope** — which credential, which systems, exposure window. +2. **Revoke immediately** — invalidate leaked token/password before cleanup discussion. +3. **Rotate downstream** — every system that trusted the credential (OpenBao, Stripe, Gitea/Forgejo, backup drop, OpenRouter, webhooks). +4. **Scan for misuse** — auth logs, Stripe dashboard, API usage anomalies, unexpected git pushes. +5. **Remove secret from exposure** — git history scrub if committed; rotate not revert. +6. **Verify separation** — confirm backup credentials still isolated (`BackupPolicy.md` CS-1). +7. **Re-deploy** — refresh Kubernetes secrets via External Secrets / OpenBao. +8. **Document** — incident note; no secret values in State Hub or evidence files. + +## References + +`AssetRegister.yaml` T0 entries · `PanicSheet.md` §4 · ops-warden credential routing \ No newline at end of file diff --git a/IncidentPlaybooks/loss-of-admin-identity.md b/IncidentPlaybooks/loss-of-admin-identity.md new file mode 100644 index 0000000..36b02b2 --- /dev/null +++ b/IncidentPlaybooks/loss-of-admin-identity.md @@ -0,0 +1,25 @@ +# Playbook: Loss of Admin Identity + +**Triggers:** locked out of registrar, Gitea, Stripe, Bubble, email, cloud; lost MFA device. + +## Steps + +1. **Identify which identity** — registrar, payment, forge, email, secrets vault. +2. **Break-glass account** — use documented secondary admin if available (see `AssetRegister.yaml` recovery_path). +3. **Offline recovery codes** — MFA reset via vendor recovery flow; requires pre-stored codes. +4. **Vendor support** — IONOS/Stripe identity verification; allow 24–48h; have domain ownership proof ready. +5. **Verify ownership** — confirm domain/registrar records unchanged; check for hostile transfer. +6. **Rotate compromised identity** — new password, new MFA, revoke active sessions. +7. **Restore dependent access** — Gitea/Forgejo admin, DNS, backup admin, OpenBao. +8. **Update registers** — record recovery paths discovered; close gaps in `AssetRegister.yaml` T0 section. +9. **Document** — evidence log; **no recovery codes in Git**. + +## Prevention (ongoing) + +- Record backup admin contacts per T0 asset. +- Store MFA recovery codes in password manager. +- Registrar lock enabled. + +## References + +`AssetRegister.yaml` T0 · `PanicSheet.md` §2 · `ResiliencePolicy.md` \ No newline at end of file diff --git a/IncidentPlaybooks/provider-outage.md b/IncidentPlaybooks/provider-outage.md new file mode 100644 index 0000000..4995c26 --- /dev/null +++ b/IncidentPlaybooks/provider-outage.md @@ -0,0 +1,19 @@ +# Playbook: Provider Outage + +**Triggers:** Bubble, Stripe, OpenRouter, IONOS DNS/email, or other vendor unavailable. + +## Steps + +1. **Confirm outage** — vendor status page, `dig`, `curl`, third-party monitor; rule out local DNS/SSH issue. +2. **Communicate** — internal status note; disable affected features in UI if possible. +3. **Graceful degradation** — queue writes; serve static fallback; read-only mode. +4. **Bubble.io** — document member-facing impact; no local DB to restore; wait for vendor. +5. **Stripe** — pause checkout if webhooks failing; queue events; reconcile on return. +6. **OpenRouter** — switch to alternate LLM provider if configured; reduce non-critical inference. +7. **DNS/email** — if IONOS down, existing TTLs may mask briefly; avoid panic changes until vendor confirms. +8. **Reconcile** — after recovery, replay webhooks, verify Stripe/Bubble state, run smoke tests. +9. **Document** — outage window, impact, follow-up (export cadence, alternate provider). + +## References + +`AssetRegister.yaml` T4 entries · `ResiliencePolicy.md` · Bubble/Stripe export gaps in `DataRegister.yaml` \ No newline at end of file diff --git a/IncidentPlaybooks/ransomware.md b/IncidentPlaybooks/ransomware.md new file mode 100644 index 0000000..2c89da8 --- /dev/null +++ b/IncidentPlaybooks/ransomware.md @@ -0,0 +1,27 @@ +# Playbook: Ransomware / Hostile Access + +**Triggers:** suspicious encryption, unknown admin sessions, altered repos, exfiltration indicators. +**Tier:** L2 / I5 — treat as breach until proven otherwise. + +## Steps + +1. **Isolate** — disconnect affected hosts from network; disable compromised accounts; preserve VM/disk snapshots for evidence. +2. **Preserve evidence** — do not wipe logs; capture auth logs, k8s audit, Gitea/Forgejo access logs, firewall logs. +3. **Revoke credentials** — rotate all tokens/passwords that may be exposed: OpenBao, Stripe, API keys, backup tokens, Gitea/Forgejo admin, SSH certs. +4. **Assess backup integrity** — verify offsite backups (Nextcloud) were not deleted; check `.last-backup` stamps; assume on-node backups on compromised host are untrusted. +5. **Build clean environment** — new machines; fresh OS; Ansible from known-good Git refs. **Never restore into compromised host.** +6. **Restore known-good data** — from backups timestamped **before** compromise; age-decrypt from offsite only. +7. **Rotate all secrets** — full rotation, not selective; update OpenBao paths; re-issue `warden` certs. +8. **Force resets** — user sessions, MFA re-enrollment where applicable. +9. **Validate** — smoke tests; compare restored repos against known-good SHAs. +10. **Document** — State Hub progress event; `TestEvidence/` incident note; update `AssetRegister.yaml` if exposure changed posture. + +## Do not + +- Pay ransom without executive decision recorded. +- Restore production DB over live data without identifying last good point. +- Paste secrets into incident notes. + +## References + +`BackupPolicy.md` §8 · `RestoreRunbooks/01-full-server-loss.md` · `PanicSheet.md` §4 \ No newline at end of file diff --git a/PanicSheet.md b/PanicSheet.md new file mode 100644 index 0000000..aebcacb --- /dev/null +++ b/PanicSheet.md @@ -0,0 +1,96 @@ +# Panic Sheet — Coulomb / Helix Recovery + +**One page. Read this first in an incident.** +**Updated:** 2026-07-07 · **Owner:** Bernd + +`no_secret_material_recorded: true` — no passwords or tokens in this document. + +--- + +## 1. Where are the backups? + +| What | Where | Encrypted | Off-node | +| --- | --- | --- | --- | +| Custodian DB + dev config | Nextcloud file drop + `~/.cache/railiance/backups/` | age | Yes (Nextcloud) | +| k3s state + Helm values | `/opt/backup/railiance/cluster/` on railiance01 | age | **No** | +| OS config (S1) | `/opt/backup/railiance/infra/` on railiance01 | age | **No** | +| Forgejo / platform (intended) | Nextcloud file drop | age | Adopted, **not wired** | +| Gitea | Manual dump only | varies | **No** | + +Details: `BackupPolicy.md`, `AssetRegister.yaml`. + +Decrypt keys: password manager first; operational keys on lost machine are gone. + +--- + +## 2. Who has access? + +| System | Admin | Recovery | +| --- | --- | --- | +| DNS / domains | Bernd | IONOS panel — backup admin **not recorded** | +| Gitea | Bernd | coulombcore | +| Forgejo | Bernd | railiance01 k3s | +| Stripe | Bernd | Dashboard — MFA recovery **not recorded** | +| Backups (offsite) | Bernd | Nextcloud admin read; upload token in OpenBao | +| Secrets | Bernd | OpenBao unseal — see platform runbooks | +| SSH to fleet | Bernd | `warden sign` (ops-warden) | + +--- + +## 3. How to rebuild (full server loss) + +1. **Stop** — do not patch a compromised host; build clean. +2. **Provision** new machine (Ubuntu). +3. **Ansible** baseline from `railiance-infra`. +4. **Secrets** — recover OpenBao unseal material; unseal; rotate if breach. +5. **Data** — restore from age-decrypted backups (see `RestoreRunbooks/01-full-server-loss.md`). +6. **k3s** — redeploy Helm releases from values backup or GitOps. +7. **Validate** — health checks, smoke tests, representative workflow. +8. **DNS** — cutover A records in IONOS when confident. +9. **Evidence** — write `TestEvidence/restore-drill-YYYY-MM-DD.md`. + +--- + +## 4. How to rotate secrets (breach or leak) + +1. Isolate affected systems. +2. Revoke exposed credentials immediately. +3. Build **clean** replacement environment. +4. Restore data from backup **before compromise time**. +5. Rotate: OpenBao paths, Stripe keys, API keys, Gitea/Forgejo tokens, backup tokens. +6. Force session/password reset where applicable. +7. Log in State Hub + `TestEvidence/`. + +Playbook: `IncidentPlaybooks/credential-leak.md`, `IncidentPlaybooks/ransomware.md`. + +--- + +## 5. How to switch DNS + +| Hostname | Production IP (2026-07-07) | Notes | +| --- | --- | --- | +| `*.coulomb.social` (railiance01 services) | `92.205.62.239` | forgejo, reuse, hub target | +| CoulombCore legacy | `92.205.130.254` | gitea until cutover | + +1. Export current zone from IONOS before changes. +2. Lower TTL ahead of planned cutover if possible. +3. Update A records; verify with `dig +short A`. +4. Validate HTTPS: `curl -fsS https:///health` or equivalent. +5. Roll back to previous A record if validation fails. + +--- + +## 6. Before any infrastructure work + +```bash +bin/railiance preflight # on dev workstation — must exit 0 +``` + +--- + +## 7. Escalation and evidence + +- Full runbook: `RestoreRunbooks/01-full-server-loss.md` +- Registers: `AssetRegister.yaml`, `DataRegister.yaml` +- Pickup queue (open backup work): `BackupPickupQueue.md` +- Forgejo restore: `railiance-infra/tools/forgejo-restore-drill.sh` \ No newline at end of file diff --git a/README.md b/README.md index 2ea7af6..d22f67d 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,52 @@ -# repo-seed +# disaster-control -A git repository template to bootstrap coulomb projects from. +Resilience Control Plane for Coulomb Social / Helix Forge — the single source +of truth for business continuity, disaster recovery, cyber recovery, and restore +test evidence. -## Bootstrap a new repo +## Purpose -1. Clone or copy this template into a new repository. -2. Run `statehub register` from the new repo root (see [docs/statehub-register.md](docs/statehub-register.md)). -3. Complete the generated bootstrap workplan (`*-0001-statehub-bootstrap.md`). -4. Sync workplans: `cd ~/state-hub && make fix-consistency REPO=`. -5. Validate with [docs/template-validation-checklist.md](docs/template-validation-checklist.md). +Know what exists, what must survive, how it is rebuilt, and whether restore +actually works. See `INTENT.md` and `ResiliencePolicy.md`. -## Registry +## Key artifacts -This repo publishes `capability.infotech.repo-template` — see -`registry/capabilities/capability.infotech.repo-template.md`. \ No newline at end of file +| Path | Purpose | +| --- | --- | +| `AssetRegister.yaml` | Infrastructure and backup stream inventory | +| `DataRegister.yaml` | Data classes and recovery mapping | +| `ResiliencePolicy.md` | Operating principle, tiers, drill cadence | +| `BackupPolicy.md` | Backup architecture and monitoring | +| `RTO-RPO-Matrix.md` | Per-system recovery targets | +| `BackupPickupQueue.md` | Cross-repo backup implementation tracker | +| `PanicSheet.md` | One-page emergency reference | +| `RestoreRunbooks/` | Recovery procedures | +| `IncidentPlaybooks/` | Incident response steps | +| `TestEvidence/` | Dated restore drill reports | + +## Off-site mirror (R10) + +This repo **must** survive loss of the primary Gitea. Mirror status: + +| Mirror | Location | Refresh | Status | +| --- | --- | --- | --- | +| Primary | `gitea.coulomb.social/coulomb/disaster-control` | git push on commit | operational | +| Off-Gitea mirror | **Not configured** | — | **open** (`DISCTL-WP-0002-T01`) | + +**Target:** at least one private mirror on infrastructure independent of +CoulombCore/Gitea (e.g. GitHub private mirror). When configured, update this +table with URL and refresh method (`git push --mirror` or scheduled sync). + +## Session orientation + +```bash +cat .custodian-brief.md +cat INTENT.md +ls workplans/ +``` + +State Hub sync after workplan edits: + +```bash +cd ~/state-hub && make fix-consistency REPO=disaster-control +``` \ No newline at end of file diff --git a/ResiliencePolicy.md b/ResiliencePolicy.md new file mode 100644 index 0000000..2f5e163 --- /dev/null +++ b/ResiliencePolicy.md @@ -0,0 +1,96 @@ +# Resilience Policy + +**Status:** adopted (initial) +**Updated:** 2026-07-07 +**Owner:** Bernd +**Scope:** Coulomb Social / Helix Forge + +## Operating principle + +> Every capability that becomes valuable must declare its source of truth, +> recovery method, backup schedule, RTO, RPO, owner, and restore test evidence. + +Servers are disposable. Data, identity, secrets, evidence, and payment records +are not. Recovery means **rebuild** (Ansible/IaC + data restore + secret +recovery + DNS cutover), not server resurrection. + +After a security breach: **never restore into the compromised machine.** Build +clean, restore known-good data, rotate credentials, preserve evidence. + +## Asset tiers + +| Tier | Definition | Requirement | +| --- | --- | --- | +| **T0** | Identity / trust / money | Recoverable even after breach; recovery path documented | +| **T1** | Irreplaceable source of truth | Strong backups, tested restore, offsite copy where feasible | +| **T2** | Runtime platforms | Rebuild from IaC + restore data | +| **T3** | Derived / rebuildable | Recompute where possible | +| **T4** | External dependencies | Export, monitor, fallback, document limits | + +Canonical inventory: `AssetRegister.yaml`, `DataRegister.yaml`. + +## Recovery objectives + +Global baseline: + +- **RPO ≤ 24h** everywhere (unless vendor-limited) +- **RTO ≤ 1 day** for full organizational recovery +- **RTO ≤ 4–8h** for code, content, and control-plane recovery + +Per-system targets: `RTO-RPO-Matrix.md`. +Backup mechanisms: `BackupPolicy.md`. + +## Drill and evidence cadence + +| Activity | Frequency | Output | +| --- | --- | --- | +| Backup job success check (M-1) | Daily | Operator review / log check | +| Backup integrity check (M-3) | Weekly | age-decrypt sample per lane | +| T1 restore drill | Quarterly | `TestEvidence/restore-drill-YYYY-MM-DD-.md` | +| T0 cold decrypt drill | Quarterly | Evidence file + register update | +| Full server rebuild drill | Quarterly | `RestoreRunbooks/01-full-server-loss.md` + evidence | +| Asset register review | Monthly | Update `AssetRegister.yaml` statuses | +| Critical secret rotation | Quarterly or after incident | Evidence note in State Hub | +| Cross-repo pickup queue review | Monthly | Update `BackupPickupQueue.md` | + +A workplan is not **finished** until at least one restore drill evidence file +exists in `TestEvidence/` for an untested T1 system. + +## Roles + +| Role | Holder | Responsibilities | +| --- | --- | --- | +| Policy owner | Bernd | Approve targets, run drills, update registers | +| Alerting owner | Bernd | Respond to backup failures (M-1–M-3) until on-call defined | +| Implementation | Owner repos | Backup automation, CNPG, cron, platform tools | + +## Control plane independence (R10) + +`disaster-control` must survive the disasters it describes: + +1. At least one git mirror off primary Gitea (see `DISCTL-WP-0002-T01`). +2. Registers, policies, and evidence readable without production infrastructure. +3. Panic sheet (`PanicSheet.md`) executable by a competent stranger. + +## Incident response + +Five playbooks under `IncidentPlaybooks/`: + +1. Ransomware / hostile access +2. Credential leak +3. Accidental delete / data corruption +4. Provider outage +5. Loss of admin identity + +Each playbook: numbered steps, one page, links to runbooks and registers. + +## Related artifacts + +| Artifact | Purpose | +| --- | --- | +| `BackupPolicy.md` | Backup architecture, encryption, monitoring | +| `RTO-RPO-Matrix.md` | Per-system RTO/RPO targets and measured drills | +| `BackupPickupQueue.md` | Cross-repo implementation tracker | +| `PanicSheet.md` | One-page emergency reference | +| `RestoreRunbooks/` | Step-by-step recovery procedures | +| `TestEvidence/` | Dated drill reports | \ No newline at end of file diff --git a/RestoreRunbooks/01-full-server-loss.md b/RestoreRunbooks/01-full-server-loss.md new file mode 100644 index 0000000..904caf7 --- /dev/null +++ b/RestoreRunbooks/01-full-server-loss.md @@ -0,0 +1,126 @@ +# Restore Runbook: Full Server Loss + +**Trigger:** disk failure, VPS deletion, hardware loss, unrecoverable OS corruption. +**RTO target:** ≤ 1 day full recovery; ≤ 4–8h for code/content/control plane. +**Owner:** Bernd + +Do **not** restore into a compromised machine. This runbook assumes hardware +loss, not active breach (see `IncidentPlaybooks/ransomware.md` for breach). + +--- + +## Prerequisites + +- [ ] Password-manager copy of age backup keys +- [ ] OpenBao unseal material (if restoring railiance01 secrets) +- [ ] Nextcloud admin or local backup cache access +- [ ] IONOS DNS access +- [ ] `warden sign` SSH access to new host +- [ ] This repo cloned (from mirror once T01 complete; until then: any copy) + +--- + +## Phase 1 — Provision clean host + +1. Provision new Ubuntu server (or recover WSL2 workstation for dev lane only). +2. Apply SSH hardening via Ansible (`railiance-infra`). +3. Install tooling: `age`, `docker`, `k3s`, `helm`, `kubectl`, `ansible`. +4. Record new host identity in `AssetRegister.yaml` after recovery. + +--- + +## Phase 2 — Secret recovery + +1. Retrieve OpenBao unseal keys from sealed recovery vault (password manager). +2. If railiance01: follow `railiance-platform` OpenBao deploy/unseal runbooks. +3. Retrieve backup upload credentials from OpenBao if re-wiring backup jobs. +4. **Do not** copy plaintext secrets into Git, chat, or State Hub. + +--- + +## Phase 3 — Data restore (by lane) + +### Dev workstation (bnt-lap001) + +1. Download latest `db-*.sql.age` and `config-*.tar.gz.age` from Nextcloud. +2. `age -d -i ~/.config/age/railiance-backup.key -o dump.sql db-*.sql.age` +3. Restore PostgreSQL `custodian` database into `infra-postgres-1`. +4. Decrypt and unpack config tar to `$HOME`. +5. Verify: `bin/railiance preflight` exits 0. + +Reference: `railiance-cluster/docs/backup-restore.md` + +### railiance01 — S1 OS config (optional) + +1. Decrypt `os-config-*.tar.gz.age` from `/opt/backup/railiance/infra/` (if disk survived) + or offsite copy when wired. +2. Restore files or re-apply Ansible playbooks (preferred). + +### railiance01 — S2 k8s runtime + +1. If rebuilding k3s: install k3s on clean host. +2. Decrypt `helm-values-*.tar.gz.age` — redeploy each release with saved values. +3. Decrypt `kubeconfig-*.yaml.age` if needed for admin access. +4. k3s `state.db` hot backup: prefer **redeploy from Git/Helm** over raw DB restore. + +Reference: `railiance-cluster/tools/cmd/railiance-backup-s2` + +### Gitea (coulombcore) + +1. If coulombcore lost: provision host, install Gitea via Ansible. +2. Restore from latest `gitea dump` or git mirrors of critical repos. +3. Verify: clone `coulomb/disaster-control`, API health. + +### Forgejo (railiance01) + +1. If automated backup wired: download latest dump from Nextcloud. +2. Run `railiance-infra/tools/forgejo-restore-drill.sh` (adapt for production namespace). +3. Verify: `GET /api/v1/version`, org repos visible. + +Evidence: `TestEvidence/restore-drill-2026-07-04-forgejo.md` + +### Platform databases + +Blocked until `BackupPickupQueue.md` items 3–4 close (platform tool missing). + +--- + +## Phase 4 — Validate services + +| Check | Command / endpoint | +| --- | --- | +| k3s nodes | `kubectl get nodes` | +| Forgejo health | `curl -fsS https://forgejo.coulomb.social/api/v1/version` | +| Gitea health | `curl -fsS https://gitea.coulomb.social/api/v1/version` | +| State Hub | `curl -fsS http://127.0.0.1:8000/state/summary` | +| Representative app | App-specific smoke (e.g. vergabe health endpoint) | + +--- + +## Phase 5 — DNS cutover + +1. Confirm all services healthy on new infrastructure. +2. Update IONOS A records per `PanicSheet.md` §5. +3. Wait for TTL propagation; verify externally (`dig @8.8.8.8`). +4. Monitor for 24h. + +--- + +## Phase 6 — Evidence report + +Write `TestEvidence/restore-drill-YYYY-MM-DD-full-server-loss.md`: + +- Start/end times (RTO measured) +- Backup artifacts used (RPO achieved) +- Systems restored vs redeployed +- Issues and follow-up actions +- Update `AssetRegister.yaml` if host identities changed + +--- + +## References + +- `PanicSheet.md` +- `BackupPolicy.md` +- `AssetRegister.yaml` +- `RTO-RPO-Matrix.md` \ No newline at end of file diff --git a/workplans/DISCTL-WP-0002-resilience-foundation.md b/workplans/DISCTL-WP-0002-resilience-foundation.md index c906977..69938e7 100644 --- a/workplans/DISCTL-WP-0002-resilience-foundation.md +++ b/workplans/DISCTL-WP-0002-resilience-foundation.md @@ -4,11 +4,11 @@ type: workplan title: "Resilience foundation: registers, policy, first tested restore" domain: infotech repo: disaster-control -status: ready -owner: claude-code +status: active +owner: grok topic_slug: custodian created: "2026-07-04" -updated: "2026-07-04" +updated: "2026-07-07" state_hub_workstream_id: "73d2ecb3-e594-4a7c-920d-8753198eb6ec" --- @@ -32,7 +32,7 @@ items 7–8 are T03/T04/T06 below; its cross-repo items are tracked by T08. ```task id: DISCTL-WP-0002-T01 -status: todo +status: progress priority: high state_hub_task_id: "f2f8648e-110c-455e-9745-6e2aa7f4fdfa" ``` @@ -46,7 +46,7 @@ mirror is refreshed. ```task id: DISCTL-WP-0002-T02 -status: todo +status: done priority: high state_hub_task_id: "99bb7f1b-3ab4-4cee-b7ff-f55f0c2b287a" ``` @@ -62,7 +62,7 @@ Every T1 entry must name its source of truth and backup mechanism (or mark it ```task id: DISCTL-WP-0002-T03 -status: todo +status: done priority: high state_hub_task_id: "a6f87f9d-5fbd-4586-bb97-62e0ce99cb27" ``` @@ -76,7 +76,7 @@ RTO ≤ 1 day full recovery, RTO ≤ 4–8h for code/content/control plane. ```task id: DISCTL-WP-0002-T04 -status: todo +status: done priority: high state_hub_task_id: "75ce1886-e53e-4382-9a39-07f638cb59a8" ``` @@ -92,7 +92,7 @@ defined schedule with a named alerting path. ```task id: DISCTL-WP-0002-T05 -status: todo +status: done priority: medium state_hub_task_id: "da73d8fd-13d0-4549-a196-c29e9f418784" ``` @@ -107,7 +107,7 @@ stranger could execute it. ```task id: DISCTL-WP-0002-T06 -status: todo +status: progress priority: high state_hub_task_id: "b08f1b02-4937-42aa-9989-042f50cc2e01" ``` @@ -125,7 +125,7 @@ workplan: no drill evidence here, not finished. ```task id: DISCTL-WP-0002-T07 -status: todo +status: done priority: medium state_hub_task_id: "b1b9df65-1dc9-45a2-89fb-c6e0655be85d" ``` @@ -139,7 +139,7 @@ page of numbered steps per the spec (§6). ```task id: DISCTL-WP-0002-T08 -status: todo +status: done priority: medium state_hub_task_id: "2c8f432e-dc63-4a6b-a838-e60eb514d848" ```