Prepare independent outage recovery and Storage Share documentation

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2

tools/bootstrap-recovery/README.md

@@ -3,6 +3,20 @@
 Owner work: NK-WP-0049. Follow
 [the exact recovery ceremony](../../docs/bootstrap-recovery-ceremony.md).
 
+For an identity/custody outage, start with the
+[Storage Share disaster recovery guide](../../docs/disaster-recovery-storage-share.md).
+`retrieve-identity-offline.py` performs fixed-provider ciphertext retrieval from
+privately entered independent provider access, without OpenBao/SSO. Its source
+receipt pins the object, strong ETag, size and digest; failed downloads are removed.
+`recovery-kit.py` builds an allowlisted standalone tools ZIP and can publish only
+three new versioned guide/ZIP/checksum files beside the backups with exact
+readback. `run-offline-recovery.sh` is the owner's private two-prompt handoff for
+publication, outage-path retrieval and the existing archive recovery check.
+Run secret prompts in the owner's own terminal, never an agent-captured PTY.
+Source/fixture checks do not replace the actual owner drill/publication receipts.
+Service restore admission remains in the
+[isolated restore packet](../../docs/identity-service-restore-packet.md).
+
 `verify-bundle.py` validates a digest-bound age-encrypted `secrets/` tar archive
 using an explicit private temporary key and registered `keys/age.pub`. It checks
 an exact required-file list, unrelated wrong-key refusal, safe isolated file

tools/bootstrap-recovery/recovery-kit.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Build a secret-free offline recovery kit; optionally publish exact new files.
+
+Publication creates only three fixed versioned objects beside identity backups,
+never changes shares, credentials, existing files, retention or runtime jobs.
+"""
+import argparse
+import base64
+from datetime import datetime, timezone
+import getpass
+import hashlib
+import importlib.util
+import io
+import json
+import os
+from pathlib import Path
+import sys
+import urllib.error
+import urllib.request
+import zipfile
+
+ROOT = Path(__file__).resolve().parents[2]
+HERE = Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location('offline', HERE / 'retrieve-identity-offline.py')
+offline = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(offline)
+offsite = offline.offsite
+FILES = [
+    'docs/disaster-recovery-storage-share.md',
+    'docs/identity-service-restore-packet.md',
+    'docs/evidence/2026-10-16-identity-backup-live/actual-offsite.json',
+    'docs/evidence/2026-10-16-identity-outage-recovery/image-observation.json',
+    'tools/bootstrap-recovery/retrieve-identity-offline.py',
+    'tools/bootstrap-recovery/identity-offsite.py',
+    'tools/bootstrap-recovery/verify-identity.py',
+    'tools/bootstrap-recovery/identity-bundle.py',
+    'tools/bootstrap-recovery/check-privacyidea-database.py',
+    'tools/bootstrap-recovery/identity-backup-profile.json',
+    'sso-mfa/bootstrap/sops-custody-unlock.sh',
+    'sso-mfa/bootstrap/sops-private.py',
+    'keys/age.pub',
+]
+
+
+def build():
+    content = {name: (ROOT / name).read_bytes() for name in FILES}
+    if any(b'AGE-SECRET-KEY-1' in value for name, value in content.items()
+           if not name.endswith(('.py', '.sh', '.md'))):
+        raise offsite.Refused()
+    sums = ''.join(hashlib.sha256(value).hexdigest() + '  ' + name + '\n'
+                   for name, value in content.items()).encode()
+    output = io.BytesIO()
+    with zipfile.ZipFile(output, 'w', compression=zipfile.ZIP_DEFLATED) as archive:
+        for name, value in {**content, 'SHA256SUMS': sums}.items():
+            entry = zipfile.ZipInfo(name, date_time=(2026, 10, 16, 0, 0, 0))
+            entry.external_attr = 0o100600 << 16
+            entry.compress_type = zipfile.ZIP_DEFLATED
+            archive.writestr(entry, value)
+    kit = output.getvalue()
+    version = hashlib.sha256(kit).hexdigest()
+    stem = 'net-kingdom-recovery-' + version[:16]
+    return {
+        stem + '-START-HERE.md': content['docs/disaster-recovery-storage-share.md'],
+        stem + '.zip': kit,
+        stem + '.zip.sha256': (version + '  ' + stem + '.zip\n').encode(),
+    }
+
+
+def publish(files, headers, client=None):
+    client = client or offsite.opener()
+    result = []
+    for name, raw in files.items():
+        url = offline.ACCOUNT_BASE + name
+        digest = hashlib.sha256(raw).hexdigest()
+        # A retry may accept an identical object after exact readback, but must
+        # never overwrite a conflicting object or retry an unknown PUT outcome.
+        present = False
+        try:
+            with client.open(urllib.request.Request(url, method='HEAD', headers=headers), timeout=30) as response:
+                version = offsite.etag(response, url, len(raw))
+                present = True
+        except urllib.error.HTTPError as error:
+            if error.code != 404:
+                raise offsite.Refused() from None
+        if not present:
+            h = dict(headers, **{'If-None-Match': '*', 'Content-Length': str(len(raw)),
+                                'Content-Type': 'application/zip' if name.endswith('.zip') else 'text/plain; charset=utf-8'})
+            with client.open(urllib.request.Request(url, data=raw, method='PUT', headers=h), timeout=60) as response:
+                if response.status != 201 or response.url != url:
+                    raise offsite.Refused()
+            with client.open(urllib.request.Request(url, method='HEAD', headers=headers), timeout=30) as response:
+                version = offsite.etag(response, url, len(raw))
+        h = dict(headers, **{'If-Match': version})
+        with client.open(urllib.request.Request(url, headers=h), timeout=60) as response:
+            if offsite.etag(response, url, len(raw)) != version:
+                raise offsite.Refused()
+            downloaded = response.read(len(raw) + 1)
+        if downloaded != raw:
+            raise offsite.Refused()
+        result.append({'name': name, 'bytes': len(raw), 'sha256': digest,
+                       'etag': version, 'readback_verified': True})
+    return result
+
+
+def main():
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--output-dir', required=True)
+    p.add_argument('--publish', action='store_true')
+    p.add_argument('--auth-source', choices=('prompt', 'attended-openbao'), default='prompt')
+    p.add_argument('--receipt', required=True)
+    p.add_argument('--offline-transfer-receipt', help='After prompted publication, retrieve the known recovered generation using the same independently held access')
+    a = p.parse_args()
+    os.umask(0o077)
+    fd = os.open(a.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    r = {'schema': 'netkingdom.recovery-kit.v1', 'status': 'failed',
+         'secret_values_exported': False, 'new_grants_created': False}
+    try:
+        if a.offline_transfer_receipt and (not a.publish or a.auth_source != 'prompt'):
+            raise offsite.Refused()
+        files = build()
+        directory = Path(a.output_dir)
+        directory.mkdir(mode=0o700, exist_ok=False)
+        for name, raw in files.items():
+            (directory / name).write_bytes(raw)
+        r.update(status='built', files=[{'name': n, 'bytes': len(v), 'sha256': hashlib.sha256(v).hexdigest()}
+                                       for n, v in files.items()])
+        if a.publish:
+            if a.auth_source == 'prompt':
+                with open('/dev/tty', 'r+') as terminal:
+                    if not terminal.isatty():
+                        raise offsite.Refused()
+                    secret = getpass.getpass('Independently held Storage Share Backup app password: ', stream=terminal)
+            else:
+                spec = importlib.util.spec_from_file_location('inventory', HERE / 'inventory-storage-share.py')
+                owner = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(owner)
+                _, credentials = owner.caller_credentials()
+                secret = credentials['BACKUP_PASSWORD']
+            if not secret or any(c in secret for c in '\r\n\x00'):
+                raise offsite.Refused()
+            try:
+                headers = {'Authorization': 'Basic ' + base64.b64encode(('Backup:' + secret).encode()).decode(),
+                           'Accept-Encoding': 'identity'}
+                r.update(status='published_readback_verified', files=publish(files, headers),
+                         folder='Backup/railiance-backups/net-kingdom-identity',
+                         provider='Storage Share', credential_source=a.auth_source)
+                if a.offline_transfer_receipt:
+                    transfer_fd = os.open(a.offline_transfer_receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+                    transfer = {'status': 'failed', 'error': 'offline_retrieval_failed_private_diagnostics_suppressed'}
+                    try:
+                        record = json.loads((ROOT / FILES[2]).read_text())
+                        transfer = offline.retrieve(record, secret, 'account')
+                    finally:
+                        with os.fdopen(transfer_fd, 'w') as out:
+                            json.dump(transfer, out, indent=2)
+                            out.write('\n')
+                    r['offline_retrieval_verified'] = transfer['status'] == 'readback_verified'
+            finally:
+                secret = None
+        r['finished_at'] = datetime.now(timezone.utc).isoformat()
+    except (Exception, KeyboardInterrupt):
+        r.update(status='failed', error='kit_operation_failed_private_diagnostics_suppressed')
+    finally:
+        with os.fdopen(fd, 'w') as output:
+            json.dump(r, output, indent=2)
+            output.write('\n')
+    # Silent: safe to use in the existing captured attended-owner conduit.
+    return int(r['status'] == 'failed')
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/bootstrap-recovery/retrieve-identity-offline.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+"""Download one reviewed ciphertext using privately entered provider access.
+
+No OpenBao, SSO, Kubernetes, credential discovery, provider writes or age key.
+Run by the operator in their own terminal, never through a recorded agent PTY.
+"""
+import argparse
+import base64
+from datetime import datetime, timezone
+import getpass
+import hashlib
+import importlib.util
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import signal
+import sys
+import tempfile
+import urllib.request
+
+HERE = Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location('offsite', HERE / 'identity-offsite.py')
+offsite = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(offsite)
+ACCOUNT_BASE = offsite.HOST + '/remote.php/dav/files/Backup/railiance-backups/net-kingdom-identity/'
+
+
+def validate(record):
+    if (record.get('schema') != 'netkingdom.identity-offsite.v1'
+            or record.get('status') != 'readback_verified'
+            or record.get('provider') != 'Storage Share'
+            or record.get('folder') != 'Backup/railiance-backups/net-kingdom-identity'
+            or not offsite.NAME.fullmatch(record.get('artifact', ''))
+            or not re.fullmatch('[a-f0-9]{64}', record.get('ciphertext_sha256', ''))
+            or type(record.get('ciphertext_bytes')) is not int
+            or not 0 < record['ciphertext_bytes'] <= offsite.LIMIT
+            or not re.fullmatch(r'"[^"\x00-\x20]{1,180}"', record.get('etag', ''))):
+        raise offsite.Refused()
+
+
+def retrieve(record, secret, mode, client=None):
+    validate(record)
+    if mode not in {'account', 'folder-share'} or not secret or any(c in secret for c in '\r\n\x00'):
+        raise offsite.Refused()
+    auth = offsite.headers(secret) if mode == 'folder-share' else {
+        'Authorization': 'Basic ' + base64.b64encode(('Backup:' + secret).encode()).decode(),
+        'Accept-Encoding': 'identity'}
+    url = (offsite.BASE if mode == 'folder-share' else ACCOUNT_BASE) + record['artifact']
+    client = client or offsite.opener()
+    directory = None
+    complete = False
+    try:
+        with client.open(urllib.request.Request(url, method='HEAD', headers=auth), timeout=30) as response:
+            if offsite.etag(response, url, record['ciphertext_bytes']) != record['etag']:
+                raise offsite.Refused()
+        directory = Path(tempfile.mkdtemp(prefix='nk49-identity-fetched-', dir='/tmp'))
+        target = directory / record['artifact']
+        auth['If-Match'] = record['etag']
+        count = 0
+        digest = hashlib.sha256()
+        with client.open(urllib.request.Request(url, headers=auth), timeout=180) as response:
+            if offsite.etag(response, url, record['ciphertext_bytes']) != record['etag']:
+                raise offsite.Refused()
+            fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+            with os.fdopen(fd, 'wb') as output:
+                while block := response.read(1024 * 1024):
+                    count += len(block)
+                    if count > record['ciphertext_bytes']:
+                        raise offsite.Refused()
+                    digest.update(block)
+                    output.write(block)
+                output.flush()
+                os.fsync(output.fileno())
+        if count != record['ciphertext_bytes'] or digest.hexdigest() != record['ciphertext_sha256']:
+            raise offsite.Refused()
+        with client.open(urllib.request.Request(url, method='HEAD', headers=auth), timeout=30) as response:
+            if offsite.etag(response, url, record['ciphertext_bytes']) != record['etag']:
+                raise offsite.Refused()
+        complete = True
+        # Closed allowlist: never copy arbitrary source-receipt fields to output.
+        result = {k: record[k] for k in ('schema', 'provider', 'folder', 'artifact',
+                  'ciphertext_bytes', 'ciphertext_sha256', 'etag')}
+        result.update(status='readback_verified', fetched_path=str(target),
+                      independent_owner_retrieval_verified=True,
+                      credential_source='independently_held_provider_access',
+                      provider_auth_mode=mode, openbao_contacted=False, sso_contacted=False,
+                      provider_writes_performed=False, age_private_key_used=False,
+                      secret_values_exported=False, object_unchanged_during_transfer=True,
+                      downloaded_at=datetime.now(timezone.utc).isoformat())
+        return result
+    finally:
+        if directory is not None and not complete:
+            shutil.rmtree(directory)
+
+
+def interrupted(signum, frame):
+    raise InterruptedError()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--source-receipt', required=True)
+    parser.add_argument('--receipt', required=True)
+    parser.add_argument('--auth', choices=('account', 'folder-share'), default='account')
+    args = parser.parse_args()
+    os.umask(0o077)
+    fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    result = {'status': 'failed', 'error': 'offline_retrieval_failed_private_diagnostics_suppressed'}
+    try:
+        record = json.loads(Path(args.source_receipt).read_text())
+        validate(record)
+        # getpass must never fall back to echoed stdin (e.g. a captured agent tool).
+        with open('/dev/tty', 'r+') as terminal:
+            if not terminal.isatty():
+                raise offsite.Refused()
+            secret = getpass.getpass('Storage Share Backup app password or exact folder-share token: ', stream=terminal)
+        try:
+            result = retrieve(record, secret, args.auth)
+        finally:
+            secret = None
+    except (Exception, KeyboardInterrupt):
+        pass
+    finally:
+        with os.fdopen(fd, 'w') as output:
+            json.dump(result, output, indent=2)
+            output.write('\n')
+    ok = result['status'] == 'readback_verified'
+    print('OK: ciphertext retrieved without OpenBao or SSO; ETag, size and SHA256 verified.' if ok
+          else 'ERROR: retrieval refused; private diagnostics suppressed. Use a new receipt path to retry.')
+    return 0 if ok else 1
+
+
+if __name__ == '__main__':
+    for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
+        signal.signal(sig, interrupted)
+    sys.exit(main())

tools/bootstrap-recovery/run-offline-recovery.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Run in the owner's own terminal. Provider password and age key are hidden inputs.
+set +x
+set -euo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd -- "$ROOT"
+umask 077
+[[ -t 0 && -t 1 ]] || { printf '%s\n' 'ERROR: run in your own interactive terminal.'; exit 1; }
+for tool in python3 age age-keygen openssl docker; do
+    command -v "$tool" >/dev/null || { printf 'ERROR: required tool unavailable: %s\n' "$tool"; exit 1; }
+done
+docker image inspect 'postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70' >/dev/null 2>&1 || {
+    printf '%s\n' 'ERROR: stage the pinned PostgreSQL image before this drill.'; exit 1;
+}
+NK49_HANDOFF_DIR="$(mktemp -d /tmp/nk49-offline-handoff-XXXXXXXX)"
+printf 'Recovery evidence directory: %s\n' "$NK49_HANDOFF_DIR"
+printf '%s\n' 'This publishes three new versioned guide/kit/checksum files beside the backups, then downloads the protected recovered generation.'
+printf '%s\n' 'Enter the independently held Backup account app password at the next hidden prompt. No OpenBao or SSO login is used.'
+python3 tools/bootstrap-recovery/recovery-kit.py \
+    --output-dir "$NK49_HANDOFF_DIR/kit" --publish \
+    --receipt "$NK49_HANDOFF_DIR/publication.json" \
+    --offline-transfer-receipt "$NK49_HANDOFF_DIR/transfer.json" || {
+        printf '%s\n' 'ERROR: publication/retrieval stopped. Inspect the safe receipts; do not paste credentials into chat.'; exit 1;
+    }
+printf '%s\n' 'Guide publication and outage-path retrieval passed. Next enter the independently held railiance-backup age key.'
+bash sso-mfa/bootstrap/sops-custody-unlock.sh \
+    --expected-recipient age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz \
+    --prompt -- \
+    python3 tools/bootstrap-recovery/verify-identity.py \
+    --transfer-receipt "$NK49_HANDOFF_DIR/transfer.json" \
+    --receipt "$NK49_HANDOFF_DIR/recovery.json"
+printf 'OK: private-provider retrieval and archive recovery passed. Safe receipts: %s\n' "$NK49_HANDOFF_DIR"
+printf '%s\n' 'Service authentication remains separate acceptance. Return only the evidence directory path and outcome, never credentials.'