Prepare isolated privacyIDEA token and audit recovery checks

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2

tools/bootstrap-recovery/README.md

@@ -29,6 +29,15 @@ grant. Actual-image synthetic tests and the real owner backup-recovery run
 passed on 2026-10-16; T16 is done. See the packet and actual LLDAP receipt for
 the exact target, resource limits, accepted scope and repeat-drill command.
 
+`run-isolated-privacyidea.sh TRANSFER_RECEIPT` is the next prepared owner step.
+It asks only for the existing backup age key. `isolated-privacyidea.py` restores
+the linked dump into a loopback-only PostgreSQL/container namespace, reads
+retained token material internally, and verifies all stored audit signatures
+plus one new isolated entry and wrong-public-key/tampering negatives. Original
+keys are preserved; audit exceptions are reported without rewriting history.
+No MFA code is consumed or native MFA acceptance claimed. Exit2 requires owner
+review of audit exceptions; see the packet for bounds and safe receipts.
+
 `verify-bundle.py` validates a digest-bound age-encrypted `secrets/` tar archive
 using an explicit private temporary key and registered `keys/age.pub`. It checks
 an exact required-file list, unrelated wrong-key refusal, safe isolated file

tools/bootstrap-recovery/isolated-privacyidea.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+"""Private native privacyIDEA token-material and audit recovery checks.
+
+This does not authenticate a user's MFA code. It restores only a disposable
+database, decrypts token material internally and verifies stored audit rows.
+"""
+import argparse
+import ast
+import base64
+from datetime import datetime, timezone
+import hashlib
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import tempfile
+import time
+import uuid
+
+# Import the already reviewed local target/transport boundary without changing
+# the accepted LLDAP executor or its source digest.
+import importlib.util
+HERE = Path(__file__).resolve().parent
+spec = importlib.util.spec_from_file_location('isolated_lldap', HERE / 'isolated-lldap.py')
+local = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(local)
+bundle = local.bundle
+IMAGE = local.CLIENT
+POSTGRES = local.verify.pg.IMAGE
+
+
+def run(args, **kwargs):
+    return local.command(args, **kwargs)
+
+
+def preflight():
+    target = local.preflight()
+    target['images'].append({'reference': POSTGRES,
+        'image_id': run(['docker', 'image', 'inspect', '--format', '{{.Id}}', POSTGRES]).decode().strip()})
+    return target
+
+
+def isolation(name, expected_network, user):
+    info = json.loads(run(['docker', 'container', 'inspect', name, '--format',
+                         '{"host":{{json .HostConfig}},"mounts":{{json .Mounts}},"user":{{json .Config.User}}}']))
+    host = info['host']
+    if (host['NetworkMode'] != expected_network or not host['ReadonlyRootfs'] or host['Privileged']
+            or host.get('Binds') or host.get('PortBindings') or 'ALL' not in host.get('CapDrop', [])
+            or 'no-new-privileges' not in host.get('SecurityOpt', []) or info['user'] != user
+            or any(m['Type'] != 'tmpfs' for m in info['mounts'])):
+        raise bundle.Refused()
+
+
+def cleanup(names):
+    for name in names:
+        present = subprocess.run(['docker', 'container', 'inspect', name],
+                                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15)
+        if present.returncode == 0:
+            run(['docker', 'rm', '--force', '--volumes', name])
+        absent = subprocess.run(['docker', 'container', 'inspect', name],
+                                stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15)
+        run(['docker', 'info', '--format', '{{.ID}}'])
+        if absent.returncode == 0:
+            raise bundle.Refused()
+
+
+def start_database(name):
+    run(['docker', 'run', '--detach', '--pull=never', '--name', name, '--network', 'none',
+         '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
+         '--user', '999:999', '--memory', '2g', '--cpus', '2', '--pids-limit', '128',
+         '--tmpfs', '/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=1073741824,uid=999,gid=999,mode=0700',
+         '--tmpfs', '/var/run/postgresql:rw,nosuid,nodev,noexec,size=16777216,uid=999,gid=999',
+         '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=999,gid=999',
+         '--env', 'POSTGRES_HOST_AUTH_METHOD=trust', '--env', 'POSTGRES_DB=recovery',
+         '--entrypoint', '/usr/bin/timeout', POSTGRES, '--kill-after=15s', '1800s',
+         '/usr/local/bin/docker-entrypoint.sh', 'postgres', '-c', 'listen_addresses=127.0.0.1'])
+    isolation(name, 'none', '999:999')
+    for _ in range(80):
+        # The entrypoint's temporary init server has TCP disabled. Require TCP
+        # readiness so we cannot mistake it for the final local-only server.
+        ready = subprocess.run(['docker', 'exec', name, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres', '-d', 'recovery'],
+                               stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10)
+        if ready.returncode == 0:
+            return
+        time.sleep(.25)
+    raise bundle.Refused()
+
+
+def start_worker(name, database, root):
+    secret = bundle.resource(root, 'mfa', 'secret', 'privacyidea-config')['data']
+    config = {'SQLALCHEMY_DATABASE_URI': 'postgresql+psycopg2://postgres@127.0.0.1:5432/recovery',
+              'PI_AUDIT_SQL_URI': 'postgresql+psycopg2://postgres@127.0.0.1:5432/recovery',
+              'SECRET_KEY': base64.b64decode(secret['PI_SECRET_KEY'], validate=True).decode(),
+              'PI_PEPPER': base64.b64decode(secret['PI_PEPPER'], validate=True).decode(),
+              'PI_ENCFILE': '/recovery/enckey', 'PI_AUDIT_KEY_PRIVATE': '/recovery/private.pem',
+              'PI_AUDIT_KEY_PUBLIC': '/recovery/public.pem', 'PI_AUDIT_NO_SIGN': False,
+              'PI_CHECK_OLD_SIGNATURES': legacy_setting(root), 'PI_NO_RESPONSE_SIGN': True,
+              'PI_LOGFILE': '/recovery/runtime.log', 'PI_LOGLEVEL': 50,
+              'PI_UUID_FILE': '/recovery/node-uuid',
+              'PI_TRUSTED_JWT': []}
+    # This deliberately does not execute archived pi.cfg or reuse its live DB URI.
+    payload = {'pi.cfg': '\n'.join(k + '=' + repr(v) for k, v in config.items()).encode()}
+    for key in ('enckey', 'private.pem', 'public.pem'):
+        payload[key] = (root / 'data/privacyidea' / key).read_bytes()
+    run(['docker', 'run', '--detach', '--pull=never', '--name', name,
+         '--network', 'container:' + database, '--read-only', '--cap-drop', 'ALL',
+         '--security-opt', 'no-new-privileges', '--user', '1000:1000',
+         '--memory', '1g', '--cpus', '2', '--pids-limit', '128',
+         '--tmpfs', '/recovery:rw,nosuid,nodev,noexec,size=33554432,uid=1000,gid=1000,mode=0700',
+         '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000',
+         '--tmpfs', '/privacyidea/etc/persistent:rw,nosuid,nodev,noexec,size=1048576,uid=1000,gid=1000,mode=0700',
+         '--env', 'PRIVACYIDEA_CONFIGFILE=/recovery/pi.cfg',
+         '--entrypoint', 'python3', IMAGE, '-c', 'import time;time.sleep(1800)'])
+    # Docker normalizes container:<name> to container:<full ID> in HostConfig.
+    database_id = run(['docker', 'container', 'inspect', database, '--format', '{{.Id}}']).decode().strip()
+    isolation(name, 'container:' + database_id, '1000:1000')
+    run(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/recovery'],
+        input=local.archive_files(payload))
+
+
+def legacy_setting(root):
+    path = root / 'resources/mfa/configmap/privacyidea-cfg.json'
+    if not path.exists():
+        return False  # Only the lower-level synthetic fixture omits ConfigMaps.
+    raw = bundle.resource(root, 'mfa', 'configmap', 'privacyidea-cfg')['data']['pi.cfg']
+    selected = False
+    for node in ast.parse(raw).body:
+        if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'PI_CHECK_OLD_SIGNATURES' for t in node.targets):
+            if not isinstance(node.value, ast.Constant) or type(node.value.value) is not bool:
+                raise bundle.Refused()
+            selected = node.value.value
+    return selected
+
+
+PROBE = r'''
+import contextlib,io,json,logging,os,sys
+logging.disable(logging.CRITICAL)
+os.umask(0o077)
+result={'status':'failed'}
+stage='network_isolation'
+try:
+    with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()):
+        from pathlib import Path
+        assert {p.name for p in Path('/sys/class/net').iterdir()}=={'lo'}
+        assert len(Path('/proc/net/route').read_text().splitlines())==1
+        stage='native_application_configuration'
+        from privacyidea.app import create_app
+        app=create_app(config_name='testing',config_file='/recovery/pi.cfg',silent=True)
+        from privacyidea.models import db,Token
+        from privacyidea.lib.security.default import DefaultSecurityModule
+        from privacyidea.lib.auditmodules.sqlaudit import Audit,LogEntry
+        from privacyidea.lib.crypto import Sign
+        import binascii
+        with app.app_context():
+            assert str(db.engine.url)=='postgresql+psycopg2://postgres@127.0.0.1:5432/recovery'
+            stage='retained_token_decryption'
+            security=DefaultSecurityModule({'file':'/recovery/enckey'})
+            wrong_path=Path('/recovery/wrong-enckey');wrong_path.write_bytes(os.urandom(96))
+            wrong=DefaultSecurityModule({'file':str(wrong_path)})
+            total=Token.query.count()
+            assert 0<total<=10000
+            checked=0;negative=0
+            for token in Token.query.yield_per(100):
+                if not token.key_enc:continue
+                encrypted=binascii.unhexlify(token.key_enc);iv=binascii.unhexlify(token.key_iv)
+                material=security.decrypt(encrypted,iv)
+                assert isinstance(material,bytes) and 0<len(material)<=4096
+                try:
+                    invalid=wrong.decrypt(encrypted,iv)
+                    assert invalid!=material
+                except (ValueError,binascii.Error):pass
+                negative+=1;checked+=1
+                del material
+            assert checked>0
+            result.update(retained_tokens_total=total,retained_tokens_with_native_decryption=checked,
+                          different_key_cannot_reproduce_material_count=negative,
+                          retained_token_material_decrypted_inside_process=True,existing_token_mfa_proven=False)
+            stage='historical_audit_verification'
+            audit=Audit(app.config)
+            allow_legacy=bool(app.config['PI_CHECK_OLD_SIGNATURES'])
+            historical=audit.session.query(LogEntry).count()
+            assert 0<historical<=2000000
+            signed=0;unsigned=0;invalid=0;legacy=0
+            for entry in audit.session.query(LogEntry).yield_per(1000):
+                if not entry.signature:
+                    unsigned+=1;continue
+                signed+=1
+                if not str(entry.signature).startswith(Sign.sig_ver+':'):legacy+=1
+                if not audit.sign_object.verify(audit._log_to_string(entry),entry.signature,allow_legacy):invalid+=1
+            result.update(historical_audit_rows=historical,historical_signed_rows=signed,
+                          historical_unsigned_rows=unsigned,historical_invalid_signatures=invalid,
+                          historical_legacy_signatures=legacy,all_stored_audit_rows_checked=True,
+                          legacy_signature_verification_enabled=allow_legacy,
+                          event_completeness_claimed=False)
+            stage='new_isolated_audit_entry'
+            audit.log({'action':'NK49 isolated recovery verification','success':True,
+                       'info':'disposable restored database only','client':'127.0.0.1'})
+            audit.finalize_log()
+            fresh=Audit(app.config)
+            assert fresh.session.query(LogEntry).count()==historical+1
+            entry=fresh.session.query(LogEntry).order_by(LogEntry.id.desc()).first()
+            message=fresh._log_to_string(entry)
+            assert fresh.sign_object.verify(message,entry.signature,False)
+            assert not fresh.sign_object.verify(message+' tampered',entry.signature,False)
+            from cryptography.hazmat.primitives.asymmetric import rsa
+            from cryptography.hazmat.primitives import serialization
+            public=rsa.generate_private_key(public_exponent=65537,key_size=2048).public_key().public_bytes(
+                serialization.Encoding.PEM,serialization.PublicFormat.SubjectPublicKeyInfo)
+            assert not Sign(public_key=public).verify(message,entry.signature,False)
+            result.update(new_audit_entry_signed_verified=True,tampered_entry_refused=True,
+                          wrong_audit_public_key_refused=True,
+                          audit_signature_continuity_proven=signed==historical and invalid==0,
+                          only_loopback_interface_verified=True,external_routes_absent=True,
+                          status='native_checks_completed')
+except Exception as error:
+    result.update(status='failed',failed_stage=stage,failure_type=type(error).__name__)
+print(json.dumps(result))
+'''
+
+
+def native_checks(root, receipt):
+    stem = 'nk49-pi-check-' + uuid.uuid4().hex
+    database = stem + '-db'
+    worker = stem + '-worker'
+    try:
+        start_database(database)
+        with (root / 'databases/privacyidea.dump').open('rb') as dump:
+            run(['docker', 'exec', '-i', database, 'pg_restore', '-U', 'postgres', '-d', 'recovery',
+                 '--no-owner', '--no-privileges', '--exit-on-error'], stdin=dump, timeout=600)
+        start_worker(worker, database, root)
+        receipt['container_isolation_verified_before_private_import'] = True
+        result = json.loads(run(['docker', 'exec', worker, 'python3', '-c', PROBE], timeout=600))
+        receipt.update(result)
+        if result.get('status') != 'native_checks_completed':
+            raise bundle.Refused()
+    finally:
+        cleanup([worker, database])
+        receipt['container_cleanup_verified'] = True
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--transfer-receipt', required=True)
+    parser.add_argument('--receipt', required=True)
+    args = parser.parse_args()
+    os.umask(0o077)
+    fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
+    result = {'schema': 'netkingdom.isolated-privacyidea-recovery.v1', 'status': 'failed',
+              'production_contacted': False, 'secret_values_exported': False,
+              'existing_token_mfa_proven': False, 'audit_signature_continuity_proven': False,
+              'container_cleanup_verified': False, 'temporary_plaintext_cleanup_verified': False,
+              'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}
+    keyfd = None
+    work = None
+    stage = 'target_preflight'
+    try:
+        result['target'] = preflight()
+        stage = 'transfer_validation'
+        transfer = json.loads(Path(args.transfer_receipt).read_text())
+        local.offline.validate(transfer)
+        if transfer.get('independent_owner_retrieval_verified') is not True:
+            raise bundle.Refused()
+        path = Path(transfer['fetched_path'])
+        bundle.private(path, max_bytes=bundle.LIMIT)
+        if path.stat().st_size != transfer['ciphertext_bytes'] or local.offline.offsite.digest(path) != transfer['ciphertext_sha256']:
+            raise bundle.Refused()
+        keyfd = local.verify.custody.private_key_fd()
+        if run(['age-keygen', '-y', f'/proc/self/fd/{keyfd}'], pass_fds=(keyfd,)).strip() != bundle.profile()['public_recipient'].encode():
+            raise bundle.Refused()
+        with tempfile.TemporaryDirectory(prefix='nk49-pi-recovery-', dir='/dev/shm') as temporary:
+            work = Path(temporary)
+            archive = work / 'archive.tar'
+            stage = 'archive_decryption'
+            with archive.open('xb') as output:
+                p = subprocess.run(['age', '-d', '-i', f'/proc/self/fd/{keyfd}', str(path)],
+                                   stdout=output, stderr=subprocess.DEVNULL, pass_fds=(keyfd,), timeout=180)
+            if p.returncode or archive.stat().st_size > bundle.LIMIT + 4 * 1024 * 1024:
+                raise bundle.Refused()
+            restored = work / 'restored';restored.mkdir(mode=0o700)
+            stage = 'archive_validation'
+            manifest, _ = bundle.restore(archive, restored)
+            result.update(ciphertext_sha256=transfer['ciphertext_sha256'],
+                          manifest_file_count=len(manifest['files']), all_manifest_hashes_verified=True)
+            stage = 'native_privacyidea_checks'
+            native_checks(restored, result)
+        result['status'] = 'isolated_privacyidea_checked'
+    except (Exception, KeyboardInterrupt):
+        result.update(status='failed', error='isolated_privacyidea_check_failed_private_diagnostics_suppressed')
+        result.setdefault('failed_stage', stage)
+    finally:
+        if keyfd is not None:os.close(keyfd)
+        if work is not None and not work.exists():result['temporary_plaintext_cleanup_verified'] = True
+        result['finished_at'] = datetime.now(timezone.utc).isoformat()
+        with os.fdopen(fd, 'w') as output:
+            json.dump(result, output, indent=2);output.write('\n')
+    if result['status'] == 'failed':
+        print('ERROR: isolated privacyIDEA check failed; review safe receipt. No production change.')
+        return 1
+    if not result['audit_signature_continuity_proven']:
+        print('REVIEW: token-material checks completed; historical audit exceptions require owner review. Cleanup passed; see safe receipt.')
+        return 2
+    print('OK: retained token material decrypted internally; historical/new audit signatures and negative checks passed; cleanup passed. Existing-token MFA remains separate.')
+    return 0
+
+
+if __name__ == '__main__':
+    for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):signal.signal(sig, local.offline.interrupted)
+    raise SystemExit(main())

tools/bootstrap-recovery/run-isolated-privacyidea.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Owner-run native audit/token-material checks. Never pass a private key in argv.
+set +x
+set -euo pipefail
+ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd -- "$ROOT"
+umask 077
+[[ $# == 1 ]] || { printf '%s\n' 'Usage: run-isolated-privacyidea.sh VERIFIED_TRANSFER_RECEIPT'; exit 1; }
+[[ -f "$1" && ! -L "$1" ]] || { printf '%s\n' 'ERROR: provide the existing verified transfer receipt.'; exit 1; }
+[[ -t 0 && -t 1 && -t 2 ]] || { printf '%s\n' 'ERROR: run in your own interactive terminal.'; exit 1; }
+for tool in python3 age age-keygen openssl docker; do
+    command -v "$tool" >/dev/null || { printf 'ERROR: required tool unavailable: %s\n' "$tool"; exit 1; }
+done
+python3 - <<'PY'
+import importlib.util
+from pathlib import Path
+p=Path('tools/bootstrap-recovery/isolated-privacyidea.py')
+spec=importlib.util.spec_from_file_location('isolated',p)
+m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
+try:
+    m.preflight()
+except Exception:
+    raise SystemExit('ERROR: exact local Docker target, pinned images or tmpfs preflight failed; no key requested.')
+print('OK: exact bnt-lap001 recovery engine, pinned images and tmpfs available.')
+PY
+NK49_PI_RUN="$(mktemp -d /tmp/nk49-isolated-privacyidea-XXXXXXXX)"
+printf 'Safe evidence directory: %s\n' "$NK49_PI_RUN"
+printf '%s\n' 'This restores the linked database and original keys only into disposable network-isolated containers.'
+printf '%s\n' 'It decrypts retained token material internally, checks every stored audit row, and creates one new isolated audit entry.'
+printf '%s\n' 'No MFA code is consumed. Invalid or unsigned history is reported, not repaired. Enter only the backup age key at the hidden prompt.'
+if bash sso-mfa/bootstrap/sops-custody-unlock.sh \
+    --expected-recipient age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz \
+    --prompt -- python3 tools/bootstrap-recovery/isolated-privacyidea.py \
+    --transfer-receipt "$1" --receipt "$NK49_PI_RUN/recovery.json"; then
+    printf 'Safe receipt: %s/recovery.json\n' "$NK49_PI_RUN"
+else
+    NK49_PI_EXIT=$?
+    printf 'Check stopped or needs review. Safe receipt: %s/recovery.json\n' "$NK49_PI_RUN"
+    exit "$NK49_PI_EXIT"
+fi
+printf '%s\n' 'Return only the outcome and evidence directory. Existing-token MFA, Authelia and KeyCape remain separate.'