From 1ab5d53bd7f56aa0528230ae484eec67312d1a93 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 16 Oct 2026 21:00:36 +0200 Subject: [PATCH] Prepare isolated privacyIDEA token and audit recovery checks Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2 --- .../isolated-privacyidea-preparation.json | 58 ++++ docs/identity-service-restore-packet.md | 56 +++- .../capability.security.bootstrap-recovery.md | 11 + tests/test_isolated_privacyidea_recovery.py | 196 +++++++++++ tools/bootstrap-recovery/README.md | 9 + .../isolated-privacyidea.py | 309 ++++++++++++++++++ .../run-isolated-privacyidea.sh | 41 +++ ...-bootstrap-secret-free-recovery-handoff.md | 90 ++++- 8 files changed, 762 insertions(+), 8 deletions(-) create mode 100644 docs/evidence/2026-10-16-identity-outage-recovery/isolated-privacyidea-preparation.json create mode 100644 tests/test_isolated_privacyidea_recovery.py create mode 100644 tools/bootstrap-recovery/isolated-privacyidea.py create mode 100644 tools/bootstrap-recovery/run-isolated-privacyidea.sh diff --git a/docs/evidence/2026-10-16-identity-outage-recovery/isolated-privacyidea-preparation.json b/docs/evidence/2026-10-16-identity-outage-recovery/isolated-privacyidea-preparation.json new file mode 100644 index 0000000..be8392e --- /dev/null +++ b/docs/evidence/2026-10-16-identity-outage-recovery/isolated-privacyidea-preparation.json @@ -0,0 +1,58 @@ +{ + "schema": "netkingdom.isolated-privacyidea-preparation.v1", + "date": "2026-10-16", + "status": "native_synthetic_verified_actual_owner_run_pending", + "sources": [ + { + "path": "tools/bootstrap-recovery/isolated-privacyidea.py", + "sha256": "4df7196678aac1ad594f32abf45c68dc21a9d2760d9e8a536a215a4a3c5d277f" + }, + { + "path": "tools/bootstrap-recovery/run-isolated-privacyidea.sh", + "sha256": "ad2d045c82f771615d8be4ab401fa86e001ca5529ad81f3cb10b1f8c25a59639" + }, + { + "path": "tests/test_isolated_privacyidea_recovery.py", + "sha256": "2e741a0d45d8cc6ada292b699dc1e2b239fb186cfa52bc3cefd3cdf2d6ddc915" + }, + { + "path": "docs/identity-service-restore-packet.md", + "sha256": "918a5b6f1c49fa8338fde8b19d0c3a44ac2cb8ae890e3b74713f7c11f10549c3" + } + ], + "target": { + "engine_name": "bnt-lap001", + "engine_id": "74f93783-6cef-43a8-a3c4-f62586cd3bc1", + "images": [ + "postgres@sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473" + ] + }, + "tests": { + "command": "NK49_RUN_ISOLATED_PRIVACYIDEA_TEST=1 python3 -m pytest -q tests/test_isolated_privacyidea_recovery.py -x", + "passed": 7, + "failed": 0, + "native_retained_token_decryption_and_original_key_new_audit": true, + "native_tampered_history_reported": true, + "native_unsigned_history_reported": true, + "native_malformed_key_fails_with_cleanup": true, + "native_full_encrypted_bundle_to_checks_and_tmpfs_cleanup": true, + "archived_config_expressions_never_executed": true, + "archived_literal_legacy_policy_retained": true + }, + "policy": "every stored audit row up to2000000, no sampling;600-second worker bound; original key/config bindings, explicit contained DB; report exceptions, do not rewrite history", + "existing_token_mfa_proven": false, + "event_completeness_claimed": false, + "actual_backup_token_material_or_audit_proven": false, + "actual_private_key_or_other_credential_received_by_agent": false, + "knowledge": { + "version": "0.2.6", + "digest": "8d1d6a9d4d07fec7b619618de7dfe2be2254c1790f85f4198c7fe19db9751c02", + "source_status": "runbooks withheld; current local owner contract, task cards and installed pinned native implementation reviewed directly; orientation only" + }, + "credential_route": { + "catalog_id": "railiance-backup-offsite-lane", + "owner": "railiance-platform", + "use": "existing independent human-held age key only; no credential fetched or new grant" + } +} diff --git a/docs/identity-service-restore-packet.md b/docs/identity-service-restore-packet.md index 0e278a1..58aaeff 100644 --- a/docs/identity-service-restore-packet.md +++ b/docs/identity-service-restore-packet.md @@ -1,7 +1,8 @@ # Isolated identity service restore packet NK-WP-0049-T15. Prepared 2026-10-16. **Actual LLDAP recovery accepted at -18:37 UTC; remaining service stages are draft and unexecuted.** +18:37 UTC. Native privacyIDEA token-material/audit stage is prepared for an +owner run; MFA, Authelia and KeyCape stages remain unexecuted.** Accountable preparer: codex. Execution/acceptance: Railiance Platform and the LLDAP, Authelia, KeyCape and privacyIDEA service owners. @@ -142,6 +143,59 @@ current production password validity, MFA or audit continuity. Do not silently reset credentials or re-enroll tokens to pass. Historical audit verification failures must be reported, not hidden with replacement keys. +## Ready owner step: privacyIDEA token material and stored audit history + +```bash +bash tools/bootstrap-recovery/run-isolated-privacyidea.sh \ + /tmp/nk49-offline-handoff-1X660a5t/transfer.json +``` + +This uses the already independently retrieved ciphertext. The owner enters only +the `railiance-backup` age key through the existing hidden helper; no password, +MFA code or provider credential is requested. The exact local Docker target and +cached images are checked first. The main executor rechecks ciphertext/recipient +and all archive hashes, with private plaintext staging on Linux tmpfs. + +`isolated-privacyidea.py` restores the linked dump into fresh PostgreSQL on +`--network none`, listening only on its loopback interface. The pinned +privacyIDEA 3.12.2 worker shares that same namespace. Both have read-only roots, +all capabilities dropped, no added privileges, fixed unprivileged UIDs, no host +mounts/ports and only tmpfs writable storage. Isolation is inspected before +private import. The database has a 2 GiB memory limit and one GiB tmpfs data +limit; the worker has a one GiB memory limit. Hold processes expire after 30 +minutes (database forced termination after a further 15 seconds); normal cleanup +removes both containers and plaintext staging. An interrupted host/process still +requires reconciliation of the exact `nk49-pi-check-*` resources. + +The worker uses original enckey/audit keys and the original pepper/session-secret +fields. It generates a narrowly scoped private config pointing only at the +isolated database, and explicitly selects that config over image defaults. +It does not execute archived Python config. A literal archived +`PI_CHECK_OLD_SIGNATURES` setting is preserved; unknown dynamic expressions are +refused. No production URI or provider-admin bearer is used, no user is enrolled, +and no original key is regenerated or replaced. + +Native checks decrypt every nonempty encrypted token-key field internally, +compare against a separate disposable wrong-key result, and emit only counts. +This proves bounded native material readback, **not existing-token MFA**; padding +and readback do not establish successful authentication. That remaining positive +and wrong-code/wrong-key authentication proof stays T19. T31 tracks the separate +actual token-material readback receipt. + +For T20, the native audit implementation verifies every stored historical row +(up to two million, otherwise refusal), reports signed/unsigned/invalid/legacy +counts, then creates and verifies one isolated audit entry using the original +signing identity. Tampering and an unrelated public key must fail. No historical +row or signature is rewritten. This checks stored rows, not event completeness. +The worker has a ten-minute execution bound; timeout is not an accepted partial +scan. Counts/signatures/keys are never replaced with a sample to claim success. + +Exit0 means these bounded checks and cleanup passed. Exit2 means checks completed +but historical audit exceptions need owner disposition; T20 must stay open. +Exit1 indicates another failure. All outcomes have a safe receipt in the printed +private evidence directory. Return only that directory and the outcome. Actual +owner results are still pending; native synthetic fixtures do not close T20/T31. + ## Unresolved inputs before execution - Named immutable disposable target, approved private browser route if required, diff --git a/registry/capabilities/capability.security.bootstrap-recovery.md b/registry/capabilities/capability.security.bootstrap-recovery.md index 766f20b..19e9b58 100644 --- a/registry/capabilities/capability.security.bootstrap-recovery.md +++ b/registry/capabilities/capability.security.bootstrap-recovery.md @@ -75,6 +75,8 @@ availability: - tools/bootstrap-recovery/run-offline-recovery.sh - tools/bootstrap-recovery/isolated-lldap.py - tools/bootstrap-recovery/run-isolated-lldap.sh + - tools/bootstrap-recovery/isolated-privacyidea.py + - tools/bootstrap-recovery/run-isolated-privacyidea.sh - docs/disaster-recovery-storage-share.md - docs/identity-service-restore-packet.md - docs/storage-share-recovery-publication.md @@ -106,6 +108,7 @@ evidence: - tests/test_identity_backup_preparation.py - tests/test_identity_offline_recovery.py - tests/test_isolated_lldap_recovery.py + - tests/test_isolated_privacyidea_recovery.py consumer_guidance: recommended_for: - Operator recovers identity, custody or data with independently held recovery material @@ -170,3 +173,11 @@ five pinned service images were saved/reloaded locally. T28/T29 record that preparation. Actual private real-backup LLDAP recovery passed on 2026-10-16; T16 is done, with original identity rows, positive/negative native bind, isolation and cleanup accepted. Other service/MFA/audit lanes remain open. + +The next prepared stage is `run-isolated-privacyidea.sh TRANSFER_RECEIPT`, with +only a hidden backup-key prompt. Its pinned native routines check internal +token-material readback and stored audit signatures, preserve original keys, +report history exceptions and remove the isolated database/worker/tmpfs. Native +synthetic tests cover valid, tampered and unsigned history, malformed keys and +encrypted-bundle recovery. Actual owner evidence remains T20/T31; existing-token +MFA remains T19 and is not implied by token-material decryption. diff --git a/tests/test_isolated_privacyidea_recovery.py b/tests/test_isolated_privacyidea_recovery.py new file mode 100644 index 0000000..c8c6957 --- /dev/null +++ b/tests/test_isolated_privacyidea_recovery.py @@ -0,0 +1,196 @@ +"""Native pinned PI/PostgreSQL recovery with synthetic token/audit material.""" +import base64 +import importlib.util +import json +import os +from pathlib import Path +import secrets +import subprocess +import textwrap +import uuid + +import pytest +from test_identity_bundle import stage +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location('pi_recovery', ROOT / 'tools/bootstrap-recovery/isolated-privacyidea.py') +m = importlib.util.module_from_spec(spec) +spec.loader.exec_module(m) + + +def test_archived_legacy_setting_cannot_execute_expressions(tmp_path): + folder = tmp_path / 'resources/mfa/configmap';folder.mkdir(parents=True) + value = {'metadata': {'namespace': 'mfa', 'name': 'privacyidea-cfg'}, + 'data': {'pi.cfg': 'PI_CHECK_OLD_SIGNATURES = __import__("os").system("false")'}} + (folder / 'privacyidea-cfg.json').write_text(json.dumps(value)) + with pytest.raises(m.bundle.Refused): + m.legacy_setting(tmp_path) + + +def test_archived_literal_legacy_setting_is_retained(tmp_path): + folder = tmp_path / 'resources/mfa/configmap';folder.mkdir(parents=True) + value = {'metadata': {'namespace': 'mfa', 'name': 'privacyidea-cfg'}, + 'data': {'pi.cfg': 'import os\nPI_CHECK_OLD_SIGNATURES = True'}} + (folder / 'privacyidea-cfg.json').write_text(json.dumps(value)) + assert m.legacy_setting(tmp_path) is True + + +@pytest.fixture +def retained_state(tmp_path): + if os.environ.get('NK49_RUN_ISOLATED_PRIVACYIDEA_TEST') != '1': + pytest.skip('explicit disposable pinned native-image fixture opt-in') + m.preflight() + os.umask(0o077) + root = tmp_path / 'retained' + keys = root / 'data/privacyidea';keys.mkdir(parents=True) + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + (keys / 'private.pem').write_bytes(key.private_bytes(serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) + (keys / 'public.pem').write_bytes(key.public_key().public_bytes(serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo)) + (keys / 'enckey').write_bytes(os.urandom(96)) + resource = root / 'resources/mfa/secret';resource.mkdir(parents=True) + (resource / 'privacyidea-config.json').write_text(json.dumps({ + 'metadata': {'namespace': 'mfa', 'name': 'privacyidea-config'}, + 'data': {name: base64.b64encode(secrets.token_bytes(32).hex().encode()).decode() + for name in ('PI_SECRET_KEY', 'PI_PEPPER')}})) + (root / 'databases').mkdir() + name = 'nk49-pi-fixture-' + uuid.uuid4().hex + database = name + '-db';worker = name + '-worker' + setup = r''' +import contextlib,io,logging,json +logging.disable(logging.CRITICAL) +with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()): + from privacyidea.app import create_app + from privacyidea.models import db + from privacyidea.lib.token import init_token + from privacyidea.lib.auditmodules.sqlaudit import Audit,LogEntry + app=create_app(config_name='testing',config_file='/recovery/pi.cfg',silent=True) + with app.app_context(): + db.create_all() + audit=Audit(app.config) + LogEntry.__table__.create(audit.engine,checkfirst=True) + for _ in range(2): + token=init_token({'type':'hotp','genkey':1,'pin':''}) + audit=Audit(app.config) + audit.log({'action':'synthetic retained fixture','success':True,'info':'synthetic only'}) + audit.finalize_log() +print(json.dumps({'synthetic_state_created':True})) +''' + try: + m.start_database(database) + m.start_worker(worker, database, root) + safe_setup = ('import traceback,sys,json\ntry:\n' + textwrap.indent(setup, ' ') + + '\nexcept Exception as error:\n' + ' frame=traceback.extract_tb(error.__traceback__)[-1]\n' + ' original=getattr(error,"orig",None)\n' + ' text=str(original).lower()\n' + ' categories=[x for x in ("connection refused","permission denied","read-only file system","no pg_hba.conf entry","password authentication failed","could not translate host name") if x in text]\n' + ' print(json.dumps({"type":type(error).__name__,"file":frame.filename.split("/")[-1],"line":frame.lineno,"sqlstate":getattr(original,"pgcode",None),"categories":categories,"frames":[f.name for f in traceback.extract_tb(error.__traceback__)]}))\n' + ' sys.exit(1)\n') + result = subprocess.run(['docker', 'exec', worker, 'python3', '-c', safe_setup], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=90) + assert result.returncode == 0, result.stdout.decode() + assert json.loads(result.stdout)['synthetic_state_created'] + dump = m.run(['docker', 'exec', database, 'pg_dump', '-U', 'postgres', '-Fc', 'recovery']) + (root / 'databases/privacyidea.dump').write_bytes(dump) + yield root, database, worker + finally: + m.cleanup([worker, database]) + + +def test_native_existing_token_material_and_audit_history_survive_restore(retained_state): + root, _, _ = retained_state + result = {} + m.native_checks(root, result) + assert result['status'] == 'native_checks_completed' + assert result['retained_tokens_with_native_decryption'] == 2 + assert result['different_key_cannot_reproduce_material_count'] == 2 + assert result['historical_audit_rows'] == result['historical_signed_rows'] == 2 + assert result['historical_invalid_signatures'] == result['historical_unsigned_rows'] == 0 + assert result['audit_signature_continuity_proven'] + assert result['new_audit_entry_signed_verified'] and result['wrong_audit_public_key_refused'] + assert result['tampered_entry_refused'] and result['container_cleanup_verified'] + assert result['container_isolation_verified_before_private_import'] + assert not result['existing_token_mfa_proven'] and not result['event_completeness_claimed'] + + +@pytest.mark.parametrize('problem', ['tampered', 'unsigned']) +def test_native_history_exceptions_reported_without_hiding_with_new_keys(retained_state, problem): + root, database, worker = retained_state + mutate = r''' +import contextlib,io,logging,sys +logging.disable(logging.CRITICAL) +with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()): + from privacyidea.app import create_app + from privacyidea.lib.auditmodules.sqlaudit import Audit,LogEntry + app=create_app(config_name='testing',config_file='/recovery/pi.cfg',silent=True) + with app.app_context(): + audit=Audit(app.config) + row=audit.session.query(LogEntry).first() + row.signature=None if sys.argv[1]=='unsigned' else 'rsa_sha256_pss:'+('00'*256) + audit.session.commit() +''' + m.run(['docker', 'exec', worker, 'python3', '-c', mutate, problem]) + (root / 'databases/privacyidea.dump').write_bytes(m.run( + ['docker', 'exec', database, 'pg_dump', '-U', 'postgres', '-Fc', 'recovery'])) + result = {} + m.native_checks(root, result) + assert result['historical_invalid_signatures'] == int(problem == 'tampered') + assert result['historical_unsigned_rows'] == int(problem == 'unsigned') + assert not result['audit_signature_continuity_proven'] + assert result['new_audit_entry_signed_verified'] and result['container_cleanup_verified'] + + +def test_native_malformed_original_encryption_key_fails_and_cleans_up(retained_state): + root, _, _ = retained_state + (root / 'data/privacyidea/enckey').write_bytes(b'synthetic-malformed-key') + result = {} + with pytest.raises(m.bundle.Refused): + m.native_checks(root, result) + assert result['failed_stage'] == 'retained_token_decryption' + assert result['container_cleanup_verified'] + assert not result.get('retained_token_material_decrypted_inside_process') + + +def test_encrypted_bundle_to_native_checks_and_tmpfs_cleanup(retained_state, stage, tmp_path, monkeypatch, capsys): + import shutil + import sys + root, _, _ = retained_state + for key in ('enckey', 'private.pem', 'public.pem'): + shutil.copyfile(root / 'data/privacyidea' / key, stage / 'data/privacyidea' / key) + shutil.copyfile(root / 'databases/privacyidea.dump', stage / 'databases/privacyidea.dump') + resource = stage / 'resources/mfa/secret/privacyidea-config.json' + value = json.loads(resource.read_text()) + value['data'] = json.loads((root / 'resources/mfa/secret/privacyidea-config.json').read_text())['data'] + resource.write_text(json.dumps(value)) + configmap = stage / 'resources/mfa/configmap/privacyidea-cfg.json' + value = json.loads(configmap.read_text()) + value['data'] = {'pi.cfg': 'PI_CHECK_OLD_SIGNATURES = False\n'} + configmap.write_text(json.dumps(value)) + key = tmp_path / 'synthetic-age.key' + m.run(['age-keygen', '-o', str(key)]) + recipient = m.run(['age-keygen', '-y', str(key)]).decode().strip() + profile = m.bundle.profile();profile['public_recipient'] = recipient + monkeypatch.setattr(m.bundle, 'profile', lambda: profile) + ciphertext = tmp_path / 'net-kingdom-identity-20261016T063159Z-0123456789ab.tar.age' + m.bundle.seal(stage, ciphertext, {'synthetic_native_fixture': True}) + transfer = tmp_path / 'transfer.json' + transfer.write_text(json.dumps({'schema': 'netkingdom.identity-offsite.v1', 'status': 'readback_verified', + 'provider': 'Storage Share', 'folder': 'Backup/railiance-backups/net-kingdom-identity', + 'artifact': ciphertext.name, 'ciphertext_bytes': ciphertext.stat().st_size, + 'ciphertext_sha256': m.local.offline.offsite.digest(ciphertext), 'etag': '"synthetic"', + 'fetched_path': str(ciphertext), 'independent_owner_retrieval_verified': True})) + monkeypatch.setattr(m.local.verify.custody, 'private_key_fd', lambda: os.open(key, os.O_RDONLY)) + receipt = tmp_path / 'receipt.json' + monkeypatch.setattr(sys, 'argv', ['isolated-privacyidea', '--transfer-receipt', str(transfer), '--receipt', str(receipt)]) + assert m.main() == 0 + result = json.loads(receipt.read_text()) + assert result['status'] == 'isolated_privacyidea_checked' and result['audit_signature_continuity_proven'] + assert result['container_cleanup_verified'] and result['temporary_plaintext_cleanup_verified'] + assert result['all_manifest_hashes_verified'] and result['retained_tokens_with_native_decryption'] == 2 + assert not result['production_contacted'] and not result['secret_values_exported'] + assert not result['existing_token_mfa_proven'] + assert 'rsa_sha256_pss:' not in capsys.readouterr().out + receipt.read_text() diff --git a/tools/bootstrap-recovery/README.md b/tools/bootstrap-recovery/README.md index deb00ce..0d7352c 100644 --- a/tools/bootstrap-recovery/README.md +++ b/tools/bootstrap-recovery/README.md @@ -29,6 +29,15 @@ grant. Actual-image synthetic tests and the real owner backup-recovery run passed on 2026-10-16; T16 is done. See the packet and actual LLDAP receipt for the exact target, resource limits, accepted scope and repeat-drill command. +`run-isolated-privacyidea.sh TRANSFER_RECEIPT` is the next prepared owner step. +It asks only for the existing backup age key. `isolated-privacyidea.py` restores +the linked dump into a loopback-only PostgreSQL/container namespace, reads +retained token material internally, and verifies all stored audit signatures +plus one new isolated entry and wrong-public-key/tampering negatives. Original +keys are preserved; audit exceptions are reported without rewriting history. +No MFA code is consumed or native MFA acceptance claimed. Exit2 requires owner +review of audit exceptions; see the packet for bounds and safe receipts. + `verify-bundle.py` validates a digest-bound age-encrypted `secrets/` tar archive using an explicit private temporary key and registered `keys/age.pub`. It checks an exact required-file list, unrelated wrong-key refusal, safe isolated file diff --git a/tools/bootstrap-recovery/isolated-privacyidea.py b/tools/bootstrap-recovery/isolated-privacyidea.py new file mode 100644 index 0000000..a086229 --- /dev/null +++ b/tools/bootstrap-recovery/isolated-privacyidea.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Private native privacyIDEA token-material and audit recovery checks. + +This does not authenticate a user's MFA code. It restores only a disposable +database, decrypts token material internally and verifies stored audit rows. +""" +import argparse +import ast +import base64 +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time +import uuid + +# Import the already reviewed local target/transport boundary without changing +# the accepted LLDAP executor or its source digest. +import importlib.util +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('isolated_lldap', HERE / 'isolated-lldap.py') +local = importlib.util.module_from_spec(spec) +spec.loader.exec_module(local) +bundle = local.bundle +IMAGE = local.CLIENT +POSTGRES = local.verify.pg.IMAGE + + +def run(args, **kwargs): + return local.command(args, **kwargs) + + +def preflight(): + target = local.preflight() + target['images'].append({'reference': POSTGRES, + 'image_id': run(['docker', 'image', 'inspect', '--format', '{{.Id}}', POSTGRES]).decode().strip()}) + return target + + +def isolation(name, expected_network, user): + info = json.loads(run(['docker', 'container', 'inspect', name, '--format', + '{"host":{{json .HostConfig}},"mounts":{{json .Mounts}},"user":{{json .Config.User}}}'])) + host = info['host'] + if (host['NetworkMode'] != expected_network or not host['ReadonlyRootfs'] or host['Privileged'] + or host.get('Binds') or host.get('PortBindings') or 'ALL' not in host.get('CapDrop', []) + or 'no-new-privileges' not in host.get('SecurityOpt', []) or info['user'] != user + or any(m['Type'] != 'tmpfs' for m in info['mounts'])): + raise bundle.Refused() + + +def cleanup(names): + for name in names: + present = subprocess.run(['docker', 'container', 'inspect', name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) + if present.returncode == 0: + run(['docker', 'rm', '--force', '--volumes', name]) + absent = subprocess.run(['docker', 'container', 'inspect', name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=15) + run(['docker', 'info', '--format', '{{.ID}}']) + if absent.returncode == 0: + raise bundle.Refused() + + +def start_database(name): + run(['docker', 'run', '--detach', '--pull=never', '--name', name, '--network', 'none', + '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', + '--user', '999:999', '--memory', '2g', '--cpus', '2', '--pids-limit', '128', + '--tmpfs', '/var/lib/postgresql/data:rw,nosuid,nodev,noexec,size=1073741824,uid=999,gid=999,mode=0700', + '--tmpfs', '/var/run/postgresql:rw,nosuid,nodev,noexec,size=16777216,uid=999,gid=999', + '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=999,gid=999', + '--env', 'POSTGRES_HOST_AUTH_METHOD=trust', '--env', 'POSTGRES_DB=recovery', + '--entrypoint', '/usr/bin/timeout', POSTGRES, '--kill-after=15s', '1800s', + '/usr/local/bin/docker-entrypoint.sh', 'postgres', '-c', 'listen_addresses=127.0.0.1']) + isolation(name, 'none', '999:999') + for _ in range(80): + # The entrypoint's temporary init server has TCP disabled. Require TCP + # readiness so we cannot mistake it for the final local-only server. + ready = subprocess.run(['docker', 'exec', name, 'pg_isready', '-h', '127.0.0.1', '-U', 'postgres', '-d', 'recovery'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10) + if ready.returncode == 0: + return + time.sleep(.25) + raise bundle.Refused() + + +def start_worker(name, database, root): + secret = bundle.resource(root, 'mfa', 'secret', 'privacyidea-config')['data'] + config = {'SQLALCHEMY_DATABASE_URI': 'postgresql+psycopg2://postgres@127.0.0.1:5432/recovery', + 'PI_AUDIT_SQL_URI': 'postgresql+psycopg2://postgres@127.0.0.1:5432/recovery', + 'SECRET_KEY': base64.b64decode(secret['PI_SECRET_KEY'], validate=True).decode(), + 'PI_PEPPER': base64.b64decode(secret['PI_PEPPER'], validate=True).decode(), + 'PI_ENCFILE': '/recovery/enckey', 'PI_AUDIT_KEY_PRIVATE': '/recovery/private.pem', + 'PI_AUDIT_KEY_PUBLIC': '/recovery/public.pem', 'PI_AUDIT_NO_SIGN': False, + 'PI_CHECK_OLD_SIGNATURES': legacy_setting(root), 'PI_NO_RESPONSE_SIGN': True, + 'PI_LOGFILE': '/recovery/runtime.log', 'PI_LOGLEVEL': 50, + 'PI_UUID_FILE': '/recovery/node-uuid', + 'PI_TRUSTED_JWT': []} + # This deliberately does not execute archived pi.cfg or reuse its live DB URI. + payload = {'pi.cfg': '\n'.join(k + '=' + repr(v) for k, v in config.items()).encode()} + for key in ('enckey', 'private.pem', 'public.pem'): + payload[key] = (root / 'data/privacyidea' / key).read_bytes() + run(['docker', 'run', '--detach', '--pull=never', '--name', name, + '--network', 'container:' + database, '--read-only', '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges', '--user', '1000:1000', + '--memory', '1g', '--cpus', '2', '--pids-limit', '128', + '--tmpfs', '/recovery:rw,nosuid,nodev,noexec,size=33554432,uid=1000,gid=1000,mode=0700', + '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000', + '--tmpfs', '/privacyidea/etc/persistent:rw,nosuid,nodev,noexec,size=1048576,uid=1000,gid=1000,mode=0700', + '--env', 'PRIVACYIDEA_CONFIGFILE=/recovery/pi.cfg', + '--entrypoint', 'python3', IMAGE, '-c', 'import time;time.sleep(1800)']) + # Docker normalizes container: to container: in HostConfig. + database_id = run(['docker', 'container', 'inspect', database, '--format', '{{.Id}}']).decode().strip() + isolation(name, 'container:' + database_id, '1000:1000') + run(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/recovery'], + input=local.archive_files(payload)) + + +def legacy_setting(root): + path = root / 'resources/mfa/configmap/privacyidea-cfg.json' + if not path.exists(): + return False # Only the lower-level synthetic fixture omits ConfigMaps. + raw = bundle.resource(root, 'mfa', 'configmap', 'privacyidea-cfg')['data']['pi.cfg'] + selected = False + for node in ast.parse(raw).body: + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == 'PI_CHECK_OLD_SIGNATURES' for t in node.targets): + if not isinstance(node.value, ast.Constant) or type(node.value.value) is not bool: + raise bundle.Refused() + selected = node.value.value + return selected + + +PROBE = r''' +import contextlib,io,json,logging,os,sys +logging.disable(logging.CRITICAL) +os.umask(0o077) +result={'status':'failed'} +stage='network_isolation' +try: + with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()): + from pathlib import Path + assert {p.name for p in Path('/sys/class/net').iterdir()}=={'lo'} + assert len(Path('/proc/net/route').read_text().splitlines())==1 + stage='native_application_configuration' + from privacyidea.app import create_app + app=create_app(config_name='testing',config_file='/recovery/pi.cfg',silent=True) + from privacyidea.models import db,Token + from privacyidea.lib.security.default import DefaultSecurityModule + from privacyidea.lib.auditmodules.sqlaudit import Audit,LogEntry + from privacyidea.lib.crypto import Sign + import binascii + with app.app_context(): + assert str(db.engine.url)=='postgresql+psycopg2://postgres@127.0.0.1:5432/recovery' + stage='retained_token_decryption' + security=DefaultSecurityModule({'file':'/recovery/enckey'}) + wrong_path=Path('/recovery/wrong-enckey');wrong_path.write_bytes(os.urandom(96)) + wrong=DefaultSecurityModule({'file':str(wrong_path)}) + total=Token.query.count() + assert 00 + result.update(retained_tokens_total=total,retained_tokens_with_native_decryption=checked, + different_key_cannot_reproduce_material_count=negative, + retained_token_material_decrypted_inside_process=True,existing_token_mfa_proven=False) + stage='historical_audit_verification' + audit=Audit(app.config) + allow_legacy=bool(app.config['PI_CHECK_OLD_SIGNATURES']) + historical=audit.session.query(LogEntry).count() + assert 0 bundle.LIMIT + 4 * 1024 * 1024: + raise bundle.Refused() + restored = work / 'restored';restored.mkdir(mode=0o700) + stage = 'archive_validation' + manifest, _ = bundle.restore(archive, restored) + result.update(ciphertext_sha256=transfer['ciphertext_sha256'], + manifest_file_count=len(manifest['files']), all_manifest_hashes_verified=True) + stage = 'native_privacyidea_checks' + native_checks(restored, result) + result['status'] = 'isolated_privacyidea_checked' + except (Exception, KeyboardInterrupt): + result.update(status='failed', error='isolated_privacyidea_check_failed_private_diagnostics_suppressed') + result.setdefault('failed_stage', stage) + finally: + if keyfd is not None:os.close(keyfd) + if work is not None and not work.exists():result['temporary_plaintext_cleanup_verified'] = True + result['finished_at'] = datetime.now(timezone.utc).isoformat() + with os.fdopen(fd, 'w') as output: + json.dump(result, output, indent=2);output.write('\n') + if result['status'] == 'failed': + print('ERROR: isolated privacyIDEA check failed; review safe receipt. No production change.') + return 1 + if not result['audit_signature_continuity_proven']: + print('REVIEW: token-material checks completed; historical audit exceptions require owner review. Cleanup passed; see safe receipt.') + return 2 + print('OK: retained token material decrypted internally; historical/new audit signatures and negative checks passed; cleanup passed. Existing-token MFA remains separate.') + return 0 + + +if __name__ == '__main__': + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):signal.signal(sig, local.offline.interrupted) + raise SystemExit(main()) diff --git a/tools/bootstrap-recovery/run-isolated-privacyidea.sh b/tools/bootstrap-recovery/run-isolated-privacyidea.sh new file mode 100644 index 0000000..17353f7 --- /dev/null +++ b/tools/bootstrap-recovery/run-isolated-privacyidea.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Owner-run native audit/token-material checks. Never pass a private key in argv. +set +x +set -euo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)" +cd -- "$ROOT" +umask 077 +[[ $# == 1 ]] || { printf '%s\n' 'Usage: run-isolated-privacyidea.sh VERIFIED_TRANSFER_RECEIPT'; exit 1; } +[[ -f "$1" && ! -L "$1" ]] || { printf '%s\n' 'ERROR: provide the existing verified transfer receipt.'; exit 1; } +[[ -t 0 && -t 1 && -t 2 ]] || { printf '%s\n' 'ERROR: run in your own interactive terminal.'; exit 1; } +for tool in python3 age age-keygen openssl docker; do + command -v "$tool" >/dev/null || { printf 'ERROR: required tool unavailable: %s\n' "$tool"; exit 1; } +done +python3 - <<'PY' +import importlib.util +from pathlib import Path +p=Path('tools/bootstrap-recovery/isolated-privacyidea.py') +spec=importlib.util.spec_from_file_location('isolated',p) +m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m) +try: + m.preflight() +except Exception: + raise SystemExit('ERROR: exact local Docker target, pinned images or tmpfs preflight failed; no key requested.') +print('OK: exact bnt-lap001 recovery engine, pinned images and tmpfs available.') +PY +NK49_PI_RUN="$(mktemp -d /tmp/nk49-isolated-privacyidea-XXXXXXXX)" +printf 'Safe evidence directory: %s\n' "$NK49_PI_RUN" +printf '%s\n' 'This restores the linked database and original keys only into disposable network-isolated containers.' +printf '%s\n' 'It decrypts retained token material internally, checks every stored audit row, and creates one new isolated audit entry.' +printf '%s\n' 'No MFA code is consumed. Invalid or unsigned history is reported, not repaired. Enter only the backup age key at the hidden prompt.' +if bash sso-mfa/bootstrap/sops-custody-unlock.sh \ + --expected-recipient age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz \ + --prompt -- python3 tools/bootstrap-recovery/isolated-privacyidea.py \ + --transfer-receipt "$1" --receipt "$NK49_PI_RUN/recovery.json"; then + printf 'Safe receipt: %s/recovery.json\n' "$NK49_PI_RUN" +else + NK49_PI_EXIT=$? + printf 'Check stopped or needs review. Safe receipt: %s/recovery.json\n' "$NK49_PI_RUN" + exit "$NK49_PI_EXIT" +fi +printf '%s\n' 'Return only the outcome and evidence directory. Existing-token MFA, Authelia and KeyCape remain separate.' diff --git a/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md b/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md index 371bf62..f3d136a 100644 --- a/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md +++ b/workplans/NK-WP-0049-bootstrap-secret-free-recovery-handoff.md @@ -28,8 +28,8 @@ These are retrospective task records of evidenced work, not additional delivery or newly granted authority. The earlier dated narrative remains in Git at `cc2dd01`; its evidence references are carried into the tasks below. -**Current position:** 19 tasks done; 2 in progress; 1 ready to start (`todo`); -7 waiting on explicit prerequisites. These are task counts, not an effort-based +**Current position:** 20 tasks done; 2 in progress; 1 ready to start (`todo`); +8 waiting on explicit prerequisites. These are task counts, not an effort-based completion percentage. Assignees are also written in task descriptions because the current Forge projector does not populate the dedicated Hub assignee field. The outage-retrieval tool and restore-packet draft are now prepared; independent outage retrieval and Storage Share publication are accepted; @@ -390,7 +390,7 @@ id: NK-WP-0049-T15 status: progress priority: high assignee: codex -depends_on: [NK-WP-0049-T08, NK-WP-0049-T13, NK-WP-0049-T27, NK-WP-0049-T28, NK-WP-0049-T29] +depends_on: [NK-WP-0049-T08, NK-WP-0049-T13, NK-WP-0049-T27, NK-WP-0049-T28, NK-WP-0049-T29, NK-WP-0049-T30] state_hub_task_id: "1d1eb903-a4b0-5bc4-a421-0b842d7f68ae" ``` @@ -493,7 +493,7 @@ id: NK-WP-0049-T19 status: wait priority: high assignee: codex -depends_on: [NK-WP-0049-T15, NK-WP-0049-T16] +depends_on: [NK-WP-0049-T15, NK-WP-0049-T16, NK-WP-0049-T31] blocking_reason: "LLDAP recovery T16 passed; await the exact isolated privacyIDEA adapter/procedure and owner-selected retained-token/MFA inputs." state_hub_task_id: "6f8af1a7-2a3f-5fa8-8202-83103ebd55b2" ``` @@ -510,6 +510,13 @@ refused, and cleanup is verified. Obtain any token/user input through attended private custody; never emit token plaintext, seeds or credentials. Native database row counts and disposable synthetic-token fixtures do not prove this outcome. + +2026-10-16 refinement: native token-material readback is separately tracked in +T31 using the prepared T30 executor. T19 retains actual existing-token MFA and +wrong-code/wrong-key authentication refusal. Internal decryption or padding +success does not authenticate a user. No MFA code is needed for T20/T31; the +selected user's private factor input and exact isolated MFA adapter remain T19. + ## Prove historical and new privacyIDEA audit-signature continuity ```task @@ -517,8 +524,8 @@ id: NK-WP-0049-T20 status: wait priority: high assignee: codex -depends_on: [NK-WP-0049-T15] -blocking_reason: "Await the accepted isolated target/procedure and owner-admitted historical/new audit verification from T15." +depends_on: [NK-WP-0049-T13, NK-WP-0049-T30] +blocking_reason: "Await the private owner run of the prepared T30 audit/token-material executor with the independent backup age key; exact local target, original-key preservation and native fixtures are prepared." state_hub_task_id: "375fa8f9-b8a5-5048-97a7-ec141aa7c45d" ``` @@ -534,6 +541,16 @@ key fails, and cleanup is recorded. Report any historical verification failure for explicit owner disposition; do not regenerate keys to hide it. Matching RSA pairs from T13 are a prerequisite, not audit-history verification. + +2026-10-16: the bounded audit stage is prepared in T30 independently of the full +multi-service T15 packet. **Unblock:** owner runs +`bash tools/bootstrap-recovery/run-isolated-privacyidea.sh /tmp/nk49-offline-handoff-1X660a5t/transfer.json` +in a private terminal and returns the safe outcome/receipt. It requests only the +backup age key, never MFA or provider credentials. All stored historical rows +are checked; unsigned/invalid history is reported for explicit owner disposition +and never rewritten. Exit2 is a review outcome, not completion. Existing-token +MFA and event completeness are not claimed by this audit stage. + ## Prove restored KeyCape issuer, policy and consumer continuity ```task @@ -827,6 +844,65 @@ acceptance remain T16; fixture success is not actual user/MFA/audit recovery. See `isolated-lldap-preparation.json` in the October16 outage evidence directory for exact test/source/target provenance and limitations. +## Prepare the bounded native privacyIDEA token-material and audit executor + +```task +id: NK-WP-0049-T30 +status: done +priority: high +assignee: codex +depends_on: [NK-WP-0049-T13, NK-WP-0049-T28] +``` + +**Accountable worker:** codex; operator retains private recovery inputs. + +Completed 2026-10-16. `isolated-privacyidea.py` and its owner launcher use the +fixed local Docker target and exact cached PostgreSQL/privacyIDEA3.12.2 images. +The linked dump, original enckey/audit pair and required secret fields are +restored only into a loopback-only, nonroot, read-only, capability-free, +no-host-mount/port, tmpfs-backed environment with inspected pre-import isolation. +Decrypted host staging is tmpfs; normal cleanup removes all private data and +owned containers, with a bounded holding lifetime if the parent is interrupted. + +The native provider decrypts token-key material internally, checks every stored +audit row, adds one isolated signed entry and checks wrong-public-key/tampering +negatives. Original policy for legacy signature verification is retained only +from literal archived configuration; archived Python is never executed. Invalid +or unsigned history remains reported, not repaired or sampled away. Native +material readback is not existing-token MFA or proof of event completeness. + +Evidence: `tests/test_isolated_privacyidea_recovery.py` and +`isolated-privacyidea-preparation.json` in the October16 outage evidence folder. +Synthetic pinned-image tests exercise valid/tampered/unsigned history, malformed +keys, encrypted archive to native restore, isolation and cleanup. Real original +material/audit acceptance remains T31/T20; actual MFA remains T19. + +## Verify retained privacyIDEA token material inside the isolated provider + +```task +id: NK-WP-0049-T31 +status: wait +priority: high +assignee: codex +depends_on: [NK-WP-0049-T13, NK-WP-0049-T30] +blocking_reason: "Await the owner's private age-key run of the prepared T30 executor against the independently retrieved actual bundle; synthetic native proof is complete." +``` + +**Accountable worker:** codex; operator supplies only the independently held age +key in the hidden helper. This task extracts actual material-readback scope from +T19, preserving T19's existing-token MFA acceptance gate and historical identity. + +**Unblock:** the same private `run-isolated-privacyidea.sh` run used for T20. +**Done when:** exact ciphertext/source/target/manifest bindings pass; the native +provider reads every nonempty encrypted token-key field using the original +enckey without exporting material; an unrelated key cannot reproduce that +material; aggregate counts and container/plaintext cleanup are recorded. + +Only bounded internal readback is accepted here. Padding/readback success does +not prove the recovered secret authenticates a retained factor; actual positive +MFA and wrong-key/wrong-code authentication refusal remain T19. No token seed, +plaintext, serial or user record belongs in the receipt, Git, State Hub or chat. + ## Accept remaining recovery and caller evidence and close the G08 handoff ```task @@ -846,7 +922,7 @@ Owner boundary: NetKingdom, Platform and affected service owners, with Custodian G08 receiving the final evidence. The completed archive/job milestones remain accepted independently of this final gate. -**Done when:** T02 and T15–T29 have their stated completed or explicitly accepted +**Done when:** T02 and T15–T31 have their stated completed or explicitly accepted handoff outcomes; service authentication/token/audit proof and current/legacy caller dispositions are accepted; the source workplan and projected tasks agree; and G08 has an exact evidence/handoff record. Record actionable residuals as live