Prepare native isolated directory recovery and offline service images

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a13eab-1e6a-7b33-9c37-372e90ada3a2
This commit is contained in:
tegwick 2026-10-16 20:33:57 +02:00
parent 31f483966c
commit d5484bb90a
9 changed files with 815 additions and 7 deletions

View file

@ -18,6 +18,16 @@ Actual owner publication/readback and outage-path archive recovery passed on
Service restore admission remains in the
[isolated restore packet](../../docs/identity-service-restore-packet.md).
The first native-service stage is `run-isolated-lldap.sh USER TRANSFER_RECEIPT`.
It preflights the exact local Docker engine/images before asking privately for
the age key and backup-era existing-user password. `isolated-lldap.py` validates
the encrypted archive, restores original directory/key material only into
network-none/read-only/tmpfs containers, proves native LDAP bind and explicit
invalid-credential denial, compares durable identity rows and removes its owned
containers/plaintext. It uses no live Kubernetes/OpenBao endpoint or provider
grant. Actual-image synthetic tests pass; a real owner run is still required
for T16. See the packet for the exact target, resource limits and private command.
`verify-bundle.py` validates a digest-bound age-encrypted `secrets/` tar archive
using an explicit private temporary key and registered `keys/age.pub`. It checks
an exact required-file list, unrelated wrong-key refusal, safe isolated file

View file

@ -0,0 +1,343 @@
#!/usr/bin/env python3
"""Owner-run native LLDAP restore from the verified identity bundle.
Private inputs remain in the owner process/pipe. Decrypted staging uses Linux
tmpfs. Containers have no network, ports, host mounts or production API access.
"""
import argparse
import base64
from datetime import datetime, timezone
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import signal
import sqlite3
import subprocess
import tarfile
import tempfile
import time
import tomllib
import uuid
HERE = Path(__file__).resolve().parent
def load(name):
spec = importlib.util.spec_from_file_location(name, HERE / (name + '.py'))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
verify = load('verify-identity')
offline = load('retrieve-identity-offline')
bundle = verify.bundle
IMAGE = 'docker.io/lldap/lldap@sha256:9e605a66c02514bfcffd1b67cafb1e98d50992216bb2871d7ae44622047dd09d'
CLIENT = 'ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473'
ENGINE_ID = '74f93783-6cef-43a8-a3c4-f62586cd3bc1'
ENGINE_NAME = 'bnt-lap001'
def command(args, **kwargs):
result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
timeout=kwargs.pop('timeout', 90), **kwargs)
if result.returncode:
raise bundle.Refused()
return result.stdout
def preflight():
contexts = json.loads(command(['docker', 'context', 'inspect']))
endpoints = [contexts[0]['Endpoints']['docker']['Host']]
if os.environ.get('DOCKER_HOST'):
endpoints.append(os.environ['DOCKER_HOST'])
if not all(url.startswith('unix://') for url in endpoints):
raise bundle.Refused()
facts = json.loads(command(['docker', 'info', '--format',
'{"ID":{{json .ID}},"Name":{{json .Name}},"OSType":{{json .OSType}}}']))
if facts['ID'] != ENGINE_ID or facts['Name'] != ENGINE_NAME or facts['OSType'] != 'linux':
raise bundle.Refused()
# /dev/shm must actually be tmpfs, not a similarly named disk directory.
mounts = Path('/proc/mounts').read_text().splitlines()
if not any(line.split()[1:3] == ['/dev/shm', 'tmpfs'] for line in mounts):
raise bundle.Refused()
images = []
for image in (IMAGE, CLIENT):
image_id = command(['docker', 'image', 'inspect', '--format', '{{.Id}}', image]).decode().strip()
images.append({'reference': image, 'image_id': image_id})
return {'engine_id': facts['ID'], 'engine_name': facts['Name'], 'images': images,
'production_contacted': False, 'network_mode': 'none', 'host_ports_published': False}
def snapshot(path):
# Include all fields of durable identity tables, including stored password
# hashes. Only counts and the equality boolean leave this process.
result = {}
with sqlite3.connect(path.as_uri() + '?mode=ro', uri=True) as db:
if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
raise bundle.Refused()
tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
for name in ('users', 'groups', 'memberships', 'user_attributes', 'group_attributes',
'user_attribute_values', 'group_attribute_values',
'user_attribute_schema', 'group_attribute_schema'):
if name in tables:
rows = db.execute('SELECT * FROM "' + name + '"').fetchall()
encoded = sorted(repr(row) for row in rows)
result[name] = {'count': len(rows), 'digest': hashlib.sha256('\n'.join(encoded).encode()).digest()}
if not {'users', 'groups', 'memberships'} <= result.keys():
raise bundle.Refused()
return result
def configuration(root):
data = root / 'data/lldap'
existing = data / 'lldap_config.toml'
config = tomllib.loads(existing.read_text()) if existing.is_file() else {}
deployment = bundle.resource(root, 'sso', 'deployment', 'lldap')
main = next(c for c in deployment['spec']['template']['spec']['containers'] if c['name'] == 'lldap')
env = {e['name']: e['value'] for e in main.get('env', []) if 'value' in e}
secret = bundle.resource(root, 'sso', 'secret', 'lldap-secrets')['data']
jwt = base64.b64decode(secret['LLDAP_JWT_SECRET'], validate=True)
password = base64.b64decode(secret['LLDAP_LDAP_USER_PASS'], validate=True)
if not jwt or not password or b'\n' in jwt or b'\n' in password:
raise bundle.Refused()
base = env.get('LLDAP_LDAP_BASE_DN', config.get('ldap_base_dn'))
user = env.get('LLDAP_LDAP_USER_DN', config.get('ldap_user_dn', 'admin'))
if not isinstance(base, str) or not base or not isinstance(user, str):
raise bundle.Refused()
chosen = {'database_url': 'sqlite:///data/users.db?mode=rwc',
'ldap_base_dn': base, 'ldap_user_dn': user,
'ldap_host': '127.0.0.1', 'http_host': '127.0.0.1',
'ldap_port': 3890, 'http_port': 17170,
'force_ldap_user_pass_reset': False, 'force_update_private_key': False}
seed = env.get('LLDAP_KEY_SEED', config.get('key_seed'))
if seed:
if not isinstance(seed, str):
raise bundle.Refused()
chosen['key_seed'] = seed
else:
value = env.get('LLDAP_KEY_FILE', config.get('key_file', '/data/private_key'))
key = PurePosixPath(value)
if not key.is_absolute():
key = PurePosixPath('/data') / key
if key.parent != PurePosixPath('/data') or not (data / key.name).is_file():
raise bundle.Refused()
chosen['key_file'] = str(key)
# JSON strings are valid TOML basic strings for these service inputs.
raw = '\n'.join(k + ' = ' + json.dumps(v) for k, v in chosen.items())
raw += '\n[smtp_options]\nenable_password_reset = false\n[ldaps_options]\nenabled = false\n'
return raw.encode(), jwt, password, base
def archive_files(files):
out = io.BytesIO()
with tarfile.open(fileobj=out, mode='w') as archive:
for name, value in files.items():
if PurePosixPath(name).is_absolute() or '..' in PurePosixPath(name).parts:
raise bundle.Refused()
entry = tarfile.TarInfo(name)
entry.size = len(value)
entry.mode = 0o600
entry.uid = entry.gid = 1000
archive.addfile(entry, io.BytesIO(value))
return out.getvalue()
PROBE = r'''
import contextlib,io,json,logging,socket,sys,time
logging.disable(logging.CRITICAL)
result={'status':'failed'}
stage='network_isolation'
try:
with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()):
from pathlib import Path
assert {p.name for p in Path('/sys/class/net').iterdir()}=={'lo'}
assert len(Path('/proc/net/route').read_text().splitlines())==1
stage='native_client_import'
from ldap3 import Server,Connection,NONE
from ldap3.utils.dn import escape_rdn
values=json.load(sys.stdin)
stage='directory_startup'
for _ in range(60):
try:
with socket.create_connection(('127.0.0.1',3890),timeout=1):pass
break
except OSError:time.sleep(.25)
else:raise RuntimeError()
dn='uid='+escape_rdn(values['user'])+',ou=people,'+values['base']
server=Server('127.0.0.1',port=3890,get_info=NONE,connect_timeout=3)
stage='existing_user_bind'
connection=Connection(server,user=dn,password=values['password'],auto_bind=True,raise_exceptions=True)
assert connection.bound
connection.unbind()
stage='wrong_password_bind'
wrong=Connection(server,user=dn,password=values['wrong_password'],auto_bind=False,raise_exceptions=False)
refused=not wrong.bind() and wrong.result.get('result')==49
wrong.unbind()
assert refused
result={'status':'verified','existing_user_native_ldap_bind':True,'wrong_password_refused':True,
'only_loopback_interface_verified':True,'external_routes_absent':True}
except Exception as error:result={'status':'failed','failed_stage':stage,'failure_type':type(error).__name__}
print(json.dumps(result))
'''
def restore_lldap(root, user, password, receipt):
before = snapshot(root / 'data/lldap/users.db')
config, jwt, admin_password, base = configuration(root)
name = 'nk49-lldap-' + uuid.uuid4().hex
client_name = name + '-client'
try:
command(['docker', 'run', '--detach', '--pull=never', '--name', name,
'--network', 'none', '--read-only', '--cap-drop', 'ALL',
'--security-opt', 'no-new-privileges', '--user', '1000:1000',
'--memory', '256m', '--cpus', '1', '--pids-limit', '128',
'--tmpfs', '/data:rw,nosuid,nodev,noexec,size=268435456,uid=1000,gid=1000,mode=0700',
'--tmpfs', '/run/recovery:rw,nosuid,nodev,noexec,size=1048576,uid=1000,gid=1000,mode=0700',
'--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000',
'--env', 'LLDAP_JWT_SECRET_FILE=/run/recovery/jwt',
'--env', 'LLDAP_LDAP_USER_PASS_FILE=/run/recovery/admin-password',
'--entrypoint', '/bin/sh', IMAGE, '-c', 'sleep 1800'])
guard = json.loads(command(['docker', 'container', 'inspect', name, '--format',
'{"host":{{json .HostConfig}},"mounts":{{json .Mounts}},"user":{{json .Config.User}}}']))
host = guard['host']
if (host['NetworkMode'] != 'none' or not host['ReadonlyRootfs'] or host['Privileged']
or host.get('Binds') or host.get('PortBindings') or 'ALL' not in host.get('CapDrop', [])
or 'no-new-privileges' not in host.get('SecurityOpt', [])
or guard['user'] != '1000:1000'
or any(mount['Type'] != 'tmpfs' for mount in guard['mounts'])):
raise bundle.Refused()
receipt['container_isolation_verified_before_private_import'] = True
payload = {p.relative_to(root / 'data/lldap').as_posix(): p.read_bytes()
for p in (root / 'data/lldap').rglob('*') if p.is_file()}
command(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/data'], input=archive_files(payload))
command(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/run/recovery'],
input=archive_files({'lldap.toml': config, 'jwt': jwt, 'admin-password': admin_password}))
# No credential values in Docker configuration, environment or argv.
command(['docker', 'exec', '--detach', name, '/bin/sh', '-c',
'echo $$ > /run/recovery/server.pid; exec /app/lldap run --config-file /run/recovery/lldap.toml > /dev/null 2>&1'])
raw = command(['docker', 'run', '--interactive', '--rm', '--name', client_name, '--pull=never', '--network', 'container:' + name,
'--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
'--memory', '256m', '--cpus', '1', '--pids-limit', '64',
'--entrypoint', 'python3', CLIENT, '-c', PROBE],
input=json.dumps({'user': user, 'password': password, 'base': base,
'wrong_password': 'nk49-wrong-' + uuid.uuid4().hex}).encode())
result = json.loads(raw)
receipt.update(result)
if result.get('status') != 'verified':
raise bundle.Refused()
command(['docker', 'exec', name, '/bin/sh', '-c', 'kill -TERM "$(cat /run/recovery/server.pid)"'])
for _ in range(60):
running = subprocess.run(['docker', 'exec', name, '/bin/sh', '-c',
'kill -0 "$(cat /run/recovery/server.pid)"'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10)
if running.returncode:
break
time.sleep(.1)
else:
raise bundle.Refused()
raw = command(['docker', 'exec', name, '/bin/sh', '-c',
'test ! -s /data/users.db-wal && cat /data/users.db'])
after_path = root / 'post-restore-users.db'
after_path.write_bytes(raw)
if snapshot(after_path) != before:
raise bundle.Refused()
receipt.update(directory_identity_rows_preserved=True,
directory_comparison_scope=sorted(before),
directory_table_counts={k: v['count'] for k, v in before.items()})
finally:
# A failed create can still have created a container (e.g. lost reply).
for owned in (client_name, name):
found = subprocess.run(['docker', 'container', 'inspect', owned], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=15)
if found.returncode == 0:
command(['docker', 'rm', '--force', '--volumes', owned])
absent = subprocess.run(['docker', 'container', 'inspect', owned], stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=15)
command(['docker', 'info', '--format', '{{.ID}}'])
if absent.returncode == 0:
raise bundle.Refused()
receipt['container_cleanup_verified'] = True
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--transfer-receipt', required=True)
parser.add_argument('--receipt', required=True)
parser.add_argument('--user', required=True)
args = parser.parse_args()
os.umask(0o077)
fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
result = {'schema': 'netkingdom.isolated-lldap-recovery.v1', 'status': 'failed',
'production_contacted': False, 'secret_values_exported': False,
'container_cleanup_verified': False, 'temporary_plaintext_cleanup_verified': False,
'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}
keyfd = None
work = None
stage = 'target_preflight'
try:
result['target'] = preflight()
stage = 'input_validation'
if not re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', args.user):
raise bundle.Refused()
transfer = json.loads(Path(args.transfer_receipt).read_text())
offline.validate(transfer)
if transfer.get('independent_owner_retrieval_verified') is not True:
raise bundle.Refused()
path = Path(transfer['fetched_path'])
bundle.private(path, max_bytes=bundle.LIMIT)
if path.stat().st_size != transfer['ciphertext_bytes'] or offline.offsite.digest(path) != transfer['ciphertext_sha256']:
raise bundle.Refused()
keyfd = verify.custody.private_key_fd()
derived = command(['age-keygen', '-y', f'/proc/self/fd/{keyfd}'], pass_fds=(keyfd,)).strip()
if derived != bundle.profile()['public_recipient'].encode():
raise bundle.Refused()
stage = 'private_existing_password_prompt'
password = offline.private_prompt('Existing account password for the isolated LLDAP check: ')
with tempfile.TemporaryDirectory(prefix='nk49-service-recovery-', dir='/dev/shm') as temporary:
work = Path(temporary)
archive = work / 'archive.tar'
stage = 'archive_decryption'
with archive.open('xb') as output:
proc = subprocess.run(['age', '-d', '-i', f'/proc/self/fd/{keyfd}', str(path)],
stdout=output, stderr=subprocess.DEVNULL, pass_fds=(keyfd,), timeout=180)
if proc.returncode or archive.stat().st_size > bundle.LIMIT + 4 * 1024 * 1024:
raise bundle.Refused()
restored = work / 'restored'
restored.mkdir(mode=0o700)
stage = 'archive_validation'
manifest, _ = bundle.restore(archive, restored)
result.update(ciphertext_sha256=transfer['ciphertext_sha256'],
manifest_file_count=len(manifest['files']), all_manifest_hashes_verified=True)
stage = 'native_directory_check'
restore_lldap(restored, args.user, password, result)
password = None
if work.exists():
raise bundle.Refused()
result.update(status='isolated_lldap_verified', temporary_plaintext_cleanup_verified=True)
except (Exception, KeyboardInterrupt):
result.update(status='failed', error='isolated_lldap_check_failed_private_diagnostics_suppressed')
result.setdefault('failed_stage', stage)
finally:
if keyfd is not None:
os.close(keyfd)
if work is not None and not work.exists():
result['temporary_plaintext_cleanup_verified'] = True
result['finished_at'] = datetime.now(timezone.utc).isoformat()
with os.fdopen(fd, 'w') as output:
json.dump(result, output, indent=2)
output.write('\n')
success = result['status'] == 'isolated_lldap_verified'
print('OK: restored LLDAP existing-user bind, wrong-password refusal, directory preservation and cleanup passed.'
if success else 'ERROR: isolated LLDAP recovery refused; review the safe receipt. No production change.')
return 0 if success else 1
if __name__ == '__main__':
for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
signal.signal(sig, offline.interrupted)
raise SystemExit(main())

View file

@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Owner-run existing-user check. No private key/password belongs in arguments.
set +x
set -euo pipefail
ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)"
cd -- "$ROOT"
umask 077
[[ $# == 2 ]] || { printf '%s\n' 'Usage: run-isolated-lldap.sh USER VERIFIED_TRANSFER_RECEIPT'; exit 1; }
[[ "$1" =~ ^[A-Za-z0-9_.-]{1,128}$ ]] || { printf '%s\n' 'ERROR: invalid username.'; exit 1; }
[[ -f "$2" && ! -L "$2" ]] || { printf '%s\n' 'ERROR: provide the existing verified transfer receipt.'; exit 1; }
[[ -t 0 && -t 1 && -t 2 ]] || { printf '%s\n' 'ERROR: run in your own interactive terminal.'; exit 1; }
for tool in python3 age age-keygen openssl docker; do
command -v "$tool" >/dev/null || { printf 'ERROR: required tool unavailable: %s\n' "$tool"; exit 1; }
done
python3 - <<'PY'
import importlib.util
from pathlib import Path
p=Path('tools/bootstrap-recovery/isolated-lldap.py')
spec=importlib.util.spec_from_file_location('isolated',p)
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
try:
m.preflight()
except Exception:
raise SystemExit('ERROR: exact local Docker target, pinned images or tmpfs preflight failed; no key requested.')
print('OK: exact bnt-lap001 recovery engine, pinned images and tmpfs available.')
PY
NK49_LLDAP_RUN="$(mktemp -d /tmp/nk49-isolated-lldap-XXXXXXXX)"
printf 'Safe evidence directory: %s\n' "$NK49_LLDAP_RUN"
printf '%s\n' 'This restores only LLDAP into disposable containers with no external network, ports or host mounts.'
printf '%s\n' 'Enter the railiance-backup age key, then the existing account password that was valid at backup capture.'
printf '%s\n' 'The check rejects wrong passwords and preserves original users, groups and memberships. It changes no production service.'
bash sso-mfa/bootstrap/sops-custody-unlock.sh \
--expected-recipient age1zvryunvjhvpkmasskauga2heeg0ztnte9ymgppvjge36ekumk50syr3tsz \
--prompt -- python3 tools/bootstrap-recovery/isolated-lldap.py \
--user "$1" --transfer-receipt "$2" --receipt "$NK49_LLDAP_RUN/recovery.json"
printf 'Safe receipt: %s/recovery.json\n' "$NK49_LLDAP_RUN"
printf '%s\n' 'Return only the outcome and evidence directory. Authelia/KeyCape/MFA/audit acceptance remains separate.'