net-kingdom/tools/bootstrap-recovery/isolated-lldap.py

#!/usr/bin/env python3
"""Owner-run native LLDAP restore from the verified identity bundle.

Private inputs remain in the owner process/pipe. Decrypted staging uses Linux
tmpfs. Containers have no network, ports, host mounts or production API access.
"""
import argparse
import base64
from datetime import datetime, timezone
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path, PurePosixPath
import re
import signal
import sqlite3
import subprocess
import tarfile
import tempfile
import time
import tomllib
import uuid

HERE = Path(__file__).resolve().parent


def load(name):
    spec = importlib.util.spec_from_file_location(name, HERE / (name + '.py'))
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


verify = load('verify-identity')
offline = load('retrieve-identity-offline')
bundle = verify.bundle
IMAGE = 'docker.io/lldap/lldap@sha256:9e605a66c02514bfcffd1b67cafb1e98d50992216bb2871d7ae44622047dd09d'
CLIENT = 'ghcr.io/gpappsoft/privacyidea-docker@sha256:af7841adad262f129e0c1d4f553af13f21cb2f4dc713533f316cfe43ed0b4473'
ENGINE_ID = '74f93783-6cef-43a8-a3c4-f62586cd3bc1'
ENGINE_NAME = 'bnt-lap001'


def command(args, **kwargs):
    result = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
                            timeout=kwargs.pop('timeout', 90), **kwargs)
    if result.returncode:
        raise bundle.Refused()
    return result.stdout


def preflight():
    contexts = json.loads(command(['docker', 'context', 'inspect']))
    endpoints = [contexts[0]['Endpoints']['docker']['Host']]
    if os.environ.get('DOCKER_HOST'):
        endpoints.append(os.environ['DOCKER_HOST'])
    if not all(url.startswith('unix://') for url in endpoints):
        raise bundle.Refused()
    facts = json.loads(command(['docker', 'info', '--format',
                               '{"ID":{{json .ID}},"Name":{{json .Name}},"OSType":{{json .OSType}}}']))
    if facts['ID'] != ENGINE_ID or facts['Name'] != ENGINE_NAME or facts['OSType'] != 'linux':
        raise bundle.Refused()
    # /dev/shm must actually be tmpfs, not a similarly named disk directory.
    mounts = Path('/proc/mounts').read_text().splitlines()
    if not any(line.split()[1:3] == ['/dev/shm', 'tmpfs'] for line in mounts):
        raise bundle.Refused()
    images = []
    for image in (IMAGE, CLIENT):
        image_id = command(['docker', 'image', 'inspect', '--format', '{{.Id}}', image]).decode().strip()
        images.append({'reference': image, 'image_id': image_id})
    return {'engine_id': facts['ID'], 'engine_name': facts['Name'], 'images': images,
            'production_contacted': False, 'network_mode': 'none', 'host_ports_published': False}


def snapshot(path):
    # Include all fields of durable identity tables, including stored password
    # hashes. Only counts and the equality boolean leave this process.
    result = {}
    with sqlite3.connect(path.as_uri() + '?mode=ro', uri=True) as db:
        if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]:
            raise bundle.Refused()
        tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")}
        for name in ('users', 'groups', 'memberships', 'user_attributes', 'group_attributes',
                     'user_attribute_values', 'group_attribute_values',
                     'user_attribute_schema', 'group_attribute_schema'):
            if name in tables:
                rows = db.execute('SELECT * FROM "' + name + '"').fetchall()
                encoded = sorted(repr(row) for row in rows)
                result[name] = {'count': len(rows), 'digest': hashlib.sha256('\n'.join(encoded).encode()).digest()}
        if not {'users', 'groups', 'memberships'} <= result.keys():
            raise bundle.Refused()
    return result


def configuration(root):
    data = root / 'data/lldap'
    existing = data / 'lldap_config.toml'
    config = tomllib.loads(existing.read_text()) if existing.is_file() else {}
    deployment = bundle.resource(root, 'sso', 'deployment', 'lldap')
    main = next(c for c in deployment['spec']['template']['spec']['containers'] if c['name'] == 'lldap')
    env = {e['name']: e['value'] for e in main.get('env', []) if 'value' in e}
    secret = bundle.resource(root, 'sso', 'secret', 'lldap-secrets')['data']
    jwt = base64.b64decode(secret['LLDAP_JWT_SECRET'], validate=True)
    password = base64.b64decode(secret['LLDAP_LDAP_USER_PASS'], validate=True)
    if not jwt or not password or b'\n' in jwt or b'\n' in password:
        raise bundle.Refused()
    base = env.get('LLDAP_LDAP_BASE_DN', config.get('ldap_base_dn'))
    user = env.get('LLDAP_LDAP_USER_DN', config.get('ldap_user_dn', 'admin'))
    if not isinstance(base, str) or not base or not isinstance(user, str):
        raise bundle.Refused()
    chosen = {'database_url': 'sqlite:///data/users.db?mode=rwc',
              'ldap_base_dn': base, 'ldap_user_dn': user,
              'ldap_host': '127.0.0.1', 'http_host': '127.0.0.1',
              'ldap_port': 3890, 'http_port': 17170,
              'force_ldap_user_pass_reset': False, 'force_update_private_key': False}
    seed = env.get('LLDAP_KEY_SEED', config.get('key_seed'))
    if seed:
        if not isinstance(seed, str):
            raise bundle.Refused()
        chosen['key_seed'] = seed
    else:
        value = env.get('LLDAP_KEY_FILE', config.get('key_file', '/data/private_key'))
        key = PurePosixPath(value)
        if not key.is_absolute():
            key = PurePosixPath('/data') / key
        if key.parent != PurePosixPath('/data') or not (data / key.name).is_file():
            raise bundle.Refused()
        chosen['key_file'] = str(key)
    # JSON strings are valid TOML basic strings for these service inputs.
    raw = '\n'.join(k + ' = ' + json.dumps(v) for k, v in chosen.items())
    raw += '\n[smtp_options]\nenable_password_reset = false\n[ldaps_options]\nenabled = false\n'
    return raw.encode(), jwt, password, base


def archive_files(files):
    out = io.BytesIO()
    with tarfile.open(fileobj=out, mode='w') as archive:
        for name, value in files.items():
            if PurePosixPath(name).is_absolute() or '..' in PurePosixPath(name).parts:
                raise bundle.Refused()
            entry = tarfile.TarInfo(name)
            entry.size = len(value)
            entry.mode = 0o600
            entry.uid = entry.gid = 1000
            archive.addfile(entry, io.BytesIO(value))
    return out.getvalue()


PROBE = r'''
import contextlib,io,json,logging,socket,sys,time
logging.disable(logging.CRITICAL)
result={'status':'failed'}
stage='network_isolation'
try:
    with contextlib.redirect_stdout(io.StringIO()),contextlib.redirect_stderr(io.StringIO()):
        from pathlib import Path
        assert {p.name for p in Path('/sys/class/net').iterdir()}=={'lo'}
        assert len(Path('/proc/net/route').read_text().splitlines())==1
        stage='native_client_import'
        from ldap3 import Server,Connection,NONE
        from ldap3.utils.dn import escape_rdn
        values=json.load(sys.stdin)
        stage='directory_startup'
        for _ in range(60):
            try:
                with socket.create_connection(('127.0.0.1',3890),timeout=1):pass
                break
            except OSError:time.sleep(.25)
        else:raise RuntimeError()
        dn='uid='+escape_rdn(values['user'])+',ou=people,'+values['base']
        server=Server('127.0.0.1',port=3890,get_info=NONE,connect_timeout=3)
        stage='existing_user_bind'
        connection=Connection(server,user=dn,password=values['password'],auto_bind=True,raise_exceptions=True)
        assert connection.bound
        connection.unbind()
        stage='wrong_password_bind'
        wrong=Connection(server,user=dn,password=values['wrong_password'],auto_bind=False,raise_exceptions=False)
        refused=not wrong.bind() and wrong.result.get('result')==49
        wrong.unbind()
        assert refused
        result={'status':'verified','existing_user_native_ldap_bind':True,'wrong_password_refused':True,
                'only_loopback_interface_verified':True,'external_routes_absent':True}
except Exception as error:result={'status':'failed','failed_stage':stage,'failure_type':type(error).__name__}
print(json.dumps(result))
'''


def restore_lldap(root, user, password, receipt):
    before = snapshot(root / 'data/lldap/users.db')
    config, jwt, admin_password, base = configuration(root)
    name = 'nk49-lldap-' + uuid.uuid4().hex
    client_name = name + '-client'
    try:
        command(['docker', 'run', '--detach', '--pull=never', '--name', name,
                 '--network', 'none', '--read-only', '--cap-drop', 'ALL',
                 '--security-opt', 'no-new-privileges', '--user', '1000:1000',
                 '--memory', '256m', '--cpus', '1', '--pids-limit', '128',
                 '--tmpfs', '/data:rw,nosuid,nodev,noexec,size=268435456,uid=1000,gid=1000,mode=0700',
                 '--tmpfs', '/run/recovery:rw,nosuid,nodev,noexec,size=1048576,uid=1000,gid=1000,mode=0700',
                 '--tmpfs', '/tmp:rw,nosuid,nodev,noexec,size=16777216,uid=1000,gid=1000',
                 '--env', 'LLDAP_JWT_SECRET_FILE=/run/recovery/jwt',
                 '--env', 'LLDAP_LDAP_USER_PASS_FILE=/run/recovery/admin-password',
                 '--entrypoint', '/bin/sh', IMAGE, '-c', 'sleep 1800'])
        guard = json.loads(command(['docker', 'container', 'inspect', name, '--format',
                                    '{"host":{{json .HostConfig}},"mounts":{{json .Mounts}},"user":{{json .Config.User}}}']))
        host = guard['host']
        if (host['NetworkMode'] != 'none' or not host['ReadonlyRootfs'] or host['Privileged']
                or host.get('Binds') or host.get('PortBindings') or 'ALL' not in host.get('CapDrop', [])
                or 'no-new-privileges' not in host.get('SecurityOpt', [])
                or guard['user'] != '1000:1000'
                or any(mount['Type'] != 'tmpfs' for mount in guard['mounts'])):
            raise bundle.Refused()
        receipt['container_isolation_verified_before_private_import'] = True
        payload = {p.relative_to(root / 'data/lldap').as_posix(): p.read_bytes()
                   for p in (root / 'data/lldap').rglob('*') if p.is_file()}
        command(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/data'], input=archive_files(payload))
        command(['docker', 'exec', '-i', name, 'tar', '-xf', '-', '-C', '/run/recovery'],
                input=archive_files({'lldap.toml': config, 'jwt': jwt, 'admin-password': admin_password}))
        # No credential values in Docker configuration, environment or argv.
        command(['docker', 'exec', '--detach', name, '/bin/sh', '-c',
                 'echo $$ > /run/recovery/server.pid; exec /app/lldap run --config-file /run/recovery/lldap.toml > /dev/null 2>&1'])
        raw = command(['docker', 'run', '--interactive', '--rm', '--name', client_name, '--pull=never', '--network', 'container:' + name,
                       '--read-only', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
                       '--memory', '256m', '--cpus', '1', '--pids-limit', '64',
                       '--entrypoint', 'python3', CLIENT, '-c', PROBE],
                      input=json.dumps({'user': user, 'password': password, 'base': base,
                                        'wrong_password': 'nk49-wrong-' + uuid.uuid4().hex}).encode())
        result = json.loads(raw)
        receipt.update(result)
        if result.get('status') != 'verified':
            raise bundle.Refused()
        command(['docker', 'exec', name, '/bin/sh', '-c', 'kill -TERM "$(cat /run/recovery/server.pid)"'])
        for _ in range(60):
            running = subprocess.run(['docker', 'exec', name, '/bin/sh', '-c',
                                      'kill -0 "$(cat /run/recovery/server.pid)"'],
                                     stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=10)
            if running.returncode:
                break
            time.sleep(.1)
        else:
            raise bundle.Refused()
        raw = command(['docker', 'exec', name, '/bin/sh', '-c',
                       'test ! -s /data/users.db-wal && cat /data/users.db'])
        after_path = root / 'post-restore-users.db'
        after_path.write_bytes(raw)
        if snapshot(after_path) != before:
            raise bundle.Refused()
        receipt.update(directory_identity_rows_preserved=True,
                       directory_comparison_scope=sorted(before),
                       directory_table_counts={k: v['count'] for k, v in before.items()})
    finally:
        # A failed create can still have created a container (e.g. lost reply).
        for owned in (client_name, name):
            found = subprocess.run(['docker', 'container', 'inspect', owned], stdout=subprocess.DEVNULL,
                                   stderr=subprocess.DEVNULL, timeout=15)
            if found.returncode == 0:
                command(['docker', 'rm', '--force', '--volumes', owned])
            absent = subprocess.run(['docker', 'container', 'inspect', owned], stdout=subprocess.DEVNULL,
                                    stderr=subprocess.DEVNULL, timeout=15)
            command(['docker', 'info', '--format', '{{.ID}}'])
            if absent.returncode == 0:
                raise bundle.Refused()
        receipt['container_cleanup_verified'] = True


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--transfer-receipt', required=True)
    parser.add_argument('--receipt', required=True)
    parser.add_argument('--user', required=True)
    args = parser.parse_args()
    os.umask(0o077)
    fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
    result = {'schema': 'netkingdom.isolated-lldap-recovery.v1', 'status': 'failed',
              'production_contacted': False, 'secret_values_exported': False,
              'container_cleanup_verified': False, 'temporary_plaintext_cleanup_verified': False,
              'source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}
    keyfd = None
    work = None
    stage = 'target_preflight'
    try:
        result['target'] = preflight()
        stage = 'input_validation'
        if not re.fullmatch(r'[A-Za-z0-9_.-]{1,128}', args.user):
            raise bundle.Refused()
        transfer = json.loads(Path(args.transfer_receipt).read_text())
        offline.validate(transfer)
        if transfer.get('independent_owner_retrieval_verified') is not True:
            raise bundle.Refused()
        path = Path(transfer['fetched_path'])
        bundle.private(path, max_bytes=bundle.LIMIT)
        if path.stat().st_size != transfer['ciphertext_bytes'] or offline.offsite.digest(path) != transfer['ciphertext_sha256']:
            raise bundle.Refused()
        keyfd = verify.custody.private_key_fd()
        derived = command(['age-keygen', '-y', f'/proc/self/fd/{keyfd}'], pass_fds=(keyfd,)).strip()
        if derived != bundle.profile()['public_recipient'].encode():
            raise bundle.Refused()
        stage = 'private_existing_password_prompt'
        password = offline.private_prompt('Existing account password for the isolated LLDAP check: ')
        with tempfile.TemporaryDirectory(prefix='nk49-service-recovery-', dir='/dev/shm') as temporary:
            work = Path(temporary)
            archive = work / 'archive.tar'
            stage = 'archive_decryption'
            with archive.open('xb') as output:
                proc = subprocess.run(['age', '-d', '-i', f'/proc/self/fd/{keyfd}', str(path)],
                                      stdout=output, stderr=subprocess.DEVNULL, pass_fds=(keyfd,), timeout=180)
            if proc.returncode or archive.stat().st_size > bundle.LIMIT + 4 * 1024 * 1024:
                raise bundle.Refused()
            restored = work / 'restored'
            restored.mkdir(mode=0o700)
            stage = 'archive_validation'
            manifest, _ = bundle.restore(archive, restored)
            result.update(ciphertext_sha256=transfer['ciphertext_sha256'],
                          manifest_file_count=len(manifest['files']), all_manifest_hashes_verified=True)
            stage = 'native_directory_check'
            restore_lldap(restored, args.user, password, result)
            password = None
        if work.exists():
            raise bundle.Refused()
        result.update(status='isolated_lldap_verified', temporary_plaintext_cleanup_verified=True)
    except (Exception, KeyboardInterrupt):
        result.update(status='failed', error='isolated_lldap_check_failed_private_diagnostics_suppressed')
        result.setdefault('failed_stage', stage)
    finally:
        if keyfd is not None:
            os.close(keyfd)
        if work is not None and not work.exists():
            result['temporary_plaintext_cleanup_verified'] = True
        result['finished_at'] = datetime.now(timezone.utc).isoformat()
        with os.fdopen(fd, 'w') as output:
            json.dump(result, output, indent=2)
            output.write('\n')
    success = result['status'] == 'isolated_lldap_verified'
    print('OK: restored LLDAP existing-user bind, wrong-password refusal, directory preservation and cleanup passed.'
          if success else 'ERROR: isolated LLDAP recovery refused; review the safe receipt. No production change.')
    return 0 if success else 1


if __name__ == '__main__':
    for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP):
        signal.signal(sig, offline.interrupted)
    raise SystemExit(main())