#!/usr/bin/env python3 """Silent attended recovery from a verified Scaleway download into isolated Docker.""" import argparse import base64 from datetime import datetime, timezone import hashlib import importlib.util import json import os from pathlib import Path import secrets import sqlite3 import subprocess import tarfile import tempfile import time import urllib.error import urllib.request from state_hub_preflight_lane import bao, data, LaneError PROBE_CONTAINER = None PROBE_IMAGE = 'python@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea' PACKAGE = Path(__file__).resolve().parents[2] / 'rapp-telemetry' spec = importlib.util.spec_from_file_location('capture', PACKAGE / 'tools/capture_recovery.py') capture = importlib.util.module_from_spec(spec) spec.loader.exec_module(capture) def cmd(argv, **kwargs): result = subprocess.run(argv, capture_output=True, timeout=90, **kwargs) if result.returncode: raise LaneError('command_failed') return result.stdout class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): return None PROBE_SCRIPT = """import json,sys,urllib.request,urllib.error class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self,*args,**kwargs):return None p=json.load(sys.stdin) r=urllib.request.Request(p['url'],headers=p['headers']) try: with urllib.request.build_opener(urllib.request.ProxyHandler({}),NoRedirect()).open(r,timeout=3) as out: print(json.dumps({'code':out.status,'body':out.read(4194304).decode()})) except urllib.error.HTTPError as e:print(json.dumps({'code':e.code,'body':''})) except Exception:print(json.dumps({'error':'probe_unavailable'})) """ def probe(base, path, headers=None): if PROBE_CONTAINER is None: raise LaneError('isolated_probe_required') result = json.loads(cmd(['docker', 'exec', '-i', PROBE_CONTAINER, 'python3', '-c', PROBE_SCRIPT], input=json.dumps({'url': base + path, 'headers': headers or {}}).encode())) if 'error' in result: raise OSError('probe_unavailable') return result['code'], result['body'].encode() def wait_health(base, path): for _ in range(90): try: if probe(base, path)[0] == 200: return except (OSError, urllib.error.URLError): pass time.sleep(1) raise LaneError('application_not_ready') def run(args, receipt, checkpoint): global PROBE_CONTAINER transfer = json.loads(args.transfer_receipt.read_text()) if transfer.get('status') != 'fetched_pending_restore' or not transfer.get('version_pinned') or not transfer.get('download_hash_matches'): raise LaneError('verified_primary_download_required') with args.source.open('rb') as source: digest = hashlib.file_digest(source, 'sha256').hexdigest() if digest != transfer['ciphertext_sha256']: raise LaneError('download_changed') policies = data(bao(['token', 'lookup', '-format=json']))['data']['policies'] if 'platform-admin' not in policies or 'root' in policies: raise LaneError('attended_operator_required') receipt.update(stage='escrow_decryption', source_sha256=digest, source_version=transfer['version_id']) checkpoint() key = data(bao(['read', '-format=json', 'platform/data/workloads/railiance/backup/offsite-lane']))['data']['data']['AGE_PRIVATE_KEY'] key_bytes = (key.strip() + '\n').encode() recipient = cmd(['age-keygen', '-y'], input=key_bytes).decode().strip() if recipient != capture.RECIPIENT: raise LaneError('escrow_recipient_mismatch') with tempfile.TemporaryDirectory(prefix='telemetry-restore-') as directory: directory = Path(directory) plain = directory / 'fetched.tar' with plain.open('xb') as out: result = subprocess.run(['age', '-d', '-i', '/dev/stdin', str(args.source)], input=key_bytes, stdout=out, stderr=subprocess.PIPE, timeout=90) if result.returncode: raise LaneError('decryption_failed') root = directory / 'restored' root.mkdir(mode=0o700) with tarfile.open(plain) as archive: capture.validate_members(archive) archive.extractall(root, filter='data') manifest = json.loads((root / 'recovery-manifest.json').read_text()) if manifest['schema'] != 'rapp-telemetry.essentials.v1': raise LaneError('wrong_manifest') images = manifest['images'] for name in ['grafana', 'alertmanager']: if '@sha256:' not in images[name]: raise LaneError('unpinned_image') cmd(['docker', 'image', 'inspect', images[name]]) receipt.update(stage='database_validation', images=images, decrypted=True) checkpoint() with sqlite3.connect('file:' + str(root / 'grafana/grafana.db') + '?mode=ro', uri=True) as db: if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': raise LaneError('sqlite_integrity_failed') counts = {'dashboards': db.execute('SELECT count(*) FROM dashboard WHERE is_folder=0').fetchone()[0], 'users': db.execute('SELECT count(*) FROM user').fetchone()[0], 'datasources': db.execute('SELECT count(*) FROM data_source').fetchone()[0]} receipt.update(sqlite_integrity='ok', legacy_database_counts=counts) if counts['users'] < 1: raise LaneError('required_user_database_content_missing') configs = json.loads((root / 'config/configmaps.json').read_text())['items'] grafana_config = next(c['data']['grafana.ini'] for c in configs if c['metadata']['name'] == 'telemetry-grafana') config = directory / 'grafana.ini' config.write_text(grafana_config) config.chmod(0o600) provisioning = directory / 'provisioning' dashboards = directory / 'dashboards' dashboards.mkdir() for part in ['dashboards', 'datasources']: (provisioning / part).mkdir(parents=True) for item in configs: labels = item['metadata'].get('labels', {}) target = None if item['metadata']['name'] == 'telemetry-grafana-config-dashboards': target = provisioning / 'dashboards' elif labels.get('grafana_dashboard') == '1': target = dashboards elif labels.get('grafana_datasource') == '1': target = provisioning / 'datasources' if target: for filename, content in item.get('data', {}).items(): if Path(filename).name != filename: raise LaneError('unsafe_config_filename') (target / filename).write_text(content) receipt['provisioning_files_restored'] = len(list(provisioning.rglob('*'))) + len(list(dashboards.iterdir())) # Notification sinks are intentionally absent in the recovery environment. am_config = directory / 'alertmanager.yaml' am_config.write_text('route:\n receiver: restore-null\nreceivers:\n - name: restore-null\n') state = list((root / 'alertmanager').rglob('silences')) if len(state) != 1 or not (state[0].parent / 'nflog').is_file(): raise LaneError('alertmanager_snapshots_missing') network = 'telemetry-recovery-' + secrets.token_hex(6) containers = [] network_created = False receipt.update(stage='isolated_startup', isolation='docker internal network; no published ports; internal probe; null notification receiver') checkpoint() try: cmd(['docker', 'network', 'create', '--internal', network]) network_created = True if not json.loads(cmd(['docker', 'network', 'inspect', network]))[0]['Internal']: raise LaneError('network_not_internal') def start(name, image, port, mounts, extra, env=()): container = network + '-' + name argv = ['docker', 'create', '--name', container, '--network', network, '--user', str(os.getuid()) + ':' + str(os.getgid()), '--read-only', '--tmpfs', '/tmp:rw,nosuid,size=64m', '--tmpfs', '/var/lib/grafana-search:rw,nosuid,size=128m,uid=' + str(os.getuid()) + ',gid=' + str(os.getgid()), '--cap-drop=ALL', '--security-opt=no-new-privileges', '--pids-limit=256', '--memory=512m', '--cpus=0.5'] for source, target, mode in mounts: argv += ['--volume', str(source) + ':' + target + ':' + mode] for item in env: argv += ['--env', item] cmd(argv + [image] + extra) containers.append(container) cmd(['docker', 'start', container]) info = json.loads(cmd(['docker', 'inspect', container]))[0] if info['HostConfig'].get('PortBindings') or set(info['NetworkSettings']['Networks']) != {network}: raise LaneError('recovery_network_mismatch') return 'http://' + container + ':' + str(port) probe_name = network + '-probe' cmd(['docker', 'create', '--name', probe_name, '--network', network, '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--memory=64m', '--pids-limit=64', PROBE_IMAGE, 'python3', '-c', 'import time; time.sleep(600)']) containers.append(probe_name) cmd(['docker', 'start', probe_name]) PROBE_CONTAINER = probe_name grafana = start('grafana', images['grafana'], 3000, [(root / 'grafana', '/var/lib/grafana', 'rw'), (config, '/etc/grafana/grafana.ini', 'ro'), (provisioning, '/etc/grafana/provisioning', 'ro'), (dashboards, '/tmp/dashboards', 'ro')], [], ['GF_PATHS_DATA=/var/lib/grafana', 'GF_PATHS_LOGS=/tmp/grafana-logs', 'GF_SERVER_HTTP_ADDR=0.0.0.0', 'GF_SERVER_DOMAIN=localhost', 'GF_SERVER_ROOT_URL=http://localhost:3000', 'GF_UNIFIED_ALERTING_ENABLED=false', 'GF_ANALYTICS_REPORTING_ENABLED=false', 'GF_PLUGINS_PREINSTALL_DISABLED=true']) alertmanager = start('alertmanager', images['alertmanager'], 9093, [(state[0].parent, '/data', 'rw'), (am_config, '/restore.yaml', 'ro')], ['--config.file=/restore.yaml', '--storage.path=/data', '--cluster.listen-address=', '--web.listen-address=0.0.0.0:9093']) receipt['stage'] = 'grafana_startup' checkpoint() wait_health(grafana, '/api/health') receipt['stage'] = 'alertmanager_startup' checkpoint() wait_health(alertmanager, '/-/ready') receipt.update(stage='application_acceptance', applications_ready=True) checkpoint() for label, headers in [('anonymous_denied', {}), ('proxy_headers_denied', {'X-WEBAUTH-USER': 'admin', 'Remote-User': 'admin'})]: if probe(grafana, '/api/user', headers)[0] != 401: raise LaneError('negative_login_failed') receipt[label] = True native = data(bao(['read', '-format=json', 'platform/data/workloads/telemetry/grafana-admin']))['data']['data'] auth = base64.b64encode((native['ADMIN_USERNAME'] + ':' + native['ADMIN_PASSWORD']).encode()).decode() headers = {'Authorization': 'Basic ' + auth} code, body = probe(grafana, '/api/user', headers) if code != 200 or not json.loads(body).get('isGrafanaAdmin'): raise LaneError('restored_admin_login_failed') for _ in range(90): code, body = probe(grafana, '/api/search?type=dash-db&limit=1000', headers) receipt['restored_dashboard_count'] = len(json.loads(body)) if code == 200 else None if receipt['restored_dashboard_count'] == args.expected_dashboard_count: break time.sleep(1) else: raise LaneError('restored_dashboard_count_mismatch') code, body = probe(grafana, '/api/datasources', headers) if code != 200: raise LaneError('restored_datasource_read_failed') sources = json.loads(body) if not any(s['type'] == 'prometheus' for s in sources) or counts['datasources'] and len(sources) != counts['datasources']: raise LaneError('restored_datasource_mismatch') receipt['restored_datasource_count'] = len(sources) code, body = probe(alertmanager, '/api/v2/silence/' + manifest['silence_id']) if code != 200 or json.loads(body)['id'] != manifest['silence_id']: raise LaneError('restored_silence_missing') receipt.update(status='verified', native_admin_login=True, dashboard_and_datasource_counts_match=True, alertmanager_fixture_restored=True, source_commit=manifest['source_commit']) finally: cleanup_errors = [] receipt['startup_diagnostics'] = [] for item in containers: try: state_info = json.loads(cmd(['docker', 'inspect', item]))[0]['State'] log_result = subprocess.run(['docker', 'logs', '--tail', '100', item], capture_output=True, timeout=10) logs = (log_result.stdout + log_result.stderr).decode(errors='replace').lower() receipt['startup_diagnostics'].append({'component': item.rsplit('-', 1)[-1], 'running': state_info['Running'], 'exit_code': state_info['ExitCode'], 'oom_killed': state_info['OOMKilled'], 'flags': [flag for flag in ['permission denied', 'read-only file system', 'database is locked', 'unable to open', 'error loading config', 'failed to start', 'provisioning', 'migration', 'panic', 'address already in use', 'no such file', 'not found', 'read-only', 'is not writable', 'error=','error:'] if flag in logs]}) except Exception: receipt['startup_diagnostics'].append({'component': item.rsplit('-', 1)[-1], 'diagnostic_unavailable': True}) for container in reversed(containers): try: cmd(['docker', 'rm', '-f', container]) except Exception: cleanup_errors.append('container_cleanup_failed') if network_created: try: cmd(['docker', 'network', 'rm', network]) except Exception: cleanup_errors.append('network_cleanup_failed') PROBE_CONTAINER = None receipt['cleanup_errors'] = cleanup_errors if cleanup_errors: raise LaneError('recovery_cleanup_failed') receipt['plaintext_removed'] = True def main(): p = argparse.ArgumentParser(description=__doc__) for name in ['source', 'transfer-receipt', 'receipt']: p.add_argument('--' + name, type=Path, required=True) p.add_argument('--expected-dashboard-count', type=int, required=True) args = p.parse_args() if args.expected_dashboard_count < 1: p.error('positive baseline dashboard count required') fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) os.close(fd) receipt = {'schema': 'platform.telemetry-recovery.v1', 'status': 'failed', 'started_at': datetime.now(timezone.utc).isoformat()} def checkpoint(): args.receipt.write_text(json.dumps(receipt, indent=2) + '\n') try: run(args, receipt, checkpoint) except Exception as error: receipt.update(status='failed', error_type=type(error).__name__, error=str(error) if isinstance(error, LaneError) else 'recovery_failed') finally: receipt['finished_at'] = datetime.now(timezone.utc).isoformat() checkpoint() return int(receipt['status'] != 'verified') if __name__ == '__main__': raise SystemExit(main())