275 lines
15 KiB
Python
275 lines
15 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""Silent attended recovery from a verified Scaleway download into isolated Docker."""
|
||
|
|
import argparse
|
||
|
|
import base64
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
import hashlib
|
||
|
|
import importlib.util
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
from pathlib import Path
|
||
|
|
import secrets
|
||
|
|
import sqlite3
|
||
|
|
import subprocess
|
||
|
|
import tarfile
|
||
|
|
import tempfile
|
||
|
|
import time
|
||
|
|
import urllib.error
|
||
|
|
import urllib.request
|
||
|
|
from state_hub_preflight_lane import bao, data, LaneError
|
||
|
|
|
||
|
|
PROBE_CONTAINER = None
|
||
|
|
PROBE_IMAGE = 'python@sha256:78387bc3881b8273120a12ebe6c1ab22b018ccc2c9adf565ae1ac9b536e184ea'
|
||
|
|
|
||
|
|
PACKAGE = Path(__file__).resolve().parents[2] / 'rapp-telemetry'
|
||
|
|
spec = importlib.util.spec_from_file_location('capture', PACKAGE / 'tools/capture_recovery.py')
|
||
|
|
capture = importlib.util.module_from_spec(spec)
|
||
|
|
spec.loader.exec_module(capture)
|
||
|
|
|
||
|
|
|
||
|
|
def cmd(argv, **kwargs):
|
||
|
|
result = subprocess.run(argv, capture_output=True, timeout=90, **kwargs)
|
||
|
|
if result.returncode:
|
||
|
|
raise LaneError('command_failed')
|
||
|
|
return result.stdout
|
||
|
|
|
||
|
|
|
||
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||
|
|
def redirect_request(self, *args, **kwargs):
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
PROBE_SCRIPT = """import json,sys,urllib.request,urllib.error
|
||
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||
|
|
def redirect_request(self,*args,**kwargs):return None
|
||
|
|
p=json.load(sys.stdin)
|
||
|
|
r=urllib.request.Request(p['url'],headers=p['headers'])
|
||
|
|
try:
|
||
|
|
with urllib.request.build_opener(urllib.request.ProxyHandler({}),NoRedirect()).open(r,timeout=3) as out:
|
||
|
|
print(json.dumps({'code':out.status,'body':out.read(4194304).decode()}))
|
||
|
|
except urllib.error.HTTPError as e:print(json.dumps({'code':e.code,'body':''}))
|
||
|
|
except Exception:print(json.dumps({'error':'probe_unavailable'}))
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
def probe(base, path, headers=None):
|
||
|
|
if PROBE_CONTAINER is None:
|
||
|
|
raise LaneError('isolated_probe_required')
|
||
|
|
result = json.loads(cmd(['docker', 'exec', '-i', PROBE_CONTAINER, 'python3', '-c', PROBE_SCRIPT], input=json.dumps({'url': base + path, 'headers': headers or {}}).encode()))
|
||
|
|
if 'error' in result:
|
||
|
|
raise OSError('probe_unavailable')
|
||
|
|
return result['code'], result['body'].encode()
|
||
|
|
|
||
|
|
|
||
|
|
def wait_health(base, path):
|
||
|
|
for _ in range(90):
|
||
|
|
try:
|
||
|
|
if probe(base, path)[0] == 200:
|
||
|
|
return
|
||
|
|
except (OSError, urllib.error.URLError):
|
||
|
|
pass
|
||
|
|
time.sleep(1)
|
||
|
|
raise LaneError('application_not_ready')
|
||
|
|
|
||
|
|
|
||
|
|
def run(args, receipt, checkpoint):
|
||
|
|
global PROBE_CONTAINER
|
||
|
|
transfer = json.loads(args.transfer_receipt.read_text())
|
||
|
|
if transfer.get('status') != 'fetched_pending_restore' or not transfer.get('version_pinned') or not transfer.get('download_hash_matches'):
|
||
|
|
raise LaneError('verified_primary_download_required')
|
||
|
|
with args.source.open('rb') as source:
|
||
|
|
digest = hashlib.file_digest(source, 'sha256').hexdigest()
|
||
|
|
if digest != transfer['ciphertext_sha256']:
|
||
|
|
raise LaneError('download_changed')
|
||
|
|
policies = data(bao(['token', 'lookup', '-format=json']))['data']['policies']
|
||
|
|
if 'platform-admin' not in policies or 'root' in policies:
|
||
|
|
raise LaneError('attended_operator_required')
|
||
|
|
receipt.update(stage='escrow_decryption', source_sha256=digest, source_version=transfer['version_id'])
|
||
|
|
checkpoint()
|
||
|
|
key = data(bao(['read', '-format=json', 'platform/data/workloads/railiance/backup/offsite-lane']))['data']['data']['AGE_PRIVATE_KEY']
|
||
|
|
key_bytes = (key.strip() + '\n').encode()
|
||
|
|
recipient = cmd(['age-keygen', '-y'], input=key_bytes).decode().strip()
|
||
|
|
if recipient != capture.RECIPIENT:
|
||
|
|
raise LaneError('escrow_recipient_mismatch')
|
||
|
|
with tempfile.TemporaryDirectory(prefix='telemetry-restore-') as directory:
|
||
|
|
directory = Path(directory)
|
||
|
|
plain = directory / 'fetched.tar'
|
||
|
|
with plain.open('xb') as out:
|
||
|
|
result = subprocess.run(['age', '-d', '-i', '/dev/stdin', str(args.source)], input=key_bytes, stdout=out, stderr=subprocess.PIPE, timeout=90)
|
||
|
|
if result.returncode:
|
||
|
|
raise LaneError('decryption_failed')
|
||
|
|
root = directory / 'restored'
|
||
|
|
root.mkdir(mode=0o700)
|
||
|
|
with tarfile.open(plain) as archive:
|
||
|
|
capture.validate_members(archive)
|
||
|
|
archive.extractall(root, filter='data')
|
||
|
|
manifest = json.loads((root / 'recovery-manifest.json').read_text())
|
||
|
|
if manifest['schema'] != 'rapp-telemetry.essentials.v1':
|
||
|
|
raise LaneError('wrong_manifest')
|
||
|
|
images = manifest['images']
|
||
|
|
for name in ['grafana', 'alertmanager']:
|
||
|
|
if '@sha256:' not in images[name]:
|
||
|
|
raise LaneError('unpinned_image')
|
||
|
|
cmd(['docker', 'image', 'inspect', images[name]])
|
||
|
|
receipt.update(stage='database_validation', images=images, decrypted=True)
|
||
|
|
checkpoint()
|
||
|
|
with sqlite3.connect('file:' + str(root / 'grafana/grafana.db') + '?mode=ro', uri=True) as db:
|
||
|
|
if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
|
||
|
|
raise LaneError('sqlite_integrity_failed')
|
||
|
|
counts = {'dashboards': db.execute('SELECT count(*) FROM dashboard WHERE is_folder=0').fetchone()[0], 'users': db.execute('SELECT count(*) FROM user').fetchone()[0], 'datasources': db.execute('SELECT count(*) FROM data_source').fetchone()[0]}
|
||
|
|
receipt.update(sqlite_integrity='ok', legacy_database_counts=counts)
|
||
|
|
if counts['users'] < 1:
|
||
|
|
raise LaneError('required_user_database_content_missing')
|
||
|
|
configs = json.loads((root / 'config/configmaps.json').read_text())['items']
|
||
|
|
grafana_config = next(c['data']['grafana.ini'] for c in configs if c['metadata']['name'] == 'telemetry-grafana')
|
||
|
|
config = directory / 'grafana.ini'
|
||
|
|
config.write_text(grafana_config)
|
||
|
|
config.chmod(0o600)
|
||
|
|
provisioning = directory / 'provisioning'
|
||
|
|
dashboards = directory / 'dashboards'
|
||
|
|
dashboards.mkdir()
|
||
|
|
for part in ['dashboards', 'datasources']:
|
||
|
|
(provisioning / part).mkdir(parents=True)
|
||
|
|
for item in configs:
|
||
|
|
labels = item['metadata'].get('labels', {})
|
||
|
|
target = None
|
||
|
|
if item['metadata']['name'] == 'telemetry-grafana-config-dashboards':
|
||
|
|
target = provisioning / 'dashboards'
|
||
|
|
elif labels.get('grafana_dashboard') == '1':
|
||
|
|
target = dashboards
|
||
|
|
elif labels.get('grafana_datasource') == '1':
|
||
|
|
target = provisioning / 'datasources'
|
||
|
|
if target:
|
||
|
|
for filename, content in item.get('data', {}).items():
|
||
|
|
if Path(filename).name != filename:
|
||
|
|
raise LaneError('unsafe_config_filename')
|
||
|
|
(target / filename).write_text(content)
|
||
|
|
receipt['provisioning_files_restored'] = len(list(provisioning.rglob('*'))) + len(list(dashboards.iterdir()))
|
||
|
|
# Notification sinks are intentionally absent in the recovery environment.
|
||
|
|
am_config = directory / 'alertmanager.yaml'
|
||
|
|
am_config.write_text('route:\n receiver: restore-null\nreceivers:\n - name: restore-null\n')
|
||
|
|
state = list((root / 'alertmanager').rglob('silences'))
|
||
|
|
if len(state) != 1 or not (state[0].parent / 'nflog').is_file():
|
||
|
|
raise LaneError('alertmanager_snapshots_missing')
|
||
|
|
network = 'telemetry-recovery-' + secrets.token_hex(6)
|
||
|
|
containers = []
|
||
|
|
network_created = False
|
||
|
|
receipt.update(stage='isolated_startup', isolation='docker internal network; no published ports; internal probe; null notification receiver')
|
||
|
|
checkpoint()
|
||
|
|
try:
|
||
|
|
cmd(['docker', 'network', 'create', '--internal', network])
|
||
|
|
network_created = True
|
||
|
|
if not json.loads(cmd(['docker', 'network', 'inspect', network]))[0]['Internal']:
|
||
|
|
raise LaneError('network_not_internal')
|
||
|
|
def start(name, image, port, mounts, extra, env=()):
|
||
|
|
container = network + '-' + name
|
||
|
|
argv = ['docker', 'create', '--name', container, '--network', network, '--user', str(os.getuid()) + ':' + str(os.getgid()), '--read-only', '--tmpfs', '/tmp:rw,nosuid,size=64m', '--tmpfs', '/var/lib/grafana-search:rw,nosuid,size=128m,uid=' + str(os.getuid()) + ',gid=' + str(os.getgid()), '--cap-drop=ALL', '--security-opt=no-new-privileges', '--pids-limit=256', '--memory=512m', '--cpus=0.5']
|
||
|
|
for source, target, mode in mounts:
|
||
|
|
argv += ['--volume', str(source) + ':' + target + ':' + mode]
|
||
|
|
for item in env:
|
||
|
|
argv += ['--env', item]
|
||
|
|
cmd(argv + [image] + extra)
|
||
|
|
containers.append(container)
|
||
|
|
cmd(['docker', 'start', container])
|
||
|
|
info = json.loads(cmd(['docker', 'inspect', container]))[0]
|
||
|
|
if info['HostConfig'].get('PortBindings') or set(info['NetworkSettings']['Networks']) != {network}:
|
||
|
|
raise LaneError('recovery_network_mismatch')
|
||
|
|
return 'http://' + container + ':' + str(port)
|
||
|
|
probe_name = network + '-probe'
|
||
|
|
cmd(['docker', 'create', '--name', probe_name, '--network', network, '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--memory=64m', '--pids-limit=64', PROBE_IMAGE, 'python3', '-c', 'import time; time.sleep(600)'])
|
||
|
|
containers.append(probe_name)
|
||
|
|
cmd(['docker', 'start', probe_name])
|
||
|
|
PROBE_CONTAINER = probe_name
|
||
|
|
grafana = start('grafana', images['grafana'], 3000, [(root / 'grafana', '/var/lib/grafana', 'rw'), (config, '/etc/grafana/grafana.ini', 'ro'), (provisioning, '/etc/grafana/provisioning', 'ro'), (dashboards, '/tmp/dashboards', 'ro')], [], ['GF_PATHS_DATA=/var/lib/grafana', 'GF_PATHS_LOGS=/tmp/grafana-logs', 'GF_SERVER_HTTP_ADDR=0.0.0.0', 'GF_SERVER_DOMAIN=localhost', 'GF_SERVER_ROOT_URL=http://localhost:3000', 'GF_UNIFIED_ALERTING_ENABLED=false', 'GF_ANALYTICS_REPORTING_ENABLED=false', 'GF_PLUGINS_PREINSTALL_DISABLED=true'])
|
||
|
|
alertmanager = start('alertmanager', images['alertmanager'], 9093, [(state[0].parent, '/data', 'rw'), (am_config, '/restore.yaml', 'ro')], ['--config.file=/restore.yaml', '--storage.path=/data', '--cluster.listen-address=', '--web.listen-address=0.0.0.0:9093'])
|
||
|
|
receipt['stage'] = 'grafana_startup'
|
||
|
|
checkpoint()
|
||
|
|
wait_health(grafana, '/api/health')
|
||
|
|
receipt['stage'] = 'alertmanager_startup'
|
||
|
|
checkpoint()
|
||
|
|
wait_health(alertmanager, '/-/ready')
|
||
|
|
receipt.update(stage='application_acceptance', applications_ready=True)
|
||
|
|
checkpoint()
|
||
|
|
for label, headers in [('anonymous_denied', {}), ('proxy_headers_denied', {'X-WEBAUTH-USER': 'admin', 'Remote-User': 'admin'})]:
|
||
|
|
if probe(grafana, '/api/user', headers)[0] != 401:
|
||
|
|
raise LaneError('negative_login_failed')
|
||
|
|
receipt[label] = True
|
||
|
|
native = data(bao(['read', '-format=json', 'platform/data/workloads/telemetry/grafana-admin']))['data']['data']
|
||
|
|
auth = base64.b64encode((native['ADMIN_USERNAME'] + ':' + native['ADMIN_PASSWORD']).encode()).decode()
|
||
|
|
headers = {'Authorization': 'Basic ' + auth}
|
||
|
|
code, body = probe(grafana, '/api/user', headers)
|
||
|
|
if code != 200 or not json.loads(body).get('isGrafanaAdmin'):
|
||
|
|
raise LaneError('restored_admin_login_failed')
|
||
|
|
for _ in range(90):
|
||
|
|
code, body = probe(grafana, '/api/search?type=dash-db&limit=1000', headers)
|
||
|
|
receipt['restored_dashboard_count'] = len(json.loads(body)) if code == 200 else None
|
||
|
|
if receipt['restored_dashboard_count'] == args.expected_dashboard_count:
|
||
|
|
break
|
||
|
|
time.sleep(1)
|
||
|
|
else:
|
||
|
|
raise LaneError('restored_dashboard_count_mismatch')
|
||
|
|
code, body = probe(grafana, '/api/datasources', headers)
|
||
|
|
if code != 200:
|
||
|
|
raise LaneError('restored_datasource_read_failed')
|
||
|
|
sources = json.loads(body)
|
||
|
|
if not any(s['type'] == 'prometheus' for s in sources) or counts['datasources'] and len(sources) != counts['datasources']:
|
||
|
|
raise LaneError('restored_datasource_mismatch')
|
||
|
|
receipt['restored_datasource_count'] = len(sources)
|
||
|
|
code, body = probe(alertmanager, '/api/v2/silence/' + manifest['silence_id'])
|
||
|
|
if code != 200 or json.loads(body)['id'] != manifest['silence_id']:
|
||
|
|
raise LaneError('restored_silence_missing')
|
||
|
|
receipt.update(status='verified', native_admin_login=True, dashboard_and_datasource_counts_match=True, alertmanager_fixture_restored=True, source_commit=manifest['source_commit'])
|
||
|
|
finally:
|
||
|
|
cleanup_errors = []
|
||
|
|
receipt['startup_diagnostics'] = []
|
||
|
|
for item in containers:
|
||
|
|
try:
|
||
|
|
state_info = json.loads(cmd(['docker', 'inspect', item]))[0]['State']
|
||
|
|
log_result = subprocess.run(['docker', 'logs', '--tail', '100', item], capture_output=True, timeout=10)
|
||
|
|
logs = (log_result.stdout + log_result.stderr).decode(errors='replace').lower()
|
||
|
|
receipt['startup_diagnostics'].append({'component': item.rsplit('-', 1)[-1], 'running': state_info['Running'], 'exit_code': state_info['ExitCode'], 'oom_killed': state_info['OOMKilled'], 'flags': [flag for flag in ['permission denied', 'read-only file system', 'database is locked', 'unable to open', 'error loading config', 'failed to start', 'provisioning', 'migration', 'panic', 'address already in use', 'no such file', 'not found', 'read-only', 'is not writable', 'error=','error:'] if flag in logs]})
|
||
|
|
except Exception:
|
||
|
|
receipt['startup_diagnostics'].append({'component': item.rsplit('-', 1)[-1], 'diagnostic_unavailable': True})
|
||
|
|
for container in reversed(containers):
|
||
|
|
try:
|
||
|
|
cmd(['docker', 'rm', '-f', container])
|
||
|
|
except Exception:
|
||
|
|
cleanup_errors.append('container_cleanup_failed')
|
||
|
|
if network_created:
|
||
|
|
try:
|
||
|
|
cmd(['docker', 'network', 'rm', network])
|
||
|
|
except Exception:
|
||
|
|
cleanup_errors.append('network_cleanup_failed')
|
||
|
|
PROBE_CONTAINER = None
|
||
|
|
receipt['cleanup_errors'] = cleanup_errors
|
||
|
|
if cleanup_errors:
|
||
|
|
raise LaneError('recovery_cleanup_failed')
|
||
|
|
receipt['plaintext_removed'] = True
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
p = argparse.ArgumentParser(description=__doc__)
|
||
|
|
for name in ['source', 'transfer-receipt', 'receipt']:
|
||
|
|
p.add_argument('--' + name, type=Path, required=True)
|
||
|
|
p.add_argument('--expected-dashboard-count', type=int, required=True)
|
||
|
|
args = p.parse_args()
|
||
|
|
if args.expected_dashboard_count < 1:
|
||
|
|
p.error('positive baseline dashboard count required')
|
||
|
|
fd = os.open(args.receipt, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||
|
|
os.close(fd)
|
||
|
|
receipt = {'schema': 'platform.telemetry-recovery.v1', 'status': 'failed', 'started_at': datetime.now(timezone.utc).isoformat()}
|
||
|
|
def checkpoint():
|
||
|
|
args.receipt.write_text(json.dumps(receipt, indent=2) + '\n')
|
||
|
|
try:
|
||
|
|
run(args, receipt, checkpoint)
|
||
|
|
except Exception as error:
|
||
|
|
receipt.update(status='failed', error_type=type(error).__name__, error=str(error) if isinstance(error, LaneError) else 'recovery_failed')
|
||
|
|
finally:
|
||
|
|
receipt['finished_at'] = datetime.now(timezone.utc).isoformat()
|
||
|
|
checkpoint()
|
||
|
|
return int(receipt['status'] != 'verified')
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
raise SystemExit(main())
|