#!/usr/bin/env python3 """Kill an isolated released Docker container during a real browser job and verify recovery.""" import argparse from datetime import datetime, timezone import json from pathlib import Path import re import subprocess import sys import time import uuid sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'client')) from rapp_playwright import Client, ClientError, collect, save def require(ok, message): if not ok: raise ClientError(message) def docker(*args): result = subprocess.run(['docker', *args], capture_output=True, text=True, timeout=90) if result.returncode: raise ClientError('docker_' + args[0] + '_failed (raw output suppressed)') return result.stdout.strip() def poll(check, description, timeout=60): deadline = time.monotonic() + timeout while time.monotonic() < deadline: value = check() if value: return value time.sleep(0.1) raise ClientError('timeout: ' + description) def verify(args): require(bool(re.fullmatch(r'forgejo\.coulomb\.social/coulomb/rapp-playwright@sha256:[a-f0-9]{64}', args.image)), 'published immutable image reference required') root = args.out.resolve() root.mkdir(parents=True, mode=0o700, exist_ok=False) data = root / 'data' data.mkdir(mode=0o700) name = 'rapp-crash-check-' + uuid.uuid4().hex[:12] cid = None report = {'started_at': datetime.now(timezone.utc).isoformat(), 'image': args.image, 'location': 'isolated-local-docker', 'signal': 'SIGKILL', 'checks': [], 'status': 'failed', 'managed_recovery_proven': False} try: cid = docker('run', '-d', '--name', name, '--read-only', '--user', '1000:1000', '--tmpfs', '/tmp:rw,size=512m', '--shm-size=256m', '--memory=1g', '--cpus=1', '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges', '--mount', 'type=bind,src=' + str(data) + ',dst=/data', '-p', '127.0.0.1::8080', '-e', 'HOST=0.0.0.0', '-e', 'RELEASE_ID=' + args.image, '-e', 'EXECUTION_LOCATION=isolated-crash-verification', args.image) require(bool(re.fullmatch('[a-f0-9]{64}', cid)), 'unexpected container identity') def connect(): binding = json.loads(docker('inspect', '--format', '{{json .NetworkSettings.Ports}}', cid))['8080/tcp'][0] require(binding['HostIp'] == '127.0.0.1', 'non_loopback_binding') return Client('http://127.0.0.1:' + binding['HostPort']) def ready(client): def probe(): try: result = client.request('/v1/preflight') return result if result.get('available') else None except ClientError: return None result = poll(probe, 'browser readiness') require(result['release'] == args.image, 'unexpected release') return result client = connect() save(root / 'before-preflight.json', ready(client)) spec = client.request('/v1/trial') def submit(specification, folder): folder.mkdir(mode=0o700) run = client.request('/v1/runs', 'POST', specification) save(folder / 'run.json', run) return run def terminal(run): def check(): result = client.request('/v1/runs/' + run['id'], token=run['token']) return result if result['status'] in ('passed', 'failed', 'cancelled', 'interrupted') else None return poll(check, 'terminal run') completed = submit(spec, root / 'completed-before') require(terminal(completed)['status'] == 'passed', 'baseline trial failed') before = collect(client, completed, root / 'completed-before') require(len(before['downloads']) == 4, 'baseline screenshot count') report['checks'].append('completed_trial_and_four_downloads') slow = {**spec, 'viewports': [{'name': 'desktop', 'width': 1280, 'height': 800}], 'steps': [{'action': 'screenshot', 'name': 'uncommitted'}, {'action': 'visible', 'selector': '#never-present-crash-test'}]} active = submit(slow, root / 'active') partial = data / active['id'] / 'desktop-0-uncommitted.png' def screenshot_in_flight(): result = client.request('/v1/runs/' + active['id'], token=active['token']) require(result['status'] not in ('passed', 'failed', 'cancelled', 'interrupted'), 'active job ended before kill') return result['status'] == 'running' and partial.is_file() and partial.stat().st_size > 0 poll(screenshot_in_flight, 'in-flight browser screenshot', timeout=30) queued = submit(spec, root / 'queued') require(client.request('/v1/runs/' + queued['id'], token=queued['token'])['status'] == 'queued', 'expected queued job') # Read actual persisted state without exposing its capability hashes. report['persisted_before_kill'] = { 'active': json.loads((data / active['id'] / 'receipt.json').read_text())['status'], 'queued': json.loads((data / queued['id'] / 'receipt.json').read_text())['status'], 'partial_png_bytes': partial.stat().st_size} require(report['persisted_before_kill']['active'] == 'running', 'active persistence missing') require(report['persisted_before_kill']['queued'] == 'queued', 'queued persistence missing') docker('kill', '--signal=KILL', cid) state = json.loads(docker('inspect', '--format', '{{json .State}}', cid)) require(not state['Running'] and state['ExitCode'] == 137 and not state['OOMKilled'], 'SIGKILL not established') report['exit_code_after_kill'] = state['ExitCode'] report['checks'].append('real_active_and_queued_jobs_killed_without_shutdown') docker('start', cid) client = connect() save(root / 'after-preflight.json', ready(client)) for label, run in [('active', active), ('queued', queued)]: result = collect(client, run, root / label) require(result['status'] == 'interrupted' and result['error'] == 'service_restarted', label + '_not_interrupted') require(result['artifacts'] == [] and result['downloads'] == [], label + '_artifacts_exposed') require(sorted(p.name for p in (data / run['id']).iterdir()) == ['receipt.json'], label + '_orphan_files') report['checks'].append('active_and_queued_interrupted_orphans_removed') try: client.request('/v1/runs/' + active['id'] + '/artifacts/desktop-0-uncommitted.png', token=active['token'], binary=True) except ClientError as exc: require(str(exc) == 'HTTP 404: not_found', 'unexpected orphan denial') else: raise ClientError('uncommitted artifact still downloadable') after_dir = root / 'completed-after' after_dir.mkdir(mode=0o700) after = collect(client, completed, after_dir) require(after['status'] == 'passed' and before['downloads'] == after['downloads'], 'completed artifact mismatch') report['completed_downloads'] = after['downloads'] report['checks'].append('completed_receipt_and_hashes_preserved') fresh = submit(spec, root / 'new-trial') require(terminal(fresh)['status'] == 'passed', 'fresh trial failed after restart') fresh_receipt = collect(client, fresh, root / 'new-trial') require(fresh_receipt['completed_steps'] == 14 and len(fresh_receipt['downloads']) == 4, 'fresh trial incomplete') report['checks'].append('fresh_keyboard_screenshot_trial_after_recovery') report['status'] = 'passed' except KeyboardInterrupt: report['error'] = 'interrupted_by_caller' except (ClientError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as exc: report['error'] = str(exc) finally: if cid: try: docker('rm', '-f', cid) report['test_container_removed'] = True except ClientError: report['test_container_removed'] = False report['cleanup_container_name'] = name report['status'] = 'failed' report['finished_at'] = datetime.now(timezone.utc).isoformat() save(root / 'verification.json', report) print(json.dumps(report, indent=2)) return 0 if report['status'] == 'passed' else 3 if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--image', required=True) parser.add_argument('--out', required=True, type=Path) try: sys.exit(verify(parser.parse_args())) except (ClientError, OSError, ValueError) as exc: print(json.dumps({'error': str(exc)}), file=sys.stderr) sys.exit(3)