rapp-playwright/tools/verify_crash_recovery.py

#!/usr/bin/env python3
"""Kill an isolated released Docker container during a real browser job and verify recovery."""
import argparse
from datetime import datetime, timezone
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import uuid

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'client'))
from rapp_playwright import Client, ClientError, collect, save


def require(ok, message):
    if not ok:
        raise ClientError(message)


def docker(*args):
    result = subprocess.run(['docker', *args], capture_output=True, text=True, timeout=90)
    if result.returncode:
        raise ClientError('docker_' + args[0] + '_failed (raw output suppressed)')
    return result.stdout.strip()


def poll(check, description, timeout=60):
    deadline = time.monotonic() + timeout
    while time.monotonic() < deadline:
        value = check()
        if value:
            return value
        time.sleep(0.1)
    raise ClientError('timeout: ' + description)


def verify(args):
    require(bool(re.fullmatch(r'forgejo\.coulomb\.social/coulomb/rapp-playwright@sha256:[a-f0-9]{64}', args.image)),
            'published immutable image reference required')
    root = args.out.resolve()
    root.mkdir(parents=True, mode=0o700, exist_ok=False)
    data = root / 'data'
    data.mkdir(mode=0o700)
    name = 'rapp-crash-check-' + uuid.uuid4().hex[:12]
    cid = None
    report = {'started_at': datetime.now(timezone.utc).isoformat(), 'image': args.image,
              'location': 'isolated-local-docker', 'signal': 'SIGKILL', 'checks': [],
              'status': 'failed', 'managed_recovery_proven': False}
    try:
        cid = docker('run', '-d', '--name', name, '--read-only', '--user', '1000:1000',
                     '--tmpfs', '/tmp:rw,size=512m', '--shm-size=256m', '--memory=1g', '--cpus=1',
                     '--cap-drop', 'ALL', '--security-opt', 'no-new-privileges',
                     '--mount', 'type=bind,src=' + str(data) + ',dst=/data',
                     '-p', '127.0.0.1::8080', '-e', 'HOST=0.0.0.0',
                     '-e', 'RELEASE_ID=' + args.image,
                     '-e', 'EXECUTION_LOCATION=isolated-crash-verification', args.image)
        require(bool(re.fullmatch('[a-f0-9]{64}', cid)), 'unexpected container identity')

        def connect():
            binding = json.loads(docker('inspect', '--format', '{{json .NetworkSettings.Ports}}', cid))['8080/tcp'][0]
            require(binding['HostIp'] == '127.0.0.1', 'non_loopback_binding')
            return Client('http://127.0.0.1:' + binding['HostPort'])

        def ready(client):
            def probe():
                try:
                    result = client.request('/v1/preflight')
                    return result if result.get('available') else None
                except ClientError:
                    return None
            result = poll(probe, 'browser readiness')
            require(result['release'] == args.image, 'unexpected release')
            return result

        client = connect()
        save(root / 'before-preflight.json', ready(client))
        spec = client.request('/v1/trial')

        def submit(specification, folder):
            folder.mkdir(mode=0o700)
            run = client.request('/v1/runs', 'POST', specification)
            save(folder / 'run.json', run)
            return run

        def terminal(run):
            def check():
                result = client.request('/v1/runs/' + run['id'], token=run['token'])
                return result if result['status'] in ('passed', 'failed', 'cancelled', 'interrupted') else None
            return poll(check, 'terminal run')

        completed = submit(spec, root / 'completed-before')
        require(terminal(completed)['status'] == 'passed', 'baseline trial failed')
        before = collect(client, completed, root / 'completed-before')
        require(len(before['downloads']) == 4, 'baseline screenshot count')
        report['checks'].append('completed_trial_and_four_downloads')
        slow = {**spec, 'viewports': [{'name': 'desktop', 'width': 1280, 'height': 800}],
                'steps': [{'action': 'screenshot', 'name': 'uncommitted'},
                          {'action': 'visible', 'selector': '#never-present-crash-test'}]}
        active = submit(slow, root / 'active')
        partial = data / active['id'] / 'desktop-0-uncommitted.png'

        def screenshot_in_flight():
            result = client.request('/v1/runs/' + active['id'], token=active['token'])
            require(result['status'] not in ('passed', 'failed', 'cancelled', 'interrupted'), 'active job ended before kill')
            return result['status'] == 'running' and partial.is_file() and partial.stat().st_size > 0

        poll(screenshot_in_flight, 'in-flight browser screenshot', timeout=30)
        queued = submit(spec, root / 'queued')
        require(client.request('/v1/runs/' + queued['id'], token=queued['token'])['status'] == 'queued', 'expected queued job')
        # Read actual persisted state without exposing its capability hashes.
        report['persisted_before_kill'] = {
            'active': json.loads((data / active['id'] / 'receipt.json').read_text())['status'],
            'queued': json.loads((data / queued['id'] / 'receipt.json').read_text())['status'],
            'partial_png_bytes': partial.stat().st_size}
        require(report['persisted_before_kill']['active'] == 'running', 'active persistence missing')
        require(report['persisted_before_kill']['queued'] == 'queued', 'queued persistence missing')
        docker('kill', '--signal=KILL', cid)
        state = json.loads(docker('inspect', '--format', '{{json .State}}', cid))
        require(not state['Running'] and state['ExitCode'] == 137 and not state['OOMKilled'], 'SIGKILL not established')
        report['exit_code_after_kill'] = state['ExitCode']
        report['checks'].append('real_active_and_queued_jobs_killed_without_shutdown')
        docker('start', cid)
        client = connect()
        save(root / 'after-preflight.json', ready(client))
        for label, run in [('active', active), ('queued', queued)]:
            result = collect(client, run, root / label)
            require(result['status'] == 'interrupted' and result['error'] == 'service_restarted', label + '_not_interrupted')
            require(result['artifacts'] == [] and result['downloads'] == [], label + '_artifacts_exposed')
            require(sorted(p.name for p in (data / run['id']).iterdir()) == ['receipt.json'], label + '_orphan_files')
        report['checks'].append('active_and_queued_interrupted_orphans_removed')
        try:
            client.request('/v1/runs/' + active['id'] + '/artifacts/desktop-0-uncommitted.png', token=active['token'], binary=True)
        except ClientError as exc:
            require(str(exc) == 'HTTP 404: not_found', 'unexpected orphan denial')
        else:
            raise ClientError('uncommitted artifact still downloadable')
        after_dir = root / 'completed-after'
        after_dir.mkdir(mode=0o700)
        after = collect(client, completed, after_dir)
        require(after['status'] == 'passed' and before['downloads'] == after['downloads'], 'completed artifact mismatch')
        report['completed_downloads'] = after['downloads']
        report['checks'].append('completed_receipt_and_hashes_preserved')
        fresh = submit(spec, root / 'new-trial')
        require(terminal(fresh)['status'] == 'passed', 'fresh trial failed after restart')
        fresh_receipt = collect(client, fresh, root / 'new-trial')
        require(fresh_receipt['completed_steps'] == 14 and len(fresh_receipt['downloads']) == 4, 'fresh trial incomplete')
        report['checks'].append('fresh_keyboard_screenshot_trial_after_recovery')
        report['status'] = 'passed'
    except KeyboardInterrupt:
        report['error'] = 'interrupted_by_caller'
    except (ClientError, OSError, ValueError, KeyError, subprocess.TimeoutExpired) as exc:
        report['error'] = str(exc)
    finally:
        if cid:
            try:
                docker('rm', '-f', cid)
                report['test_container_removed'] = True
            except ClientError:
                report['test_container_removed'] = False
                report['cleanup_container_name'] = name
                report['status'] = 'failed'
        report['finished_at'] = datetime.now(timezone.utc).isoformat()
        save(root / 'verification.json', report)
    print(json.dumps(report, indent=2))
    return 0 if report['status'] == 'passed' else 3


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--image', required=True)
    parser.add_argument('--out', required=True, type=Path)
    try:
        sys.exit(verify(parser.parse_args()))
    except (ClientError, OSError, ValueError) as exc:
        print(json.dumps({'error': str(exc)}), file=sys.stderr)
        sys.exit(3)