"""Compile and exercise the actual composed Forgejo packages on isolated PG16.""" from pathlib import Path import argparse import hashlib import json import shutil import subprocess import time HERE = Path(__file__).resolve().parent CLAIMS = HERE.parent / 'forgejo-keycape-claims-r1' parser = argparse.ArgumentParser() parser.add_argument('action', choices=['build', 'suite', 'db-regressions', 'combined-regressions', 'prepare', 'crash', 'recover-prepared', 'recover-applied', 'recover-inverse']) parser.add_argument('--workspace', type=Path, required=True) parser.add_argument('--attempt', type=int, default=1) parser.add_argument('--runtime-attempt', type=int, default=2) parser.add_argument('--control', choices=['none', 'omit-postgres-lock', 'omit-private-context'], default='none') parser.add_argument('--source-workspace', type=Path) parser.add_argument('--module-cache', type=Path) parser.add_argument('--go-cache', type=Path) args = parser.parse_args() work = args.workspace.resolve() assert work.is_relative_to('/tmp') runtime_suffix = '' if args.runtime_attempt == 1 else '-attempt' + str(args.runtime_attempt) runtime = json.loads((HERE / ('database-runtime' + runtime_suffix + '.json')).read_text()) runtime_work = Path('/tmp/custodian-forgejo-postgres-r1' + runtime_suffix) env = dict(PATH='/usr/bin:/bin', HOME=str(runtime_work / 'home'), DOCKER_CONFIG=str(runtime_work / 'docker-config'), DOCKER_HOST='unix:///var/run/docker.sock') metadata = subprocess.check_output(['docker', 'inspect', '--format', '{{json .}}', runtime['container_id']], env=env) item = json.loads(metadata) assert item['State']['Running'] and item['Image'] == runtime['image_id'] assert item['Config']['Labels']['custodian.source-test'] == runtime['trial'] assert item['HostConfig']['NetworkMode'] == 'none' and not item['HostConfig']['Privileged'] assert not item['HostConfig']['PortBindings'] base = json.loads((CLAIMS / 'consumer-test-attempt1.json').read_text()) if args.source_workspace: source_work = args.source_workspace.resolve() assert source_work.is_relative_to('/tmp') and source_work.is_dir() assert args.module_cache and args.go_cache for cache in [args.module_cache, args.go_cache]: assert cache.resolve().is_relative_to('/tmp') and cache.is_dir() source = source_work / 'source' manifest = json.loads((HERE / 'composed-source-inventory.json').read_text()) else: source = Path(base['workspace']) / 'source' manifest = json.loads((CLAIMS / 'composed-source-inventory.json').read_text()) for row in manifest['files']: assert hashlib.sha256((source / row['path']).read_bytes()).hexdigest() == row['after_sha256'], row['path'] if args.action == 'build': assert not work.exists(), 'new compilation workspace required; retain prior trials' work.mkdir() shutil.copytree(source / 'models/auth', work / 'auth') for name in ['source_operation_postgres_test.go', 'source_operation_postgres_export_test.go']: shutil.copyfile(HERE / name, work / 'auth' / name) if (HERE / 'consumer-source/models/db').is_dir(): shutil.copytree(source / 'models/db', work / 'db') for path in (HERE / 'consumer-source/models/db').glob('*.go'): shutil.copyfile(path, work / 'db' / path.name) control = {'mutation': args.control} if args.control != 'none': file = work / 'auth/source_operation.go' original = file.read_text() if args.control == 'omit-private-context': before = 'db.WithTx(db.WithSensitiveSQL(ctx), func(tx context.Context) error {' after = 'db.WithTx(ctx, func(tx context.Context) error {' else: before = '''\t\t\tif _, err := db.GetEngine(tx).Exec(`LOCK TABLE login_source_operation, login_source, "user", external_login_user IN EXCLUSIVE MODE`); err != nil { \t\t\t\treturn err \t\t\t}''' after = '\t\t\t// Controlled test mutation: omit the PostgreSQL table lock.' assert original.count(before) == 1 modified = original.replace(before, after) file.write_text(modified) control.update(original_source_sha256='sha256:' + hashlib.sha256(original.encode()).hexdigest(), controlled_source_sha256='sha256:' + hashlib.sha256(modified.encode()).hexdigest(), production_source_changed=False) (work / 'controlled-mutation.json').write_text(json.dumps(control, indent=2) + '\n') pins = {p.name: 'sha256:' + hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((work / 'auth').glob('*.go'))} (work / 'source-pins.json').write_text(json.dumps(pins, indent=2) + '\n') else: assert (work / 'models-auth.test').is_file() pins = json.loads((work / 'source-pins.json').read_text()) for name, value in pins.items(): assert 'sha256:' + hashlib.sha256((work / 'auth' / name).read_bytes()).hexdigest() == value control = json.loads((work / 'controlled-mutation.json').read_text()) if (work / 'controlled-mutation.json').exists() else {'mutation': 'none'} assert control['mutation'] == 'none' or args.action in ['build', 'suite'] db_pins = {p.name: 'sha256:' + hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((work / 'db').glob('*.go'))} if args.source_workspace: argv = ['bwrap', '--unshare-net', '--unshare-pid', '--die-with-parent', '--new-session'] for name in ['/usr', '/lib', '/lib64']: argv += ['--ro-bind', name, name] argv += ['--symlink', 'usr/bin', '/bin', '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp', '--bind', str(source_work), '/work', '--ro-bind', str(args.module_cache.resolve()), '/work/modules', '--bind', str(args.go_cache.resolve()), '/work/gocache', '--ro-bind', '/etc/ssl/certs', '/etc/ssl/certs', '--chdir', '/work/source', '--clearenv'] for key, value in dict(PATH='/work/go/bin:/usr/bin:/bin', HOME='/work/home', GOPATH='/work/gopath', GOTOOLCHAIN='local', GOPROXY='off', GOSUMDB='off', GOVCS='*:off', GOMODCACHE='/work/modules', GOCACHE='/work/gocache', GOMAXPROCS='4', CGO_ENABLED='1', GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL='/work/home/test.gitconfig', GIT_TERMINAL_PROMPT='0').items(): argv += ['--setenv', key, value] else: argv = base['argv'][:base['argv'].index('--')] argv += ['--ro-bind', str(work / 'auth'), '/work/source/models/auth', '--bind', str(work), '/pg-work', '--dir', '/run', '--ro-bind', str(runtime_work / 'socket'), '/run/source-operation-postgres', '--setenv', 'CUSTODIAN_PG_SOCKET', '/run/source-operation-postgres'] if (work / 'db').is_dir(): argv += ['--ro-bind', str(work / 'db'), '/work/source/models/db'] if args.action == 'combined-regressions': # The real replacement test deliberately consumes a leaf. Each run needs # its own synthetic read-only mount and writable replacement alias. fixtures = work / ('descriptor-fixtures-attempt' + str(args.attempt)) assert not fixtures.exists() fixtures.mkdir() for name, value in [('race', 'A' * 64), ('race-replacement', 'B' * 64)]: file = fixtures / name file.write_text(value) file.chmod(0o400) argv += ['--dir', '/run', '--ro-bind', str(fixtures), '/run/forgejo-oidc', '--bind', str(fixtures), '/run/writable-input'] if args.action == 'build': command = ['go', 'test', '-c', '-p', '4', '-tags', 'sqlite sqlite_unlock_notify source_operation_postgres', '-o', '/pg-work/models-auth.test', './models/auth'] elif args.action == 'db-regressions': command = ['go', 'test', '-p', '1', '-json', '-tags', 'sqlite sqlite_unlock_notify', './models/db', '-run', 'TestSensitiveSQL|TestTracingContext', '-count=1'] elif args.action == 'combined-regressions': command = ['go', 'test', '-p', '1', '-json', '-tags', 'sqlite sqlite_unlock_notify', './services/auth/source/oauth2', './cmd', './models/auth', './models/db', '-run', 'TestAccessEngine|TestVerifiedApplicationIdentity|TestRequired|TestCurrent|TestPrivateSecret|TestSourceOperation|TestSensitiveSQL|TestTracingContext', '-count=1'] else: if args.action == 'suite': selector = '^TestSourceOperationPostgres$' else: argv += ['--setenv', 'CUSTODIAN_PG_STEP', args.action] selector = '^TestSourceOperationPostgresProcessRecovery$' binary_args = ['/pg-work/models-auth.test', '-test.v', '-test.run=' + selector, '-test.count=1'] command = binary_args if args.action == 'crash' else ['go', 'tool', 'test2json', '-t', '-p', 'forgejo.org/models/auth', *binary_args] argv += ['--', *command] stem = 'postgres-' + args.action + '-attempt' + str(args.attempt) receipt = HERE / (stem + '.json') assert not receipt.exists() start = time.monotonic() with (HERE / (stem + '.log')).open('xb') as output: result = subprocess.run(argv, stdout=output, stderr=subprocess.STDOUT, timeout=600) expected = 73 if args.action == 'crash' else (1 if args.action == 'suite' and control['mutation'] != 'none' else 0) data = dict(action=args.action, returncode=result.returncode, expected_returncode=expected, outcome_matches=result.returncode == expected, seconds=round(time.monotonic() - start, 2), argv=argv, workspace=str(work), source_pins=pins, database_source_pins=db_pins, original_composite_patch='sha256:' + manifest['patch_sha256'], runtime=runtime['trial'], image_id=runtime['image_id'], controlled_mutation=control, network='disabled; only the exact disposable database Unix socket is bound', production_database=False, actual_authenticated_operation=False, real_credentials=False) if (work / 'models-auth.test').is_file(): data['test_binary_sha256'] = 'sha256:' + hashlib.sha256((work / 'models-auth.test').read_bytes()).hexdigest() receipt.write_text(json.dumps(data, indent=2) + '\n') print(json.dumps({k: data[k] for k in ['action', 'returncode', 'expected_returncode', 'outcome_matches', 'seconds']})) raise SystemExit(0 if data['outcome_matches'] else 1)