163 lines
10 KiB
Python
163 lines
10 KiB
Python
"""Compile and exercise the actual composed Forgejo packages on isolated PG16."""
|
|
from pathlib import Path
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import time
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
CLAIMS = HERE.parent / 'forgejo-keycape-claims-r1'
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('action', choices=['build', 'suite', 'db-regressions', 'combined-regressions', 'prepare', 'crash', 'recover-prepared', 'recover-applied', 'recover-inverse'])
|
|
parser.add_argument('--workspace', type=Path, required=True)
|
|
parser.add_argument('--attempt', type=int, default=1)
|
|
parser.add_argument('--runtime-attempt', type=int, default=2)
|
|
parser.add_argument('--control', choices=['none', 'omit-postgres-lock', 'omit-private-context'], default='none')
|
|
parser.add_argument('--source-workspace', type=Path)
|
|
parser.add_argument('--module-cache', type=Path)
|
|
parser.add_argument('--go-cache', type=Path)
|
|
args = parser.parse_args()
|
|
work = args.workspace.resolve()
|
|
assert work.is_relative_to('/tmp')
|
|
runtime_suffix = '' if args.runtime_attempt == 1 else '-attempt' + str(args.runtime_attempt)
|
|
runtime = json.loads((HERE / ('database-runtime' + runtime_suffix + '.json')).read_text())
|
|
runtime_work = Path('/tmp/custodian-forgejo-postgres-r1' + runtime_suffix)
|
|
env = dict(PATH='/usr/bin:/bin', HOME=str(runtime_work / 'home'),
|
|
DOCKER_CONFIG=str(runtime_work / 'docker-config'), DOCKER_HOST='unix:///var/run/docker.sock')
|
|
metadata = subprocess.check_output(['docker', 'inspect', '--format', '{{json .}}', runtime['container_id']], env=env)
|
|
item = json.loads(metadata)
|
|
assert item['State']['Running'] and item['Image'] == runtime['image_id']
|
|
assert item['Config']['Labels']['custodian.source-test'] == runtime['trial']
|
|
assert item['HostConfig']['NetworkMode'] == 'none' and not item['HostConfig']['Privileged']
|
|
assert not item['HostConfig']['PortBindings']
|
|
|
|
base = json.loads((CLAIMS / 'consumer-test-attempt1.json').read_text())
|
|
if args.source_workspace:
|
|
source_work = args.source_workspace.resolve()
|
|
assert source_work.is_relative_to('/tmp') and source_work.is_dir()
|
|
assert args.module_cache and args.go_cache
|
|
for cache in [args.module_cache, args.go_cache]:
|
|
assert cache.resolve().is_relative_to('/tmp') and cache.is_dir()
|
|
source = source_work / 'source'
|
|
manifest = json.loads((HERE / 'composed-source-inventory.json').read_text())
|
|
else:
|
|
source = Path(base['workspace']) / 'source'
|
|
manifest = json.loads((CLAIMS / 'composed-source-inventory.json').read_text())
|
|
for row in manifest['files']:
|
|
assert hashlib.sha256((source / row['path']).read_bytes()).hexdigest() == row['after_sha256'], row['path']
|
|
if args.action == 'build':
|
|
assert not work.exists(), 'new compilation workspace required; retain prior trials'
|
|
work.mkdir()
|
|
shutil.copytree(source / 'models/auth', work / 'auth')
|
|
for name in ['source_operation_postgres_test.go', 'source_operation_postgres_export_test.go']:
|
|
shutil.copyfile(HERE / name, work / 'auth' / name)
|
|
if (HERE / 'consumer-source/models/db').is_dir():
|
|
shutil.copytree(source / 'models/db', work / 'db')
|
|
for path in (HERE / 'consumer-source/models/db').glob('*.go'):
|
|
shutil.copyfile(path, work / 'db' / path.name)
|
|
control = {'mutation': args.control}
|
|
if args.control != 'none':
|
|
file = work / 'auth/source_operation.go'
|
|
original = file.read_text()
|
|
if args.control == 'omit-private-context':
|
|
before = 'db.WithTx(db.WithSensitiveSQL(ctx), func(tx context.Context) error {'
|
|
after = 'db.WithTx(ctx, func(tx context.Context) error {'
|
|
else:
|
|
before = '''\t\t\tif _, err := db.GetEngine(tx).Exec(`LOCK TABLE login_source_operation, login_source, "user", external_login_user IN EXCLUSIVE MODE`); err != nil {
|
|
\t\t\t\treturn err
|
|
\t\t\t}'''
|
|
after = '\t\t\t// Controlled test mutation: omit the PostgreSQL table lock.'
|
|
assert original.count(before) == 1
|
|
modified = original.replace(before, after)
|
|
file.write_text(modified)
|
|
control.update(original_source_sha256='sha256:' + hashlib.sha256(original.encode()).hexdigest(),
|
|
controlled_source_sha256='sha256:' + hashlib.sha256(modified.encode()).hexdigest(),
|
|
production_source_changed=False)
|
|
(work / 'controlled-mutation.json').write_text(json.dumps(control, indent=2) + '\n')
|
|
pins = {p.name: 'sha256:' + hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((work / 'auth').glob('*.go'))}
|
|
(work / 'source-pins.json').write_text(json.dumps(pins, indent=2) + '\n')
|
|
else:
|
|
assert (work / 'models-auth.test').is_file()
|
|
pins = json.loads((work / 'source-pins.json').read_text())
|
|
for name, value in pins.items():
|
|
assert 'sha256:' + hashlib.sha256((work / 'auth' / name).read_bytes()).hexdigest() == value
|
|
control = json.loads((work / 'controlled-mutation.json').read_text()) if (work / 'controlled-mutation.json').exists() else {'mutation': 'none'}
|
|
assert control['mutation'] == 'none' or args.action in ['build', 'suite']
|
|
db_pins = {p.name: 'sha256:' + hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted((work / 'db').glob('*.go'))}
|
|
|
|
if args.source_workspace:
|
|
argv = ['bwrap', '--unshare-net', '--unshare-pid', '--die-with-parent', '--new-session']
|
|
for name in ['/usr', '/lib', '/lib64']:
|
|
argv += ['--ro-bind', name, name]
|
|
argv += ['--symlink', 'usr/bin', '/bin', '--proc', '/proc', '--dev', '/dev', '--tmpfs', '/tmp',
|
|
'--bind', str(source_work), '/work', '--ro-bind', str(args.module_cache.resolve()), '/work/modules',
|
|
'--bind', str(args.go_cache.resolve()), '/work/gocache', '--ro-bind', '/etc/ssl/certs', '/etc/ssl/certs',
|
|
'--chdir', '/work/source', '--clearenv']
|
|
for key, value in dict(PATH='/work/go/bin:/usr/bin:/bin', HOME='/work/home', GOPATH='/work/gopath',
|
|
GOTOOLCHAIN='local', GOPROXY='off', GOSUMDB='off', GOVCS='*:off',
|
|
GOMODCACHE='/work/modules', GOCACHE='/work/gocache', GOMAXPROCS='4', CGO_ENABLED='1',
|
|
GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL='/work/home/test.gitconfig',
|
|
GIT_TERMINAL_PROMPT='0').items():
|
|
argv += ['--setenv', key, value]
|
|
else:
|
|
argv = base['argv'][:base['argv'].index('--')]
|
|
argv += ['--ro-bind', str(work / 'auth'), '/work/source/models/auth',
|
|
'--bind', str(work), '/pg-work', '--dir', '/run',
|
|
'--ro-bind', str(runtime_work / 'socket'), '/run/source-operation-postgres',
|
|
'--setenv', 'CUSTODIAN_PG_SOCKET', '/run/source-operation-postgres']
|
|
if (work / 'db').is_dir():
|
|
argv += ['--ro-bind', str(work / 'db'), '/work/source/models/db']
|
|
if args.action == 'combined-regressions':
|
|
# The real replacement test deliberately consumes a leaf. Each run needs
|
|
# its own synthetic read-only mount and writable replacement alias.
|
|
fixtures = work / ('descriptor-fixtures-attempt' + str(args.attempt))
|
|
assert not fixtures.exists()
|
|
fixtures.mkdir()
|
|
for name, value in [('race', 'A' * 64), ('race-replacement', 'B' * 64)]:
|
|
file = fixtures / name
|
|
file.write_text(value)
|
|
file.chmod(0o400)
|
|
argv += ['--dir', '/run', '--ro-bind', str(fixtures), '/run/forgejo-oidc',
|
|
'--bind', str(fixtures), '/run/writable-input']
|
|
if args.action == 'build':
|
|
command = ['go', 'test', '-c', '-p', '4', '-tags', 'sqlite sqlite_unlock_notify source_operation_postgres',
|
|
'-o', '/pg-work/models-auth.test', './models/auth']
|
|
elif args.action == 'db-regressions':
|
|
command = ['go', 'test', '-p', '1', '-json', '-tags', 'sqlite sqlite_unlock_notify',
|
|
'./models/db', '-run', 'TestSensitiveSQL|TestTracingContext', '-count=1']
|
|
elif args.action == 'combined-regressions':
|
|
command = ['go', 'test', '-p', '1', '-json', '-tags', 'sqlite sqlite_unlock_notify',
|
|
'./services/auth/source/oauth2', './cmd', './models/auth', './models/db', '-run',
|
|
'TestAccessEngine|TestVerifiedApplicationIdentity|TestRequired|TestCurrent|TestPrivateSecret|TestSourceOperation|TestSensitiveSQL|TestTracingContext', '-count=1']
|
|
else:
|
|
if args.action == 'suite':
|
|
selector = '^TestSourceOperationPostgres$'
|
|
else:
|
|
argv += ['--setenv', 'CUSTODIAN_PG_STEP', args.action]
|
|
selector = '^TestSourceOperationPostgresProcessRecovery$'
|
|
binary_args = ['/pg-work/models-auth.test', '-test.v', '-test.run=' + selector, '-test.count=1']
|
|
command = binary_args if args.action == 'crash' else ['go', 'tool', 'test2json', '-t', '-p', 'forgejo.org/models/auth', *binary_args]
|
|
argv += ['--', *command]
|
|
stem = 'postgres-' + args.action + '-attempt' + str(args.attempt)
|
|
receipt = HERE / (stem + '.json')
|
|
assert not receipt.exists()
|
|
start = time.monotonic()
|
|
with (HERE / (stem + '.log')).open('xb') as output:
|
|
result = subprocess.run(argv, stdout=output, stderr=subprocess.STDOUT, timeout=600)
|
|
expected = 73 if args.action == 'crash' else (1 if args.action == 'suite' and control['mutation'] != 'none' else 0)
|
|
data = dict(action=args.action, returncode=result.returncode, expected_returncode=expected,
|
|
outcome_matches=result.returncode == expected, seconds=round(time.monotonic() - start, 2),
|
|
argv=argv, workspace=str(work), source_pins=pins,
|
|
database_source_pins=db_pins,
|
|
original_composite_patch='sha256:' + manifest['patch_sha256'],
|
|
runtime=runtime['trial'], image_id=runtime['image_id'],
|
|
controlled_mutation=control,
|
|
network='disabled; only the exact disposable database Unix socket is bound',
|
|
production_database=False, actual_authenticated_operation=False, real_credentials=False)
|
|
if (work / 'models-auth.test').is_file():
|
|
data['test_binary_sha256'] = 'sha256:' + hashlib.sha256((work / 'models-auth.test').read_bytes()).hexdigest()
|
|
receipt.write_text(json.dumps(data, indent=2) + '\n')
|
|
print(json.dumps({k: data[k] for k in ['action', 'returncode', 'expected_returncode', 'outcome_matches', 'seconds']}))
|
|
raise SystemExit(0 if data['outcome_matches'] else 1)
|