335 lines
17 KiB
Python
335 lines
17 KiB
Python
"""Synthetic PostgreSQL journal compatibility; LDAP remains a declared test double."""
|
|
import concurrent.futures
|
|
from dataclasses import replace
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import threading
|
|
import uuid
|
|
|
|
ROOT = Path('/fixture')
|
|
sys.path[:0] = [str(ROOT / 'source'), str(ROOT / 'source/tests')]
|
|
from diagnostic_protocol import emit_stage, error_metadata
|
|
CURRENT_STAGE = 'probe_entry'
|
|
|
|
def stage(value):
|
|
global CURRENT_STAGE
|
|
CURRENT_STAGE = value
|
|
emit_stage(value)
|
|
|
|
stage('probe_entry')
|
|
try:
|
|
import psycopg
|
|
from group_provenance import GroupProvenanceUnavailable
|
|
from group_provenance_postgres import GroupJournalBinding, PostgresGroupProvenance, psycopg_connector, journal_genesis
|
|
from test_service_groups import Directory, req
|
|
|
|
EPOCH = 'bfbb8939-e88c-4f9c-a5c6-14ba3e399ddf'
|
|
CONTEXT = GroupJournalBinding('tenant:synthetic:one', 'synthetic-pg-directory', 'sha256:' + '1' * 64,
|
|
'sha256:' + '2' * 64, 'synthetic-principal', EPOCH)
|
|
except Exception as error:
|
|
print(json.dumps({'status': 'failed', 'failure': 'synthetic_pg_oracle_failed',
|
|
'diagnostic': error_metadata(error, CURRENT_STAGE)}), flush=True)
|
|
raise SystemExit(0)
|
|
|
|
|
|
def require(value):
|
|
if not value:
|
|
raise RuntimeError('postgres_fixture_oracle_failed')
|
|
|
|
|
|
def info(admin=False, database=None):
|
|
values = json.loads((ROOT / 'private.json').read_text())
|
|
database = database or ((ROOT / 'active-database').read_text() if (ROOT / 'active-database').exists() else 'postgres')
|
|
require(database in {'postgres', 'fixture_fork'})
|
|
return {'host': '127.0.0.1', 'port': 5432, 'dbname': database, 'user': 'fixture_admin' if admin else 'fixture_app',
|
|
'password': values['admin_password'] if admin else values['runtime_password'], 'sslmode': 'disable'}
|
|
|
|
|
|
class FileCheckpoint:
|
|
"""Fixture-only independently retained checkpoint; never a native owner grant."""
|
|
def __init__(self):
|
|
self.path = ROOT / 'checkpoint.json'
|
|
def _apply(self, epoch, directory, previous, previous_digest, current=None, current_digest=None):
|
|
with (ROOT / 'checkpoint.lock').open('a+') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
value = json.loads(self.path.read_text())
|
|
require(value == {'epoch': epoch, 'directory': directory, 'head': previous, 'digest': previous_digest})
|
|
if current is not None:
|
|
temporary = ROOT / ('checkpoint-' + uuid.uuid4().hex)
|
|
with temporary.open('x') as out:
|
|
json.dump({'epoch': epoch, 'directory': directory, 'head': current, 'digest': current_digest}, out)
|
|
out.flush(); os.fsync(out.fileno())
|
|
os.replace(temporary, self.path)
|
|
fd = os.open(ROOT, os.O_RDONLY | os.O_DIRECTORY)
|
|
try: os.fsync(fd)
|
|
finally: os.close(fd)
|
|
return True
|
|
def check(self, epoch, directory, head, digest):
|
|
return self._apply(epoch, directory, head, digest)
|
|
def advance(self, epoch, directory, previous, previous_digest, current, current_digest):
|
|
return self._apply(epoch, directory, previous, previous_digest, current, current_digest)
|
|
|
|
|
|
def store(context=CONTEXT, checkpoint=None, database=None):
|
|
return PostgresGroupProvenance(connect=psycopg_connector(connection_info=lambda: info(database=database)), binding=context,
|
|
continuity=checkpoint or FileCheckpoint())
|
|
|
|
|
|
def refused(call):
|
|
try: call()
|
|
except (ValueError, RuntimeError): return
|
|
raise RuntimeError('postgres_fixture_expected_refusal_missing')
|
|
|
|
|
|
def snapshot(directory):
|
|
# Directory._gql(deleteGroup) removes the visible group but retains its
|
|
# empty backing set. Reject any nonempty orphan; compare every current
|
|
# group and its complete membership. Copy mappings so later mutations
|
|
# cannot change the recorded prestate in memory before serialization.
|
|
current = set(directory.groups.values())
|
|
require(all(not users for gid, users in directory.members.items() if gid not in current))
|
|
return {'users': sorted(directory.users), 'groups': dict(directory.groups),
|
|
'members': {str(gid): sorted(directory.members[gid]) for gid in current}}
|
|
|
|
|
|
def restore_directory(value):
|
|
directory = Directory()
|
|
directory.users = set(value['users'])
|
|
directory.groups = value['groups']
|
|
directory.members = {int(k): set(v) for k, v in value['members'].items()}
|
|
directory.group_provenance = store()
|
|
return directory
|
|
|
|
|
|
def bootstrap():
|
|
stage('bootstrap_connect_wait')
|
|
last_connection_error = None
|
|
for _ in range(80):
|
|
try:
|
|
db = psycopg.connect(**info(admin=True), connect_timeout=1)
|
|
db.close(); break
|
|
except Exception as error:
|
|
last_connection_error = error
|
|
time.sleep(.25)
|
|
else: raise RuntimeError('synthetic_postgres_not_ready') from last_connection_error
|
|
stage('bootstrap_connected')
|
|
from psycopg import sql
|
|
with psycopg.connect(**info(admin=True)) as db:
|
|
stage('bootstrap_schema')
|
|
db.execute('CREATE SCHEMA identity_provisioner')
|
|
stage('bootstrap_migration')
|
|
db.execute((ROOT / 'source/migrations/group_journal_postgres_v1.sql').read_text())
|
|
stage('bootstrap_epoch_insert')
|
|
db.execute('INSERT INTO identity_provisioner.group_journal_meta VALUES (true, %s, %s, 0, %s, false)', (EPOCH, CONTEXT.directory, journal_genesis(EPOCH, CONTEXT.directory)))
|
|
stage('bootstrap_role_create')
|
|
db.execute(sql.SQL('CREATE ROLE fixture_app LOGIN PASSWORD {}').format(sql.Literal(info()['password'])))
|
|
stage('bootstrap_runtime_grants')
|
|
db.execute('GRANT USAGE ON SCHEMA identity_provisioner TO fixture_app')
|
|
db.execute('GRANT SELECT, INSERT, UPDATE ON identity_provisioner.group_receipts TO fixture_app')
|
|
db.execute('GRANT SELECT ON identity_provisioner.group_journal_meta TO fixture_app')
|
|
db.execute('GRANT UPDATE (head,head_digest) ON identity_provisioner.group_journal_meta TO fixture_app')
|
|
stage('bootstrap_checkpoint_init')
|
|
(ROOT / 'checkpoint.json').write_text(json.dumps({'epoch': EPOCH, 'directory': CONTEXT.directory, 'head': 0, 'digest': journal_genesis(EPOCH, CONTEXT.directory)}))
|
|
stage('bootstrap_directory_init')
|
|
directory = Directory()
|
|
directory.members[3].add('tegwick')
|
|
before = snapshot(directory)
|
|
directory.group_provenance = store()
|
|
stage('bootstrap_ensure')
|
|
result = directory.ensure_service_groups(req())
|
|
stage('bootstrap_ensure_readback')
|
|
require(result.status == 'groups_ensured')
|
|
require({x['name'] for x in result.created_groups} == {'forge-coulomb-owners', 'forge-coulomb-readers'})
|
|
require({(x['name'], x['username']) for x in result.created_memberships} == {('forge-coulomb-owners', 'tegwick')})
|
|
(ROOT / 'directory.json').write_text(json.dumps({'before': before, 'after': snapshot(directory)}))
|
|
stage('bootstrap_runtime_ddl_refusal')
|
|
with psycopg.connect(**info()) as db:
|
|
try: db.execute('CREATE TABLE identity_provisioner.runtime_ddl_must_fail (id integer)')
|
|
except psycopg.errors.InsufficientPrivilege: pass
|
|
else: raise RuntimeError('runtime_ddl_not_denied')
|
|
stage('bootstrap_complete')
|
|
return {'actual_postgresql_journal_ensure': True, 'explicit_scope_context_retained': True,
|
|
'confirmed_inventory_excludes_preexisting': True, 'runtime_ddl_denied': True}
|
|
|
|
|
|
def restart():
|
|
stage('restart_load_snapshot')
|
|
saved = json.loads((ROOT / 'directory.json').read_text())
|
|
stage('restart_reopen_store')
|
|
directory = restore_directory(saved['after'])
|
|
stage('restart_replay_ensure')
|
|
require(directory.ensure_service_groups(req()).resumed)
|
|
require(not directory.calls)
|
|
stage('restart_revoke')
|
|
result = directory.revoke_service_groups(req(ensure_idempotency_key=req()['idempotency_key'],
|
|
ensure_correlation_id=req()['correlation_id'], delete_if_empty=True))
|
|
stage('restart_revoke_readback')
|
|
require(result.status == 'groups_revoked_and_deleted')
|
|
stage('restart_prestate_equality')
|
|
require(snapshot(directory) == saved['before'])
|
|
stage('restart_foreign_context')
|
|
foreign = store(replace(CONTEXT, principal='other-synthetic-principal'))
|
|
refused(lambda: foreign.get(req()['idempotency_key'], directory._group_binding(req()['groups'], req()['members'], req()['correlation_id'])))
|
|
stage('restart_complete')
|
|
return {'separate_process_postgres_restart_and_revoke': True, 'preexisting_shared_state_preserved': True,
|
|
'foreign_context_receipt_refused': True}
|
|
|
|
|
|
def concurrency():
|
|
binding = '4' * 64
|
|
def reserve(_):
|
|
try: return store().reserve('synthetic-concurrent-reservation', binding)
|
|
except GroupProvenanceUnavailable: return None
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
results = list(pool.map(reserve, range(24)))
|
|
created = [r[0] for r in results if r is not None and r[1]]
|
|
require(len(created) == 1)
|
|
receipt = created[0]
|
|
receipt['state'] = 'refused'
|
|
store().save(receipt)
|
|
stale = json.loads(json.dumps(receipt))
|
|
store().save(receipt)
|
|
head = json.loads((ROOT / 'checkpoint.json').read_text())['head']
|
|
refused(lambda: store().save(stale))
|
|
require(json.loads((ROOT / 'checkpoint.json').read_text())['head'] == head)
|
|
with psycopg.connect(**info()) as db:
|
|
require(db.execute('SELECT count(*) FROM identity_provisioner.group_receipts WHERE key=%s', ('synthetic-concurrent-reservation',)).fetchone()[0] == 1)
|
|
return {'concurrent_original_reservation_exactly_once': True, 'stale_version_update_refused_without_head_advance': True}
|
|
|
|
|
|
def fork():
|
|
# Clone actual PostgreSQL state at the exact same head/digest. Both callers
|
|
# must pass the old external check before either commits a divergent body.
|
|
with psycopg.connect(**{**info(admin=True), 'dbname': 'template1'}, autocommit=True) as db:
|
|
db.execute('CREATE DATABASE fixture_fork TEMPLATE postgres')
|
|
barrier = threading.Barrier(2)
|
|
class SimultaneousCheckpoint(FileCheckpoint):
|
|
def check(self, *args):
|
|
result = super().check(*args)
|
|
barrier.wait(timeout=10)
|
|
return result
|
|
def attempt(database):
|
|
candidate = store(database=database, checkpoint=SimultaneousCheckpoint())
|
|
try:
|
|
record, fresh = candidate.reserve('synthetic-fork-' + database, '8' * 64)
|
|
require(fresh)
|
|
return database, record
|
|
except GroupProvenanceUnavailable:
|
|
return database, None
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(attempt, ('postgres', 'fixture_fork')))
|
|
winners = [(database, record) for database, record in results if record is not None]
|
|
require(len(winners) == 1)
|
|
winner, record = winners[0]
|
|
loser = next(database for database, result in results if result is None)
|
|
markers = {}
|
|
for database in ('postgres', 'fixture_fork'):
|
|
with psycopg.connect(**info(admin=True, database=database)) as db:
|
|
markers[database] = db.execute('SELECT head,head_digest FROM identity_provisioner.group_journal_meta').fetchone()
|
|
require(markers[winner][0] == markers[loser][0] and markers[winner][1] != markers[loser][1])
|
|
store(database=winner).check()
|
|
refused(lambda: store(database=loser).check())
|
|
retained = json.loads((ROOT / 'checkpoint.json').read_text())
|
|
require((retained['head'], retained['digest']) == markers[winner])
|
|
# No directory call was attempted by this direct reservation fixture. Retain
|
|
# terminal refusal for that key before continuing against the winner only.
|
|
record['state'] = 'refused'
|
|
store(database=winner).save(record)
|
|
(ROOT / 'active-database').write_text(winner)
|
|
return {'concurrent_divergent_clones_same_counter_verified': True,
|
|
'independent_exact_transition_cas_selects_one_winner': True,
|
|
'reopened_loser_clone_refused_at_same_counter': True}
|
|
|
|
|
|
def restored():
|
|
# Actual pg_restore has rolled SQL back while this checkpoint survived.
|
|
with psycopg.connect(**info()) as db:
|
|
epoch, directory, head, quarantine = db.execute('SELECT epoch::text,directory,head,quarantined FROM identity_provisioner.group_journal_meta').fetchone()
|
|
require(epoch == EPOCH and directory == CONTEXT.directory and quarantine is False)
|
|
require(head < json.loads((ROOT / 'checkpoint.json').read_text())['head'])
|
|
refused(lambda: store().check())
|
|
refused(lambda: store().reserve('no-replay-after-restore', '6' * 64))
|
|
return {'actual_pg_restore_behind_checkpoint_detected': True, 'restored_store_cannot_gain_replay_authority': True}
|
|
|
|
|
|
def lost_ack():
|
|
class LoseAcknowledgment(FileCheckpoint):
|
|
def advance(self, *args):
|
|
super().advance(*args)
|
|
raise RuntimeError('synthetic-lost-ack-after-durable-checkpoint')
|
|
directory = Directory()
|
|
directory.group_provenance = store(checkpoint=LoseAcknowledgment())
|
|
refused(lambda: directory.ensure_service_groups(req(idempotency_key='synthetic-lost-sql-ack-key')))
|
|
require(not directory.calls)
|
|
directory.group_provenance = store()
|
|
refused(lambda: directory.ensure_service_groups(req(idempotency_key='synthetic-lost-sql-ack-key')))
|
|
require(not directory.calls)
|
|
receipt = store().get('synthetic-lost-sql-ack-key', directory._group_binding(req()['groups'], req()['members'], req()['correlation_id']))
|
|
require(receipt['state'] == 'pending')
|
|
refused(lambda: store().reserve('unrelated-after-unresolved', '7' * 64))
|
|
# Definite no-directory-call refusal is disposed only inside this generated
|
|
# fixture, before moving to the next synthetic fault. No native recovery API.
|
|
receipt['state'] = 'refused'
|
|
store().save(receipt)
|
|
directory = Directory()
|
|
directory.group_provenance = store()
|
|
original = directory._gql
|
|
def lose_directory_response(token, query, variables):
|
|
result = original(token, query, variables)
|
|
if 'addUserToGroup' in query:
|
|
raise RuntimeError('synthetic-directory-response-lost')
|
|
return result
|
|
directory._gql = lose_directory_response
|
|
payload = req(idempotency_key='synthetic-lost-directory-response')
|
|
refused(lambda: directory.ensure_service_groups(payload))
|
|
checked = snapshot(directory)
|
|
directory.group_provenance = store()
|
|
refused(lambda: directory.ensure_service_groups(payload))
|
|
require(snapshot(directory) == checked)
|
|
record = store().get(payload['idempotency_key'], directory._group_binding(payload['groups'], payload['members'], payload['correlation_id']))
|
|
require(record['state'] == 'uncertain' and record['intent']['operation'] == 'add_member')
|
|
class LostCommitReply:
|
|
def __init__(self): self.db = psycopg.connect(**info())
|
|
def __enter__(self): self.db.__enter__(); return self
|
|
def __exit__(self, typ, value, traceback):
|
|
result = self.db.__exit__(typ, value, traceback)
|
|
if typ is None: raise OSError('synthetic-reply-lost-after-actual-sql-commit')
|
|
return result
|
|
def cursor(self): return self.db.cursor()
|
|
def close(self): self.db.close()
|
|
broken = PostgresGroupProvenance(connect=LostCommitReply, binding=CONTEXT, continuity=FileCheckpoint())
|
|
checkpoint = json.loads((ROOT / 'checkpoint.json').read_text())['head']
|
|
refused(lambda: broken.save(record))
|
|
with psycopg.connect(**info()) as db:
|
|
committed_head = db.execute('SELECT head FROM identity_provisioner.group_journal_meta').fetchone()[0]
|
|
require(committed_head == checkpoint + 1)
|
|
require(json.loads((ROOT / 'checkpoint.json').read_text())['head'] == checkpoint)
|
|
refused(lambda: store().check())
|
|
return {'lost_checkpoint_ack_retains_pending_reservation': True, 'reopened_uncertainty_refuses_ldap_effects': True,
|
|
'active_fence_survives_process_reopen': True, 'lost_synthetic_directory_response_retains_postgres_intent': True,
|
|
'lost_sql_commit_reply_retains_committed_head': True, 'reopened_stale_checkpoint_refuses_continuation': True}
|
|
|
|
|
|
def main():
|
|
try:
|
|
stage('loopback_check')
|
|
require({p.name for p in Path('/sys/class/net').iterdir()} == {'lo'})
|
|
require(len(Path('/proc/net/route').read_text().splitlines()) == 1)
|
|
checks = {'only_loopback_no_external_routes': True}
|
|
stage('phase_dispatch')
|
|
checks.update({'bootstrap': bootstrap, 'restart': restart, 'concurrency': concurrency,
|
|
'fork': fork, 'restored': restored, 'lost-ack': lost_ack}[sys.argv[1]]())
|
|
stage('phase_complete')
|
|
print(json.dumps({'status': 'passed', 'checks': checks, 'psycopg_version': psycopg.__version__, 'active_database': info()['dbname']}))
|
|
except Exception as error:
|
|
print(json.dumps({'status': 'failed', 'failure': 'synthetic_pg_oracle_failed',
|
|
'diagnostic': error_metadata(error, CURRENT_STAGE)}), flush=True)
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|