debf981097
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a0e3c3-621b-7350-9f77-50a8d3ee7657
147 lines
8 KiB
Python
147 lines
8 KiB
Python
"""Hash-pinned native recovery receipts, with original completion timestamps."""
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
from service_assurance import timestamp
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def pinned_receipt(entry, root):
|
|
path = (root / entry['path']).resolve()
|
|
if not path.is_relative_to((root / 'docs/evidence').resolve()):
|
|
raise ValueError('receipt outside evidence directory')
|
|
raw = path.read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != entry['sha256']:
|
|
raise ValueError('receipt drift')
|
|
return json.loads(raw)
|
|
|
|
|
|
def operation_times(receipt, now):
|
|
start, finish = (timestamp(receipt[key]) for key in ('started_at', 'finished_at'))
|
|
if not start <= finish <= now:
|
|
raise ValueError('invalid receipt chronology')
|
|
return start, finish
|
|
|
|
|
|
def primary_archive(receipt, now):
|
|
"""Accept only explicit full primary transfers, including a verified versioned GET."""
|
|
operation_times(receipt, now)
|
|
if (receipt['schema'] != 'platform.forgejo-primary-archive.v1'
|
|
or receipt['status'] != 'primary_fetched_pending_application_restore'
|
|
or receipt['stage'] != 'transfer_verified'
|
|
or receipt['archive_profile'] != 'full'
|
|
or not receipt['destination'].startswith(
|
|
's3://railiance-platform-pg-backup/platform-pg/application-archives/forgejo/')
|
|
or not re.fullmatch(r'[0-9a-f]{64}', receipt['ciphertext_sha256'])
|
|
or not all(receipt.get(key) is True for key in
|
|
('multipart_completed', 'version_pinned', 'download_hash_matches'))
|
|
or type(receipt['ciphertext_bytes']) is not int
|
|
or receipt['ciphertext_bytes'] <= 0
|
|
or any(type(receipt[key]) is not int or receipt[key] != receipt['ciphertext_bytes']
|
|
for key in ('uploaded_bytes', 'downloaded_bytes'))):
|
|
raise ValueError('primary archive not accepted')
|
|
|
|
|
|
def archive_signal(entry, receipt, now, root):
|
|
if entry['signal'] == 'offsite.upload':
|
|
primary_archive(receipt, now)
|
|
else:
|
|
start, _ = operation_times(receipt, now)
|
|
if (receipt['schema'] != 'platform.forgejo-isolated-restore.v1'
|
|
or receipt['status'] != 'restored'
|
|
or receipt['archive_profile'] != 'full'
|
|
or receipt['source_provider'] != 'Scaleway'
|
|
or receipt['stage'] != 'package_blob_recovery'
|
|
or not all(receipt.get(key) is True for key in
|
|
('database_import', 'application_health', 'cleanup'))
|
|
or not receipt['repositories_verified']
|
|
or type(receipt['package_blobs_verified']) is not int
|
|
or receipt['package_blobs_verified'] < 0
|
|
or type(receipt['database_counts']['package_blobs']) is not int
|
|
or receipt['package_blobs_verified'] != receipt['database_counts']['package_blobs']):
|
|
raise ValueError('full application recovery not accepted')
|
|
decryption = pinned_receipt(entry['decryption'], root)
|
|
transfer = pinned_receipt(entry['transfer'], root)
|
|
primary_archive(transfer, now)
|
|
decrypt_start, decrypt_finish = operation_times(decryption, now)
|
|
if (decryption['schema'] != 'platform.forgejo-primary-decryption.v1'
|
|
or decryption['status'] != 'primary_fetched_pending_application_restore'
|
|
or decryption['archive_profile'] != 'full'
|
|
or decryption['decrypted'] is not True
|
|
or decryption['download_hash_matches'] is not True
|
|
or not re.fullmatch(r'[0-9a-f]{64}', decryption['plaintext_sha256'])
|
|
or receipt['transfer_receipt_sha256'] != entry['decryption']['sha256']
|
|
or decryption['transfer_receipt_sha256'] != entry['transfer']['sha256']
|
|
or not timestamp(transfer['finished_at']) <= decrypt_start <= decrypt_finish <= start
|
|
or receipt['offsite_artifact'] != transfer['destination']
|
|
or decryption['destination'] != transfer['destination']
|
|
or decryption['ciphertext_bytes'] != transfer['ciphertext_bytes']
|
|
or any(r['ciphertext_sha256'] != transfer['ciphertext_sha256']
|
|
for r in (receipt, decryption))):
|
|
raise ValueError('recovery provenance mismatch')
|
|
return {'result': 'pass', 'observed_at': receipt['finished_at']}
|
|
|
|
|
|
def recovery_signals(now, root=ROOT):
|
|
index = json.loads((root / 'assurance/recovery-evidence.json').read_text())
|
|
if index['schema'] != 'railiance-platform.recovery-evidence.v1':
|
|
raise ValueError('unknown recovery index')
|
|
signals = {}
|
|
for entry in index['receipts']:
|
|
signal = entry['signal']
|
|
if signal in signals or signal not in ('apps-pg.restore', 'forgejo-db.restore',
|
|
'openbao.snapshot', 'offsite.upload', 'offsite.restore'):
|
|
raise ValueError('unexpected recovery signal')
|
|
sample = {'result': 'unavailable', 'observed_at': now.isoformat()}
|
|
try:
|
|
if signal.startswith('offsite.'):
|
|
signals[signal] = archive_signal(entry, pinned_receipt(entry, root), now, root)
|
|
continue
|
|
path = (root / entry['path']).resolve()
|
|
allowed = [root / 'docs/evidence']
|
|
if signal == 'openbao.snapshot':
|
|
allowed.append(root / 'reviews')
|
|
if not any(path.is_relative_to(directory.resolve()) for directory in allowed):
|
|
raise ValueError('receipt outside evidence directory')
|
|
raw = path.read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != entry['sha256']:
|
|
raise ValueError('receipt drift')
|
|
receipt = json.loads(raw)
|
|
if signal == 'openbao.snapshot':
|
|
required = ('snapshot_created', 'source_initialized', 'source_unsealed',
|
|
'snapshot_encrypted', 'encrypted_copy_off_host',
|
|
'encryption_verified', 'hash_verified', 'no_secret_material_recorded')
|
|
if (receipt.get('receipt_version') != 1
|
|
or receipt.get('source_cluster') != 'railiance01'
|
|
or receipt.get('source_namespace') != 'openbao'
|
|
or receipt.get('cluster_id') != 'fd28df5d-98ec-57dd-42ec-9b3e4f4e53bf'
|
|
or not all(receipt.get(key) is True for key in required)
|
|
or not receipt.get('encrypted_location_ref', '').startswith('offhost-custody:')):
|
|
raise ValueError('snapshot not accepted')
|
|
for key in ('snapshot_sha256', 'encrypted_snapshot_sha256'):
|
|
value = receipt.get(key, '')
|
|
if not re.fullmatch(r'sha256:[0-9a-f]{64}', value):
|
|
raise ValueError('snapshot hash missing')
|
|
if timestamp(receipt['created_at']) > now:
|
|
raise ValueError('future snapshot')
|
|
signals[signal] = {'result': 'pass', 'observed_at': receipt['created_at']}
|
|
continue
|
|
cell = signal.removesuffix('.restore')
|
|
if (receipt['schema'] != 'platform.scaleway-primary-restore.v1'
|
|
or receipt['primary_destination'] != f's3://railiance-platform-pg-backup/platform-pg/{cell}/'
|
|
or receipt['source'] != 'Scaleway Barman base backup and WAL'
|
|
or receipt['stage'] != 'database_acceptance'
|
|
or receipt['status'] != 'verified'
|
|
or receipt['cleanup'] is not True
|
|
or receipt['production_ready'] is not True):
|
|
raise ValueError('receipt not accepted')
|
|
completed = timestamp(receipt['finished_at'])
|
|
if not timestamp(receipt['started_at']) <= completed <= now:
|
|
raise ValueError('invalid receipt chronology')
|
|
sample = {'result': 'pass', 'observed_at': receipt['finished_at']}
|
|
except (OSError, ValueError, KeyError, TypeError, AttributeError):
|
|
pass
|
|
signals[signal] = sample
|
|
return signals
|