Diagnose attended custody finish failures with safe staged receipts
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1029d-2c93-7500-a0e0-e3e089668b29
This commit is contained in:
codex
parent
b8f8a7ada6
commit
33ff82210e
8 changed files with 397 additions and 34 deletions
|
|
@ -19,22 +19,23 @@ from migrate_nextcloud_backup_account import HOST, OPERATOR_PATH, LANE, quota, r
|
|||
|
||||
VALIDATOR_DIGEST='9156e399c8d92a45b817bfc3738c89843487ce57e1718ed747f804f95cc719bb'
|
||||
SCHEMA_DIGEST='bb6cd6da5dc48eddcf118ae78c2cacfafd5f860098fd8a8368fe6bc845776d07'
|
||||
INDEPENDENT_ESCROW=Path('/home/worsch/.config/age/railiance-backup.key')
|
||||
|
||||
|
||||
def checked_bundle(state_path, validator):
|
||||
state=json.loads(state_path.read_text())
|
||||
if state.get('pending') or validator.validate(state['document']) or state['record_id']!=RECORD_ID or digest(state['document'])!=state['digest']:
|
||||
raise ValueError('retained_state_invalid')
|
||||
raise FinishFailure('retained_state_invalid')
|
||||
history=[]
|
||||
for path in sorted((state_path.parent/'history').glob('*.json')):
|
||||
if path.is_symlink() or path.stat().st_size>262144:
|
||||
raise ValueError('history_invalid')
|
||||
raise FinishFailure('history_invalid')
|
||||
receipt=json.loads(path.read_text())
|
||||
if validator.validate(receipt['document']) or receipt['record_id']!=RECORD_ID or receipt['digest']!=digest(receipt['document']):
|
||||
raise ValueError('history_invalid')
|
||||
raise FinishFailure('history_invalid')
|
||||
history.append(receipt)
|
||||
if state not in history:
|
||||
raise ValueError('latest_history_missing')
|
||||
raise FinishFailure('latest_history_missing')
|
||||
return state,json.dumps({'baseline':state,'history':history},sort_keys=True,separators=(',',':')).encode()
|
||||
|
||||
|
||||
|
|
@ -54,78 +55,169 @@ def historical_config_comparison(config_path, values):
|
|||
"historical_credential_status":"comparison-unavailable"}
|
||||
|
||||
|
||||
def roundtrip(bundle, lane, operator, network=request, get_quota=quota):
|
||||
def roundtrip(bundle, lane, operator, network=None, get_quota=None, progress=lambda stage, **details: None):
|
||||
network=request if network is None else network
|
||||
get_quota=quota if get_quota is None else get_quota
|
||||
progress('provider_contract')
|
||||
values=lane['data']
|
||||
if operator['BACKUP_USERNAME']!='Backup':
|
||||
raise ValueError('governed_owner_mismatch')
|
||||
raise FinishFailure('governed_owner_mismatch')
|
||||
auth=(operator['BACKUP_USERNAME'],operator['BACKUP_PASSWORD'])
|
||||
upload=values['NC_WEBDAV_URL']
|
||||
if not upload.startswith(HOST+'/public.php/dav/filesdrop/') or values['NC_WEBDAV_TOKEN']!=upload.rsplit('/',1)[-1]:
|
||||
raise ValueError('current_lane_contract_mismatch')
|
||||
raise FinishFailure('current_lane_contract_mismatch')
|
||||
key=(values['AGE_PRIVATE_KEY'].strip()+'\n').encode()
|
||||
def process(command,payload):
|
||||
result=subprocess.run(command,input=payload,capture_output=True,timeout=30)
|
||||
try:
|
||||
result=subprocess.run(command,input=payload,capture_output=True,timeout=30)
|
||||
except FileNotFoundError:
|
||||
raise FinishFailure('crypto_binary_unavailable') from None
|
||||
except subprocess.TimeoutExpired:
|
||||
raise FinishFailure('crypto_command_timeout') from None
|
||||
if result.returncode:
|
||||
raise ValueError('protected_process_failed')
|
||||
raise FinishFailure('crypto_command_failed')
|
||||
return result.stdout
|
||||
progress('derive_age_recipient')
|
||||
recipient=process(['age-keygen','-y','/dev/stdin'],key).decode().strip()
|
||||
progress('encrypt_history')
|
||||
encrypted=process(['age','-r',recipient],bundle)
|
||||
progress('independent_key_escrow')
|
||||
if not INDEPENDENT_ESCROW.is_file():
|
||||
raise FinishFailure('independent_key_escrow_missing')
|
||||
independent_recipient=process(['age-keygen','-y',str(INDEPENDENT_ESCROW)],b'').decode().strip()
|
||||
matched=hmac.compare_digest(recipient,independent_recipient)
|
||||
progress('independent_key_escrow',independent_escrow_recipient_matches_current_lane=matched)
|
||||
if not matched:
|
||||
raise FinishFailure('independent_key_escrow_mismatch')
|
||||
progress('provider_quota')
|
||||
if len(encrypted)>1048576 or get_quota(auth)['available_bytes']<len(encrypted):
|
||||
raise ValueError('backup_size_or_quota_invalid')
|
||||
raise FinishFailure('backup_size_or_quota_invalid')
|
||||
name='root-custody-history-'+datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')+'-'+uuid.uuid4().hex+'.json.age'
|
||||
progress('provider_upload',artifact=name)
|
||||
status,_=network(upload+'/forgejo/'+name,'PUT',encrypted,(values['NC_WEBDAV_TOKEN'],''),{'Content-Type':'application/octet-stream','If-None-Match':'*'})
|
||||
progress('provider_upload',upload_http_status=status)
|
||||
if status!=201:
|
||||
raise ValueError('new_artifact_upload_failed')
|
||||
raise FinishFailure('new_artifact_upload_failed')
|
||||
progress('provider_download')
|
||||
status,fetched=network(HOST+'/remote.php/dav/files/Backup/railiance-backups/forgejo/'+name,'GET',auth=auth)
|
||||
if status!=200 or hashlib.sha256(fetched).digest()!=hashlib.sha256(encrypted).digest():
|
||||
raise ValueError('provider_readback_mismatch')
|
||||
progress('provider_download',readback_http_status=status)
|
||||
if status!=200:
|
||||
raise FinishFailure('provider_download_failed')
|
||||
progress('provider_ciphertext_check')
|
||||
if hashlib.sha256(fetched).digest()!=hashlib.sha256(encrypted).digest():
|
||||
raise FinishFailure('provider_ciphertext_mismatch')
|
||||
progress('decrypt_and_compare_history')
|
||||
with tempfile.TemporaryDirectory(prefix='root-custody-recovery-') as tmp:
|
||||
cipher=Path(tmp)/'history.json.age'
|
||||
cipher.write_bytes(fetched)
|
||||
cipher.chmod(0o600)
|
||||
restored=process(['age','-d','-i','/dev/stdin',str(cipher)],key)
|
||||
restored=process(['age','-d','-i',str(INDEPENDENT_ESCROW),str(cipher)],b'')
|
||||
if restored!=bundle:
|
||||
raise ValueError('decrypted_history_mismatch')
|
||||
return {'provider':'existing-governed-nextcloud-backup','artifact':name,'upload_http_status':201,'readback_http_status':200,'ciphertext_sha256':hashlib.sha256(encrypted).hexdigest(),'decrypted_history_matches':True,'lane_kv_version':lane['metadata']['version'],'local_historical_rclone_used':False}
|
||||
raise FinishFailure('decrypted_history_mismatch')
|
||||
return {'provider':'existing-governed-nextcloud-backup','artifact':name,'upload_http_status':201,'readback_http_status':200,'ciphertext_sha256':hashlib.sha256(encrypted).hexdigest(),'decrypted_history_matches':True,'lane_kv_version':lane['metadata']['version'],'local_historical_rclone_used':False,'independent_escrow_recipient_matches_current_lane':True,'independent_escrow_decryption_verified':True}
|
||||
|
||||
|
||||
def main():
|
||||
STAGES = {
|
||||
'startup', 'attended_envelope', 'operator_endpoint', 'canon_pins', 'validator_import',
|
||||
'publisher_lock', 'retained_history', 'current_source_read', 'current_baseline_check',
|
||||
'backup_lane_read', 'backup_operator_read', 'historical_share_comparison',
|
||||
'provider_contract', 'derive_age_recipient', 'encrypt_history', 'independent_key_escrow', 'provider_quota',
|
||||
'provider_upload', 'provider_download', 'provider_ciphertext_check', 'decrypt_and_compare_history', 'complete',
|
||||
}
|
||||
|
||||
|
||||
class FinishFailure(ValueError):
|
||||
def __init__(self, code):
|
||||
self.code = code
|
||||
|
||||
|
||||
ERROR_CODES = {'stage_failed', 'attended_envelope_required', 'approved_operator_endpoint_required',
|
||||
'canon_pin_mismatch', 'current_backup_lane_incident', 'unexpected_stage',
|
||||
'crypto_binary_unavailable','crypto_command_timeout','crypto_command_failed','independent_key_escrow_missing','independent_key_escrow_mismatch',
|
||||
'provider_download_failed','provider_ciphertext_mismatch','decrypted_history_mismatch',
|
||||
'retained_state_invalid','history_invalid','latest_history_missing',
|
||||
'governed_owner_mismatch','current_lane_contract_mismatch','backup_size_or_quota_invalid','new_artifact_upload_failed'}
|
||||
|
||||
|
||||
def main(receipt_reserved=False):
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument('--state',type=Path,required=True)
|
||||
p.add_argument('--net-kingdom',type=Path,required=True)
|
||||
p.add_argument('--receipt',type=Path,required=True)
|
||||
args=p.parse_args()
|
||||
result={'ok':False,'stage':'startup','status':'started',
|
||||
'recorded_at':datetime.now(timezone.utc).isoformat()}
|
||||
def progress(stage, **details):
|
||||
if stage not in STAGES:
|
||||
raise FinishFailure('unexpected_stage')
|
||||
result['stage']=stage
|
||||
for key,value in details.items():
|
||||
if key in {'upload_http_status','readback_http_status'} and type(value) is int and 100 <= value <= 599:
|
||||
result[key]=value
|
||||
elif key=='independent_escrow_recipient_matches_current_lane' and type(value) is bool:
|
||||
result[key]=value
|
||||
elif key=='artifact' and isinstance(value,str) and value.startswith('root-custody-history-') and value.endswith('.json.age'):
|
||||
result[key]=value
|
||||
atomic_json(args.receipt,result)
|
||||
try:
|
||||
if args.receipt.exists() or args.receipt.is_symlink():
|
||||
if args.receipt.is_symlink():
|
||||
return 1
|
||||
require_attended()
|
||||
if not receipt_reserved:
|
||||
fd=os.open(args.receipt,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600)
|
||||
os.close(fd)
|
||||
progress('attended_envelope')
|
||||
try:
|
||||
require_attended()
|
||||
except Exception:
|
||||
raise FinishFailure('attended_envelope_required') from None
|
||||
progress('operator_endpoint')
|
||||
if os.getenv('BAO_ADDR')!='http://127.0.0.1:18200':
|
||||
return 1
|
||||
raise FinishFailure('approved_operator_endpoint_required')
|
||||
progress('canon_pins')
|
||||
source=args.net_kingdom/'tools/custody-record/validate.py'
|
||||
schema=args.net_kingdom/'canon/schemas/custody-record_v0.1.schema.json'
|
||||
if hashlib.sha256(source.read_bytes()).hexdigest()!=VALIDATOR_DIGEST or hashlib.sha256(schema.read_bytes()).hexdigest()!=SCHEMA_DIGEST:
|
||||
return 1
|
||||
raise FinishFailure('canon_pin_mismatch')
|
||||
progress('validator_import')
|
||||
spec=importlib.util.spec_from_file_location('custody_validator',source)
|
||||
validator=importlib.util.module_from_spec(spec);spec.loader.exec_module(validator)
|
||||
import fcntl
|
||||
progress('publisher_lock')
|
||||
with (args.state.parent/'publisher.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX)
|
||||
progress('retained_history')
|
||||
state,bundle=checked_bundle(args.state,validator)
|
||||
bao=Bao()
|
||||
check_baseline(bao.read('platform/data/custody/net-kingdom/platform-root'),state,validator)
|
||||
# Only current owner-held lane; no cached rclone config or caller token.
|
||||
progress('current_source_read')
|
||||
actual=bao.read('platform/data/custody/net-kingdom/platform-root')
|
||||
progress('current_baseline_check')
|
||||
check_baseline(actual,state,validator)
|
||||
result.update(record_id=RECORD_ID,revision=state['revision'],kv_version=state['kv_version'],
|
||||
digest=state['digest'],current_live_baseline_matches=True)
|
||||
progress('backup_lane_read')
|
||||
lane=bao.read(LANE)
|
||||
progress('backup_operator_read')
|
||||
operator=bao.read(OPERATOR_PATH)['data']
|
||||
progress('historical_share_comparison')
|
||||
comparison=historical_config_comparison(Path('/home/worsch/.config/rclone/rclone.conf'),lane['data'])
|
||||
result.update(comparison)
|
||||
if comparison['historical_share_matches_current_lane'] is True:
|
||||
atomic_json(args.receipt,{'ok':False,'current_live_baseline_matches':True,**comparison,
|
||||
'error':'current_backup_lane_requires_coordinated_existing_incident_recovery','recorded_at':datetime.now(timezone.utc).isoformat()})
|
||||
return 1
|
||||
receipt={**roundtrip(bundle,lane,operator),**comparison}
|
||||
atomic_json(args.receipt,{'ok':True,'scope':'current-owner-read-and-governed-history-provider-roundtrip','record_id':RECORD_ID,'revision':state['revision'],'kv_version':state['kv_version'],'digest':state['digest'],'current_live_baseline_matches':True,'history_entries':len(json.loads(bundle)['history']),'recorded_at':datetime.now(timezone.utc).isoformat(),**receipt})
|
||||
raise FinishFailure('current_backup_lane_incident')
|
||||
receipt=roundtrip(bundle,lane,operator,progress=progress)
|
||||
result.update(ok=True,status='verified',stage='complete',finished_at=datetime.now(timezone.utc).isoformat(),
|
||||
scope='current-owner-read-and-governed-history-provider-roundtrip',
|
||||
history_entries=len(json.loads(bundle)['history']),**receipt)
|
||||
atomic_json(args.receipt,result)
|
||||
return 0
|
||||
except Exception:
|
||||
# Server/credential/HTTP/process diagnostics are never disclosed.
|
||||
except FileExistsError:
|
||||
# Never change an old success receipt or claim it belongs to this attempt.
|
||||
return 1
|
||||
except Exception as error:
|
||||
result.update(status='failed',finished_at=datetime.now(timezone.utc).isoformat(),error_code=error.code if isinstance(error,FinishFailure) and error.code in ERROR_CODES else 'stage_failed')
|
||||
try:
|
||||
atomic_json(args.receipt,result)
|
||||
except Exception:
|
||||
pass
|
||||
return 1
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue