From a867ec269a5260d8e494ad225ecdd642cbdb4679 Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 6 Sep 2026 14:53:15 +0200 Subject: [PATCH] Record archive recovery lifecycle and validate receipt provenance Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a06ecb-456a-71c2-b41e-0755d336e883 --- docs/service-assurance.md | 14 +++ history/2026-09-06-archive-receipt-review.md | 35 +++++++ scripts/decrypt_primary_forgejo_archive.py | 19 +++- scripts/restore_forgejo_offsite_locally.py | 18 +++- scripts/scaleway_forgejo_archive.py | 5 +- scripts/verify_nextcloud_offsite_restore.py | 4 +- tests/test_archive_receipt_lifecycle.py | 99 +++++++++++++++++++ .../RPF-WP-0036-platform-service-assurance.md | 13 +++ ...-0038-forgejo-scaleway-primary-coverage.md | 7 ++ 9 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 history/2026-09-06-archive-receipt-review.md create mode 100644 tests/test_archive_receipt_lifecycle.py diff --git a/docs/service-assurance.md b/docs/service-assurance.md index fe292d5..d00304b 100644 --- a/docs/service-assurance.md +++ b/docs/service-assurance.md @@ -115,3 +115,17 @@ Update pins only after reviewing replacement evidence. Native database recovery does not attest full application or essentials recovery. Legacy archive receipts without completion timestamps remain manual evidence; no timestamp is inferred from file modification time. Automatic cadence and alert delivery remain pending. + +New archive runs record UTC `started_at` and terminal `finished_at` in transfer, +decryption and isolated restore receipts, including failures. A timestamp alone +is not success: consumers must validate status, provider/profile, hashes and +cleanup. Restore completion is recorded after scratch cleanup, and cleanup +failure sets status to `failed`. Decryption and restore also record SHA-256 of +the exact input receipt bytes as `transfer_receipt_sha256`. + +Decryption now retains `platform.forgejo-primary-decryption.v1` and its own +operation times; older decryption receipts used the transfer schema through an +overwrite bug. The restore tool explicitly accepts both forms, with verified +hash/decryption flags. Existing historical receipts are unchanged. These producer +fixes enable future dated archive evidence; automatic archive adapters, fresh +end-to-end receipts and recurring execution are still pending. diff --git a/history/2026-09-06-archive-receipt-review.md b/history/2026-09-06-archive-receipt-review.md new file mode 100644 index 0000000..e06be9c --- /dev/null +++ b/history/2026-09-06-archive-receipt-review.md @@ -0,0 +1,35 @@ +# Repository review and archive receipt repair — 2026-09-06 + +The checkout was clean at 445f136. Inbox was empty. Source still has seven open +plans: WP-0038 active and six blocked on the explicit gates reviewed in +`2026-09-06-blocked-workplan-progress.md`. The dated generated brief includes +retired aliases; it is not evidence of additional actionable work. + +Selected local work under WP-0036-T03 / WP-0038-T04 supports INTENT's tested, +observable recovery requirement. Review found three concrete defects: + +1. Archive transfer and restore producers omitted operation times, preventing + reliable freshness checks. All four producers now record UTC start and + terminal finish times, including failed operations. +2. Primary decryption copied the whole transfer receipt over its own schema. + It now copies only needed provenance fields and keeps its own schema/times. +3. Failed scratch cleanup could return nonzero while retaining a successful + restore status. The receipt now records failure as well. + +Restore/decryption bind the exact input receipt bytes by SHA-256. Restore +requires recognized provider receipt schemas, verified decryption/download +claims and matching plaintext hash before running Docker. Both the historical +primary schema and the corrected decryption schema remain supported. These +hashes bind local evidence; they are not signatures or independent attestations. + +Validation: 31 tests passed across receipt lifecycle, offsite boundaries, +backup tiers, native recovery adapters and archive integrity. Tests exercise +failed validation before credentials/Docker, terminal timestamps, cleanup +failure, provenance mismatch, schema preservation, primary legacy/new receipts, +and both archive profiles. No live backup, credential retrieval, cleanup of +existing backups or scheduler mutation was required. + +Remaining: historical receipts are not backdated, and no fresh end-to-end +archive proof was claimed. Automatic archive evidence adapters and durable +scheduled delivery/retention remain open. All existing whole-plan statuses +remain accurate; this implementation does not satisfy external approval gates. diff --git a/scripts/decrypt_primary_forgejo_archive.py b/scripts/decrypt_primary_forgejo_archive.py index 3682bff..4f0dda3 100644 --- a/scripts/decrypt_primary_forgejo_archive.py +++ b/scripts/decrypt_primary_forgejo_archive.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Silent attended decryption of the hash-verified primary archive download.""" import argparse +from datetime import datetime, timezone import hashlib import json import os @@ -16,12 +17,15 @@ def digest(path): def main(): p=argparse.ArgumentParser(description=__doc__) for name in ['source','output','transfer-receipt','receipt']: p.add_argument('--'+name,required=True,type=Path) - a=p.parse_args();receipt={'schema':'platform.forgejo-primary-decryption.v1','status':'failed'} + a=p.parse_args();receipt={'schema':'platform.forgejo-primary-decryption.v1','status':'failed','started_at':datetime.now(timezone.utc).isoformat()} fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) try: - transfer=json.loads(a.transfer_receipt.read_text()) - if (transfer.get('status')!='primary_fetched_pending_application_restore' - or not transfer.get('download_hash_matches') + raw=a.transfer_receipt.read_bytes() + transfer=json.loads(raw) + receipt['transfer_receipt_sha256']=hashlib.sha256(raw).hexdigest() + if (transfer.get('schema')!='platform.forgejo-primary-archive.v1' + or transfer.get('status')!='primary_fetched_pending_application_restore' + or transfer.get('download_hash_matches') is not True or digest(a.source)!=transfer.get('ciphertext_sha256')): raise ValueError('verified_primary_download_required') value=data(bao(['read','-format=json','platform/data/workloads/railiance/backup/offsite-lane']))['data']['data']['AGE_PRIVATE_KEY'] @@ -29,10 +33,15 @@ def main(): a.output.chmod(0o600) result=subprocess.run(['age','-d','-i','/dev/stdin',str(a.source)],input=(value.strip()+'\n').encode(),stdout=target,stderr=subprocess.PIPE,timeout=1200) if result.returncode: raise ValueError('primary_decryption_failed') - receipt.update(transfer,decrypted=True,plaintext_sha256=digest(a.output)) + # Preserve this operation's schema and times, not the transfer's. + for key in ('destination','ciphertext_sha256','ciphertext_bytes','download_hash_matches','archive_profile'): + if key in transfer: receipt[key]=transfer[key] + receipt.update(status='primary_fetched_pending_application_restore', + decrypted=True,plaintext_sha256=digest(a.output)) except Exception: receipt['error']='bounded_primary_decryption_failed' finally: + receipt['finished_at']=datetime.now(timezone.utc).isoformat() with os.fdopen(fd,'w') as output: json.dump(receipt,output,indent=2) return int(receipt['status']=='failed') if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/restore_forgejo_offsite_locally.py b/scripts/restore_forgejo_offsite_locally.py index fe733bc..6ad0ab3 100644 --- a/scripts/restore_forgejo_offsite_locally.py +++ b/scripts/restore_forgejo_offsite_locally.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Restore a fetched Forgejo archive on a disposable, internal Docker network.""" import argparse +from datetime import datetime, timezone import configparser import hashlib import json @@ -189,13 +190,19 @@ def main(): p.add_argument('--archive',required=True,type=Path);p.add_argument('--receipt',required=True) p.add_argument('--profile',choices=['full','essentials'],default='full') p.add_argument('--transfer-receipt',required=True,type=Path) - a=p.parse_args();receipt={'schema':'platform.forgejo-isolated-restore.v1','status':'failed','images':[FORGE,POSTGRES]} + a=p.parse_args();receipt={'schema':'platform.forgejo-isolated-restore.v1','status':'failed','started_at':datetime.now(timezone.utc).isoformat(),'images':[FORGE,POSTGRES]} fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) try: - transfer=json.loads(a.transfer_receipt.read_text()) - nextcloud = (transfer.get('status') == 'offsite_fetched_pending_isolated_restore' + raw=a.transfer_receipt.read_bytes() + transfer=json.loads(raw) + receipt['transfer_receipt_sha256']=hashlib.sha256(raw).hexdigest() + nextcloud = (transfer.get('schema') == 'platform.real-offsite-recovery.v1' + and transfer.get('decrypted') is True + and transfer.get('upload_http_status') == 201 + and transfer.get('status') == 'offsite_fetched_pending_isolated_restore' and transfer.get('download_http_status') == 200) - primary = (transfer.get('status') == 'primary_fetched_pending_application_restore' + primary = (transfer.get('schema') in ('platform.forgejo-primary-archive.v1', 'platform.forgejo-primary-decryption.v1') + and transfer.get('status') == 'primary_fetched_pending_application_restore' and transfer.get('download_hash_matches') is True and transfer.get('decrypted') is True) if (not (nextcloud or primary) or transfer.get('plaintext_sha256') != file_digest(a.archive)): @@ -205,10 +212,13 @@ def main(): receipt['ciphertext_sha256']=transfer['ciphertext_sha256'] if transfer.get('archive_profile','full')!=a.profile: raise ValueError('profile_provenance_mismatch') run(a.archive,receipt,a.profile) + if receipt.get('cleanup') is not True: + raise ValueError('restore_cleanup_incomplete') except Exception: receipt['status']='failed' receipt['error']='isolated_restore_failed' finally: + receipt['finished_at']=datetime.now(timezone.utc).isoformat() with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2) return int(receipt['status'] not in ('restored','restored_essentials') or not receipt.get('cleanup')) diff --git a/scripts/scaleway_forgejo_archive.py b/scripts/scaleway_forgejo_archive.py index 6f88f4d..4cd87da 100644 --- a/scripts/scaleway_forgejo_archive.py +++ b/scripts/scaleway_forgejo_archive.py @@ -70,7 +70,7 @@ def main(): p.add_argument('--output',required=True,type=Path);p.add_argument('--receipt',required=True,type=Path) p.add_argument('--kubeconfig',required=True) a=p.parse_args() - receipt={'schema':'platform.forgejo-primary-archive.v1','status':'running'} + receipt={'schema':'platform.forgejo-primary-archive.v1','status':'running','started_at':datetime.now(timezone.utc).isoformat()} fd=os.open(a.receipt,os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o600);os.close(fd) def checkpoint(): a.receipt.write_text(json.dumps(receipt,indent=2)+'\n') try: @@ -99,6 +99,9 @@ def main(): receipt['exception_type']=type(error).__name__ receipt.update(status='failed',error='bounded_primary_archive_transfer_failed');checkpoint() return 1 + finally: + receipt['finished_at']=datetime.now(timezone.utc).isoformat() + checkpoint() return 0 if __name__=='__main__': raise SystemExit(main()) diff --git a/scripts/verify_nextcloud_offsite_restore.py b/scripts/verify_nextcloud_offsite_restore.py index e903d46..03ea5da 100644 --- a/scripts/verify_nextcloud_offsite_restore.py +++ b/scripts/verify_nextcloud_offsite_restore.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Silent attended transfer of a real encrypted backup for isolated recovery.""" import argparse +from datetime import datetime, timezone import base64 import urllib.request import shutil @@ -70,10 +71,11 @@ def main(): p.add_argument('--receipt',required=True) a=p.parse_args() fd=os.open(a.receipt,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600) - receipt={'schema':'platform.real-offsite-recovery.v1','status':'failed'} + receipt={'schema':'platform.real-offsite-recovery.v1','status':'failed','started_at':datetime.now(timezone.utc).isoformat()} try: run(a.source,a.directory,receipt) except Exception as error: receipt['error']=str(error) if isinstance(error,LaneError) else 'internal_error' finally: + receipt['finished_at']=datetime.now(timezone.utc).isoformat() with os.fdopen(fd,'w') as f: json.dump(receipt,f,indent=2) return int(receipt['status']=='failed') diff --git a/tests/test_archive_receipt_lifecycle.py b/tests/test_archive_receipt_lifecycle.py new file mode 100644 index 0000000..dad33f5 --- /dev/null +++ b/tests/test_archive_receipt_lifecycle.py @@ -0,0 +1,99 @@ +"""Exercise receipt CLI boundaries without credentials, providers or Docker.""" +from datetime import datetime +import hashlib +import json +from pathlib import Path +import sys +from unittest.mock import patch +import pytest +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +import restore_forgejo_offsite_locally as restore +import verify_nextcloud_offsite_restore as secondary +import scaleway_forgejo_archive as primary +import decrypt_primary_forgejo_archive as decrypt + + +def times(receipt): + start, finish = (datetime.fromisoformat(receipt[key]) for key in ('started_at', 'finished_at')) + assert start.tzinfo is not None and finish.tzinfo is not None + assert start <= finish + + +def provenance(archive): + return dict(schema='platform.real-offsite-recovery.v1', + status='offsite_fetched_pending_isolated_restore', decrypted=True, + upload_http_status=201, download_http_status=200, artifact='fixture.zip.age', + ciphertext_sha256='a'*64, plaintext_sha256=hashlib.sha256(archive.read_bytes()).hexdigest()) + + +@pytest.mark.parametrize('case', ['success', 'cleanup_failure', 'wrong_schema', 'not_decrypted', 'wrong_hash']) +def test_restore_receipt_binds_provenance_and_finishes_after_cleanup(tmp_path, case): + archive=tmp_path/'archive.zip'; archive.write_bytes(b'fixture') + source=tmp_path/'transfer.json'; output=tmp_path/'receipt.json' + evidence=provenance(archive) + if case=='wrong_schema': evidence['schema']='other' + if case=='not_decrypted': evidence['decrypted']=False + if case=='wrong_hash': evidence['plaintext_sha256']='b'*64 + source.write_text(json.dumps(evidence)) + def run(archive, receipt, profile): + assert 'finished_at' not in receipt + receipt.update(status='restored',cleanup=case!='cleanup_failure') + with patch.object(sys,'argv',['restore','--archive',str(archive),'--receipt',str(output),'--transfer-receipt',str(source)]), patch.object(restore,'run',side_effect=run) as runner: + result=restore.main() + receipt=json.loads(output.read_text()); times(receipt) + assert receipt['transfer_receipt_sha256']==hashlib.sha256(source.read_bytes()).hexdigest() + assert result == (0 if case=='success' else 1) + assert receipt['status']==('restored' if case=='success' else 'failed') + if case in ('wrong_schema','not_decrypted','wrong_hash'): runner.assert_not_called() + + +@pytest.mark.parametrize('success', [False, True]) +def test_secondary_records_both_terminal_outcomes(tmp_path, success): + output=tmp_path/'receipt.json' + def run(source, directory, receipt): + if not success: raise RuntimeError('PRIVATE_CANARY') + receipt['status']='offsite_fetched_pending_isolated_restore' + with patch.object(sys,'argv',['transfer','--source','unused','--directory','unused','--receipt',str(output)]), patch.object(secondary,'run',side_effect=run): + assert secondary.main()==(0 if success else 1) + receipt=json.loads(output.read_text()); times(receipt) + assert 'PRIVATE_CANARY' not in output.read_text() + + +def test_primary_validation_failure_has_terminal_time_without_credentials(tmp_path): + source=tmp_path/'source.json';source.write_text('{}');output=tmp_path/'receipt.json' + with patch.object(sys,'argv',['primary','--source','missing','--source-receipt',str(source),'--output',str(tmp_path/'out'),'--receipt',str(output),'--kubeconfig','unused']), patch.object(primary,'assert_cluster') as cluster: + assert primary.main()==1 + cluster.assert_not_called() + times(json.loads(output.read_text())) + + +def test_decryption_keeps_own_schema_and_times(tmp_path): + encrypted=tmp_path/'encrypted';encrypted.write_bytes(b'fixture') + source=tmp_path/'source.json';source.write_text(json.dumps(dict(schema='platform.forgejo-primary-archive.v1',status='primary_fetched_pending_application_restore',download_hash_matches=True,ciphertext_sha256=hashlib.sha256(b'fixture').hexdigest(),started_at='2000-01-01T00:00:00Z',finished_at='2000-01-01T00:00:01Z'))) + output=tmp_path/'receipt.json' + with patch.object(sys,'argv',['decrypt','--source',str(encrypted),'--output',str(tmp_path/'plain'),'--transfer-receipt',str(source),'--receipt',str(output)]), patch.object(decrypt,'bao'), patch.object(decrypt,'data',return_value={'data':{'data':{'AGE_PRIVATE_KEY':'fixture'}}}), patch.object(decrypt.subprocess,'run') as command: + command.return_value.returncode=0 + assert decrypt.main()==0 + receipt=json.loads(output.read_text());times(receipt) + assert receipt['schema']=='platform.forgejo-primary-decryption.v1' + assert not receipt['started_at'].startswith('2000') + assert receipt['transfer_receipt_sha256']==hashlib.sha256(source.read_bytes()).hexdigest() + + +@pytest.mark.parametrize('schema', ['platform.forgejo-primary-archive.v1', 'platform.forgejo-primary-decryption.v1']) +@pytest.mark.parametrize('profile', ['full', 'essentials']) +def test_primary_restore_accepts_legacy_and_distinct_decryption_receipts(tmp_path, schema, profile): + archive=tmp_path/'archive.zip';archive.write_bytes(b'fixture') + source=tmp_path/'transfer.json';output=tmp_path/'receipt.json' + source.write_text(json.dumps(dict(schema=schema, + status='primary_fetched_pending_application_restore', decrypted=True, + download_hash_matches=True, archive_profile=profile, + destination='s3://railiance-platform-pg-backup/platform-pg/application-archives/forgejo/fixture.zip.age', + ciphertext_sha256='a'*64, plaintext_sha256=hashlib.sha256(archive.read_bytes()).hexdigest()))) + def run(archive, receipt, selected): + assert selected==profile + receipt.update(status='restored' if profile=='full' else 'restored_essentials',cleanup=True) + with patch.object(sys,'argv',['restore','--archive',str(archive),'--receipt',str(output),'--transfer-receipt',str(source),'--profile',profile]), patch.object(restore,'run',side_effect=run): + assert restore.main()==0 + receipt=json.loads(output.read_text()); times(receipt) + assert receipt['source_provider']=='Scaleway' diff --git a/workplans/RPF-WP-0036-platform-service-assurance.md b/workplans/RPF-WP-0036-platform-service-assurance.md index f8da056..36d042c 100644 --- a/workplans/RPF-WP-0036-platform-service-assurance.md +++ b/workplans/RPF-WP-0036-platform-service-assurance.md @@ -278,3 +278,16 @@ reports 16 healthy, six missing and one stale ESO refresh signal. The aggregate crossed the one-hour diagnostic boundary; subsequent metadata inspection found all 27 ExternalSecrets Ready with newer refreshes. Cadence/grace acceptance is still needed; no outage or successful alert transport is inferred. + +## Archive receipt producer repair — 2026-09-06 + +T03 progressed further: primary/secondary transfer, primary decryption and +isolated restore now emit their own start/finish timestamps, including failures. +Decryption preserves its schema instead of inheriting the transfer's. Restore +checks receipt schema and decryption proof, hashes the exact input receipt and +marks failed cleanup as failed recovery. This closes the producer timestamp gap +for future runs; old evidence remains unchanged. Native adapters remain the only +automatic recovery adapters pending reviewed fresh archive receipts and adapter +implementation. Validation and residual gates are recorded in +`history/2026-09-06-archive-receipt-review.md`; T03 remains waiting on its wider +cadence/custody/acceptance gates. diff --git a/workplans/RPF-WP-0038-forgejo-scaleway-primary-coverage.md b/workplans/RPF-WP-0038-forgejo-scaleway-primary-coverage.md index 6fd53c6..f29cdd3 100644 --- a/workplans/RPF-WP-0038-forgejo-scaleway-primary-coverage.md +++ b/workplans/RPF-WP-0038-forgejo-scaleway-primary-coverage.md @@ -106,3 +106,10 @@ live expiration or cron cutover performed. T04 remains in progress for durable scheduled caller/dependency binding, canonical verified inventory, fresh quota checks and owner retention activation. See `history/2026-09-06-backup-tiers-implementation.md`. + +Receipt follow-up, September 6: new archive transfer/decryption/restore runs now +record operation timestamps and provenance receipt hashes. Cleanup failure cannot +leave a successful restore status. Historical primary decryption receipts remain +accepted explicitly, while new decryption retains its own schema. No repeat +upload, expiry or scheduled caller change was made; T04 remains in progress for +the existing durable caller, inventory, quota and retention gates.