diff --git a/deploy/backup/coulomb-social-paired-backup.service b/deploy/backup/coulomb-social-paired-backup.service index 4f68658..8affb50 100644 --- a/deploy/backup/coulomb-social-paired-backup.service +++ b/deploy/backup/coulomb-social-paired-backup.service @@ -13,7 +13,11 @@ StateDirectoryMode=0700 EnvironmentFile=/etc/coulomb-social-backup/primary.env # Public recipient only, delivered after independent Coulomb custody verification. EnvironmentFile=/etc/coulomb-social-backup/recipient.env +# Catch up an unmirrored primary before allowing another unattended capture. +# New helper/runtime installation remains subject to the operating review. +ExecStartPre=/opt/railiance-platform/backup-venv/bin/python /opt/coulomb-social-backup/mirror-paired-backup.py --execute ExecStart=/usr/bin/python3 /opt/coulomb-social-backup/paired-backup.py --execute +ExecStartPost=/opt/railiance-platform/backup-venv/bin/python /opt/coulomb-social-backup/mirror-paired-backup.py --execute TimeoutStartSec=30min TimeoutStopSec=5min KillMode=mixed diff --git a/deploy/backup/mirror-platform-runtime.json b/deploy/backup/mirror-platform-runtime.json new file mode 100644 index 0000000..96b742d --- /dev/null +++ b/deploy/backup/mirror-platform-runtime.json @@ -0,0 +1,11 @@ +{ + "files": { + "application_recovery_identities.py": "20f56255ed6eeec4aafe33f142bc3ee03f623d4957a7b0cabb741d340f15f79d", + "application_recovery_mirror.py": "d48ab95e89b9eeae36269d853ebe4c44c22e1f57d4e14224f03d0b9aeeb34178", + "application_recovery_resources.py": "ab70ec341e738272730eb0daf76840d0bca807caeb3c4e4c2357f162fbed4498", + "primary_application_archive.py": "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d" + }, + "owner": "railiance-platform", + "schema": "coulomb.mirror-platform-runtime.v1", + "source_commit": "09f4fd33cbeefecb833998246cf53848bb1e5d8d" +} diff --git a/docs/release/backup-operating-preflight-2026-10-30.json b/docs/release/backup-operating-preflight-2026-10-30.json index fa46e4d..a78d346 100644 --- a/docs/release/backup-operating-preflight-2026-10-30.json +++ b/docs/release/backup-operating-preflight-2026-10-30.json @@ -1,4 +1,47 @@ { + "followup_checker_sha256": "c003871fe91c5c31788fd9bfc50aef3b2a275c06c5a42c2c3165b92d5ea80db9", + "followup_exact_unit_verification": { + "exit_code": 0, + "status": "verified", + "target_sdk_executable": true, + "temporary_review_files_removed": true, + "unit_sha256": { + "coulomb-social-backup-freshness.service": "aaed68bc4bce752eb55a22c1b14bf74df306f1b0c017bd5bdfaff0f0943878d8", + "coulomb-social-backup-freshness.timer": "75d1d0721c07fdd0703910c06bb8a678396440e7d95411a37ee7de4938f3ac31", + "coulomb-social-paired-backup.service": "dd0049cbbc0e8adc7ce48d39b7ab2d1689b432cac094e9654308abcaabbb3486", + "coulomb-social-paired-backup.timer": "5e87235e66f89a64a972562d6f163f62420628fc44fb7fd8d603a96c347dfbb9" + }, + "units_installed": false, + "units_started": false + }, + "followup_observation": { + "backup_process_holds_lock": false, + "capture_age_seconds": 6930.156, + "free_bytes": 118889779200, + "maximum_state_bytes": 8589934592, + "minimum_free_bytes": 8589934592, + "notification_sent": false, + "observed_at": "2026-10-29T23:18:16.156469+00:00", + "operator_reception_proven": false, + "pending_count": 0, + "provider_contacted": false, + "reasons": [ + "matching_mirror_evidence_missing_or_invalid" + ], + "recorded_readback_only": true, + "schema": "coulomb.paired-backup-health.v1", + "state_bytes": 2123252, + "state_modified": false, + "status": "unhealthy" + }, + "followup_target_sdk": { + "boto3": "1.40.18", + "botocore": "1.40.76", + "conditional_metadata_put_supported": true, + "credentials_read": false, + "interpreter_target": "/usr/bin/python3.12", + "provider_contacted": false + }, "limits_owner_admitted": false, "native_checker_sha256": "e30047d5759b1f200516dad09dec2368f88da505109f601165ead192a786fde4", "node_files_installed": false, diff --git a/docs/release/backup-operating-preflight-2026-10-30.md b/docs/release/backup-operating-preflight-2026-10-30.md index 29ff814..9657fc8 100644 --- a/docs/release/backup-operating-preflight-2026-10-30.md +++ b/docs/release/backup-operating-preflight-2026-10-30.md @@ -47,6 +47,9 @@ scheduled checker and the proposed new one have not been installed by this work. or independent reader belongs on the worker. Promote last-mirror only after the matching version/hash readback, and block another unattended capture while the previous point lacks its required mirror/receipt. + The source worker, pinned dependency manifest and pre/post service hooks are + now prepared in mirror-worker-operating-review-2026-10-30.md. Its synthetic + integration is verified; native publication/retrieval is still outstanding. 2. Complete the rolling-retention/headroom contract, provider prefix expiry compatibility and pre-capture budget/reserve guard. Preserve protected kit and restored point outside rolling expiry. Historical private copies still await @@ -64,3 +67,8 @@ pending failures hidden by a fresh point, active/stalled locks, capacity and symlink refusal. They do not prove provider availability, received alerts or recurring operation. The existing storage release packet remains unchanged. The full `make check` gate passed 273 tests, including mandatory browser journeys. + +Follow-up: the updated checker additionally requires retained public-receipt +metadata. Its exact source hash and read-only native observation, plus exact +candidate-unit verification on the target node, are appended to the JSON proof; +the initial observation and source hash above remain historical evidence. diff --git a/docs/release/mirror-worker-operating-review-2026-10-30.md b/docs/release/mirror-worker-operating-review-2026-10-30.md new file mode 100644 index 0000000..88093f2 --- /dev/null +++ b/docs/release/mirror-worker-operating-review-2026-10-30.md @@ -0,0 +1,91 @@ +# Recurring mirror and independent receipt preparation + +The paired capture previously advanced its primary receipt without a recurring +mirror step. The first manually verified recovery mirror and protected kit remain +valid, but they do not mirror later daily points. The new worker prepares that +missing operating step; it has not been installed or run against the provider. + +## Result and retry boundary + +`scripts/mirror-paired-backup.py` previews without credentials, state access or +provider contact. Execute requires the existing own publisher environment; it +does not fall back to SDK credential discovery or read reader/custodian material. +It verifies the exact primary receipt and local ciphertext, then uses the same +backup lock to exclude capture/mirror overlap. + +The platform's existing mirror adapter and immutable multipart engine are reused, +with four owner-source hashes pinned in `deploy/backup/mirror-platform-runtime.json`. +Install those exact files from railiance-platform commit09f4fd33 into its existing +scripts directory during the reviewed operating delivery. The dedicated locked +backup Python environment is reused; no SDK belongs in the product image. +All pins are checked before imports, client construction or state modification. + +The mirror preserves the primary key/version/hash, image, public recipient and +paired-store hashes. The multipart engine conditionally creates the ciphertext +and downloads its exact version. A local per-binding journal preserves uncertainty +for retry. A fresh encrypted readback scratch directory is removed afterward; +no retained ciphertext or historical plaintext copy is deleted. + +The worker then uploads a strictly selected public receipt beneath the same own +`archives/` prefix. Arbitrary local fields are excluded. Its key includes the +receipt payload hash; PUT uses IfNoneMatch=* and the returned immutable version +is downloaded and compared byte for byte. On an uncertain PUT reply, retry adopts +only matching existing metadata and reads that exact version. Changed retained +versions or bindings refuse. Only after both readbacks does an atomic checkpoint +advance last-mirror. Primary inventory/last-success remain owned by the original +capture worker and are unchanged here. No application restore is claimed. + +The candidate service retries the latest mirror in ExecStartPre, before allowing +another unattended capture, then mirrors the new point in ExecStartPost. A mirror +failure fails the unit and prevents its next pre-capture step from silently +advancing past that failure. The capture helper's existing pending review guards +remain in force. The hourly checker now also requires the retained receipt's +key/hash/version/size and readback flag, so ciphertext-only mirror metadata cannot +certify operating health. Timers remain disabled/absent on the node. + +## Verification + +The reproducible development rehearsal executes the actual hash-pinned platform +engine against an in-memory synthetic store, without constructing a provider +client or reading credentials: + +```sh +uv run python scripts/rehearse-backup-mirror.py \ + --platform-scripts /private/reviewed-railiance-platform/scripts \ + --output /tmp/mirror-worker-rehearsal.json +``` + +`mirror-worker-rehearsal-2026-10-30.json` binds the worker, checker, driver, fixture +sources and all platform sources. Ordinary success and lost metadata PUT reply +both produce exactly one ciphertext multipart completion and one conditional +receipt PUT. Retry verifies existing versions without duplicate retained objects; +the primary is unchanged and only the own archive prefix is used. This is +synthetic integration evidence, not native provider execution. + +53 focused cases cover the backup, monitor and worker, including stale/foreign +metadata, lost replies, corrupt readback, version changes, interrupted final +checkpoint, concurrent locks, private-field filtering, runtime tampering and +missing own credentials. +The full `make check` gate passed 291 tests, including mandatory browser journeys. + +Exact candidate units passed systemd-analyze verification on railiance01 using +temporary review files that were removed; no unit was installed or started. +The installed SDK uses boto3 1.40.18 / botocore 1.40.76 and exposes conditional +metadata PUT. Its interpreter resolves to /usr/bin/python3.12, compatible with +ProtectHome. The updated checker was also rerun read-only against the real state; +it still identifies missing worker mirror evidence. Native metadata/source hashes +are retained in `backup-operating-preflight-2026-10-30.json`. + +## Remaining operating return + +Native publisher/mirror/receipt publication and independent reader retrieval of +that exact receipt still need the reviewed operating execution. Then bind +received failed/pending/stale alerts, finish rolling provider retention and the +pre-capture capacity guard, resolve historical-copy disposition, install exact +reviewed sources/units and prove service runs before enabling approved timers. +Local state remains bounded only by the proposed monitor threshold; this worker +does not enforce provider retention or a pre-capture budget. + +Existing custody/schedule approvals remain valid. This preparation changes no +grant, credential, provider object, installed helper, active timer or production +application. The separate storage release packet and its digest are unchanged. diff --git a/docs/release/mirror-worker-rehearsal-2026-10-30.json b/docs/release/mirror-worker-rehearsal-2026-10-30.json new file mode 100644 index 0000000..ed9e164 --- /dev/null +++ b/docs/release/mirror-worker-rehearsal-2026-10-30.json @@ -0,0 +1,49 @@ +{ + "checker_sha256": "c003871fe91c5c31788fd9bfc50aef3b2a275c06c5a42c2c3165b92d5ea80db9", + "credentials_read": false, + "driver_sha256": "26d60db0675b76dc0e305a7296c727eddd8ac3b280f93a48cfb4e883d5e4a8e8", + "fixture_sha256": { + "test_backup_health.py": "6d9397ff77d00f0d17ca8a6cb1ec81f09c8859ad450b1b2a673d19914b90b6f8", + "test_backup_mirror_worker.py": "827649cbc1a2bc56c883cf45bae2d700d0e7a5886721d3917c9fc5ac57127589" + }, + "node_files_changed": false, + "provider_contacted": false, + "runtime_manifest": { + "files": { + "application_recovery_identities.py": "20f56255ed6eeec4aafe33f142bc3ee03f623d4957a7b0cabb741d340f15f79d", + "application_recovery_mirror.py": "d48ab95e89b9eeae36269d853ebe4c44c22e1f57d4e14224f03d0b9aeeb34178", + "application_recovery_resources.py": "ab70ec341e738272730eb0daf76840d0bca807caeb3c4e4c2357f162fbed4498", + "primary_application_archive.py": "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d" + }, + "owner": "railiance-platform", + "schema": "coulomb.mirror-platform-runtime.v1", + "source_commit": "09f4fd33cbeefecb833998246cf53848bb1e5d8d" + }, + "scenarios": [ + { + "ciphertext_multipart_completions": 1, + "exact_version_hash_readbacks": true, + "last_mirror_only_after_full_readback": true, + "lost_receipt_put_reply": false, + "metadata_conditional_puts": 1, + "only_coulomb_archive_prefix_used": true, + "primary_unchanged": true, + "retained_objects": 2 + }, + { + "ciphertext_multipart_completions": 1, + "exact_version_hash_readbacks": true, + "last_mirror_only_after_full_readback": true, + "lost_receipt_put_reply": true, + "metadata_conditional_puts": 1, + "only_coulomb_archive_prefix_used": true, + "primary_unchanged": true, + "retained_objects": 2 + } + ], + "schema": "coulomb.mirror-worker-rehearsal.v1", + "status": "pinned_engine_synthetic_integration_passed", + "synthetic_store": true, + "timers_activated": false, + "worker_sha256": "f1209356b6ccdfcf434d5806eb104a11e1493caf6d6138986bf067116648e64a" +} diff --git a/scripts/check-paired-backup-health.py b/scripts/check-paired-backup-health.py index 1861e3a..71810b4 100644 --- a/scripts/check-paired-backup-health.py +++ b/scripts/check-paired-backup-health.py @@ -92,7 +92,7 @@ def validate_primary(receipt): return captured -def validate_mirror(primary, mirrored): +def validate_mirror(primary, mirrored, *, require_retained_receipt=False): validate_primary(mirrored) ref = mirrored["recovery_mirror"] if (any(mirrored[k] != primary[k] for k in BINDING) @@ -104,6 +104,18 @@ def validate_mirror(primary, mirrored): or not isinstance(ref.get("version_id"), str) or not ref["version_id"] or ref["version_id"] == "null"): raise Refused("mirror_binding_differs") + if require_retained_receipt: + digest = ref.get("receipt_sha256", "") + version = ref.get("receipt_version_id") + if (not isinstance(digest, str) or not re.fullmatch(r"[a-f0-9]{64}", digest) + or ref.get("receipt_object_key") != "archives/" + primary["object_key"] + + "." + digest + ".receipt.json" + or not isinstance(version, str) or not version or version == "null" + or any(c in version for c in "\r\n") + or type(ref.get("receipt_bytes")) is not int + or not 0 < ref["receipt_bytes"] <= 65536 + or ref.get("receipt_readback_verified") is not True): + raise Refused("retained_mirror_receipt_required") def check(state, mirror_receipt, now, *, min_free_bytes, max_state_bytes, @@ -182,7 +194,7 @@ def check(state, mirror_receipt, now, *, min_free_bytes, max_state_bytes, or digest != receipt["ciphertext_sha256"]): reasons.append("local_ciphertext_differs") try: - validate_mirror(receipt, read_json(mirror_receipt)) + validate_mirror(receipt, read_json(mirror_receipt), require_retained_receipt=True) report["matching_mirror_readback_recorded"] = True except (OSError, ValueError, KeyError, TypeError, AttributeError, OverflowError): reasons.append("matching_mirror_evidence_missing_or_invalid") diff --git a/scripts/mirror-paired-backup.py b/scripts/mirror-paired-backup.py new file mode 100644 index 0000000..da074fd --- /dev/null +++ b/scripts/mirror-paired-backup.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Mirror the latest verified pair and retain its public recovery receipt. + +Preview has no effects. Execute uses only this worker's existing AWS environment, +the hash-pinned platform engine, and the dedicated Coulomb archives prefix. +It never captures, decrypts, deletes retained objects or reads recovery credentials. +""" + +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import importlib +import importlib.util +import json +import os +import signal +import stat +import sys +import tempfile +from pathlib import Path + +spec = importlib.util.spec_from_file_location( + "csoc_backup_health", Path(__file__).with_name("check-paired-backup-health.py") +) +health = importlib.util.module_from_spec(spec) +spec.loader.exec_module(health) +Refused = health.Refused +FILES = { + "primary_application_archive.py", + "application_recovery_mirror.py", + "application_recovery_resources.py", + "application_recovery_identities.py", +} +PUBLIC_FIELDS = ( + "schema", + "status", + "application", + "captured_at", + "local_archive", + "destination", + "object_key", + "version_id", + "ciphertext_sha256", + "ciphertext_bytes", + "encrypted_sha256", + "encrypted_bytes", + "image", + "age_recipient", + "database_sha256", + "content_sha256", + "immutable_create", + "remote_retrieval_verified", +) +MIRROR_FIELDS = ( + "project_id", + "bucket", + "object_key", + "version_id", + "ciphertext_sha256", + "ciphertext_bytes", + "hash_readback_verified", +) + + +def canonical(value): + return ( + json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n" + ).encode("utf-8") + + +def private_directory(path): + info = path.lstat() + if ( + not stat.S_ISDIR(info.st_mode) + or info.st_uid != os.geteuid() + or info.st_mode & 0o077 + or path.resolve() != path.absolute() + ): + raise Refused("private_owner_directory_required") + + +def load_runtime(directory, manifest_path): + manifest = json.loads(manifest_path.read_bytes()) + if ( + manifest.get("schema") != "coulomb.mirror-platform-runtime.v1" + or set(manifest["files"]) != FILES + or directory.resolve() != directory.absolute() + ): + raise Refused("reviewed_platform_runtime_required") + for name, expected in manifest["files"].items(): + path = directory / name + info = path.lstat() + if ( + not stat.S_ISREG(info.st_mode) + or info.st_uid != os.geteuid() + or info.st_mode & 0o022 + or hashlib.sha256(path.read_bytes()).hexdigest() != expected + ): + raise Refused("platform_runtime_hash_differs") + loaded = sys.modules.get(path.stem) + if loaded is not None and Path(loaded.__file__).resolve() != path: + raise Refused("platform_module_already_loaded_elsewhere") + sys.path.insert(0, str(directory)) + return importlib.import_module("application_recovery_mirror") + + +def public_receipt(primary, mirrored): + health.validate_primary(primary) + health.validate_mirror(primary, mirrored) + # Never upload arbitrary private fields present in local input/journal files. + return dict( + {key: primary[key] for key in PUBLIC_FIELDS}, + application_restore_verified=False, + recovery_mirror={key: mirrored["recovery_mirror"][key] for key in MIRROR_FIELDS}, + ) + + +def retain_receipt(client, receipt, journal, save): + """Conditionally publish bounded metadata, then GET the exact returned version.""" + payload = canonical(receipt) + if len(payload) > 65536: + raise Refused("bounded_public_receipt_required") + digest = hashlib.sha256(payload).hexdigest() + key = "archives/" + receipt["object_key"] + "." + digest + ".receipt.json" + binding = {"object_key": key, "sha256": digest, "bytes": len(payload)} + previous = journal.get("public_receipt") + if previous is not None and any(previous.get(k) != v for k, v in binding.items()): + raise Refused("pending_receipt_binding_differs") + journal.setdefault("public_receipt", binding) + save() + try: + head = client.head_object(Bucket=health.MIRROR_BUCKET, Key=key) + except Exception as error: + if getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode") != 404: + raise + head = None + if head is None: + if previous and previous.get("version_id"): + raise Refused("retained_receipt_disappeared") + # On uncertain completion, leave this binding and adopt only an exact + # matching HEAD on retry. Never blindly PUT another retained version. + created = client.put_object( + Bucket=health.MIRROR_BUCKET, + Key=key, + Body=payload, + ContentType="application/json", + Metadata={"sha256": digest, "application": "coulomb-social"}, + IfNoneMatch="*", + ) + version = created.get("VersionId") + else: + if ( + head.get("ContentLength") != len(payload) + or head.get("Metadata", {}).get("sha256") != digest + ): + raise Refused("retained_receipt_binding_differs") + version = head.get("VersionId") + if ( + not isinstance(version, str) + or not version + or version == "null" + or "\n" in version + or "\r" in version + ): + raise Refused("immutable_receipt_version_required") + if previous and previous.get("version_id", version) != version: + raise Refused("retained_receipt_version_changed") + journal["public_receipt"]["version_id"] = version + save() + response = client.get_object(Bucket=health.MIRROR_BUCKET, Key=key, VersionId=version) + with response["Body"] as stream: + readback = stream.read(len(payload) + 1) + if ( + response.get("VersionId") != version + or response.get("ContentLength") != len(payload) + or readback != payload + ): + raise Refused("public_receipt_readback_differs") + journal["public_receipt"]["readback_verified"] = True + save() + return dict(binding, version_id=version, readback_verified=True) + + +def mirror_latest(state, engine, client): + private_directory(state) + lock_path = state / "backup.lock" + if lock_path.exists() or lock_path.is_symlink(): + health.regular(lock_path) + fd = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as lock: + health.regular(lock_path) + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise Refused("backup_or_mirror_already_running") from None + primary = health.read_json(state / "last-success.json") + original = canonical(primary) + health.validate_primary(primary) + source = state / primary["local_archive"] + info = health.regular(source) + with source.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + if info.st_size != primary["ciphertext_bytes"] or digest != primary["ciphertext_sha256"]: + raise Refused("primary_ciphertext_differs") + directory = state / "mirror-jobs" + directory.mkdir(mode=0o700, exist_ok=True) + private_directory(directory) + identity = {key: primary[key] for key in health.BINDING} + job = directory / (hashlib.sha256(canonical(identity)).hexdigest() + ".json") + journal = health.read_json(job) if job.exists() or job.is_symlink() else {} + + def save(): + engine.checkpoint(job, journal) + + with tempfile.TemporaryDirectory(prefix="mirror-readback-", dir=state) as temporary: + mirrored = engine.mirror( + client, source, Path(temporary) / "ciphertext.age", primary, journal, save + ) + receipt = public_receipt(primary, mirrored) + reference = retain_receipt(client, receipt, journal, save) + if canonical(health.read_json(state / "last-success.json")) != original: + raise Refused("primary_changed_during_mirror") + receipt["recovery_mirror"].update( + receipt_object_key=reference["object_key"], + receipt_version_id=reference["version_id"], + receipt_sha256=reference["sha256"], + receipt_bytes=reference["bytes"], + receipt_readback_verified=True, + ) + engine.checkpoint(state / "last-mirror.json", receipt) + return { + "status": "mirrored_with_retained_receipt", + "primary_recaptured": False, + "ciphertext_readback_verified": True, + "receipt_readback_verified": True, + "recovery_key_read": False, + "independent_reader_used": False, + "retained_objects_deleted": False, + "application_restore_verified": False, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute", action="store_true") + parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/coulomb-social-backup")) + parser.add_argument( + "--platform-scripts", type=Path, default=Path("/opt/railiance-platform/scripts") + ) + parser.add_argument( + "--runtime-manifest", + type=Path, + default=Path(__file__).with_name("mirror-platform-runtime.json"), + ) + args = parser.parse_args() + if not args.execute: + print( + json.dumps({"status": "preview", "provider_contacted": False, "state_modified": False}) + ) + return + engine = load_runtime(args.platform_scripts, args.runtime_manifest) + pair = { + name: os.environ.get(name, "") for name in ("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY") + } + if any(not value or any(c in value for c in "\r\n") for value in pair.values()): + raise Refused("own_publisher_environment_required") + client = engine.s3(pair, health.PROJECT) + try: + print(json.dumps(mirror_latest(args.state_dir, engine, client))) + finally: + client.close() + + +if __name__ == "__main__": + try: + + def terminate(signum, frame): + raise Refused("mirror_interrupted") + + signal.signal(signal.SIGTERM, terminate) + main() + except Exception as error: + print( + json.dumps({"status": "mirror_requires_review", "failure_type": type(error).__name__}) + ) + raise SystemExit(1) diff --git a/scripts/rehearse-backup-mirror.py b/scripts/rehearse-backup-mirror.py new file mode 100644 index 0000000..74bd091 --- /dev/null +++ b/scripts/rehearse-backup-mirror.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Rehearse the pinned platform engine using development-only synthetic fixtures. + +No provider client is created and no credentials are read. This proves adapter +integration, not IAM, native provider behavior or recurring service operation. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from tests.test_backup_health import health_state # noqa: E402 +from tests.test_backup_mirror_worker import MemoryS3 # noqa: E402 + + +def fixture(directory, worker): + state, primary, _ = health_state.__wrapped__(directory, worker.health) + data = b"age-encryption.org/v1\n" + b"SYNTHETIC CIPHERTEXT ONLY\n" + (state / primary["local_archive"]).write_bytes(data) + primary.update(ciphertext_sha256=hashlib.sha256(data).hexdigest(), + encrypted_sha256=hashlib.sha256(data).hexdigest(), + ciphertext_bytes=len(data), encrypted_bytes=len(data)) + (state / "last-success.json").write_text(json.dumps(primary)) + (state / "last-mirror.json").unlink() + return state, primary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--platform-scripts", type=Path, required=True) + parser.add_argument("--runtime-manifest", type=Path, + default=ROOT / "deploy/backup/mirror-platform-runtime.json") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists() or args.output.is_symlink(): + raise ValueError("Fresh proof output required") + source = ROOT / "scripts/mirror-paired-backup.py" + spec = importlib.util.spec_from_file_location("integration_worker", source) + worker = importlib.util.module_from_spec(spec) + spec.loader.exec_module(worker) + runtime = worker.load_runtime(args.platform_scripts, args.runtime_manifest) + scenarios = [] + for lost_reply in [False, True]: + with tempfile.TemporaryDirectory(prefix="csoc-mirror-engine-") as directory: + state, primary = fixture(Path(directory), worker) + client = MemoryS3() + client.lose_put_reply = lost_reply + before = (state / "last-success.json").read_bytes() + if lost_reply: + try: + worker.mirror_latest(state, runtime, client) + except TimeoutError: + pass + else: + raise AssertionError("Lost reply was not exercised") + assert not (state / "last-mirror.json").exists() + result = worker.mirror_latest(state, runtime, client) + mirrored = json.loads((state / "last-mirror.json").read_bytes()) + worker.health.validate_mirror(primary, mirrored, require_retained_receipt=True) + ref = mirrored["recovery_mirror"] + held = client.objects[ref["receipt_object_key"]]["Body"] + assert hashlib.sha256(held).hexdigest() == ref["receipt_sha256"] + assert client.objects[ref["object_key"]]["Body"] == ( + state / primary["local_archive"]).read_bytes() + assert (state / "last-success.json").read_bytes() == before + assert len(client.objects) == 2 + assert sum(op == "complete" for op, _ in client.calls) == 1 + assert sum(op == "put" for op, _ in client.calls) == 1 + assert all(a["Bucket"] == worker.health.MIRROR_BUCKET for _, a in client.calls) + assert all(a["Key"].startswith("archives/" + primary["object_key"]) + for _, a in client.calls) + assert result["receipt_readback_verified"] + scenarios.append({ + "lost_receipt_put_reply": lost_reply, "ciphertext_multipart_completions": 1, + "metadata_conditional_puts": 1, "retained_objects": 2, + "primary_unchanged": True, "exact_version_hash_readbacks": True, + "last_mirror_only_after_full_readback": True, + "only_coulomb_archive_prefix_used": True, + }) + proof = { + "schema": "coulomb.mirror-worker-rehearsal.v1", + "status": "pinned_engine_synthetic_integration_passed", + "worker_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "checker_sha256": hashlib.sha256( + (ROOT / "scripts/check-paired-backup-health.py").read_bytes()).hexdigest(), + "driver_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "fixture_sha256": {name: hashlib.sha256((ROOT / "tests" / name).read_bytes()).hexdigest() + for name in ["test_backup_health.py", "test_backup_mirror_worker.py"]}, + "runtime_manifest": json.loads(args.runtime_manifest.read_bytes()), + "scenarios": scenarios, "synthetic_store": True, "provider_contacted": False, + "credentials_read": False, "node_files_changed": False, "timers_activated": False, + } + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(proof, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({"status": proof["status"], "scenarios": len(scenarios), + "provider_contacted": False})) + + +if __name__ == "__main__": + main() diff --git a/tests/test_backup_health.py b/tests/test_backup_health.py index 8405675..0e0922e 100644 --- a/tests/test_backup_health.py +++ b/tests/test_backup_health.py @@ -56,6 +56,10 @@ def health_state(tmp_path, monitor): "object_key": "archives/" + primary["object_key"], "version_id": "mirror-version", "ciphertext_sha256": digest, "ciphertext_bytes": len(ciphertext), "hash_readback_verified": True, + "receipt_sha256": "e" * 64, "receipt_bytes": 1900, + "receipt_version_id": "metadata-version", "receipt_readback_verified": True, + "receipt_object_key": "archives/" + primary["object_key"] + "." + "e" * 64 + + ".receipt.json", } save(state / "last-success.json", primary) save(state / "last-mirror.json", mirrored) @@ -109,6 +113,17 @@ def test_missing_or_forged_readback_and_corrupted_ciphertext_fail(monitor, healt assert "matching_mirror_evidence_missing_or_invalid" in check(monitor, state)["reasons"] +@pytest.mark.parametrize("field", ["receipt_readback_verified", "receipt_version_id", + "receipt_object_key", "receipt_sha256"]) +def test_ciphertext_mirror_without_its_retained_receipt_is_not_operating_health( + monitor, health_state, field, +): + state, _, mirrored = health_state + del mirrored["recovery_mirror"][field] + save(state / "last-mirror.json", mirrored) + assert "matching_mirror_evidence_missing_or_invalid" in check(monitor, state)["reasons"] + + def test_recent_success_does_not_hide_abandoned_pending_capture(monitor, health_state): state, _, _ = health_state pending = state / "pending-synthetic" diff --git a/tests/test_backup_mirror_worker.py b/tests/test_backup_mirror_worker.py new file mode 100644 index 0000000..71d80f9 --- /dev/null +++ b/tests/test_backup_mirror_worker.py @@ -0,0 +1,331 @@ +"""Mirror/receipt failures must block promotion and preserve exact retry bindings.""" + +import copy +import fcntl +import hashlib +import importlib.util +import io +import json +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from tests.test_backup_health import health_state as health_state +from tests.test_backup_health import monitor as monitor + + +class Missing(Exception): + response = {"ResponseMetadata": {"HTTPStatusCode": 404}} + + +class MemoryS3: + """Synthetic object store; no credentials, provider URL or network.""" + + def __init__(self): + self.objects = {} + self.calls = [] + self.lose_put_reply = False + self.corrupt_get = False + self.missing_version = False + self.uploads = {} + + def create_multipart_upload(self, **args): + self.calls.append(("create", args)) + identity = str(len(self.uploads) + 1) + self.uploads[identity] = {"args": args, "parts": {}} + return {"UploadId": identity} + + def upload_part(self, **args): + self.calls.append(("part", args)) + self.uploads[args["UploadId"]]["parts"][args["PartNumber"]] = args["Body"] + return {"ETag": "synthetic-part-" + str(args["PartNumber"])} + + def complete_multipart_upload(self, **args): + self.calls.append(("complete", args)) + assert args["IfNoneMatch"] == "*" and args["Key"] not in self.objects + upload = self.uploads[args["UploadId"]] + version = "version-" + str(len(self.objects) + 1) + self.objects[args["Key"]] = dict( + upload["args"], + Body=b"".join(upload["parts"][i] for i in sorted(upload["parts"])), + VersionId=version, + ) + return {"VersionId": version} + + def abort_multipart_upload(self, **args): + self.calls.append(("abort", args)) + self.uploads.pop(args["UploadId"], None) + + def head_object(self, **args): + self.calls.append(("head", args)) + if args["Key"] not in self.objects: + raise Missing() + obj = self.objects[args["Key"]] + return { + "ContentLength": len(obj["Body"]), + "VersionId": obj["VersionId"], + "Metadata": obj["Metadata"], + } + + def put_object(self, **args): + self.calls.append(("put", args)) + assert args["IfNoneMatch"] == "*" + assert args["Key"] not in self.objects + version = "null" if self.missing_version else "version-" + str(len(self.objects) + 1) + self.objects[args["Key"]] = dict(args, VersionId=version) + if self.lose_put_reply: + self.lose_put_reply = False + raise TimeoutError("synthetic lost completion reply") + return {"VersionId": version} + + def get_object(self, **args): + self.calls.append(("get", args)) + obj = self.objects[args["Key"]] + assert args["VersionId"] == obj["VersionId"] + body = b"damaged" if self.corrupt_get else obj["Body"] + return { + "ContentLength": len(obj["Body"]), + "VersionId": obj["VersionId"], + "Body": io.BytesIO(body), + } + + +@pytest.fixture +def worker(): + source = Path(__file__).resolve().parents[1] / "scripts/mirror-paired-backup.py" + spec = importlib.util.spec_from_file_location("mirror_worker", source) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def engine(health_state): + _, _, mirrored = health_state + calls = [] + + def checkpoint(path, data): + # Real platform checkpoint is separately hash-pinned and native-verified. + path.write_text(json.dumps(data)) + path.chmod(0o600) + + def mirror(client, source, readback, primary, journal, save): + calls.append((source, primary["object_key"], primary["version_id"])) + journal["ciphertext_engine_completed"] = True + save() + return dict(primary, recovery_mirror=copy.deepcopy(mirrored["recovery_mirror"])) + + return SimpleNamespace(checkpoint=checkpoint, mirror=mirror, calls=calls) + + +def run(worker, health_state, engine, client): + return worker.mirror_latest(health_state[0], engine, client) + + +def test_only_promotes_after_conditional_receipt_put_and_exact_version_readback( + worker, + health_state, + engine, +): + state, primary, _ = health_state + before = (state / "last-success.json").read_bytes() + client = MemoryS3() + result = run(worker, health_state, engine, client) + assert result["status"] == "mirrored_with_retained_receipt" + assert not result["primary_recaptured"] and not result["recovery_key_read"] + promoted = json.loads((state / "last-mirror.json").read_bytes()) + ref = promoted["recovery_mirror"] + assert ref["receipt_readback_verified"] + assert len(client.objects) == 1 + payload = client.objects[ref["receipt_object_key"]]["Body"] + assert hashlib.sha256(payload).hexdigest() == ref["receipt_sha256"] + stored = json.loads(payload) + worker.health.validate_mirror(primary, stored) + assert "receipt_object_key" not in stored["recovery_mirror"] # No recursive self-hash. + assert (state / "last-success.json").read_bytes() == before + assert (state / "last-mirror.json").stat().st_mode & 0o777 == 0o600 + assert not list(state.glob("mirror-readback-*")) + assert [operation for operation, _ in client.calls] == ["head", "put", "get"] + + +def test_lost_receipt_put_reply_reuses_object_and_never_advances_old_pointer( + worker, + health_state, + engine, +): + state, _, _ = health_state + old = (state / "last-mirror.json").read_bytes() + client = MemoryS3() + client.lose_put_reply = True + with pytest.raises(TimeoutError): + run(worker, health_state, engine, client) + assert (state / "last-mirror.json").read_bytes() == old + assert len(client.objects) == 1 and list(state.glob("mirror-jobs/*.json")) + assert run(worker, health_state, engine, client)["receipt_readback_verified"] + assert [op for op, _ in client.calls].count("put") == 1 + assert len(engine.calls) == 2 and engine.calls[0] == engine.calls[1] + + +@pytest.mark.parametrize("failure", ["body", "version"]) +def test_invalid_receipt_readback_does_not_promote(worker, health_state, engine, failure): + state, _, _ = health_state + old = (state / "last-mirror.json").read_bytes() + client = MemoryS3() + client.corrupt_get = failure == "body" + client.missing_version = failure == "version" + with pytest.raises(ValueError): + run(worker, health_state, engine, client) + assert (state / "last-mirror.json").read_bytes() == old + assert list(state.glob("mirror-jobs/*.json")) + + +def test_changed_retained_receipt_version_requires_review(worker, health_state, engine): + state, _, _ = health_state + client = MemoryS3() + run(worker, health_state, engine, client) + old = (state / "last-mirror.json").read_bytes() + next(iter(client.objects.values()))["VersionId"] = "unreviewed-replacement" + with pytest.raises(ValueError, match="version_changed"): + run(worker, health_state, engine, client) + assert (state / "last-mirror.json").read_bytes() == old + + +def test_unknown_private_fields_are_not_uploaded(worker, health_state, engine): + state, primary, _ = health_state + primary["unrelated_private_data"] = "SYNTHETIC MUST STAY LOCAL" + (state / "last-success.json").write_text(json.dumps(primary)) + client = MemoryS3() + run(worker, health_state, engine, client) + assert b"SYNTHETIC MUST STAY LOCAL" not in next(iter(client.objects.values()))["Body"] + + +def test_backup_lock_excludes_mirror_before_provider_calls(worker, health_state, engine): + state, _, _ = health_state + path = state / "backup.lock" + path.touch(mode=0o600) + client = MemoryS3() + with path.open("r") as stream: + fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(ValueError, match="already_running"): + run(worker, health_state, engine, client) + assert not client.calls and not engine.calls + + +def test_tampered_ciphertext_refuses_before_publication(worker, health_state, engine): + state, primary, _ = health_state + (state / primary["local_archive"]).write_bytes(b"tampered") + client = MemoryS3() + with pytest.raises(ValueError, match="ciphertext_differs"): + run(worker, health_state, engine, client) + assert not client.calls and not engine.calls + + +def test_preview_needs_no_runtime_credentials_or_state(worker, tmp_path): + environment = {k: v for k, v in os.environ.items() if not k.startswith("AWS_")} + result = subprocess.run( + [sys.executable, worker.__file__, "--state-dir", str(tmp_path / "absent")], + env=environment, + capture_output=True, + text=True, + ) + assert result.returncode == 0 + assert json.loads(result.stdout) == { + "status": "preview", + "provider_contacted": False, + "state_modified": False, + } + assert not list(tmp_path.iterdir()) + + +def test_runtime_hash_drift_refuses_before_importing_any_owner_module(worker, tmp_path): + directory = tmp_path / "platform" + directory.mkdir() + files = {} + for name in worker.FILES: + source = directory / name + source.write_text('raise RuntimeError("must not import")\n') + files[name] = hashlib.sha256(source.read_bytes()).hexdigest() + manifest = tmp_path / "runtime.json" + manifest.write_text( + json.dumps({"schema": "coulomb.mirror-platform-runtime.v1", "files": files}) + ) + (directory / "application_recovery_mirror.py").write_text("changed\n") + with pytest.raises(ValueError, match="runtime_hash_differs"): + worker.load_runtime(directory, manifest) + + +def test_failed_final_checkpoint_retries_without_duplicate_retained_versions( + worker, + health_state, + engine, +): + state, _, _ = health_state + old = (state / "last-mirror.json").read_bytes() + checkpoint = engine.checkpoint + + def fail_promotion(path, value): + if path.name == "last-mirror.json": + raise OSError("synthetic unavailable local checkpoint") + checkpoint(path, value) + + engine.checkpoint = fail_promotion + client = MemoryS3() + with pytest.raises(OSError): + run(worker, health_state, engine, client) + assert (state / "last-mirror.json").read_bytes() == old + engine.checkpoint = checkpoint + assert run(worker, health_state, engine, client)["receipt_readback_verified"] + assert sum(op == "put" for op, _ in client.calls) == 1 + + +def test_incorrect_engine_result_refuses_metadata_upload(worker, health_state, engine): + state, primary, mirrored = health_state + old = (state / "last-mirror.json").read_bytes() + mirrored["recovery_mirror"]["object_key"] = "unrelated" + engine.mirror = lambda *args: dict(primary, recovery_mirror=mirrored["recovery_mirror"]) + client = MemoryS3() + with pytest.raises(ValueError): + run(worker, health_state, engine, client) + assert not client.calls and (state / "last-mirror.json").read_bytes() == old + + +def test_execute_without_own_environment_refuses_before_client_or_state( + worker, + monkeypatch, + tmp_path, +): + for name in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]: + monkeypatch.delenv(name, raising=False) + + def unexpected(*args): + raise AssertionError("No client or credential fallback allowed") + + monkeypatch.setattr(worker, "load_runtime", lambda *args: SimpleNamespace(s3=unexpected)) + monkeypatch.setattr( + sys, "argv", [worker.__file__, "--execute", "--state-dir", str(tmp_path / "absent")] + ) + with pytest.raises(ValueError, match="own_publisher_environment_required"): + worker.main() + assert not list(tmp_path.iterdir()) + + +def test_service_retries_prior_mirror_before_capture_and_mirrors_the_new_point_afterward(): + root = Path(__file__).resolve().parents[1] + service = (root / "deploy/backup/coulomb-social-paired-backup.service").read_text() + commands = [line for line in service.splitlines() if line.startswith("ExecStart")] + assert len(commands) == 3 + assert ( + commands[0].startswith("ExecStartPre=") + and "mirror-paired-backup.py --execute" in commands[0] + ) + assert commands[1].startswith("ExecStart=") and "paired-backup.py --execute" in commands[1] + assert ( + commands[2].startswith("ExecStartPost=") + and "mirror-paired-backup.py --execute" in commands[2] + ) + assert "EnvironmentFile=/etc/coulomb-social-backup/primary.env" in service + assert "reader" not in service and "custodian" not in service diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 22f3d58..3787b55 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -202,6 +202,21 @@ and proposed capacity limits: docs/release/backup-operating-preflight-2026-10-30 native metadata/hash proof is its JSON companion. Received monitoring, retention, pre-capture capacity guard and service-contract returns remain incomplete. +Recurring mirror follow-through: prepared mirror-paired-backup.py using the +hash-pinned platform engine and only the own publisher environment. It preserves +primary key/version/hash retry binding, conditionally retains a filtered public +receipt in the own archive prefix, verifies both exact readbacks and only then +promotes last-mirror. Candidate service hooks retry the previous mirror before +capture and mirror the new point afterward. The monitor now requires the retained +receipt as well as ciphertext evidence. Actual pinned-engine synthetic integration +proves normal/lost-reply paths without duplicate retained objects; 53 focused +cases pass. Exact units verify on the target with temporary review files only; +installed SDK supports the conditional PUT. No provider call, credential read, +installed helper change, unit start or timer activation. Review and proof: +docs/release/mirror-worker-operating-review-2026-10-30.md and +mirror-worker-rehearsal-2026-10-30.json. Native publication/independent retrieval, +received alerts, retention and pre-capture capacity remain required. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.