"""Mirror/receipt failures must block promotion and preserve exact retry bindings.""" import copy import fcntl import hashlib import importlib.util import io import json import os import subprocess import sys from pathlib import Path from types import SimpleNamespace import pytest from tests.test_backup_health import health_state as health_state from tests.test_backup_health import monitor as monitor class Missing(Exception): response = {"ResponseMetadata": {"HTTPStatusCode": 404}} class MemoryS3: """Synthetic object store; no credentials, provider URL or network.""" def __init__(self): self.objects = {} self.calls = [] self.lose_put_reply = False self.corrupt_get = False self.missing_version = False self.uploads = {} def create_multipart_upload(self, **args): self.calls.append(("create", args)) identity = str(len(self.uploads) + 1) self.uploads[identity] = {"args": args, "parts": {}} return {"UploadId": identity} def upload_part(self, **args): self.calls.append(("part", args)) self.uploads[args["UploadId"]]["parts"][args["PartNumber"]] = args["Body"] return {"ETag": "synthetic-part-" + str(args["PartNumber"])} def complete_multipart_upload(self, **args): self.calls.append(("complete", args)) assert args["IfNoneMatch"] == "*" and args["Key"] not in self.objects upload = self.uploads[args["UploadId"]] version = "version-" + str(len(self.objects) + 1) self.objects[args["Key"]] = dict( upload["args"], Body=b"".join(upload["parts"][i] for i in sorted(upload["parts"])), VersionId=version, ) return {"VersionId": version} def abort_multipart_upload(self, **args): self.calls.append(("abort", args)) self.uploads.pop(args["UploadId"], None) def head_object(self, **args): self.calls.append(("head", args)) if args["Key"] not in self.objects: raise Missing() obj = self.objects[args["Key"]] return { "ContentLength": len(obj["Body"]), "VersionId": obj["VersionId"], "Metadata": obj["Metadata"], } def put_object(self, **args): self.calls.append(("put", args)) assert args["IfNoneMatch"] == "*" assert args["Key"] not in self.objects version = "null" if self.missing_version else "version-" + str(len(self.objects) + 1) self.objects[args["Key"]] = dict(args, VersionId=version) if self.lose_put_reply: self.lose_put_reply = False raise TimeoutError("synthetic lost completion reply") return {"VersionId": version} def get_object(self, **args): self.calls.append(("get", args)) obj = self.objects[args["Key"]] assert args["VersionId"] == obj["VersionId"] body = b"damaged" if self.corrupt_get else obj["Body"] return { "ContentLength": len(obj["Body"]), "VersionId": obj["VersionId"], "Body": io.BytesIO(body), } @pytest.fixture def worker(monkeypatch): source = Path(__file__).resolve().parents[1] / "scripts/mirror-paired-backup.py" monkeypatch.syspath_prepend(str(source.parent)) spec = importlib.util.spec_from_file_location("mirror_worker", source) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) monkeypatch.setattr(module.capacity, "active_file_limit", lambda: module.capacity.FILE_LIMIT) monkeypatch.setattr(module.capacity, "disk_free", lambda path: 128 * 1024**3) monkeypatch.setattr(module.capacity, "free_inodes", lambda path: 1000000) return module @pytest.fixture def engine(health_state): _, _, mirrored = health_state calls = [] def checkpoint(path, data): # Real platform checkpoint is separately hash-pinned and native-verified. path.write_text(json.dumps(data)) path.chmod(0o600) def mirror(client, source, readback, primary, journal, save): calls.append((source, primary["object_key"], primary["version_id"])) journal["ciphertext_engine_completed"] = True save() return dict(primary, recovery_mirror=copy.deepcopy(mirrored["recovery_mirror"])) return SimpleNamespace(checkpoint=checkpoint, mirror=mirror, calls=calls) def run(worker, health_state, engine, client): return worker.mirror_latest(health_state[0], engine, client) def test_only_promotes_after_conditional_receipt_put_and_exact_version_readback( worker, health_state, engine, ): state, primary, _ = health_state before = (state / "last-success.json").read_bytes() client = MemoryS3() result = run(worker, health_state, engine, client) assert result["status"] == "mirrored_with_retained_receipt" assert not result["primary_recaptured"] and not result["recovery_key_read"] promoted = json.loads((state / "last-mirror.json").read_bytes()) ref = promoted["recovery_mirror"] assert ref["receipt_readback_verified"] assert len(client.objects) == 1 payload = client.objects[ref["receipt_object_key"]]["Body"] assert hashlib.sha256(payload).hexdigest() == ref["receipt_sha256"] stored = json.loads(payload) worker.health.validate_mirror(primary, stored) assert "receipt_object_key" not in stored["recovery_mirror"] # No recursive self-hash. assert (state / "last-success.json").read_bytes() == before assert (state / "last-mirror.json").stat().st_mode & 0o777 == 0o600 assert not list(state.glob("mirror-readback-*")) assert [operation for operation, _ in client.calls] == ["head", "put", "get"] def test_lost_receipt_put_reply_reuses_object_and_never_advances_old_pointer( worker, health_state, engine, ): state, _, _ = health_state old = (state / "last-mirror.json").read_bytes() client = MemoryS3() client.lose_put_reply = True with pytest.raises(TimeoutError): run(worker, health_state, engine, client) assert (state / "last-mirror.json").read_bytes() == old assert len(client.objects) == 1 and list(state.glob("mirror-jobs/*.json")) assert run(worker, health_state, engine, client)["receipt_readback_verified"] assert [op for op, _ in client.calls].count("put") == 1 assert len(engine.calls) == 2 and engine.calls[0] == engine.calls[1] @pytest.mark.parametrize("failure", ["body", "version"]) def test_invalid_receipt_readback_does_not_promote(worker, health_state, engine, failure): state, _, _ = health_state old = (state / "last-mirror.json").read_bytes() client = MemoryS3() client.corrupt_get = failure == "body" client.missing_version = failure == "version" with pytest.raises(ValueError): run(worker, health_state, engine, client) assert (state / "last-mirror.json").read_bytes() == old assert list(state.glob("mirror-jobs/*.json")) def test_changed_retained_receipt_version_requires_review(worker, health_state, engine): state, _, _ = health_state client = MemoryS3() run(worker, health_state, engine, client) old = (state / "last-mirror.json").read_bytes() next(iter(client.objects.values()))["VersionId"] = "unreviewed-replacement" with pytest.raises(ValueError, match="version_changed"): run(worker, health_state, engine, client) assert (state / "last-mirror.json").read_bytes() == old def test_unknown_private_fields_are_not_uploaded(worker, health_state, engine): state, primary, _ = health_state primary["unrelated_private_data"] = "SYNTHETIC MUST STAY LOCAL" (state / "last-success.json").write_text(json.dumps(primary)) client = MemoryS3() run(worker, health_state, engine, client) assert b"SYNTHETIC MUST STAY LOCAL" not in next(iter(client.objects.values()))["Body"] def test_backup_lock_excludes_mirror_before_provider_calls(worker, health_state, engine): state, _, _ = health_state path = state / "backup.lock" path.touch(mode=0o600) client = MemoryS3() with path.open("r") as stream: fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB) with pytest.raises(ValueError, match="already_running"): run(worker, health_state, engine, client) assert not client.calls and not engine.calls def test_tampered_ciphertext_refuses_before_publication(worker, health_state, engine): state, primary, _ = health_state (state / primary["local_archive"]).write_bytes(b"tampered") client = MemoryS3() with pytest.raises(ValueError, match="ciphertext_differs"): run(worker, health_state, engine, client) assert not client.calls and not engine.calls def test_capacity_failure_refuses_mirror_before_provider_calls( worker, health_state, engine, monkeypatch, ): monkeypatch.setattr(worker.capacity, "disk_free", lambda path: 1) client = MemoryS3() with pytest.raises(ValueError, match="capacity_review_required"): run(worker, health_state, engine, client) assert not client.calls and not engine.calls def test_preview_needs_no_runtime_credentials_or_state(worker, tmp_path): environment = {k: v for k, v in os.environ.items() if not k.startswith("AWS_")} result = subprocess.run( [sys.executable, worker.__file__, "--state-dir", str(tmp_path / "absent")], env=environment, capture_output=True, text=True, ) assert result.returncode == 0 assert json.loads(result.stdout) == { "status": "preview", "provider_contacted": False, "state_modified": False, } assert not list(tmp_path.iterdir()) def test_runtime_hash_drift_refuses_before_importing_any_owner_module(worker, tmp_path): directory = tmp_path / "platform" directory.mkdir() files = {} for name in worker.FILES: source = directory / name source.write_text('raise RuntimeError("must not import")\n') files[name] = hashlib.sha256(source.read_bytes()).hexdigest() manifest = tmp_path / "runtime.json" manifest.write_text( json.dumps({"schema": "coulomb.mirror-platform-runtime.v1", "files": files}) ) (directory / "application_recovery_mirror.py").write_text("changed\n") with pytest.raises(ValueError, match="runtime_hash_differs"): worker.load_runtime(directory, manifest) def test_failed_final_checkpoint_retries_without_duplicate_retained_versions( worker, health_state, engine, ): state, _, _ = health_state old = (state / "last-mirror.json").read_bytes() checkpoint = engine.checkpoint def fail_promotion(path, value): if path.name == "last-mirror.json": raise OSError("synthetic unavailable local checkpoint") checkpoint(path, value) engine.checkpoint = fail_promotion client = MemoryS3() with pytest.raises(OSError): run(worker, health_state, engine, client) assert (state / "last-mirror.json").read_bytes() == old engine.checkpoint = checkpoint assert run(worker, health_state, engine, client)["receipt_readback_verified"] assert sum(op == "put" for op, _ in client.calls) == 1 def test_incorrect_engine_result_refuses_metadata_upload(worker, health_state, engine): state, primary, mirrored = health_state old = (state / "last-mirror.json").read_bytes() mirrored["recovery_mirror"]["object_key"] = "unrelated" engine.mirror = lambda *args: dict(primary, recovery_mirror=mirrored["recovery_mirror"]) client = MemoryS3() with pytest.raises(ValueError): run(worker, health_state, engine, client) assert not client.calls and (state / "last-mirror.json").read_bytes() == old def test_execute_without_own_environment_refuses_before_client_or_state( worker, monkeypatch, tmp_path, ): for name in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]: monkeypatch.delenv(name, raising=False) def unexpected(*args): raise AssertionError("No client or credential fallback allowed") monkeypatch.setattr(worker, "load_runtime", lambda *args: SimpleNamespace(s3=unexpected)) monkeypatch.setattr( sys, "argv", [worker.__file__, "--execute", "--state-dir", str(tmp_path / "absent")] ) with pytest.raises(ValueError, match="own_publisher_environment_required"): worker.main() assert not list(tmp_path.iterdir()) def test_service_retries_prior_mirror_before_capture_and_mirrors_the_new_point_afterward(): root = Path(__file__).resolve().parents[1] service = (root / "deploy/backup/coulomb-social-paired-backup.service").read_text() commands = [line for line in service.splitlines() if line.startswith("ExecStart")] assert len(commands) == 3 assert ( commands[0].startswith("ExecStartPre=") and "mirror-paired-backup.py --execute" in commands[0] ) assert commands[1].startswith("ExecStart=") and "paired-backup.py --execute" in commands[1] assert ( commands[2].startswith("ExecStartPost=") and "mirror-paired-backup.py --execute" in commands[2] ) assert "EnvironmentFile=/etc/coulomb-social-backup/primary.env" in service assert "reader" not in service and "custodian" not in service