tegwick
11cab7298b
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m27s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a17f16-d99c-7ea3-95f0-8212897fba6b
345 lines
13 KiB
Python
345 lines
13 KiB
Python
"""Mirror/receipt failures must block promotion and preserve exact retry bindings."""
|
|
|
|
import copy
|
|
import fcntl
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from tests.test_backup_health import health_state as health_state
|
|
from tests.test_backup_health import monitor as monitor
|
|
|
|
|
|
class Missing(Exception):
|
|
response = {"ResponseMetadata": {"HTTPStatusCode": 404}}
|
|
|
|
|
|
class MemoryS3:
|
|
"""Synthetic object store; no credentials, provider URL or network."""
|
|
|
|
def __init__(self):
|
|
self.objects = {}
|
|
self.calls = []
|
|
self.lose_put_reply = False
|
|
self.corrupt_get = False
|
|
self.missing_version = False
|
|
self.uploads = {}
|
|
|
|
def create_multipart_upload(self, **args):
|
|
self.calls.append(("create", args))
|
|
identity = str(len(self.uploads) + 1)
|
|
self.uploads[identity] = {"args": args, "parts": {}}
|
|
return {"UploadId": identity}
|
|
|
|
def upload_part(self, **args):
|
|
self.calls.append(("part", args))
|
|
self.uploads[args["UploadId"]]["parts"][args["PartNumber"]] = args["Body"]
|
|
return {"ETag": "synthetic-part-" + str(args["PartNumber"])}
|
|
|
|
def complete_multipart_upload(self, **args):
|
|
self.calls.append(("complete", args))
|
|
assert args["IfNoneMatch"] == "*" and args["Key"] not in self.objects
|
|
upload = self.uploads[args["UploadId"]]
|
|
version = "version-" + str(len(self.objects) + 1)
|
|
self.objects[args["Key"]] = dict(
|
|
upload["args"],
|
|
Body=b"".join(upload["parts"][i] for i in sorted(upload["parts"])),
|
|
VersionId=version,
|
|
)
|
|
return {"VersionId": version}
|
|
|
|
def abort_multipart_upload(self, **args):
|
|
self.calls.append(("abort", args))
|
|
self.uploads.pop(args["UploadId"], None)
|
|
|
|
def head_object(self, **args):
|
|
self.calls.append(("head", args))
|
|
if args["Key"] not in self.objects:
|
|
raise Missing()
|
|
obj = self.objects[args["Key"]]
|
|
return {
|
|
"ContentLength": len(obj["Body"]),
|
|
"VersionId": obj["VersionId"],
|
|
"Metadata": obj["Metadata"],
|
|
}
|
|
|
|
def put_object(self, **args):
|
|
self.calls.append(("put", args))
|
|
assert args["IfNoneMatch"] == "*"
|
|
assert args["Key"] not in self.objects
|
|
version = "null" if self.missing_version else "version-" + str(len(self.objects) + 1)
|
|
self.objects[args["Key"]] = dict(args, VersionId=version)
|
|
if self.lose_put_reply:
|
|
self.lose_put_reply = False
|
|
raise TimeoutError("synthetic lost completion reply")
|
|
return {"VersionId": version}
|
|
|
|
def get_object(self, **args):
|
|
self.calls.append(("get", args))
|
|
obj = self.objects[args["Key"]]
|
|
assert args["VersionId"] == obj["VersionId"]
|
|
body = b"damaged" if self.corrupt_get else obj["Body"]
|
|
return {
|
|
"ContentLength": len(obj["Body"]),
|
|
"VersionId": obj["VersionId"],
|
|
"Body": io.BytesIO(body),
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def worker(monkeypatch):
|
|
source = Path(__file__).resolve().parents[1] / "scripts/mirror-paired-backup.py"
|
|
monkeypatch.syspath_prepend(str(source.parent))
|
|
spec = importlib.util.spec_from_file_location("mirror_worker", source)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
monkeypatch.setattr(module.capacity, "active_file_limit", lambda: module.capacity.FILE_LIMIT)
|
|
monkeypatch.setattr(module.capacity, "disk_free", lambda path: 128 * 1024**3)
|
|
monkeypatch.setattr(module.capacity, "free_inodes", lambda path: 1000000)
|
|
return module
|
|
|
|
|
|
@pytest.fixture
|
|
def engine(health_state):
|
|
_, _, mirrored = health_state
|
|
calls = []
|
|
|
|
def checkpoint(path, data):
|
|
# Real platform checkpoint is separately hash-pinned and native-verified.
|
|
path.write_text(json.dumps(data))
|
|
path.chmod(0o600)
|
|
|
|
def mirror(client, source, readback, primary, journal, save):
|
|
calls.append((source, primary["object_key"], primary["version_id"]))
|
|
journal["ciphertext_engine_completed"] = True
|
|
save()
|
|
return dict(primary, recovery_mirror=copy.deepcopy(mirrored["recovery_mirror"]))
|
|
|
|
return SimpleNamespace(checkpoint=checkpoint, mirror=mirror, calls=calls)
|
|
|
|
|
|
def run(worker, health_state, engine, client):
|
|
return worker.mirror_latest(health_state[0], engine, client)
|
|
|
|
|
|
def test_only_promotes_after_conditional_receipt_put_and_exact_version_readback(
|
|
worker,
|
|
health_state,
|
|
engine,
|
|
):
|
|
state, primary, _ = health_state
|
|
before = (state / "last-success.json").read_bytes()
|
|
client = MemoryS3()
|
|
result = run(worker, health_state, engine, client)
|
|
assert result["status"] == "mirrored_with_retained_receipt"
|
|
assert not result["primary_recaptured"] and not result["recovery_key_read"]
|
|
promoted = json.loads((state / "last-mirror.json").read_bytes())
|
|
ref = promoted["recovery_mirror"]
|
|
assert ref["receipt_readback_verified"]
|
|
assert len(client.objects) == 1
|
|
payload = client.objects[ref["receipt_object_key"]]["Body"]
|
|
assert hashlib.sha256(payload).hexdigest() == ref["receipt_sha256"]
|
|
stored = json.loads(payload)
|
|
worker.health.validate_mirror(primary, stored)
|
|
assert "receipt_object_key" not in stored["recovery_mirror"] # No recursive self-hash.
|
|
assert (state / "last-success.json").read_bytes() == before
|
|
assert (state / "last-mirror.json").stat().st_mode & 0o777 == 0o600
|
|
assert not list(state.glob("mirror-readback-*"))
|
|
assert [operation for operation, _ in client.calls] == ["head", "put", "get"]
|
|
|
|
|
|
def test_lost_receipt_put_reply_reuses_object_and_never_advances_old_pointer(
|
|
worker,
|
|
health_state,
|
|
engine,
|
|
):
|
|
state, _, _ = health_state
|
|
old = (state / "last-mirror.json").read_bytes()
|
|
client = MemoryS3()
|
|
client.lose_put_reply = True
|
|
with pytest.raises(TimeoutError):
|
|
run(worker, health_state, engine, client)
|
|
assert (state / "last-mirror.json").read_bytes() == old
|
|
assert len(client.objects) == 1 and list(state.glob("mirror-jobs/*.json"))
|
|
assert run(worker, health_state, engine, client)["receipt_readback_verified"]
|
|
assert [op for op, _ in client.calls].count("put") == 1
|
|
assert len(engine.calls) == 2 and engine.calls[0] == engine.calls[1]
|
|
|
|
|
|
@pytest.mark.parametrize("failure", ["body", "version"])
|
|
def test_invalid_receipt_readback_does_not_promote(worker, health_state, engine, failure):
|
|
state, _, _ = health_state
|
|
old = (state / "last-mirror.json").read_bytes()
|
|
client = MemoryS3()
|
|
client.corrupt_get = failure == "body"
|
|
client.missing_version = failure == "version"
|
|
with pytest.raises(ValueError):
|
|
run(worker, health_state, engine, client)
|
|
assert (state / "last-mirror.json").read_bytes() == old
|
|
assert list(state.glob("mirror-jobs/*.json"))
|
|
|
|
|
|
def test_changed_retained_receipt_version_requires_review(worker, health_state, engine):
|
|
state, _, _ = health_state
|
|
client = MemoryS3()
|
|
run(worker, health_state, engine, client)
|
|
old = (state / "last-mirror.json").read_bytes()
|
|
next(iter(client.objects.values()))["VersionId"] = "unreviewed-replacement"
|
|
with pytest.raises(ValueError, match="version_changed"):
|
|
run(worker, health_state, engine, client)
|
|
assert (state / "last-mirror.json").read_bytes() == old
|
|
|
|
|
|
def test_unknown_private_fields_are_not_uploaded(worker, health_state, engine):
|
|
state, primary, _ = health_state
|
|
primary["unrelated_private_data"] = "SYNTHETIC MUST STAY LOCAL"
|
|
(state / "last-success.json").write_text(json.dumps(primary))
|
|
client = MemoryS3()
|
|
run(worker, health_state, engine, client)
|
|
assert b"SYNTHETIC MUST STAY LOCAL" not in next(iter(client.objects.values()))["Body"]
|
|
|
|
|
|
def test_backup_lock_excludes_mirror_before_provider_calls(worker, health_state, engine):
|
|
state, _, _ = health_state
|
|
path = state / "backup.lock"
|
|
path.touch(mode=0o600)
|
|
client = MemoryS3()
|
|
with path.open("r") as stream:
|
|
fcntl.flock(stream, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
with pytest.raises(ValueError, match="already_running"):
|
|
run(worker, health_state, engine, client)
|
|
assert not client.calls and not engine.calls
|
|
|
|
|
|
def test_tampered_ciphertext_refuses_before_publication(worker, health_state, engine):
|
|
state, primary, _ = health_state
|
|
(state / primary["local_archive"]).write_bytes(b"tampered")
|
|
client = MemoryS3()
|
|
with pytest.raises(ValueError, match="ciphertext_differs"):
|
|
run(worker, health_state, engine, client)
|
|
assert not client.calls and not engine.calls
|
|
|
|
|
|
def test_capacity_failure_refuses_mirror_before_provider_calls(
|
|
worker, health_state, engine, monkeypatch,
|
|
):
|
|
monkeypatch.setattr(worker.capacity, "disk_free", lambda path: 1)
|
|
client = MemoryS3()
|
|
with pytest.raises(ValueError, match="capacity_review_required"):
|
|
run(worker, health_state, engine, client)
|
|
assert not client.calls and not engine.calls
|
|
|
|
|
|
def test_preview_needs_no_runtime_credentials_or_state(worker, tmp_path):
|
|
environment = {k: v for k, v in os.environ.items() if not k.startswith("AWS_")}
|
|
result = subprocess.run(
|
|
[sys.executable, worker.__file__, "--state-dir", str(tmp_path / "absent")],
|
|
env=environment,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0
|
|
assert json.loads(result.stdout) == {
|
|
"status": "preview",
|
|
"provider_contacted": False,
|
|
"state_modified": False,
|
|
}
|
|
assert not list(tmp_path.iterdir())
|
|
|
|
|
|
def test_runtime_hash_drift_refuses_before_importing_any_owner_module(worker, tmp_path):
|
|
directory = tmp_path / "platform"
|
|
directory.mkdir()
|
|
files = {}
|
|
for name in worker.FILES:
|
|
source = directory / name
|
|
source.write_text('raise RuntimeError("must not import")\n')
|
|
files[name] = hashlib.sha256(source.read_bytes()).hexdigest()
|
|
manifest = tmp_path / "runtime.json"
|
|
manifest.write_text(
|
|
json.dumps({"schema": "coulomb.mirror-platform-runtime.v1", "files": files})
|
|
)
|
|
(directory / "application_recovery_mirror.py").write_text("changed\n")
|
|
with pytest.raises(ValueError, match="runtime_hash_differs"):
|
|
worker.load_runtime(directory, manifest)
|
|
|
|
|
|
def test_failed_final_checkpoint_retries_without_duplicate_retained_versions(
|
|
worker,
|
|
health_state,
|
|
engine,
|
|
):
|
|
state, _, _ = health_state
|
|
old = (state / "last-mirror.json").read_bytes()
|
|
checkpoint = engine.checkpoint
|
|
|
|
def fail_promotion(path, value):
|
|
if path.name == "last-mirror.json":
|
|
raise OSError("synthetic unavailable local checkpoint")
|
|
checkpoint(path, value)
|
|
|
|
engine.checkpoint = fail_promotion
|
|
client = MemoryS3()
|
|
with pytest.raises(OSError):
|
|
run(worker, health_state, engine, client)
|
|
assert (state / "last-mirror.json").read_bytes() == old
|
|
engine.checkpoint = checkpoint
|
|
assert run(worker, health_state, engine, client)["receipt_readback_verified"]
|
|
assert sum(op == "put" for op, _ in client.calls) == 1
|
|
|
|
|
|
def test_incorrect_engine_result_refuses_metadata_upload(worker, health_state, engine):
|
|
state, primary, mirrored = health_state
|
|
old = (state / "last-mirror.json").read_bytes()
|
|
mirrored["recovery_mirror"]["object_key"] = "unrelated"
|
|
engine.mirror = lambda *args: dict(primary, recovery_mirror=mirrored["recovery_mirror"])
|
|
client = MemoryS3()
|
|
with pytest.raises(ValueError):
|
|
run(worker, health_state, engine, client)
|
|
assert not client.calls and (state / "last-mirror.json").read_bytes() == old
|
|
|
|
|
|
def test_execute_without_own_environment_refuses_before_client_or_state(
|
|
worker,
|
|
monkeypatch,
|
|
tmp_path,
|
|
):
|
|
for name in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]:
|
|
monkeypatch.delenv(name, raising=False)
|
|
|
|
def unexpected(*args):
|
|
raise AssertionError("No client or credential fallback allowed")
|
|
|
|
monkeypatch.setattr(worker, "load_runtime", lambda *args: SimpleNamespace(s3=unexpected))
|
|
monkeypatch.setattr(
|
|
sys, "argv", [worker.__file__, "--execute", "--state-dir", str(tmp_path / "absent")]
|
|
)
|
|
with pytest.raises(ValueError, match="own_publisher_environment_required"):
|
|
worker.main()
|
|
assert not list(tmp_path.iterdir())
|
|
|
|
|
|
def test_service_retries_prior_mirror_before_capture_and_mirrors_the_new_point_afterward():
|
|
root = Path(__file__).resolve().parents[1]
|
|
service = (root / "deploy/backup/coulomb-social-paired-backup.service").read_text()
|
|
commands = [line for line in service.splitlines() if line.startswith("ExecStart")]
|
|
assert len(commands) == 3
|
|
assert (
|
|
commands[0].startswith("ExecStartPre=")
|
|
and "mirror-paired-backup.py --execute" in commands[0]
|
|
)
|
|
assert commands[1].startswith("ExecStart=") and "paired-backup.py --execute" in commands[1]
|
|
assert (
|
|
commands[2].startswith("ExecStartPost=")
|
|
and "mirror-paired-backup.py --execute" in commands[2]
|
|
)
|
|
assert "EnvironmentFile=/etc/coulomb-social-backup/primary.env" in service
|
|
assert "reader" not in service and "custodian" not in service
|