tegwick
235f13c0d2
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m27s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
329 lines
13 KiB
Python
329 lines
13 KiB
Python
"""Synthetic expiry must preserve custody and recover ambiguous single-version deletes."""
|
|
|
|
import copy
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from tests.test_backup_health import NOW
|
|
from tests.test_backup_health import health_state as health_state
|
|
from tests.test_backup_health import monitor as monitor
|
|
from tests.test_backup_retention_observation import ProviderError, ReadOnlyStore
|
|
from tests.test_backup_retention_review import evidence as evidence
|
|
from tests.test_backup_retention_review import reviewer as reviewer
|
|
|
|
RUNTIME_MANIFEST = (
|
|
Path(__file__).resolve().parents[1] / "deploy/backup/mirror-platform-runtime.json"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def executor(monkeypatch):
|
|
scripts = Path(__file__).resolve().parents[1] / "scripts"
|
|
monkeypatch.syspath_prepend(str(scripts))
|
|
spec = importlib.util.spec_from_file_location(
|
|
"mirror_expiry", scripts / "expire-backup-mirror.py"
|
|
)
|
|
value = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(value)
|
|
return value
|
|
|
|
|
|
class Store(ReadOnlyStore):
|
|
def head_object(self, Bucket, Key, VersionId):
|
|
if (Bucket, Key, VersionId) not in self.heads:
|
|
raise ProviderError(404, "NoSuchVersion")
|
|
return super().head_object(Bucket, Key, VersionId)
|
|
|
|
def remove(self, identity):
|
|
bucket, key, version = identity
|
|
self.buckets[bucket]["versions"] = [
|
|
v
|
|
for v in self.buckets[bucket]["versions"]
|
|
if (v["Key"], v["VersionId"]) != (key, version)
|
|
]
|
|
self.heads.pop(identity, None)
|
|
self.payloads.pop(identity, None)
|
|
|
|
|
|
class ExpiryCaller:
|
|
"""No get/list/put methods: reads must use the independent reader."""
|
|
|
|
def __init__(self, store):
|
|
self.store = store
|
|
self.calls = []
|
|
self.failure = None
|
|
self.after_delete = None
|
|
|
|
def delete_object(self, *, Bucket, Key, VersionId):
|
|
self.calls.append((Bucket, Key, VersionId))
|
|
if self.failure == "before":
|
|
self.failure = None
|
|
raise TimeoutError("synthetic uncertain request")
|
|
if self.failure != "no_effect":
|
|
self.store.remove((Bucket, Key, VersionId))
|
|
if self.after_delete:
|
|
self.after_delete()
|
|
if self.failure == "after":
|
|
self.failure = None
|
|
raise TimeoutError("synthetic lost reply")
|
|
return {"ResponseMetadata": {"HTTPStatusCode": 204}, "VersionId": VersionId}
|
|
|
|
|
|
@pytest.fixture
|
|
def scenario(executor, evidence, monkeypatch):
|
|
inventory, recovery, observation, selection = evidence
|
|
for name in list(inventory[0]["recovery_mirror"]):
|
|
if name.startswith("receipt_"):
|
|
del inventory[0]["recovery_mirror"][name]
|
|
store = Store(evidence)
|
|
payload = b"synthetic encrypted paired point"
|
|
ref = inventory[0]["recovery_mirror"]
|
|
store.payloads[ref["bucket"], ref["object_key"], ref["version_id"]] = payload
|
|
|
|
def add_version(bucket, key, version, data):
|
|
row = {"Key": key, "VersionId": version, "Size": len(data)}
|
|
head = {
|
|
"bucket": bucket,
|
|
"object_key": key,
|
|
"version_id": version,
|
|
"bytes": len(data),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
}
|
|
next(b for b in observation["observations"] if b["bucket"] == bucket)["versions"].append(
|
|
row
|
|
)
|
|
observation["version_heads"].append(head)
|
|
store.buckets[bucket]["versions"].append(dict(row))
|
|
store.heads[bucket, key, version] = {
|
|
"VersionId": version,
|
|
"ContentLength": len(data),
|
|
"Metadata": {"sha256": head["sha256"]},
|
|
}
|
|
store.payloads[bucket, key, version] = data
|
|
|
|
for minutes, label in [(30, "candidate"), (0, "newest")]:
|
|
point = copy.deepcopy(inventory[0])
|
|
stamp = NOW - timedelta(minutes=minutes)
|
|
name = stamp.strftime("%Y%m%dT%H%M%SZ-") + "f" * 24 + ".tar.age"
|
|
key = executor.health.PREFIX + stamp.strftime("%Y/%m/%d/") + name
|
|
point.update(
|
|
captured_at=stamp.isoformat(),
|
|
local_archive=name,
|
|
object_key=key,
|
|
destination="s3://" + executor.observer.PRIMARY + "/" + key,
|
|
version_id=label + "-primary",
|
|
)
|
|
mirror = point["recovery_mirror"]
|
|
mirror.update(object_key="archives/" + key, version_id=label + "-mirror")
|
|
add_version(executor.observer.PRIMARY, key, point["version_id"], payload)
|
|
add_version(executor.MIRROR, mirror["object_key"], mirror["version_id"], payload)
|
|
if label == "candidate":
|
|
sidecar = executor.canonical(executor.observer.worker.public_receipt(point, point))
|
|
digest = hashlib.sha256(sidecar).hexdigest()
|
|
mirror.update(
|
|
receipt_object_key=mirror["object_key"] + "." + digest + ".receipt.json",
|
|
receipt_version_id="candidate-receipt",
|
|
receipt_bytes=len(sidecar),
|
|
receipt_sha256=digest,
|
|
receipt_readback_verified=True,
|
|
)
|
|
add_version(executor.MIRROR, mirror["receipt_object_key"], "candidate-receipt", sidecar)
|
|
inventory.append(point)
|
|
identity = {
|
|
k: point[k]
|
|
for k in ("object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes")
|
|
}
|
|
selection["expiration_candidates" if label == "candidate" else "keep"].append(identity)
|
|
# Preserve a synthetic boundary fixture despite its not being a generation.
|
|
add_version(executor.MIRROR, "archives/synthetic-fixture", "fixture", b"fixture")
|
|
monkeypatch.setattr(
|
|
executor.reviewer,
|
|
"load_owner",
|
|
lambda _: SimpleNamespace(plan=lambda *args: copy.deepcopy(selection)),
|
|
)
|
|
proposal = executor.prepare(
|
|
inventory, recovery, observation, Path("unused"), NOW, RUNTIME_MANIFEST
|
|
)
|
|
digest = executor.sha(executor.canonical(proposal))
|
|
executor.validate(proposal, digest, Path("unused"), NOW, RUNTIME_MANIFEST)
|
|
journal = executor.new_journal(digest, len(proposal["targets"]))
|
|
return proposal, journal, store, ExpiryCaller(store)
|
|
|
|
|
|
def test_exact_two_version_expiry_preserves_kept_unknown_primary_and_kit(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
before = set(store.heads)
|
|
snapshots = []
|
|
result = executor.execute(
|
|
store, caller, proposal, journal, lambda: snapshots.append(copy.deepcopy(journal))
|
|
)
|
|
expected = [executor.key(t) for t in proposal["targets"]]
|
|
assert caller.calls == expected
|
|
assert set(store.heads) == before - set(expected)
|
|
assert result["versions_absent_verified"] == 2 and journal["status"] == "completed"
|
|
assert any(s["steps"] == ["intent", "pending"] for s in snapshots)
|
|
assert any(s["steps"] == ["absent_verified", "intent"] for s in snapshots)
|
|
assert all(bucket == executor.MIRROR and version for bucket, _, version in caller.calls)
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert caller.calls == expected # Terminal replay verifies; it does not delete again.
|
|
|
|
|
|
@pytest.mark.parametrize("failure,expected_calls", [("before", 3), ("after", 2)])
|
|
def test_ambiguous_delete_reconciles_before_retry(executor, scenario, failure, expected_calls):
|
|
proposal, journal, store, caller = scenario
|
|
durable = []
|
|
|
|
def save():
|
|
durable.append(copy.deepcopy(journal))
|
|
|
|
caller.failure = failure
|
|
with pytest.raises(TimeoutError):
|
|
executor.execute(store, caller, proposal, journal, save)
|
|
assert durable[-1]["steps"] == ["intent", "pending"]
|
|
resumed = copy.deepcopy(durable[-1])
|
|
executor.execute(store, caller, proposal, resumed, lambda: None)
|
|
assert len(caller.calls) == expected_calls and resumed["status"] == "completed"
|
|
|
|
|
|
def test_durable_checkpoint_failure_prevents_delete(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
|
|
def full_disk():
|
|
raise OSError("synthetic disk full")
|
|
|
|
with pytest.raises(OSError):
|
|
executor.execute(store, caller, proposal, journal, full_disk)
|
|
assert caller.calls == []
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"change",
|
|
["missing_keep", "missing_unknown", "new_version", "hash", "body", "lifecycle", "versioning"],
|
|
)
|
|
def test_changed_custody_refuses_all_expiry(executor, scenario, change):
|
|
proposal, journal, store, caller = scenario
|
|
keep = executor.key(proposal["keep"][0])
|
|
if change == "missing_keep":
|
|
store.remove(keep)
|
|
elif change == "missing_unknown":
|
|
store.remove((executor.MIRROR, "archives/synthetic-fixture", "fixture"))
|
|
elif change == "new_version":
|
|
store.buckets[executor.MIRROR]["versions"].append(
|
|
{"Key": "archives/new", "VersionId": "new", "Size": 1}
|
|
)
|
|
elif change == "hash":
|
|
store.heads[keep]["Metadata"]["sha256"] = "0" * 64
|
|
elif change == "body":
|
|
store.payloads[keep] = b"changed"
|
|
elif change == "lifecycle":
|
|
store.buckets[executor.KIT]["lifecycle"]["Rules"] = [{"Status": "Enabled"}]
|
|
else:
|
|
store.buckets[executor.MIRROR]["versioning"] = {"Status": "Suspended"}
|
|
with pytest.raises(ValueError):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert caller.calls == []
|
|
|
|
|
|
def test_missing_candidate_without_own_intent_is_not_adopted(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
store.remove(executor.key(proposal["targets"][0]))
|
|
with pytest.raises(ValueError, match="inventory_changed"):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert caller.calls == []
|
|
|
|
|
|
def test_provider_success_without_actual_absence_stops_before_receipt(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
caller.failure = "no_effect"
|
|
with pytest.raises(ValueError, match="deletion_not_independently_verified"):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert len(caller.calls) == 1 and journal["steps"] == ["intent", "pending"]
|
|
|
|
|
|
def test_keep_loss_after_first_delete_stops_remaining_actions(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
caller.after_delete = lambda: store.remove(executor.key(proposal["keep"][0]))
|
|
with pytest.raises(ValueError, match="inventory_changed"):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert len(caller.calls) == 1 and journal["steps"] == ["intent", "pending"]
|
|
|
|
|
|
def test_forbidden_is_not_proof_of_deleted_version(executor, scenario):
|
|
proposal, journal, store, caller = scenario
|
|
target = proposal["targets"][0]
|
|
store.remove(executor.key(target))
|
|
journal["steps"][0] = "intent"
|
|
original = store.head_object
|
|
|
|
def denied(**kwargs):
|
|
if kwargs["Key"] == target["object_key"]:
|
|
raise ProviderError(403, "AccessDenied")
|
|
return original(**kwargs)
|
|
|
|
store.head_object = denied
|
|
with pytest.raises(ProviderError):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert caller.calls == []
|
|
|
|
|
|
@pytest.mark.parametrize("change", ["digest", "source", "runtime", "target", "protected", "future"])
|
|
def test_capsule_mutation_is_rejected(executor, scenario, change):
|
|
proposal, journal, _, _ = scenario
|
|
expected = journal["proposal_digest"]
|
|
if change == "digest":
|
|
expected = "0" * 64
|
|
elif change == "source":
|
|
proposal["sources"]["expire-backup-mirror.py"] = "0" * 64
|
|
elif change == "runtime":
|
|
proposal["runtime_manifest_sha256"] = "0" * 64
|
|
elif change == "target":
|
|
proposal["targets"][0]["bucket"] = executor.KIT
|
|
elif change == "protected":
|
|
proposal["recovery"]["database_sha256"] = "0" * 64
|
|
else:
|
|
proposal["created_at"] = (NOW + timedelta(hours=1)).isoformat()
|
|
if change != "digest":
|
|
expected = executor.sha(executor.canonical(proposal))
|
|
with pytest.raises(ValueError):
|
|
executor.validate(proposal, expected, Path("unused"), NOW, RUNTIME_MANIFEST)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"steps", [["pending", "absent_verified"], ["intent", "intent"], ["unknown", "pending"]]
|
|
)
|
|
def test_out_of_order_journal_refuses_without_provider_calls(executor, scenario, steps):
|
|
proposal, journal, store, caller = scenario
|
|
journal["steps"] = steps
|
|
with pytest.raises(ValueError):
|
|
executor.execute(store, caller, proposal, journal, lambda: None)
|
|
assert store.calls == [] and caller.calls == []
|
|
|
|
|
|
def test_checkpoint_is_private_and_refuses_linked_journal(executor, tmp_path):
|
|
target = tmp_path / "journal.json"
|
|
executor.checkpoint(target, {"status": "synthetic"})
|
|
assert target.stat().st_mode & 0o777 == 0o600
|
|
assert json.loads(target.read_text()) == {"status": "synthetic"}
|
|
link = tmp_path / "link.json"
|
|
link.symlink_to(target)
|
|
with pytest.raises(ValueError):
|
|
executor.checkpoint(link, {"status": "wrong"})
|
|
assert json.loads(target.read_text()) == {"status": "synthetic"}
|
|
|
|
|
|
def test_default_preview_needs_no_provider_or_credentials():
|
|
script = Path(__file__).resolve().parents[1] / "scripts/expire-backup-mirror.py"
|
|
result = subprocess.run([sys.executable, str(script)], capture_output=True, check=True)
|
|
assert json.loads(result.stdout) == {
|
|
"status": "preview",
|
|
"provider_contacted": False,
|
|
"deletion_admitted": False,
|
|
}
|