diff --git a/docs/release/generation-retention-review-2026-10-30.json b/docs/release/generation-retention-review-2026-10-30.json new file mode 100644 index 0000000..add2099 --- /dev/null +++ b/docs/release/generation-retention-review-2026-10-30.json @@ -0,0 +1,105 @@ +{ + "age_window_counterexample": { + "fixed_age_window_preserves_generation_policy": false, + "input_generations": 15, + "oldest_selected_age_days": 47, + "selected_generations_older_than_30_days": 13, + "synthetic_source_gap": true + }, + "blockers": [], + "bucket_bytes": { + "railiance-coulomb-social-archives": 2120470, + "railiance-coulomb-social-recovery": 277590204, + "railiance-platform-pg-backup": 2120392 + }, + "bucket_version_counts": { + "railiance-coulomb-social-archives": 3, + "railiance-coulomb-social-recovery": 3, + "railiance-platform-pg-backup": 1 + }, + "credential_values_printed": false, + "deletion_capability": false, + "expiration_candidates": 0, + "generation_selection_valid": true, + "head_observed_at": "2026-10-30T16:24:22.736307+00:00", + "keep_versions": 1, + "known_version_heads": [ + { + "bucket": "railiance-coulomb-social-archives", + "bytes": 2120392, + "last_modified": "2026-10-29T21:25:56+00:00", + "object_key": "archives/platform-pg/application-archives/coulomb-social/2026/10/29/20261029T212246Z-fd128164c7bb22a3a6c42acf.tar.age", + "role": "mirror", + "sha256": "acbf92eda5074761bb6d00170e6e459e787359afcd377f2f18324ba17496510a", + "version_id": "1793309156692541" + }, + { + "bucket": "railiance-coulomb-social-recovery", + "bytes": 277585920, + "last_modified": "2026-10-29T21:39:31+00:00", + "object_key": "kits/db05c8f3b5587dc1fc09a5352c317dd3357527dbe72366ab7ca1041db578c834/recovery-kit.tar", + "role": "protected_kit", + "sha256": "db05c8f3b5587dc1fc09a5352c317dd3357527dbe72366ab7ca1041db578c834", + "version_id": "1793309971213507" + }, + { + "bucket": "railiance-coulomb-social-recovery", + "bytes": 4237, + "last_modified": "2026-10-29T21:47:56+00:00", + "object_key": "kits/receipts/a77477cac53086b83ad2cb512973ae0d1dfecba028a29a92e06a1bcfe5287f29/recovery-proof.json", + "role": "protected_proof", + "sha256": "a77477cac53086b83ad2cb512973ae0d1dfecba028a29a92e06a1bcfe5287f29", + "version_id": "1793310476676486" + }, + { + "bucket": "railiance-platform-pg-backup", + "bytes": 2120392, + "last_modified": "2026-10-29T21:23:40+00:00", + "object_key": "platform-pg/application-archives/coulomb-social/2026/10/29/20261029T212246Z-fd128164c7bb22a3a6c42acf.tar.age", + "role": "primary", + "sha256": "acbf92eda5074761bb6d00170e6e459e787359afcd377f2f18324ba17496510a", + "version_id": "1793309020705518" + } + ], + "native_lifecycle_rules": { + "railiance-coulomb-social-archives": [], + "railiance-coulomb-social-recovery": [], + "railiance-platform-pg-backup": [ + { + "Expiration": { + "Days": 30 + }, + "Filter": { + "Prefix": "" + }, + "ID": "retain-30-days", + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + }, + "Status": "Enabled" + } + ] + }, + "observed_at": "2026-10-29T23:40:50.119394+00:00", + "owner_source_sha256": { + "application_archive_retention_plan.py": "08bc57308f53bbfd0d58fdbb9b5671dd4285e31c5b5137e201524df606aa3f9f", + "primary_application_archive.py": "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d" + }, + "personal_session_used": false, + "private_age_identity_read": false, + "private_review_sha256": "30a353f937c5e8182be1a54a5ec5f6ca192c661e48a05d616b5598eb509f15ea", + "protected_kit_and_proof_versions_verified": true, + "provider_mutations": false, + "provider_observation_sha256": "25a03e48aa866fcc87b89363dabc7a34727b71aa17cceefca119b5a9fb8cb566", + "ready_to_execute": false, + "reviewer_sha256": "929c0416add6bfddee5bce9e6aaeb436d2d9de8084db48d061f66186f7127794", + "routine_api_credentials_used": true, + "schema": "coulomb.native-generation-retention-review.v1", + "source_sha256": { + "inventory": "a0cbf99d0e87b2993eb301a5034763e94169f3cc3589eaba12c3a275eaf19fd1", + "observation": "25a03e48aa866fcc87b89363dabc7a34727b71aa17cceefca119b5a9fb8cb566", + "recovery": "801744d29c320a811cac5931fd9ea835afd5760bf203729cd91a82c17a93ad23" + }, + "status": "selection_reviewed_execution_not_admitted", + "timers_activated": false +} diff --git a/docs/release/generation-retention-review-2026-10-30.md b/docs/release/generation-retention-review-2026-10-30.md new file mode 100644 index 0000000..72f39a7 --- /dev/null +++ b/docs/release/generation-retention-review-2026-10-30.md @@ -0,0 +1,92 @@ +# Exact-generation retention review + +The 14-daily/four-weekly generation policy is preserved. A 30-day mirror lifecycle +must not replace that selector: after a capture gap, selected generations can be +older than 30 days. This review adds no deletion capability or live lifecycle rule. + +## Native provider evidence + +Routine scoped publisher/reader APIs read version inventories, versioning and +lifecycle configuration. Exact-version HEAD metadata was compared with the +previous independently verified ciphertext/kit/proof readbacks. No private age +identity, personal Console session or sibling credential was used. + +| Storage scope | Native current state | +| --- | --- | +| Primary Coulomb prefix in railiance-platform-pg-backup | One version, 2,120,392 bytes. Bucket-wide enabled current/noncurrent expiry at 30 days applies to this prefix. | +| Independent archives bucket, archives/ | Three versions, 2,120,470 bytes. One known application point plus two admission fixtures. Versioning enabled; no lifecycle rules. | +| Protected recovery bucket, kits/ | Three versions, 277,590,204 bytes. Exact complete kit and independent proof versions match their recorded hashes/sizes; versioning enabled, no lifecycle rules. | + +Inventories are complete for these exact prefixes, with no delete markers. +The original immutable primary version, corresponding mirror version, protected +kit and protected proof all have matching version/hash/size metadata. This is a +fresh metadata check linked to earlier real retrieval/recovery proofs; the complete +277 MB kit was not downloaded again merely to inspect retention. + +Public evidence and source/input hashes: +`generation-retention-review-2026-10-30.json`. Full observation and compiled plan +remain private outside Git. The independent mirror is the generation-retention +surface even after primary age expiry; the logical primary key/version/hash still +identifies the captured point, while retrieval uses its distinct mirror version. +The last recovered point also has its complete kit/proof outside rolling expiry. + +## Deterministic review + +`scripts/review-backup-retention.py` loads the existing platform selector only +after verifying its source and primary-helper hashes. It checks fresh complete +provider observations, exact mirrored versions, protected kit/proof custody, +recovery image/recipient/both-store bindings, disjoint selection and newest-point +protection. Partial, stale, changed or ambiguous evidence refuses. Unknown objects +are counted and preserved; admission fixtures are not expiration candidates. + +```sh +uv run python scripts/review-backup-retention.py \ + --inventory /private/verified-mirrored-inventory.json \ + --recovery docs/release/provider-kit-recovery-2026-10-29.json \ + --observation /private/provider-observation-with-heads.json \ + --platform-scripts /private/reviewed-railiance-platform/scripts \ + --output /private/generation-review.json +``` + +The actual plan keeps one available generation and has zero expiration candidates. +This startup inventory does not establish a full 14-day/four-week history. +For any later expiration candidate the compiler additionally requires its +independently retained receipt's exact version/hash/size; it cannot orphan unknown +or unverified recovery metadata. It grants no execution and never contacts the +provider, deletes data or changes custody. + +Validation: 15 focused evidence/mapping cases and the full `make check` gate pass +(319 tests, including mandatory browser journeys). + +The same pinned selector was exercised with a synthetic capture gap. Of 15 input +points, it kept 13 older than 30 days, with the oldest selected point 47 days old. +An age-only mirror rule would violate the generation selection. The existing +primary's time-based lifecycle is recorded as a separate storage constraint, +not falsely treated as exact generation enforcement. + +Scaleway documents prefix-filtered lifecycle rules and separate current/noncurrent +expiration. Those are age-based actions, with asynchronous application; they do +not express this cross-object generation selector. See the official +[lifecycle API specification](https://www.scaleway.com/en/docs/object-storage/api-cli/lifecycle-rules-api/) +and [lifecycle timing FAQ](https://www.scaleway.com/en/docs/object-storage/faq/). + +## Remaining operating action + +Retain the mirror without blanket age expiry. Prepare a separately admitted +mirror-only exact-version expiry executor using the fresh selector return. It +must preserve newest/daily/weekly/recovered points, verify protected custody and +the complete keep set immediately before execution, retain checkpoints for partial +or uncertain outcomes and verify the kept versions afterward. No primary or kit +deletion belongs in that executor. + +Its future caller needs only the dedicated mirror prefix's listing/read/version +expiry scope. Native denials must cover writes, unversioned deletion, primary and +protected-kit access, foreign prefixes, lifecycle/bucket administration, escrow +and IAM. The scheduled publisher and independent reader retain their deletion +denials. No caller/key/grant or expiration executor was created by this review; +additional expiry authority still needs its exact reviewed admission. + +Current zero-candidate review is evidence, not recurring retention enforcement. +Received alerts, operating delivery/native mirror-and-receipt return, bounded +capacity admission and historical-copy disposition remain activation gates. +Existing custody and schedule approvals are unchanged. diff --git a/scripts/review-backup-retention.py b/scripts/review-backup-retention.py new file mode 100644 index 0000000..c843a9d --- /dev/null +++ b/scripts/review-backup-retention.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Compile an offline exact-version review; no expiry or provider capability. + +The owner planner selects 14 daily/four weekly logical points. This consumer maps +them to independently held mirror versions and checks the protected recovery kit. +Primary age-based expiry is not authority to expire the independent generations. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import re +import sys +from datetime import UTC, datetime, timedelta +from pathlib import Path + +spec = importlib.util.spec_from_file_location( + "retention_health", Path(__file__).with_name("check-paired-backup-health.py") +) +health = importlib.util.module_from_spec(spec) +spec.loader.exec_module(health) +PRIMARY = "railiance-platform-pg-backup" +KIT = "railiance-coulomb-social-recovery" +PLANNER_SHA256 = "08bc57308f53bbfd0d58fdbb9b5671dd4285e31c5b5137e201524df606aa3f9f" +ARCHIVE_SHA256 = "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d" +OWNER_FILES = { + "application_archive_retention_plan.py": PLANNER_SHA256, + "primary_application_archive.py": ARCHIVE_SHA256, +} + + +def identity(receipt): + return tuple( + receipt[k] for k in ("object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes") + ) + + +def review(inventory, recovery, observed, selection, now): + if ( + observed.get("schema") != "coulomb.provider-retention-observation.v1" + or observed.get("provider_mutations") is not False + ): + raise ValueError("Read-only provider observation required") + for stamp in [observed["observed_at"], observed["head_observed_at"]]: + age = (now - health.timestamp(stamp)).total_seconds() + if not 0 <= age <= 26 * 3600: + raise ValueError("Fresh version/policy observation required") + buckets = {item["bucket"]: item for item in observed["observations"]} + if len(buckets) != 3 or set(buckets) != {PRIMARY, health.MIRROR_BUCKET, KIT}: + raise ValueError("Exact application bucket observations required") + for bucket, prefix in [ + (PRIMARY, health.PREFIX), + (health.MIRROR_BUCKET, "archives/"), + (KIT, "kits/"), + ]: + item = buckets[bucket] + if ( + item.get("prefix") != prefix + or item.get("inventory_complete") is not True + or item["versioning"].get("Status") != "Enabled" + or item["lifecycle"].get("status") != "observed" + ): + raise ValueError("Complete versioned prefix and lifecycle evidence required") + if ( + selection.get("application") != "coulomb-social" + or selection.get("status") != "reviewed" + or selection.get("daily_generations") != 14 + or selection.get("weekly_generations") != 4 + or recovery.get("status") != "application_restored_verified" + or recovery.get("application_restore_verified") is not True + or recovery.get("application") != "coulomb-social" + or recovery.get("application_data_images_and_recovery_tools_from_provider_kit") is not True + ): + raise ValueError("Exact owner selection and protected app recovery required") + heads = {(h["bucket"], h["object_key"], h["version_id"]): h for h in observed["version_heads"]} + if len(heads) != len(observed["version_heads"]): + raise ValueError("Unambiguous exact-version heads required") + versions = {} + for bucket, item in buckets.items(): + for row in item["versions"]: + key = (bucket, row["Key"], row["VersionId"]) + if key in versions or not row["Key"].startswith(item["prefix"]): + raise ValueError("Unambiguous prefix version inventory required") + versions[key] = row + + def require_version(bucket, key, version, digest, size): + found = versions.get((bucket, key, version)) + head = heads.get((bucket, key, version)) + if ( + found is None + or head is None + or found["Size"] != size + or head["bytes"] != size + or head["sha256"] != digest + ): + raise ValueError("Exact retained version/hash/size evidence required") + + for name in ["protected_kit", "protected_recovery_proof"]: + ref = recovery[name] + if ( + ref.get("bucket") != KIT + or ref.get("project_id") != health.PROJECT + or not ref["object_key"].startswith("kits/") + or ref.get("automatic_expiry") is not False + or not re.fullmatch(r"[a-f0-9]{64}", ref.get("sha256", "")) + or type(ref.get("bytes")) is not int + or ref["bytes"] <= 0 + ): + raise ValueError("Separate nonexpiring recovery custody required") + expected = ( + "kits/" + ref["sha256"] + "/recovery-kit.tar" + if name == "protected_kit" + else "kits/receipts/" + ref["sha256"] + "/recovery-proof.json" + ) + if ref["object_key"] != expected: + raise ValueError("Hash-addressed protected object required") + require_version(KIT, ref["object_key"], ref["version_id"], ref["sha256"], ref["bytes"]) + if recovery["protected_recovery_proof"].get("independent_readback_verified") is not True: + raise ValueError("Independently retained recovery proof required") + protected = identity(recovery) + rows = {} + for receipt in inventory: + health.validate_primary(receipt) + health.validate_mirror(receipt, receipt) + item = identity(receipt) + if item in rows: + raise ValueError("Duplicate logical point") + rows[item] = receipt + keep = {identity(r) for r in selection["keep"]} + candidates = {identity(r) for r in selection["expiration_candidates"]} + if ( + not keep + or keep & candidates + or keep | candidates != set(rows) + or protected not in keep + or identity(selection["protected"]) != protected + ): + raise ValueError("Complete disjoint selection preserving recovered point required") + newest = max(rows, key=lambda item: (health.timestamp(rows[item]["captured_at"]), item)) + if newest not in keep: + raise ValueError("Newest generation must be preserved") + if any( + rows[protected][key] != recovery.get(key) + for key in ( + "image", + "age_recipient", + "database_sha256", + "content_sha256", + ) + ): + raise ValueError("Protected recovery binding differs from selected point") + blockers = [] + if buckets[KIT]["lifecycle"].get("Rules"): + blockers.append("protected_bucket_has_lifecycle_rules") + if any( + r.get("Status") == "Enabled" for r in buckets[health.MIRROR_BUCKET]["lifecycle"]["Rules"] + ): + blockers.append("mirror_lifecycle_cannot_guarantee_generation_selection") + mirror_keep, mirror_candidates = [], [] + primary_missing = 0 + for item, receipt in rows.items(): + ref = receipt["recovery_mirror"] + require_version( + health.MIRROR_BUCKET, + ref["object_key"], + ref["version_id"], + receipt["ciphertext_sha256"], + receipt["ciphertext_bytes"], + ) + target = { + "bucket": health.MIRROR_BUCKET, + "object_key": ref["object_key"], + "version_id": ref["version_id"], + "ciphertext_sha256": receipt["ciphertext_sha256"], + "ciphertext_bytes": receipt["ciphertext_bytes"], + "logical_primary_identity": dict( + zip(["object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes"], item) + ), + } + if (PRIMARY, receipt["object_key"], receipt["version_id"]) not in versions: + primary_missing += 1 # Mirror retains the logical point after primary age expiry. + else: + require_version( + PRIMARY, + receipt["object_key"], + receipt["version_id"], + receipt["ciphertext_sha256"], + receipt["ciphertext_bytes"], + ) + if item in keep: + mirror_keep.append(target) + else: + # Expiry review cannot orphan an unverified independent receipt. + health.validate_mirror(receipt, receipt, require_retained_receipt=True) + require_version( + health.MIRROR_BUCKET, + ref["receipt_object_key"], + ref["receipt_version_id"], + ref["receipt_sha256"], + ref["receipt_bytes"], + ) + target["associated_receipt"] = { + k: ref[k] + for k in ( + "receipt_object_key", + "receipt_version_id", + "receipt_sha256", + "receipt_bytes", + ) + } + mirror_candidates.append(target) + # This review intentionally grants no execution. Separate custody/boundaries + # and fresh keep-set revalidation are needed before any exact-version expiry. + return { + "schema": "coulomb.generation-retention-review.v1", + "status": "reviewed", + "daily_generations": 14, + "weekly_generations": 4, + "protected_point_in_mirror_keep": True, + "protected_kit_and_proof_versions_verified": True, + "keep": sorted(mirror_keep, key=lambda r: (r["object_key"], r["version_id"])), + "expiration_candidates": sorted( + mirror_candidates, key=lambda r: (r["object_key"], r["version_id"]) + ), + "primary_versions_already_absent": primary_missing, + "primary_lifecycle_observed": buckets[PRIMARY]["lifecycle"]["Rules"], + "bucket_version_counts": {k: len(v["versions"]) for k, v in sorted(buckets.items())}, + "bucket_bytes": { + k: sum(x["Size"] for x in v["versions"]) for k, v in sorted(buckets.items()) + }, + "blockers": sorted(blockers), + "generation_selection_valid": not blockers, + "unknown_objects_preserved": True, + "provider_mutations": False, + "local_expiry": False, + "ready_to_execute": False, + "deletion_capability": False, + "activation_gate": "Separate mirror-only exact-version expiry admission/execution, " + "received monitoring and fresh protected/keep-set verification remain required.", + } + + +def load_owner(directory): + for name, digest in OWNER_FILES.items(): + path = directory / name + if path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != digest: + raise ValueError("Owner planner source differs") + existing = sys.modules.get("primary_application_archive") + if ( + existing + and Path(existing.__file__).resolve() + != (directory / "primary_application_archive.py").resolve() + ): + raise ValueError("Owner helper already loaded elsewhere") + sys.path.insert(0, str(directory)) + spec = importlib.util.spec_from_file_location( + "retention_owner", directory / "application_archive_retention_plan.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def age_window_counterexample(owner, now): + """Show why an age window cannot replace the admitted generation selector.""" + inventory = [] + for days in [1, *range(35, 49)]: + stamp = now - timedelta(days=days) + suffix = hashlib.sha256(stamp.isoformat().encode()).hexdigest()[:24] + inventory.append( + { + "application": "coulomb-social", + "status": "offsite_fetched_verified", + "remote_retrieval_verified": True, + "captured_at": stamp.isoformat(), + "object_key": health.PREFIX + + stamp.strftime("%Y/%m/%d/%Y%m%dT%H%M%SZ-") + + suffix + + ".tar.age", + "version_id": "synthetic-" + str(days), + "ciphertext_sha256": "a" * 64, + "ciphertext_bytes": 1, + } + ) + recovery = dict( + inventory[0], application_restore_verified=True, status="application_restored_verified" + ) + selected = owner.plan("coulomb-social", inventory, recovery, now) + by_key = {r["object_key"]: r for r in inventory} + ages = [ + (now - health.timestamp(by_key[r["object_key"]]["captured_at"])).days + for r in selected["keep"] + ] + return { + "synthetic_source_gap": True, + "input_generations": len(inventory), + "selected_generations_older_than_30_days": sum(age > 30 for age in ages), + "oldest_selected_age_days": max(ages), + "fixed_age_window_preserves_generation_policy": not any(age > 30 for age in ages), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ["inventory", "recovery", "observation", "platform-scripts", "output"]: + parser.add_argument("--" + name, type=Path, required=True) + args = parser.parse_args() + output = args.output.resolve() + if output.is_relative_to(Path(__file__).resolve().parents[1]): + raise ValueError("Private plan must stay outside checkout") + sources = { + name: getattr(args, name).read_bytes() for name in ["inventory", "recovery", "observation"] + } + inputs = {name: json.loads(raw) for name, raw in sources.items()} + now = datetime.now(UTC) + owner = load_owner(args.platform_scripts) + selection = owner.plan("coulomb-social", inputs["inventory"], inputs["recovery"], now) + result = review(inputs["inventory"], inputs["recovery"], inputs["observation"], selection, now) + result["age_window_counterexample"] = age_window_counterexample(owner, now) + result["source_sha256"] = { + name: hashlib.sha256(raw).hexdigest() for name, raw in sources.items() + } + result["owner_source_sha256"] = OWNER_FILES + fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(result, stream, indent=2, sort_keys=True) + stream.write("\n") + print( + json.dumps( + { + "status": result["status"], + "keep_versions": len(result["keep"]), + "expiration_candidates": len(result["expiration_candidates"]), + "blockers": result["blockers"], + "ready_to_execute": False, + "provider_mutations": False, + } + ) + ) + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, TypeError, OSError, AttributeError): + raise SystemExit("Retention review refused; inspect private inputs locally.") diff --git a/tests/test_backup_retention_review.py b/tests/test_backup_retention_review.py new file mode 100644 index 0000000..4aa6412 --- /dev/null +++ b/tests/test_backup_retention_review.py @@ -0,0 +1,301 @@ +"""Exact independent generations survive primary age expiry and foreign objects.""" + +import copy +import importlib.util +from datetime import timedelta +from pathlib import Path + +import pytest + +from tests.test_backup_health import NOW +from tests.test_backup_health import health_state as health_state +from tests.test_backup_health import monitor as monitor + + +@pytest.fixture +def reviewer(): + path = Path(__file__).resolve().parents[1] / "scripts/review-backup-retention.py" + spec = importlib.util.spec_from_file_location("retention_review", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def evidence(reviewer, health_state): + _, receipt, mirrored = health_state + recovery = dict( + receipt, + status="application_restored_verified", + application_restore_verified=True, + application_data_images_and_recovery_tools_from_provider_kit=True, + ) + recovery["protected_kit"] = { + "project_id": reviewer.health.PROJECT, + "bucket": reviewer.KIT, + "object_key": "kits/" + "a" * 64 + "/recovery-kit.tar", + "version_id": "kit-version", + "sha256": "a" * 64, + "bytes": 100, + "automatic_expiry": False, + } + recovery["protected_recovery_proof"] = { + "project_id": reviewer.health.PROJECT, + "bucket": reviewer.KIT, + "object_key": "kits/receipts/" + "b" * 64 + "/recovery-proof.json", + "version_id": "proof-version", + "sha256": "b" * 64, + "bytes": 80, + "automatic_expiry": False, + "independent_readback_verified": True, + } + roles = [ + ( + reviewer.PRIMARY, + receipt["object_key"], + receipt["version_id"], + receipt["ciphertext_sha256"], + receipt["ciphertext_bytes"], + ), + ( + reviewer.health.MIRROR_BUCKET, + mirrored["recovery_mirror"]["object_key"], + mirrored["recovery_mirror"]["version_id"], + receipt["ciphertext_sha256"], + receipt["ciphertext_bytes"], + ), + ] + for name in ["protected_kit", "protected_recovery_proof"]: + ref = recovery[name] + roles.append( + (ref["bucket"], ref["object_key"], ref["version_id"], ref["sha256"], ref["bytes"]) + ) + buckets = [] + for bucket, prefix in [ + (reviewer.PRIMARY, reviewer.health.PREFIX), + (reviewer.health.MIRROR_BUCKET, "archives/"), + (reviewer.KIT, "kits/"), + ]: + rules = ( + [ + { + "ID": "existing-global-30-day", + "Status": "Enabled", + "Filter": {"Prefix": ""}, + "Expiration": {"Days": 30}, + } + ] + if bucket == reviewer.PRIMARY + else [] + ) + buckets.append( + { + "bucket": bucket, + "prefix": prefix, + "inventory_complete": True, + "versioning": {"Status": "Enabled"}, + "lifecycle": {"status": "observed", "Rules": rules}, + "versions": [ + {"Key": k, "VersionId": v, "Size": n, "IsLatest": True} + for b, k, v, h, n in roles + if b == bucket + ], + } + ) + observed = { + "schema": "coulomb.provider-retention-observation.v1", + "provider_mutations": False, + "observed_at": NOW.isoformat(), + "head_observed_at": NOW.isoformat(), + "observations": buckets, + "version_heads": [ + {"bucket": b, "object_key": k, "version_id": v, "sha256": h, "bytes": n} + for b, k, v, h, n in roles + ], + } + point = { + k: receipt[k] for k in ["object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes"] + } + selection = { + "application": "coulomb-social", + "status": "reviewed", + "daily_generations": 14, + "weekly_generations": 4, + "protected": point, + "keep": [point], + "expiration_candidates": [], + } + return [mirrored], recovery, observed, selection + + +def test_review_maps_logical_point_to_independent_version_without_expiry(reviewer, evidence): + before = copy.deepcopy(evidence) + plan = reviewer.review(*evidence, NOW) + assert plan["generation_selection_valid"] and not plan["blockers"] + assert len(plan["keep"]) == 1 and not plan["expiration_candidates"] + assert plan["keep"][0]["bucket"] == reviewer.health.MIRROR_BUCKET + assert plan["protected_kit_and_proof_versions_verified"] + assert not plan["ready_to_execute"] and not plan["deletion_capability"] + assert evidence == before + + +def test_primary_version_expiry_does_not_discard_held_mirror_generation(reviewer, evidence): + for bucket in evidence[2]["observations"]: + if bucket["bucket"] == reviewer.PRIMARY: + bucket["versions"] = [] + evidence[2]["version_heads"] = [ + h for h in evidence[2]["version_heads"] if h["bucket"] != reviewer.PRIMARY + ] + plan = reviewer.review(*evidence, NOW) + assert plan["generation_selection_valid"] and plan["primary_versions_already_absent"] == 1 + assert len(plan["keep"]) == 1 + + +@pytest.mark.parametrize( + "bucket_role,reason", + [ + ("mirror", "mirror_lifecycle_cannot_guarantee_generation_selection"), + ("kit", "protected_bucket_has_lifecycle_rules"), + ], +) +def test_fixed_age_expiry_cannot_certify_generation_or_protected_custody( + reviewer, + evidence, + bucket_role, + reason, +): + target = reviewer.KIT if bucket_role == "kit" else reviewer.health.MIRROR_BUCKET + for bucket in evidence[2]["observations"]: + if bucket["bucket"] == target: + bucket["lifecycle"]["Rules"] = [{"Status": "Enabled", "Expiration": {"Days": 30}}] + plan = reviewer.review(*evidence, NOW) + assert not plan["generation_selection_valid"] and reason in plan["blockers"] + + +@pytest.mark.parametrize( + "failure", + [ + "partial", + "stale", + "hash", + "kit_version", + "protected_selection", + "recovery_binding", + "independent_proof", + ], +) +def test_weak_or_changed_provider_recovery_evidence_refuses(reviewer, evidence, failure): + if failure == "partial": + evidence[2]["observations"][0]["inventory_complete"] = False + elif failure == "stale": + evidence[2]["head_observed_at"] = (NOW - timedelta(hours=27)).isoformat() + elif failure == "hash": + evidence[2]["version_heads"][1]["sha256"] = "c" * 64 + elif failure == "kit_version": + evidence[1]["protected_kit"]["version_id"] = "unreviewed-version" + elif failure == "recovery_binding": + evidence[1]["database_sha256"] = "f" * 64 + elif failure == "independent_proof": + evidence[1]["protected_recovery_proof"]["independent_readback_verified"] = False + else: + evidence[3]["keep"] = [] + with pytest.raises(ValueError): + reviewer.review(*evidence, NOW) + + +def test_unknown_objects_are_counted_and_preserved(reviewer, evidence): + mirror = next( + b for b in evidence[2]["observations"] if b["bucket"] == reviewer.health.MIRROR_BUCKET + ) + mirror["versions"].append( + { + "Key": "archives/unassigned-admission-fixture", + "VersionId": "fixture", + "Size": 123, + "IsLatest": True, + } + ) + plan = reviewer.review(*evidence, NOW) + assert plan["unknown_objects_preserved"] and not plan["expiration_candidates"] + assert plan["bucket_version_counts"][reviewer.health.MIRROR_BUCKET] == 2 + + +def test_owner_source_tampering_refuses_before_import(reviewer, tmp_path): + for name in reviewer.OWNER_FILES: + (tmp_path / name).write_text('raise AssertionError("must not execute")\n') + with pytest.raises(ValueError, match="Owner planner source differs"): + reviewer.load_owner(tmp_path) + + +@pytest.mark.parametrize("sidecar_present", [True, False]) +def test_expiry_mapping_requires_exact_independent_receipt_evidence( + reviewer, + evidence, + sidecar_present, +): + # Exercise mapping of a preselected candidate; selection itself is supplied + # by the hash-pinned owner planner in the CLI, never by an expiry caller. + older = copy.deepcopy(evidence[0][0]) + stamp = NOW - timedelta(days=40) + name = stamp.strftime("%Y%m%dT%H%M%SZ-") + "f" * 24 + ".tar.age" + key = reviewer.health.PREFIX + stamp.strftime("%Y/%m/%d/") + name + older.update( + captured_at=stamp.isoformat(), + local_archive=name, + object_key=key, + destination="s3://" + reviewer.PRIMARY + "/" + key, + version_id="older-primary", + ) + ref = older["recovery_mirror"] + ref.update( + object_key="archives/" + key, + version_id="older-mirror", + receipt_object_key="archives/" + key + "." + ref["receipt_sha256"] + ".receipt.json", + receipt_version_id="older-sidecar", + ) + evidence[0].append(older) + evidence[3]["expiration_candidates"].append( + {k: older[k] for k in ["object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes"]} + ) + mirror = next( + b for b in evidence[2]["observations"] if b["bucket"] == reviewer.health.MIRROR_BUCKET + ) + triples = [ + ( + ref["object_key"], + ref["version_id"], + older["ciphertext_sha256"], + older["ciphertext_bytes"], + ) + ] + if sidecar_present: + triples.append( + ( + ref["receipt_object_key"], + ref["receipt_version_id"], + ref["receipt_sha256"], + ref["receipt_bytes"], + ) + ) + for object_key, version, digest, size in triples: + mirror["versions"].append({"Key": object_key, "VersionId": version, "Size": size}) + evidence[2]["version_heads"].append( + { + "bucket": reviewer.health.MIRROR_BUCKET, + "object_key": object_key, + "version_id": version, + "sha256": digest, + "bytes": size, + } + ) + if not sidecar_present: + with pytest.raises(ValueError, match="Exact retained version"): + reviewer.review(*evidence, NOW) + else: + plan = reviewer.review(*evidence, NOW) + assert len(plan["expiration_candidates"]) == 1 + target = plan["expiration_candidates"][0] + assert target["bucket"] == reviewer.health.MIRROR_BUCKET + assert target["version_id"] == "older-mirror" + assert target["associated_receipt"]["receipt_version_id"] == "older-sidecar" + assert not plan["ready_to_execute"] diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 4be22d9..82d12bc 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -118,6 +118,21 @@ equal readback and real age recipient validation pass. No private identity or recovery API pair was installed there. Receipt: docs/release/backup-public-recipient-delivery-2026-10-29.json. +2026-10-30 retention reconciliation: scoped routine API observations prove the +primary bucket's global current/noncurrent 30-day expiry covers Coulomb, while +independent archive and protected-kit buckets have no lifecycle rules. Complete +prefix inventories and four exact-version HEADs match known primary/mirror/kit/ +proof hashes and sizes. No private identity or personal session was used, and no +provider mutation occurred. Added an offline reviewer using the hash-pinned owner +14-daily/four-weekly selector, validating complete fresh evidence and independent +protected recovery bindings. Actual plan keeps one point, with zero expiration +candidates. A real-selector synthetic gap keeps 13 points older than 30 days +(oldest47days), so blanket 30-day mirror expiry cannot replace generation +selection. Preserve that policy; a separately admitted mirror-only exact-version +expiry executor remains needed, with publisher/reader deletion denials intact. +Review/proof: docs/release/generation-retention-review-2026-10-30.{md,json}. +This is not recurring retention enforcement or additional expiry admission. + ## T02 — Activate and prove the first encrypted paired recovery point ```task