coulomb-social/scripts/review-backup-retention.py

#!/usr/bin/env python3
"""Compile an offline exact-version review; no expiry or provider capability.

The owner planner selects 14 daily/four weekly logical points. This consumer maps
them to independently held mirror versions and checks the protected recovery kit.
Primary age-based expiry is not authority to expire the independent generations.
"""

from __future__ import annotations

import argparse
import hashlib
import importlib.util
import json
import os
import re
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path

spec = importlib.util.spec_from_file_location(
    "retention_health", Path(__file__).with_name("check-paired-backup-health.py")
)
health = importlib.util.module_from_spec(spec)
spec.loader.exec_module(health)
PRIMARY = "railiance-platform-pg-backup"
KIT = "railiance-coulomb-social-recovery"
PLANNER_SHA256 = "08bc57308f53bbfd0d58fdbb9b5671dd4285e31c5b5137e201524df606aa3f9f"
ARCHIVE_SHA256 = "a3f0a61e869a6012edb4b5288be14e3b0c4a9a1273b090320ab146d12e5aa94d"
OWNER_FILES = {
    "application_archive_retention_plan.py": PLANNER_SHA256,
    "primary_application_archive.py": ARCHIVE_SHA256,
}


def identity(receipt):
    return tuple(
        receipt[k] for k in ("object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes")
    )


def review(inventory, recovery, observed, selection, now):
    if (
        observed.get("schema") != "coulomb.provider-retention-observation.v1"
        or observed.get("provider_mutations") is not False
    ):
        raise ValueError("Read-only provider observation required")
    for stamp in [observed["observed_at"], observed["head_observed_at"]]:
        age = (now - health.timestamp(stamp)).total_seconds()
        if not 0 <= age <= 26 * 3600:
            raise ValueError("Fresh version/policy observation required")
    buckets = {item["bucket"]: item for item in observed["observations"]}
    if len(buckets) != 3 or set(buckets) != {PRIMARY, health.MIRROR_BUCKET, KIT}:
        raise ValueError("Exact application bucket observations required")
    for bucket, prefix in [
        (PRIMARY, health.PREFIX),
        (health.MIRROR_BUCKET, "archives/"),
        (KIT, "kits/"),
    ]:
        item = buckets[bucket]
        if (
            item.get("prefix") != prefix
            or item.get("inventory_complete") is not True
            or item["versioning"].get("Status") != "Enabled"
            or item["lifecycle"].get("status") != "observed"
        ):
            raise ValueError("Complete versioned prefix and lifecycle evidence required")
    if (
        selection.get("application") != "coulomb-social"
        or selection.get("status") != "reviewed"
        or selection.get("daily_generations") != 14
        or selection.get("weekly_generations") != 4
        or recovery.get("status") != "application_restored_verified"
        or recovery.get("application_restore_verified") is not True
        or recovery.get("application") != "coulomb-social"
        or recovery.get("application_data_images_and_recovery_tools_from_provider_kit") is not True
    ):
        raise ValueError("Exact owner selection and protected app recovery required")
    heads = {(h["bucket"], h["object_key"], h["version_id"]): h for h in observed["version_heads"]}
    if len(heads) != len(observed["version_heads"]):
        raise ValueError("Unambiguous exact-version heads required")
    versions = {}
    for bucket, item in buckets.items():
        for row in item["versions"]:
            key = (bucket, row["Key"], row["VersionId"])
            if key in versions or not row["Key"].startswith(item["prefix"]):
                raise ValueError("Unambiguous prefix version inventory required")
            versions[key] = row

    def require_version(bucket, key, version, digest, size):
        found = versions.get((bucket, key, version))
        head = heads.get((bucket, key, version))
        if (
            found is None
            or head is None
            or found["Size"] != size
            or head["bytes"] != size
            or head["sha256"] != digest
        ):
            raise ValueError("Exact retained version/hash/size evidence required")

    for name in ["protected_kit", "protected_recovery_proof"]:
        ref = recovery[name]
        if (
            ref.get("bucket") != KIT
            or ref.get("project_id") != health.PROJECT
            or not ref["object_key"].startswith("kits/")
            or ref.get("automatic_expiry") is not False
            or not re.fullmatch(r"[a-f0-9]{64}", ref.get("sha256", ""))
            or type(ref.get("bytes")) is not int
            or ref["bytes"] <= 0
        ):
            raise ValueError("Separate nonexpiring recovery custody required")
        expected = (
            "kits/" + ref["sha256"] + "/recovery-kit.tar"
            if name == "protected_kit"
            else "kits/receipts/" + ref["sha256"] + "/recovery-proof.json"
        )
        if ref["object_key"] != expected:
            raise ValueError("Hash-addressed protected object required")
        require_version(KIT, ref["object_key"], ref["version_id"], ref["sha256"], ref["bytes"])
    if recovery["protected_recovery_proof"].get("independent_readback_verified") is not True:
        raise ValueError("Independently retained recovery proof required")
    protected = identity(recovery)
    rows = {}
    for receipt in inventory:
        health.validate_primary(receipt)
        health.validate_mirror(receipt, receipt)
        item = identity(receipt)
        if item in rows:
            raise ValueError("Duplicate logical point")
        rows[item] = receipt
    keep = {identity(r) for r in selection["keep"]}
    candidates = {identity(r) for r in selection["expiration_candidates"]}
    if (
        not keep
        or keep & candidates
        or keep | candidates != set(rows)
        or protected not in keep
        or identity(selection["protected"]) != protected
    ):
        raise ValueError("Complete disjoint selection preserving recovered point required")
    newest = max(rows, key=lambda item: (health.timestamp(rows[item]["captured_at"]), item))
    if newest not in keep:
        raise ValueError("Newest generation must be preserved")
    if any(
        rows[protected][key] != recovery.get(key)
        for key in (
            "image",
            "age_recipient",
            "database_sha256",
            "content_sha256",
        )
    ):
        raise ValueError("Protected recovery binding differs from selected point")
    blockers = []
    if buckets[KIT]["lifecycle"].get("Rules"):
        blockers.append("protected_bucket_has_lifecycle_rules")
    if any(
        r.get("Status") == "Enabled" for r in buckets[health.MIRROR_BUCKET]["lifecycle"]["Rules"]
    ):
        blockers.append("mirror_lifecycle_cannot_guarantee_generation_selection")
    mirror_keep, mirror_candidates = [], []
    primary_missing = 0
    for item, receipt in rows.items():
        ref = receipt["recovery_mirror"]
        require_version(
            health.MIRROR_BUCKET,
            ref["object_key"],
            ref["version_id"],
            receipt["ciphertext_sha256"],
            receipt["ciphertext_bytes"],
        )
        target = {
            "bucket": health.MIRROR_BUCKET,
            "object_key": ref["object_key"],
            "version_id": ref["version_id"],
            "ciphertext_sha256": receipt["ciphertext_sha256"],
            "ciphertext_bytes": receipt["ciphertext_bytes"],
            "logical_primary_identity": dict(
                zip(["object_key", "version_id", "ciphertext_sha256", "ciphertext_bytes"], item)
            ),
        }
        if (PRIMARY, receipt["object_key"], receipt["version_id"]) not in versions:
            primary_missing += 1  # Mirror retains the logical point after primary age expiry.
        else:
            require_version(
                PRIMARY,
                receipt["object_key"],
                receipt["version_id"],
                receipt["ciphertext_sha256"],
                receipt["ciphertext_bytes"],
            )
        if item in keep:
            mirror_keep.append(target)
        else:
            # Expiry review cannot orphan an unverified independent receipt.
            health.validate_mirror(receipt, receipt, require_retained_receipt=True)
            require_version(
                health.MIRROR_BUCKET,
                ref["receipt_object_key"],
                ref["receipt_version_id"],
                ref["receipt_sha256"],
                ref["receipt_bytes"],
            )
            target["associated_receipt"] = {
                k: ref[k]
                for k in (
                    "receipt_object_key",
                    "receipt_version_id",
                    "receipt_sha256",
                    "receipt_bytes",
                )
            }
            mirror_candidates.append(target)
    # This review intentionally grants no execution. Separate custody/boundaries
    # and fresh keep-set revalidation are needed before any exact-version expiry.
    return {
        "schema": "coulomb.generation-retention-review.v1",
        "status": "reviewed",
        "daily_generations": 14,
        "weekly_generations": 4,
        "protected_point_in_mirror_keep": True,
        "protected_kit_and_proof_versions_verified": True,
        "keep": sorted(mirror_keep, key=lambda r: (r["object_key"], r["version_id"])),
        "expiration_candidates": sorted(
            mirror_candidates, key=lambda r: (r["object_key"], r["version_id"])
        ),
        "primary_versions_already_absent": primary_missing,
        "primary_lifecycle_observed": buckets[PRIMARY]["lifecycle"]["Rules"],
        "bucket_version_counts": {k: len(v["versions"]) for k, v in sorted(buckets.items())},
        "bucket_bytes": {
            k: sum(x["Size"] for x in v["versions"]) for k, v in sorted(buckets.items())
        },
        "blockers": sorted(blockers),
        "generation_selection_valid": not blockers,
        "unknown_objects_preserved": True,
        "provider_mutations": False,
        "local_expiry": False,
        "ready_to_execute": False,
        "deletion_capability": False,
        "activation_gate": "Separate mirror-only exact-version expiry admission/execution, "
        "received monitoring and fresh protected/keep-set verification remain required.",
    }


def load_owner(directory):
    for name, digest in OWNER_FILES.items():
        path = directory / name
        if path.is_symlink() or hashlib.sha256(path.read_bytes()).hexdigest() != digest:
            raise ValueError("Owner planner source differs")
    existing = sys.modules.get("primary_application_archive")
    if (
        existing
        and Path(existing.__file__).resolve()
        != (directory / "primary_application_archive.py").resolve()
    ):
        raise ValueError("Owner helper already loaded elsewhere")
    sys.path.insert(0, str(directory))
    spec = importlib.util.spec_from_file_location(
        "retention_owner", directory / "application_archive_retention_plan.py"
    )
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def age_window_counterexample(owner, now):
    """Show why an age window cannot replace the admitted generation selector."""
    inventory = []
    for days in [1, *range(35, 49)]:
        stamp = now - timedelta(days=days)
        suffix = hashlib.sha256(stamp.isoformat().encode()).hexdigest()[:24]
        inventory.append(
            {
                "application": "coulomb-social",
                "status": "offsite_fetched_verified",
                "remote_retrieval_verified": True,
                "captured_at": stamp.isoformat(),
                "object_key": health.PREFIX
                + stamp.strftime("%Y/%m/%d/%Y%m%dT%H%M%SZ-")
                + suffix
                + ".tar.age",
                "version_id": "synthetic-" + str(days),
                "ciphertext_sha256": "a" * 64,
                "ciphertext_bytes": 1,
            }
        )
    recovery = dict(
        inventory[0], application_restore_verified=True, status="application_restored_verified"
    )
    selected = owner.plan("coulomb-social", inventory, recovery, now)
    by_key = {r["object_key"]: r for r in inventory}
    ages = [
        (now - health.timestamp(by_key[r["object_key"]]["captured_at"])).days
        for r in selected["keep"]
    ]
    return {
        "synthetic_source_gap": True,
        "input_generations": len(inventory),
        "selected_generations_older_than_30_days": sum(age > 30 for age in ages),
        "oldest_selected_age_days": max(ages),
        "fixed_age_window_preserves_generation_policy": not any(age > 30 for age in ages),
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    for name in ["inventory", "recovery", "observation", "platform-scripts", "output"]:
        parser.add_argument("--" + name, type=Path, required=True)
    args = parser.parse_args()
    output = args.output.resolve()
    if output.is_relative_to(Path(__file__).resolve().parents[1]):
        raise ValueError("Private plan must stay outside checkout")
    sources = {
        name: getattr(args, name).read_bytes() for name in ["inventory", "recovery", "observation"]
    }
    inputs = {name: json.loads(raw) for name, raw in sources.items()}
    now = datetime.now(UTC)
    owner = load_owner(args.platform_scripts)
    selection = owner.plan("coulomb-social", inputs["inventory"], inputs["recovery"], now)
    result = review(inputs["inventory"], inputs["recovery"], inputs["observation"], selection, now)
    result["age_window_counterexample"] = age_window_counterexample(owner, now)
    result["source_sha256"] = {
        name: hashlib.sha256(raw).hexdigest() for name, raw in sources.items()
    }
    result["owner_source_sha256"] = OWNER_FILES
    fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
    with os.fdopen(fd, "w") as stream:
        json.dump(result, stream, indent=2, sort_keys=True)
        stream.write("\n")
    print(
        json.dumps(
            {
                "status": result["status"],
                "keep_versions": len(result["keep"]),
                "expiration_candidates": len(result["expiration_candidates"]),
                "blockers": result["blockers"],
                "ready_to_execute": False,
                "provider_mutations": False,
            }
        )
    )


if __name__ == "__main__":
    try:
        main()
    except (ValueError, KeyError, TypeError, OSError, AttributeError):
        raise SystemExit("Retention review refused; inspect private inputs locally.")