tegwick
51a527128a
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m47s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
335 lines
13 KiB
Python
335 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Read current version/custody evidence for the offline generation reviewer.
|
|
|
|
Preview is offline. --execute performs only scoped provider reads and publishes
|
|
one new private observation. It never captures, decrypts, deletes or changes IAM.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
|
|
spec = importlib.util.spec_from_file_location(
|
|
"retention_mirror", Path(__file__).with_name("mirror-paired-backup.py")
|
|
)
|
|
worker = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(worker)
|
|
health = worker.health
|
|
Refused = health.Refused
|
|
PRIMARY = "railiance-platform-pg-backup"
|
|
KIT = "railiance-coulomb-social-recovery"
|
|
SCOPES = {PRIMARY: health.PREFIX, health.MIRROR_BUCKET: "archives/", KIT: "kits/"}
|
|
|
|
|
|
def status(error):
|
|
return getattr(error, "response", {}).get("ResponseMetadata", {}).get("HTTPStatusCode")
|
|
|
|
|
|
def version_id(value):
|
|
if (
|
|
not isinstance(value, str)
|
|
or not value
|
|
or value == "null"
|
|
or any(c in value for c in "\r\n")
|
|
):
|
|
raise Refused("immutable_version_required")
|
|
return value
|
|
|
|
|
|
def observe_bucket(client, bucket):
|
|
prefix = SCOPES[bucket]
|
|
versioning = client.get_bucket_versioning(Bucket=bucket)
|
|
if versioning.get("Status") != "Enabled":
|
|
raise Refused("enabled_versioning_required")
|
|
try:
|
|
lifecycle = client.get_bucket_lifecycle_configuration(Bucket=bucket)
|
|
rules = lifecycle["Rules"]
|
|
except Exception as error:
|
|
# A bucket itself being missing or inaccessible is not "no lifecycle".
|
|
code = getattr(error, "response", {}).get("Error", {}).get("Code")
|
|
if status(error) != 404 or code != "NoSuchLifecycleConfiguration":
|
|
raise
|
|
rules = []
|
|
if not isinstance(rules, list):
|
|
raise Refused("lifecycle_rules_required")
|
|
versions, markers, seen, identities, arguments = [], [], set(), set(), {}
|
|
for _ in range(100):
|
|
page = client.list_object_versions(Bucket=bucket, Prefix=prefix, MaxKeys=1000, **arguments)
|
|
for field, destination in [("Versions", versions), ("DeleteMarkers", markers)]:
|
|
for row in page.get(field, []):
|
|
key, version = row["Key"], version_id(row["VersionId"])
|
|
if not isinstance(key, str) or not key.startswith(prefix):
|
|
raise Refused("version_outside_application_prefix")
|
|
if (key, version) in identities:
|
|
raise Refused("duplicate_or_changing_version_inventory")
|
|
identities.add((key, version))
|
|
item = {"Key": key, "VersionId": version}
|
|
if field == "Versions":
|
|
if type(row.get("Size")) is not int or row["Size"] < 0:
|
|
raise Refused("version_size_required")
|
|
item["Size"] = row["Size"]
|
|
if "IsLatest" in row:
|
|
item["IsLatest"] = row["IsLatest"]
|
|
destination.append(item)
|
|
if page.get("IsTruncated") is False:
|
|
return {
|
|
"bucket": bucket,
|
|
"prefix": prefix,
|
|
"inventory_complete": True,
|
|
"versioning": {"Status": "Enabled"},
|
|
"lifecycle": {"status": "observed", "Rules": rules},
|
|
"versions": versions,
|
|
"delete_markers": markers,
|
|
}
|
|
if page.get("IsTruncated") is not True:
|
|
raise Refused("explicit_pagination_completion_required")
|
|
cursor = page.get("NextKeyMarker"), page.get("NextVersionIdMarker")
|
|
if not isinstance(cursor[0], str) or not cursor[0].startswith(prefix) or cursor in seen:
|
|
raise Refused("invalid_or_repeated_version_cursor")
|
|
seen.add(cursor)
|
|
arguments = {"KeyMarker": cursor[0]}
|
|
if cursor[1] is not None:
|
|
arguments["VersionIdMarker"] = version_id(cursor[1])
|
|
raise Refused("version_inventory_limit_exceeded")
|
|
|
|
|
|
def observe(primary_client, reader, inventory, recovery, now=None):
|
|
now = now or datetime.now(UTC)
|
|
if not isinstance(inventory, list) or not inventory:
|
|
raise Refused("nonempty_verified_inventory_required")
|
|
expected = {}
|
|
|
|
def add(bucket, key, version, digest, size):
|
|
version_id(version)
|
|
if (
|
|
bucket not in SCOPES
|
|
or not isinstance(key, str)
|
|
or not key.startswith(SCOPES[bucket])
|
|
or not isinstance(digest, str)
|
|
or len(digest) != 64
|
|
or any(c not in "0123456789abcdef" for c in digest)
|
|
or type(size) is not int
|
|
or size <= 0
|
|
):
|
|
raise Refused("exact_version_binding_required")
|
|
identity = bucket, key, version
|
|
if identity in expected and expected[identity] != (digest, size):
|
|
raise Refused("conflicting_version_binding")
|
|
expected[identity] = digest, size
|
|
|
|
for receipt in inventory:
|
|
health.validate_mirror(receipt, receipt)
|
|
add(
|
|
PRIMARY,
|
|
receipt["object_key"],
|
|
receipt["version_id"],
|
|
receipt["ciphertext_sha256"],
|
|
receipt["ciphertext_bytes"],
|
|
)
|
|
ref = receipt["recovery_mirror"]
|
|
add(
|
|
health.MIRROR_BUCKET,
|
|
ref["object_key"],
|
|
ref["version_id"],
|
|
ref["ciphertext_sha256"],
|
|
ref["ciphertext_bytes"],
|
|
)
|
|
if "receipt_object_key" in ref:
|
|
health.validate_mirror(receipt, receipt, require_retained_receipt=True)
|
|
add(
|
|
health.MIRROR_BUCKET,
|
|
ref["receipt_object_key"],
|
|
ref["receipt_version_id"],
|
|
ref["receipt_sha256"],
|
|
ref["receipt_bytes"],
|
|
)
|
|
if (
|
|
recovery.get("status") != "application_restored_verified"
|
|
or recovery.get("application") != "coulomb-social"
|
|
or recovery.get("application_restore_verified") is not True
|
|
):
|
|
raise Refused("verified_application_recovery_required")
|
|
for name in ["protected_kit", "protected_recovery_proof"]:
|
|
ref = recovery[name]
|
|
suffix = "/recovery-kit.tar" if name == "protected_kit" else "/recovery-proof.json"
|
|
prefix = "kits/" if name == "protected_kit" else "kits/receipts/"
|
|
if (
|
|
ref.get("bucket") != KIT
|
|
or ref.get("project_id") != health.PROJECT
|
|
or ref.get("automatic_expiry") is not False
|
|
or ref.get("object_key") != prefix + ref["sha256"] + suffix
|
|
):
|
|
raise Refused("protected_custody_binding_required")
|
|
add(KIT, ref["object_key"], ref["version_id"], ref["sha256"], ref["bytes"])
|
|
|
|
clients = {PRIMARY: primary_client, health.MIRROR_BUCKET: reader, KIT: reader}
|
|
observations = [observe_bucket(clients[bucket], bucket) for bucket in SCOPES]
|
|
versions = {
|
|
(b["bucket"], v["Key"], v["VersionId"]): v for b in observations for v in b["versions"]
|
|
}
|
|
heads = []
|
|
for (bucket, key, version), (digest, size) in sorted(expected.items()):
|
|
listed = versions.get((bucket, key, version))
|
|
if listed is None and bucket == PRIMARY:
|
|
continue # Primary age expiry never removes an independent mirror generation.
|
|
if listed is None or listed["Size"] != size:
|
|
raise Refused("retained_version_missing_or_changed")
|
|
value = clients[bucket].head_object(Bucket=bucket, Key=key, VersionId=version)
|
|
if (
|
|
value.get("VersionId") != version
|
|
or value.get("ContentLength") != size
|
|
or value.get("Metadata", {}).get("sha256") != digest
|
|
or value.get("DeleteMarker")
|
|
):
|
|
raise Refused("exact_version_head_differs")
|
|
heads.append(
|
|
{
|
|
"bucket": bucket,
|
|
"object_key": key,
|
|
"version_id": version,
|
|
"sha256": digest,
|
|
"bytes": size,
|
|
}
|
|
)
|
|
|
|
sidecars = 0
|
|
for receipt in inventory:
|
|
ref = receipt["recovery_mirror"]
|
|
if "receipt_object_key" not in ref:
|
|
continue
|
|
response = reader.get_object(
|
|
Bucket=health.MIRROR_BUCKET,
|
|
Key=ref["receipt_object_key"],
|
|
VersionId=ref["receipt_version_id"],
|
|
)
|
|
with response["Body"] as stream:
|
|
payload = stream.read(ref["receipt_bytes"] + 1)
|
|
if (
|
|
response.get("VersionId") != ref["receipt_version_id"]
|
|
or response.get("ContentLength") != ref["receipt_bytes"]
|
|
or len(payload) != ref["receipt_bytes"]
|
|
or hashlib.sha256(payload).hexdigest() != ref["receipt_sha256"]
|
|
or payload != worker.canonical(worker.public_receipt(receipt, receipt))
|
|
):
|
|
raise Refused("independent_public_receipt_readback_differs")
|
|
sidecars += 1
|
|
return {
|
|
"schema": "coulomb.provider-retention-observation.v1",
|
|
"observed_at": now.isoformat(),
|
|
"head_observed_at": datetime.now(UTC).isoformat(),
|
|
"observations": observations,
|
|
"version_heads": heads,
|
|
"independent_public_receipts_fetched_verified": sidecars,
|
|
"archive_bodies_refetched": False,
|
|
"protected_kit_body_refetched": False,
|
|
"provider_mutations": False,
|
|
"credentials_printed": False,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--execute", action="store_true", help="Read provider metadata and receipts"
|
|
)
|
|
for name in [
|
|
"inventory",
|
|
"recovery",
|
|
"credentials-directory",
|
|
"platform-scripts",
|
|
"runtime-manifest",
|
|
"output",
|
|
]:
|
|
parser.add_argument("--" + name, type=Path)
|
|
args = parser.parse_args()
|
|
if not args.execute:
|
|
print(json.dumps({"status": "preview", "provider_contacted": False}))
|
|
return
|
|
if any(
|
|
getattr(args, name) is None
|
|
for name in [
|
|
"inventory",
|
|
"recovery",
|
|
"credentials_directory",
|
|
"platform_scripts",
|
|
"runtime_manifest",
|
|
"output",
|
|
]
|
|
):
|
|
raise Refused("explicit_observation_inputs_required")
|
|
worker.load_runtime(args.platform_scripts, args.runtime_manifest)
|
|
from application_recovery_identities import private_directory, private_json
|
|
from application_recovery_resources import s3
|
|
|
|
private_directory(args.credentials_directory)
|
|
private_directory(args.output.parent)
|
|
if args.output.exists() or args.output.is_symlink():
|
|
raise Refused("fresh_observation_output_required")
|
|
inputs = {name: getattr(args, name).read_bytes() for name in ["inventory", "recovery"]}
|
|
# Use explicit static credentials: no SDK ambient identity or metadata fallback.
|
|
import boto3
|
|
from botocore.config import Config
|
|
|
|
pair = private_json(args.credentials_directory / "publisher-pair.json")
|
|
if not all(
|
|
isinstance(pair.get(k), str) and pair[k].strip()
|
|
for k in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"]
|
|
):
|
|
raise Refused("own_publisher_pair_required")
|
|
primary = boto3.client(
|
|
"s3",
|
|
endpoint_url="https://s3.nl-ams.scw.cloud",
|
|
region_name="nl-ams",
|
|
aws_access_key_id=pair["AWS_ACCESS_KEY_ID"],
|
|
aws_secret_access_key=pair["AWS_SECRET_ACCESS_KEY"],
|
|
config=Config(
|
|
signature_version="s3v4",
|
|
connect_timeout=15,
|
|
read_timeout=30,
|
|
retries={"max_attempts": 0},
|
|
s3={"addressing_style": "path"},
|
|
),
|
|
)
|
|
reader = None
|
|
try:
|
|
reader = s3(private_json(args.credentials_directory / "reader-pair.json"), health.PROJECT)
|
|
result = observe(
|
|
primary, reader, json.loads(inputs["inventory"]), json.loads(inputs["recovery"])
|
|
)
|
|
finally:
|
|
primary.close()
|
|
if reader is not None:
|
|
reader.close()
|
|
result["source_sha256"] = {
|
|
name: hashlib.sha256(value).hexdigest() for name, value in inputs.items()
|
|
}
|
|
fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
with os.fdopen(fd, "w") as stream:
|
|
json.dump(result, stream, indent=2, sort_keys=True)
|
|
stream.write("\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "provider_observation_completed",
|
|
"provider_mutations": False,
|
|
"exact_version_heads": len(result["version_heads"]),
|
|
"independent_receipts_verified": result[
|
|
"independent_public_receipts_fetched_verified"
|
|
],
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except Exception as error:
|
|
print(json.dumps({"status": "observation_refused", "failure_type": type(error).__name__}))
|
|
raise SystemExit(1)
|