#!/usr/bin/env python3 """Export private client data from an isolated retained restore, without source writes.""" from __future__ import annotations import argparse import hashlib import json import os import re import runpy import subprocess import tempfile import uuid from pathlib import Path CHECKS = r''' import hashlib, json, os, zipfile, django from pathlib import Path django.setup() from django.db import connection from coulomb_social.apps.members.client_export import export_client_dataset from coulomb_social.apps.spaces.models import Space assert connection.vendor == "postgresql" root = Path("/recovered-content") tenant = os.environ["EXPORT_TENANT"] def files(): return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() for p in root.rglob("*") if p.is_file()} def sql(): with connection.cursor() as c: c.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY 1") tables = [r[0] for r in c.fetchall()] rows = {} for table in tables: c.execute("SELECT * FROM " + connection.ops.quote_name(table)) rows[table] = sorted(json.dumps(r, default=str) for r in c.fetchall()) return rows before_files, before_sql = files(), sql() output = Path("/export/client.zip") result = export_client_dataset(tenant, root, output) with zipfile.ZipFile(output) as archive: manifest = json.loads(archive.read("manifest.json")) slugs = set(Space.objects.filter(tenant_id=tenant).values_list("slug", flat=True)) expected = {name: digest for name, digest in before_files.items() if len(Path(name).parts) >= 3 and Path(name).parts[0] in {"spaces", ".revisions"} and Path(name).parts[1] in slugs} assert manifest["tenant_id"] == tenant assert manifest["files_sha256"] == expected assert set(archive.namelist()) == {"manifest.json"} | {"content/" + n for n in expected} assert all(hashlib.sha256(archive.read("content/" + n)).hexdigest() == h for n, h in expected.items()) assert all("password" not in row for row in manifest["data"]["records"]["members.user"]) assert manifest["offboarding_complete"] is False assert output.stat().st_mode & 0o777 == 0o600 assert sql() == before_sql and files() == before_files print(json.dumps({"status": "recovered_client_export_verified", **result, "all_source_tables_compared": len(before_sql), "all_source_files_compared": len(before_files), "archive_member_hashes_verified": True, "selected_client_content_complete_locally": True, "source_sql_and_content_unchanged": True, "external_bindings_require_review": len(manifest["external_bindings"]), "sessions_and_credentials_excluded": True})) ''' def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--restore-receipt", type=Path, required=True) parser.add_argument("--tenant-id", required=True) parser.add_argument("--image", required=True) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists() or args.output.is_symlink(): raise ValueError("Fresh receipt output required") previous = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py"))) run, hashes = previous["run"], previous["hashes"] raw = args.restore_receipt.read_bytes() state = json.loads(raw) if (state["status"] != "restored_verified" or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"]) or state["network"] != state["database_container"] + "-net"): raise ValueError("Isolated retained restore required") net = json.loads(run(["docker", "network", "inspect", state["network"]]))[0] db = json.loads(run(["docker", "inspect", state["database_container"]]))[0] if (net["Internal"] is not True or not db["State"]["Running"] or set(net["Containers"]) != {db["Id"]} or set(db["NetworkSettings"]["Networks"]) != {state["network"]} or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"] or any(db["NetworkSettings"]["Ports"].values()) or db["Image"] != state["postgres_image_id"]): raise ValueError("Recovery isolation mismatch") image = run(["docker", "image", "inspect", args.image, "--format", "{{.Id}}"]).decode().strip() if not re.fullmatch(r"sha256:[a-f0-9]{64}", image): raise ValueError("Locally retained image required") root = Path(state["restored_directory"]) / "content" if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to( Path("/tmp")): raise ValueError("Real temporary recovered content required") before = hashes(root) module = Path(__file__).resolve().parents[1] / "coulomb_social/apps/members/client_export.py" source = module.read_bytes() with tempfile.TemporaryDirectory(prefix="csoc-client-export-") as temporary: work = Path(temporary) (work / "client_export.py").write_bytes(source) (work / "export").mkdir(mode=0o700) name = "csoc-export-" + uuid.uuid4().hex command = ["docker", "run", "--rm", "--name", name, "--network", state["network"], "--read-only", "--tmpfs", "/tmp", "--cap-drop=ALL", "--security-opt=no-new-privileges", "--user", f"{os.getuid()}:{os.getgid()}", "--entrypoint", "/app/.venv/bin/python", "-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base", "-e", "DATABASE_URL=postgres://postgres@recovery-db/coulomb_social_restore", "-e", "EXPORT_TENANT=" + args.tenant_id, "-e", "CONTENT_ROOT=/recovered-content", "-e", "FORGEJO_BASE_URL=", "-e", "PYTHONDONTWRITEBYTECODE=1", "-v", str(root) + ":/recovered-content:ro", "-v", str(work / "client_export.py") + ":/app/coulomb_social/apps/members/client_export.py:ro", "-v", str(work / "export") + ":/export", image, "-c", CHECKS] try: result = subprocess.run(command, capture_output=True, timeout=180) finally: subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30, check=False) if result.returncode: fd, diagnostic = tempfile.mkstemp(prefix="csoc-client-export-private-diagnostics-") with os.fdopen(fd, "wb") as stream: stream.write(result.stdout + result.stderr) print(json.dumps({"private_diagnostics": diagnostic})) raise ValueError("Private export check failed") rows = [json.loads(line) for line in result.stdout.splitlines() if line.startswith(b"{")] if len(rows) != 1 or hashes(root) != before: raise ValueError("Single result and preserved source files required") proof = {**rows[0], "dataset_schema": rows[0]["schema"], "schema": "coulomb.client-export-rehearsal.v1", "module_sha256": hashlib.sha256(source).hexdigest(), "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "restore_receipt_sha256": hashlib.sha256(raw).hexdigest(), "candidate_image_id": image, "source_overlay_used": True, "private_temporary_export_removed": True, "network_internal": True, "published_ports": 0, "production_access": False, "production_writes": False} fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(fd, "w") as stream: json.dump(proof, stream, indent=2) stream.write("\n") print(json.dumps(proof)) if __name__ == "__main__": try: main() except (ValueError, OSError, KeyError, subprocess.SubprocessError): raise SystemExit("Client export rehearsal refused; inspect private diagnostics locally.")