tegwick
54eef1c4b5
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 6m41s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
154 lines
7.9 KiB
Python
154 lines
7.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Export private client data from an isolated retained restore, without source writes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import runpy
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
CHECKS = r'''
|
|
import hashlib, json, os, zipfile, django
|
|
from pathlib import Path
|
|
django.setup()
|
|
from django.db import connection
|
|
from coulomb_social.apps.members.client_export import export_client_dataset
|
|
from coulomb_social.apps.spaces.models import Space
|
|
assert connection.vendor == "postgresql"
|
|
root = Path("/recovered-content")
|
|
tenant = os.environ["EXPORT_TENANT"]
|
|
def files():
|
|
return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in root.rglob("*") if p.is_file()}
|
|
def sql():
|
|
with connection.cursor() as c:
|
|
c.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY 1")
|
|
tables = [r[0] for r in c.fetchall()]
|
|
rows = {}
|
|
for table in tables:
|
|
c.execute("SELECT * FROM " + connection.ops.quote_name(table))
|
|
rows[table] = sorted(json.dumps(r, default=str) for r in c.fetchall())
|
|
return rows
|
|
before_files, before_sql = files(), sql()
|
|
output = Path("/export/client.zip")
|
|
result = export_client_dataset(tenant, root, output)
|
|
with zipfile.ZipFile(output) as archive:
|
|
manifest = json.loads(archive.read("manifest.json"))
|
|
slugs = set(Space.objects.filter(tenant_id=tenant).values_list("slug", flat=True))
|
|
expected = {name: digest for name, digest in before_files.items()
|
|
if len(Path(name).parts) >= 3
|
|
and Path(name).parts[0] in {"spaces", ".revisions"}
|
|
and Path(name).parts[1] in slugs}
|
|
assert manifest["tenant_id"] == tenant
|
|
assert manifest["files_sha256"] == expected
|
|
assert set(archive.namelist()) == {"manifest.json"} | {"content/" + n for n in expected}
|
|
assert all(hashlib.sha256(archive.read("content/" + n)).hexdigest() == h
|
|
for n, h in expected.items())
|
|
assert all("password" not in row for row in manifest["data"]["records"]["members.user"])
|
|
assert manifest["offboarding_complete"] is False
|
|
assert output.stat().st_mode & 0o777 == 0o600
|
|
assert sql() == before_sql and files() == before_files
|
|
print(json.dumps({"status": "recovered_client_export_verified", **result,
|
|
"all_source_tables_compared": len(before_sql), "all_source_files_compared": len(before_files),
|
|
"archive_member_hashes_verified": True, "selected_client_content_complete_locally": True,
|
|
"source_sql_and_content_unchanged": True,
|
|
"external_bindings_require_review": len(manifest["external_bindings"]),
|
|
"sessions_and_credentials_excluded": True}))
|
|
'''
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--restore-receipt", type=Path, required=True)
|
|
parser.add_argument("--tenant-id", required=True)
|
|
parser.add_argument("--image", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists() or args.output.is_symlink():
|
|
raise ValueError("Fresh receipt output required")
|
|
previous = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py")))
|
|
run, hashes = previous["run"], previous["hashes"]
|
|
raw = args.restore_receipt.read_bytes()
|
|
state = json.loads(raw)
|
|
if (state["status"] != "restored_verified"
|
|
or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"])
|
|
or state["network"] != state["database_container"] + "-net"):
|
|
raise ValueError("Isolated retained restore required")
|
|
net = json.loads(run(["docker", "network", "inspect", state["network"]]))[0]
|
|
db = json.loads(run(["docker", "inspect", state["database_container"]]))[0]
|
|
if (net["Internal"] is not True or not db["State"]["Running"]
|
|
or set(net["Containers"]) != {db["Id"]}
|
|
or set(db["NetworkSettings"]["Networks"]) != {state["network"]}
|
|
or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"]
|
|
or any(db["NetworkSettings"]["Ports"].values())
|
|
or db["Image"] != state["postgres_image_id"]):
|
|
raise ValueError("Recovery isolation mismatch")
|
|
image = run(["docker", "image", "inspect", args.image, "--format", "{{.Id}}"]).decode().strip()
|
|
if not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
|
|
raise ValueError("Locally retained image required")
|
|
root = Path(state["restored_directory"]) / "content"
|
|
if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to(
|
|
Path("/tmp")):
|
|
raise ValueError("Real temporary recovered content required")
|
|
before = hashes(root)
|
|
module = Path(__file__).resolve().parents[1] / "coulomb_social/apps/members/client_export.py"
|
|
source = module.read_bytes()
|
|
with tempfile.TemporaryDirectory(prefix="csoc-client-export-") as temporary:
|
|
work = Path(temporary)
|
|
(work / "client_export.py").write_bytes(source)
|
|
(work / "export").mkdir(mode=0o700)
|
|
name = "csoc-export-" + uuid.uuid4().hex
|
|
command = ["docker", "run", "--rm", "--name", name, "--network", state["network"],
|
|
"--read-only", "--tmpfs", "/tmp", "--cap-drop=ALL",
|
|
"--security-opt=no-new-privileges", "--user", f"{os.getuid()}:{os.getgid()}",
|
|
"--entrypoint", "/app/.venv/bin/python",
|
|
"-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base",
|
|
"-e", "DATABASE_URL=postgres://postgres@recovery-db/coulomb_social_restore",
|
|
"-e", "EXPORT_TENANT=" + args.tenant_id,
|
|
"-e", "CONTENT_ROOT=/recovered-content", "-e", "FORGEJO_BASE_URL=",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-v", str(root) + ":/recovered-content:ro",
|
|
"-v", str(work / "client_export.py")
|
|
+ ":/app/coulomb_social/apps/members/client_export.py:ro",
|
|
"-v", str(work / "export") + ":/export", image, "-c", CHECKS]
|
|
try:
|
|
result = subprocess.run(command, capture_output=True, timeout=180)
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True,
|
|
timeout=30, check=False)
|
|
if result.returncode:
|
|
fd, diagnostic = tempfile.mkstemp(prefix="csoc-client-export-private-diagnostics-")
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(result.stdout + result.stderr)
|
|
print(json.dumps({"private_diagnostics": diagnostic}))
|
|
raise ValueError("Private export check failed")
|
|
rows = [json.loads(line) for line in result.stdout.splitlines() if line.startswith(b"{")]
|
|
if len(rows) != 1 or hashes(root) != before:
|
|
raise ValueError("Single result and preserved source files required")
|
|
proof = {**rows[0], "dataset_schema": rows[0]["schema"],
|
|
"schema": "coulomb.client-export-rehearsal.v1",
|
|
"module_sha256": hashlib.sha256(source).hexdigest(),
|
|
"runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
"restore_receipt_sha256": hashlib.sha256(raw).hexdigest(),
|
|
"candidate_image_id": image, "source_overlay_used": True,
|
|
"private_temporary_export_removed": True, "network_internal": True,
|
|
"published_ports": 0, "production_access": False, "production_writes": False}
|
|
fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "w") as stream:
|
|
json.dump(proof, stream, indent=2)
|
|
stream.write("\n")
|
|
print(json.dumps(proof))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, OSError, KeyError, subprocess.SubprocessError):
|
|
raise SystemExit("Client export rehearsal refused; inspect private diagnostics locally.")
|