tegwick
ee67b6717b
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 7m19s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
233 lines
12 KiB
Python
233 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove date-journal crash recovery on a disposable clone of a recovered pair.
|
|
|
|
No production endpoint is accepted. The retained recovery database/files are
|
|
observed before/after; SQL commits occur only in a fresh temporary database clone.
|
|
Private page/SQL values never appear in the public result.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import runpy
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
# Share the existing reviewed private SQL/file observation helpers, without
|
|
# invoking the old rehearsal's main or its transaction/file mutation section.
|
|
PREVIOUS = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py")))
|
|
COMMON = PREVIOUS["CHECKS"].split("before_files = files()", 1)[0]
|
|
SNAPSHOT = COMMON.replace("patched = prepare_index_date_patch(plan, original)", "") + '''
|
|
print(json.dumps({"tables_sha256": hashlib.sha256(json.dumps(
|
|
digest(database()), sort_keys=True).encode()).hexdigest(), "files": files()}))
|
|
'''
|
|
CHECKS = COMMON + r'''
|
|
import os, subprocess, sys
|
|
from coulomb_social.apps.spaces.date_backfill_journal import (
|
|
apply_date_patch, reconcile_date_patch)
|
|
|
|
before_files, before = files(), database()
|
|
prior = source_timestamp(plan["observed_sql_created_at"])
|
|
desired = source_timestamp(plan["source_created_at"])
|
|
assert prior != desired # this exact cohort needs a SQL date correction
|
|
selected = [r for r in before["spaces_space"] if str(r["id"]) == selection["space_id"]]
|
|
assert len(selected) == 1 and selected[0]["created_at"] == prior
|
|
expected = copy.deepcopy(before)
|
|
next(r for r in expected["spaces_space"]
|
|
if str(r["id"]) == selection["space_id"])["created_at"] = desired
|
|
expected_files = dict(before_files)
|
|
expected_files[str(index.relative_to(root))] = hashlib.sha256(patched).hexdigest()
|
|
|
|
child = """
|
|
import django, json, os
|
|
from pathlib import Path
|
|
django.setup()
|
|
from coulomb_social.apps.spaces.date_backfill_journal import apply_date_patch
|
|
apply_date_patch(Path('/rehearsal-content'), json.loads(Path('/review/plan.json').read_bytes()),
|
|
Path(os.environ['JOURNAL']), backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'],
|
|
checkpoint=lambda phase: os._exit(73) if phase == os.environ['CRASH_PHASE'] else None)
|
|
"""
|
|
phases = []
|
|
for phase in ["intent", "sql", "file", "commit"]:
|
|
journal = Path("/journal") / (phase + ".json")
|
|
result = subprocess.run([sys.executable, "-c", child], capture_output=True, timeout=45,
|
|
env={**os.environ, "JOURNAL": str(journal), "CRASH_PHASE": phase})
|
|
assert result.returncode == 73
|
|
assert json.loads(journal.read_bytes())["state"] == "pending"
|
|
verdict = reconcile_date_patch(root, journal)
|
|
committed = phase == "commit"
|
|
assert verdict == ("completed" if committed else "rolled_back")
|
|
assert digest(database()) == digest(expected if committed else before)
|
|
assert files() == (expected_files if committed else before_files)
|
|
phases.append({"phase": phase, "child_exit": 73, "reconciled": verdict,
|
|
"all_tables_and_files_match_expected": True})
|
|
# Explicitly reverse only the disposable clone for the next scenario.
|
|
if committed:
|
|
assert Space.objects.filter(pk=selection["space_id"], created_at=desired).update(
|
|
created_at=prior) == 1
|
|
index.write_bytes(original)
|
|
assert digest(database()) == digest(before) and files() == before_files
|
|
|
|
final = Path("/journal/success.json")
|
|
apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
assert json.loads(final.read_bytes())["state"] == "completed"
|
|
try:
|
|
apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
|
|
except (ValueError, FileExistsError):
|
|
pass
|
|
else:
|
|
raise AssertionError("old plan replay unexpectedly succeeded")
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
print(json.dumps({"status": "recovered_pair_journal_verified", "phases": phases,
|
|
"successful_commit_verified": True, "replay_refused": True,
|
|
"only_selected_sql_date_and_index_changed": True,
|
|
"all_tables_compared": len(before), "all_files_compared": len(before_files),
|
|
"grants_and_local_edits_preserved": True}))
|
|
'''
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--restore-receipt", type=Path, required=True)
|
|
parser.add_argument("--plan", type=Path, required=True)
|
|
parser.add_argument("--candidate-image", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists() or args.output.is_symlink():
|
|
raise ValueError("Fresh output required")
|
|
raw_receipt = args.restore_receipt.read_bytes()
|
|
state = json.loads(raw_receipt)
|
|
if (state["status"] != "restored_verified"
|
|
or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"])
|
|
or state["network"] != state["database_container"] + "-net"):
|
|
raise ValueError("Isolated recovery receipt required")
|
|
run, hashes = PREVIOUS["run"], PREVIOUS["hashes"]
|
|
network = json.loads(run(["docker", "network", "inspect", state["network"]]))[0]
|
|
db = json.loads(run(["docker", "inspect", state["database_container"]]))[0]
|
|
if (network["Internal"] is not True or not db["State"]["Running"]
|
|
or set(db["NetworkSettings"]["Networks"]) != {state["network"]}
|
|
or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"]
|
|
or any(db["NetworkSettings"]["Ports"].values())
|
|
or db["Image"] != state["postgres_image_id"]):
|
|
raise ValueError("Recovery isolation mismatch")
|
|
# The alias used by the app must identify only the inspected database.
|
|
for identity in network["Containers"]:
|
|
if identity == db["Id"]:
|
|
continue
|
|
other = json.loads(run(["docker", "inspect", identity]))[0]
|
|
if "recovery-db" in (other["NetworkSettings"]["Networks"][state["network"]].get(
|
|
"Aliases") or []):
|
|
raise ValueError("Ambiguous recovery database alias")
|
|
image = run(["docker", "image", "inspect", args.candidate_image,
|
|
"--format", "{{.Id}}"]).decode().strip()
|
|
if not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
|
|
raise ValueError("Locally retained image required")
|
|
root = Path(state["restored_directory"]) / "content"
|
|
if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to(
|
|
Path("/tmp")):
|
|
raise ValueError("Real temporary recovery directory required")
|
|
original_hashes = hashes(root)
|
|
repo = Path(__file__).resolve().parents[1]
|
|
modules = {name: (repo / "coulomb_social/apps/spaces" / name).read_bytes()
|
|
for name in ["date_backfill.py", "date_backfill_journal.py"]}
|
|
plan_bytes = args.plan.read_bytes()
|
|
clone = "csoc_date_" + uuid.uuid4().hex
|
|
with tempfile.TemporaryDirectory(prefix="csoc-date-journal-rehearsal-") as temporary:
|
|
work = Path(temporary)
|
|
shutil.copytree(root, work / "content")
|
|
(work / "plan.json").write_bytes(plan_bytes)
|
|
(work / "journal").mkdir(mode=0o700)
|
|
for name, data in modules.items():
|
|
(work / name).write_bytes(data)
|
|
|
|
def application(database, code):
|
|
name = "csoc-date-journal-" + uuid.uuid4().hex
|
|
command = ["docker", "run", "--rm", "--name", name,
|
|
"--network", state["network"], "--read-only", "--tmpfs", "/tmp",
|
|
"--cap-drop=ALL", "--security-opt=no-new-privileges",
|
|
"--user", f"{os.getuid()}:{os.getgid()}",
|
|
"--entrypoint", "/app/.venv/bin/python",
|
|
"-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base",
|
|
"-e", f"DATABASE_URL=postgres://postgres@recovery-db/{database}",
|
|
"-e", "CONTENT_ROOT=/rehearsal-content", "-e", "FORGEJO_BASE_URL=",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", "BACKUP_RECEIPT_SHA256=" + sha(raw_receipt),
|
|
"-v", str(work / "content") + ":/rehearsal-content",
|
|
"-v", str(work / "plan.json") + ":/review/plan.json:ro",
|
|
"-v", str(work / "journal") + ":/journal"]
|
|
for member in modules:
|
|
command += ["-v", str(work / member)
|
|
+ ":/app/coulomb_social/apps/spaces/" + member + ":ro"]
|
|
try:
|
|
result = subprocess.run(command + [image, "-c", code], capture_output=True,
|
|
timeout=240)
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True,
|
|
timeout=30, check=False)
|
|
if result.returncode:
|
|
fd, path = tempfile.mkstemp(prefix="csoc-journal-private-diagnostics-")
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(result.stdout + result.stderr)
|
|
print(json.dumps({"private_diagnostics": path}))
|
|
raise ValueError("Private application check failed")
|
|
rows = [json.loads(line) for line in result.stdout.splitlines()
|
|
if line.startswith(b"{")]
|
|
if len(rows) != 1:
|
|
raise ValueError("Single result required")
|
|
return rows[0]
|
|
|
|
before = application("coulomb_social_restore", SNAPSHOT)
|
|
# A PostgreSQL template clone keeps every original row/sequence while
|
|
# isolating all subsequent commits from the retained recovery database.
|
|
run(["docker", "exec", state["database_container"], "createdb", "-U", "postgres",
|
|
"-T", "coulomb_social_restore", clone])
|
|
try:
|
|
verdict = application(clone, CHECKS)
|
|
after = application("coulomb_social_restore", SNAPSHOT)
|
|
# SNAPSHOT reads the disposable content copy; the source DB itself
|
|
# must be identical. Original files are checked separately below.
|
|
if before["tables_sha256"] != after["tables_sha256"]:
|
|
raise ValueError("Original recovery SQL changed")
|
|
if hashes(root) != original_hashes:
|
|
raise ValueError("Original recovery files changed")
|
|
finally:
|
|
run(["docker", "exec", state["database_container"], "dropdb", "-U", "postgres",
|
|
"--force", clone])
|
|
proof = {
|
|
"schema": "coulomb.recovered-date-journal-rehearsal.v1", **verdict,
|
|
"candidate_image_id": image, "source_overlay_used": True,
|
|
"modules_sha256": {k: sha(v) for k, v in modules.items()},
|
|
"runner_sha256": sha(Path(__file__).read_bytes()),
|
|
"shared_runner_sha256": sha(Path(__file__).with_name(
|
|
"rehearse-source-date-backfill.py").read_bytes()),
|
|
"private_plan_sha256": sha(plan_bytes), "restore_receipt_sha256": sha(raw_receipt),
|
|
"network_internal": True, "published_ports": 0,
|
|
"original_recovered_sql_and_files_unchanged": True,
|
|
"disposable_sql_clone_removed": True,
|
|
"production_access": False, "production_writes": False,
|
|
"supervised_live_executor_prepared": False,
|
|
}
|
|
fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "w") as stream:
|
|
json.dump(proof, stream, indent=2)
|
|
stream.write("\n")
|
|
print(json.dumps(proof))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, OSError, KeyError, subprocess.SubprocessError):
|
|
raise SystemExit("Journal rehearsal refused; inspect private inputs/diagnostics locally.")
|