tegwick
2344b2d18b
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 8m0s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
374 lines
18 KiB
Python
374 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Prove date-journal crash recovery on a disposable clone of a recovered pair.
|
|
|
|
No production endpoint is accepted. The retained recovery database/files are
|
|
observed before/after; SQL commits occur only in a fresh temporary database clone.
|
|
Private page/SQL values never appear in the public result.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import runpy
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
# Share the existing reviewed private SQL/file observation helpers, without
|
|
# invoking the old rehearsal's main or its transaction/file mutation section.
|
|
PREVIOUS = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py")))
|
|
COMMON = PREVIOUS["CHECKS"].split("before_files = files()", 1)[0]
|
|
SNAPSHOT = COMMON.replace("patched = prepare_index_date_patch(plan, original)", "") + '''
|
|
print(json.dumps({"tables_sha256": hashlib.sha256(json.dumps(
|
|
digest(database()), sort_keys=True).encode()).hexdigest(), "files": files()}))
|
|
'''
|
|
CHECKS = COMMON + r'''
|
|
import os, subprocess, sys
|
|
from coulomb_social.apps.spaces.date_backfill_journal import (
|
|
apply_date_patch, reconcile_date_patch)
|
|
|
|
before_files, before = files(), database()
|
|
prior = source_timestamp(plan["observed_sql_created_at"])
|
|
desired = source_timestamp(plan["source_created_at"])
|
|
assert prior != desired # this exact cohort needs a SQL date correction
|
|
selected = [r for r in before["spaces_space"] if str(r["id"]) == selection["space_id"]]
|
|
assert len(selected) == 1 and selected[0]["created_at"] == prior
|
|
expected = copy.deepcopy(before)
|
|
next(r for r in expected["spaces_space"]
|
|
if str(r["id"]) == selection["space_id"])["created_at"] = desired
|
|
expected_files = dict(before_files)
|
|
expected_files[str(index.relative_to(root))] = hashlib.sha256(patched).hexdigest()
|
|
|
|
child = """
|
|
import django, json, os
|
|
from pathlib import Path
|
|
django.setup()
|
|
from coulomb_social.apps.spaces.date_backfill_journal import apply_date_patch
|
|
apply_date_patch(Path('/rehearsal-content'), json.loads(Path('/review/plan.json').read_bytes()),
|
|
Path(os.environ['JOURNAL']), backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'],
|
|
checkpoint=lambda phase: os._exit(73) if phase == os.environ['CRASH_PHASE'] else None)
|
|
"""
|
|
phases = []
|
|
for phase in ["intent", "sql", "file", "commit"]:
|
|
journal = Path("/journal") / (phase + ".json")
|
|
result = subprocess.run([sys.executable, "-c", child], capture_output=True, timeout=45,
|
|
env={**os.environ, "JOURNAL": str(journal), "CRASH_PHASE": phase})
|
|
assert result.returncode == 73
|
|
assert json.loads(journal.read_bytes())["state"] == "pending"
|
|
verdict = reconcile_date_patch(root, journal)
|
|
committed = phase == "commit"
|
|
assert verdict == ("completed" if committed else "rolled_back")
|
|
assert digest(database()) == digest(expected if committed else before)
|
|
assert files() == (expected_files if committed else before_files)
|
|
phases.append({"phase": phase, "child_exit": 73, "reconciled": verdict,
|
|
"all_tables_and_files_match_expected": True})
|
|
# Explicitly reverse only the disposable clone for the next scenario.
|
|
if committed:
|
|
assert Space.objects.filter(pk=selection["space_id"], created_at=desired).update(
|
|
created_at=prior) == 1
|
|
index.write_bytes(original)
|
|
assert digest(database()) == digest(before) and files() == before_files
|
|
|
|
final = Path("/journal/success.json")
|
|
apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
assert json.loads(final.read_bytes())["state"] == "completed"
|
|
try:
|
|
apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
|
|
except (ValueError, FileExistsError):
|
|
pass
|
|
else:
|
|
raise AssertionError("old plan replay unexpectedly succeeded")
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
print(json.dumps({"status": "recovered_pair_journal_verified", "phases": phases,
|
|
"successful_commit_verified": True, "replay_refused": True,
|
|
"only_selected_sql_date_and_index_changed": True,
|
|
"all_tables_compared": len(before), "all_files_compared": len(before_files),
|
|
"grants_and_local_edits_preserved": True}))
|
|
'''
|
|
|
|
GUARDED_CHECKS = COMMON + r'''
|
|
import os, subprocess, sys, psycopg
|
|
from datetime import timedelta
|
|
from coulomb_social.apps.spaces.date_backfill_operation import (
|
|
execute_date_operation, source_fingerprint, _content_lock)
|
|
|
|
backup = json.loads(Path("/review/backup.json").read_bytes())
|
|
backup = backup.get("receipt", backup)
|
|
# A historical metadata-check clock, never evidence of a new live backup.
|
|
review_now = source_timestamp(backup["completed_at"]) + timedelta(seconds=1)
|
|
baseline = source_fingerprint(root)
|
|
Path("/journal/baseline.json").write_text(json.dumps(baseline))
|
|
before_files, before = files(), database()
|
|
prior = source_timestamp(plan["observed_sql_created_at"])
|
|
desired = source_timestamp(plan["source_created_at"])
|
|
assert prior != desired
|
|
expected = copy.deepcopy(before)
|
|
target = next(r for r in expected["spaces_space"] if str(r["id"]) == selection["space_id"])
|
|
assert target["created_at"] == prior
|
|
target["created_at"] = desired
|
|
expected_files = dict(before_files)
|
|
expected_files[str(index.relative_to(root))] = hashlib.sha256(patched).hexdigest()
|
|
|
|
def execute(journal, **options):
|
|
return execute_date_operation(root, plan, journal, baseline, backup,
|
|
image=backup["image"], recipient=backup["age_recipient"], now=review_now, **options)
|
|
|
|
child = """
|
|
import django, json, os
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
django.setup()
|
|
from coulomb_social.apps.spaces.bubble_transfer import source_timestamp
|
|
from coulomb_social.apps.spaces.date_backfill_operation import execute_date_operation
|
|
backup = json.loads(Path('/review/backup.json').read_bytes())
|
|
backup = backup.get('receipt', backup)
|
|
execute_date_operation(Path('/rehearsal-content'),
|
|
json.loads(Path('/review/plan.json').read_bytes()), Path(os.environ['JOURNAL']),
|
|
json.loads(Path('/journal/baseline.json').read_bytes()), backup,
|
|
image=backup['image'], recipient=backup['age_recipient'],
|
|
now=source_timestamp(backup['completed_at']) + timedelta(seconds=1),
|
|
checkpoint=lambda phase: os._exit(73) if phase == os.environ['CRASH_PHASE'] else None)
|
|
"""
|
|
phases = []
|
|
for phase in ["intent", "sql", "file", "commit"]:
|
|
journal = Path("/journal") / (phase + ".json")
|
|
result = subprocess.run([sys.executable, "-c", child], capture_output=True, timeout=45,
|
|
env={**os.environ, "JOURNAL": str(journal), "CRASH_PHASE": phase})
|
|
assert result.returncode == 73
|
|
assert json.loads(journal.read_bytes())["state"] == "pending"
|
|
# Recovery binds the original point, even a day later. It never captures a
|
|
# replacement point from an uncertain combination of SQL and file state.
|
|
proof = execute_date_operation(root, plan, journal, baseline, backup,
|
|
image=backup["image"], recipient=backup["age_recipient"],
|
|
now=review_now + timedelta(days=1), reconcile=True)
|
|
committed = phase == "commit"
|
|
assert proof["status"] == ("completed" if committed else "rolled_back")
|
|
assert proof["whole_store_postflight"]
|
|
assert digest(database()) == digest(expected if committed else before)
|
|
assert files() == (expected_files if committed else before_files)
|
|
phases.append({"phase": phase, "child_exit": 73, "reconciled": proof["status"],
|
|
"all_tables_and_files_match_expected": True})
|
|
if committed:
|
|
assert Space.objects.filter(pk=selection["space_id"], created_at=desired).update(
|
|
created_at=prior) == 1
|
|
index.write_bytes(original)
|
|
assert digest(database()) == digest(before) and files() == before_files
|
|
|
|
for kind in ["sql", "file", "writer"]:
|
|
journal = Path("/journal") / ("refused-" + kind + ".json")
|
|
old_title = Space.objects.get(pk=selection["space_id"]).title
|
|
stray = root / "unrelated-guard-fixture.bin"
|
|
try:
|
|
if kind == "sql":
|
|
Space.objects.filter(pk=selection["space_id"]).update(title="synthetic drift")
|
|
elif kind == "file":
|
|
stray.write_bytes(b"synthetic unrelated drift")
|
|
try:
|
|
if kind == "writer":
|
|
with _content_lock(root):
|
|
execute(journal)
|
|
else:
|
|
execute(journal)
|
|
except (ValueError, BlockingIOError):
|
|
pass
|
|
else:
|
|
raise AssertionError("whole-store drift or busy writer unexpectedly accepted")
|
|
assert not journal.exists()
|
|
finally:
|
|
if kind == "sql":
|
|
Space.objects.filter(pk=selection["space_id"]).update(title=old_title)
|
|
stray.unlink(missing_ok=True)
|
|
assert digest(database()) == digest(before) and files() == before_files
|
|
|
|
lock_proved = []
|
|
def probe_lock(phase):
|
|
if phase == "sql":
|
|
with psycopg.connect(os.environ['DATABASE_URL']) as other:
|
|
with other.cursor() as cursor:
|
|
cursor.execute("SET lock_timeout='100ms'")
|
|
try:
|
|
cursor.execute("UPDATE spaces_space SET title=title WHERE id=%s",
|
|
(selection["space_id"],))
|
|
except psycopg.errors.LockNotAvailable:
|
|
lock_proved.append(True)
|
|
else:
|
|
raise AssertionError("unrelated SQL writer was not blocked")
|
|
|
|
final = Path("/journal/success.json")
|
|
proof = execute(final, checkpoint=probe_lock)
|
|
assert lock_proved == [True]
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
try:
|
|
execute(final)
|
|
except (ValueError, FileExistsError):
|
|
pass
|
|
else:
|
|
raise AssertionError("guarded operation replay unexpectedly accepted")
|
|
assert digest(database()) == digest(expected) and files() == expected_files
|
|
print(json.dumps({"status": "guarded_recovered_date_operation_verified", "phases": phases,
|
|
"successful_commit_verified": True, "replay_refused": True,
|
|
"only_selected_sql_date_and_index_changed": True,
|
|
"all_tables_compared": len(before), "all_files_compared": len(before_files),
|
|
"grants_and_local_edits_preserved": True, "whole_store_drift_refused": True,
|
|
"busy_content_writer_refused": True, "concurrent_sql_writer_blocked": True,
|
|
"recovery_retains_original_old_backup_binding": True,
|
|
"historical_backup_metadata_clock_used": True,
|
|
"fresh_native_backup_or_external_fence_proved": False}))
|
|
'''
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--restore-receipt", type=Path, required=True)
|
|
parser.add_argument("--plan", type=Path, required=True)
|
|
parser.add_argument("--candidate-image", required=True)
|
|
parser.add_argument("--backup-receipt", type=Path,
|
|
help="Enable whole-store operation checks with historical metadata clock")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
if args.output.exists() or args.output.is_symlink():
|
|
raise ValueError("Fresh output required")
|
|
raw_receipt = args.restore_receipt.read_bytes()
|
|
state = json.loads(raw_receipt)
|
|
if (state["status"] != "restored_verified"
|
|
or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"])
|
|
or state["network"] != state["database_container"] + "-net"):
|
|
raise ValueError("Isolated recovery receipt required")
|
|
run, hashes = PREVIOUS["run"], PREVIOUS["hashes"]
|
|
network = json.loads(run(["docker", "network", "inspect", state["network"]]))[0]
|
|
db = json.loads(run(["docker", "inspect", state["database_container"]]))[0]
|
|
if (network["Internal"] is not True or not db["State"]["Running"]
|
|
or set(db["NetworkSettings"]["Networks"]) != {state["network"]}
|
|
or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"]
|
|
or any(db["NetworkSettings"]["Ports"].values())
|
|
or db["Image"] != state["postgres_image_id"]):
|
|
raise ValueError("Recovery isolation mismatch")
|
|
# The alias used by the app must identify only the inspected database.
|
|
for identity in network["Containers"]:
|
|
if identity == db["Id"]:
|
|
continue
|
|
other = json.loads(run(["docker", "inspect", identity]))[0]
|
|
if "recovery-db" in (other["NetworkSettings"]["Networks"][state["network"]].get(
|
|
"Aliases") or []):
|
|
raise ValueError("Ambiguous recovery database alias")
|
|
image = run(["docker", "image", "inspect", args.candidate_image,
|
|
"--format", "{{.Id}}"]).decode().strip()
|
|
if not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
|
|
raise ValueError("Locally retained image required")
|
|
root = Path(state["restored_directory"]) / "content"
|
|
if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to(
|
|
Path("/tmp")):
|
|
raise ValueError("Real temporary recovery directory required")
|
|
original_hashes = hashes(root)
|
|
repo = Path(__file__).resolve().parents[1]
|
|
modules = {name: (repo / "coulomb_social/apps/spaces" / name).read_bytes()
|
|
for name in ["date_backfill.py", "date_backfill_journal.py"]}
|
|
if args.backup_receipt:
|
|
modules["date_backfill_operation.py"] = (
|
|
repo / "coulomb_social/apps/spaces/date_backfill_operation.py").read_bytes()
|
|
backup_bytes = args.backup_receipt.read_bytes() if args.backup_receipt else None
|
|
plan_bytes = args.plan.read_bytes()
|
|
clone = "csoc_date_" + uuid.uuid4().hex
|
|
with tempfile.TemporaryDirectory(prefix="csoc-date-journal-rehearsal-") as temporary:
|
|
work = Path(temporary)
|
|
shutil.copytree(root, work / "content")
|
|
(work / "plan.json").write_bytes(plan_bytes)
|
|
if backup_bytes:
|
|
(work / "backup.json").write_bytes(backup_bytes)
|
|
(work / "journal").mkdir(mode=0o700)
|
|
for name, data in modules.items():
|
|
(work / name).write_bytes(data)
|
|
|
|
def application(database, code):
|
|
name = "csoc-date-journal-" + uuid.uuid4().hex
|
|
command = ["docker", "run", "--rm", "--name", name,
|
|
"--network", state["network"], "--read-only", "--tmpfs", "/tmp",
|
|
"--cap-drop=ALL", "--security-opt=no-new-privileges",
|
|
"--user", f"{os.getuid()}:{os.getgid()}",
|
|
"--entrypoint", "/app/.venv/bin/python",
|
|
"-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base",
|
|
"-e", f"DATABASE_URL=postgres://postgres@recovery-db/{database}",
|
|
"-e", "CONTENT_ROOT=/rehearsal-content", "-e", "FORGEJO_BASE_URL=",
|
|
"-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
"-e", "BACKUP_RECEIPT_SHA256=" + sha(raw_receipt),
|
|
"-v", str(work / "content") + ":/rehearsal-content",
|
|
"-v", str(work / "plan.json") + ":/review/plan.json:ro",
|
|
"-v", str(work / "journal") + ":/journal"]
|
|
if backup_bytes:
|
|
command += ["-v", str(work / "backup.json") + ":/review/backup.json:ro"]
|
|
for member in modules:
|
|
command += ["-v", str(work / member)
|
|
+ ":/app/coulomb_social/apps/spaces/" + member + ":ro"]
|
|
try:
|
|
result = subprocess.run(command + [image, "-c", code], capture_output=True,
|
|
timeout=240)
|
|
finally:
|
|
subprocess.run(["docker", "rm", "-f", name], capture_output=True,
|
|
timeout=30, check=False)
|
|
if result.returncode:
|
|
fd, path = tempfile.mkstemp(prefix="csoc-journal-private-diagnostics-")
|
|
with os.fdopen(fd, "wb") as stream:
|
|
stream.write(result.stdout + result.stderr)
|
|
print(json.dumps({"private_diagnostics": path}))
|
|
raise ValueError("Private application check failed")
|
|
rows = [json.loads(line) for line in result.stdout.splitlines()
|
|
if line.startswith(b"{")]
|
|
if len(rows) != 1:
|
|
raise ValueError("Single result required")
|
|
return rows[0]
|
|
|
|
before = application("coulomb_social_restore", SNAPSHOT)
|
|
# A PostgreSQL template clone keeps every original row/sequence while
|
|
# isolating all subsequent commits from the retained recovery database.
|
|
run(["docker", "exec", state["database_container"], "createdb", "-U", "postgres",
|
|
"-T", "coulomb_social_restore", clone])
|
|
try:
|
|
verdict = application(clone, GUARDED_CHECKS if backup_bytes else CHECKS)
|
|
after = application("coulomb_social_restore", SNAPSHOT)
|
|
# SNAPSHOT reads the disposable content copy; the source DB itself
|
|
# must be identical. Original files are checked separately below.
|
|
if before["tables_sha256"] != after["tables_sha256"]:
|
|
raise ValueError("Original recovery SQL changed")
|
|
if hashes(root) != original_hashes:
|
|
raise ValueError("Original recovery files changed")
|
|
finally:
|
|
run(["docker", "exec", state["database_container"], "dropdb", "-U", "postgres",
|
|
"--force", clone])
|
|
proof = {
|
|
"schema": "coulomb.recovered-date-journal-rehearsal.v1", **verdict,
|
|
"candidate_image_id": image, "source_overlay_used": True,
|
|
"modules_sha256": {k: sha(v) for k, v in modules.items()},
|
|
"runner_sha256": sha(Path(__file__).read_bytes()),
|
|
"shared_runner_sha256": sha(Path(__file__).with_name(
|
|
"rehearse-source-date-backfill.py").read_bytes()),
|
|
"private_plan_sha256": sha(plan_bytes), "restore_receipt_sha256": sha(raw_receipt),
|
|
"historical_backup_receipt_sha256": sha(backup_bytes) if backup_bytes else None,
|
|
"network_internal": True, "published_ports": 0,
|
|
"original_recovered_sql_and_files_unchanged": True,
|
|
"disposable_sql_clone_removed": True,
|
|
"production_access": False, "production_writes": False,
|
|
"supervised_live_executor_prepared": False,
|
|
}
|
|
fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
with os.fdopen(fd, "w") as stream:
|
|
json.dump(proof, stream, indent=2)
|
|
stream.write("\n")
|
|
print(json.dumps(proof))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except (ValueError, OSError, KeyError, subprocess.SubprocessError):
|
|
raise SystemExit("Journal rehearsal refused; inspect private inputs/diagnostics locally.")
|