diff --git a/docs/release/source-date-journal-real-pair-2026-10-30.json b/docs/release/source-date-journal-real-pair-2026-10-30.json new file mode 100644 index 0000000..449569c --- /dev/null +++ b/docs/release/source-date-journal-real-pair-2026-10-30.json @@ -0,0 +1,53 @@ +{ + "schema": "coulomb.recovered-date-journal-rehearsal.v1", + "status": "recovered_pair_journal_verified", + "phases": [ + { + "phase": "intent", + "child_exit": 73, + "reconciled": "rolled_back", + "all_tables_and_files_match_expected": true + }, + { + "phase": "sql", + "child_exit": 73, + "reconciled": "rolled_back", + "all_tables_and_files_match_expected": true + }, + { + "phase": "file", + "child_exit": 73, + "reconciled": "rolled_back", + "all_tables_and_files_match_expected": true + }, + { + "phase": "commit", + "child_exit": 73, + "reconciled": "completed", + "all_tables_and_files_match_expected": true + } + ], + "successful_commit_verified": true, + "replay_refused": true, + "only_selected_sql_date_and_index_changed": true, + "all_tables_compared": 13, + "all_files_compared": 82, + "grants_and_local_edits_preserved": true, + "candidate_image_id": "sha256:512819463a1902b43a615beee14be7bf97e6a05e016385082b9380692ad1a8cd", + "source_overlay_used": true, + "modules_sha256": { + "date_backfill.py": "84ce5d3369389e1af9ab999aad3804456fd2f66e654e290496be6930cd4885e4", + "date_backfill_journal.py": "f3eea06fb9f444d183c06f72ec95dd05e2be576cc8d40f93c935eea819f6d1be" + }, + "runner_sha256": "0bc6b21adadc1ed2a10b136495e76fd89d1af23134dda51bcb9fe0a53bd8e6a2", + "shared_runner_sha256": "812fc4b6325ea1b4c9c3698ad1175fe5a95d77ce5c1dc1631174f46e93bbb956", + "private_plan_sha256": "44369d5abbf92d3e6d52e909cdc1e1c3a90b8817be7027c3a43846fbfc381ef8", + "restore_receipt_sha256": "5f745cdae0a34c4a98f0e4429ceea49475b4e5cd9d2f8f09dea6be000e200258", + "network_internal": true, + "published_ports": 0, + "original_recovered_sql_and_files_unchanged": true, + "disposable_sql_clone_removed": true, + "production_access": false, + "production_writes": false, + "supervised_live_executor_prepared": false +} diff --git a/docs/release/source-date-journal-review-2026-10-30.md b/docs/release/source-date-journal-review-2026-10-30.md index a77b3b9..d55f83f 100644 --- a/docs/release/source-date-journal-review-2026-10-30.md +++ b/docs/release/source-date-journal-review-2026-10-30.md @@ -42,6 +42,29 @@ data, a temporary Docker container and a loopback-only port; it did not connect to any production/recovered database or provider. The full `make check` gate is recorded in the companion evidence JSON. +The retained independently recovered pair now also passes the journal rehearsal. +`scripts/rehearse-source-date-journal.py` verifies the actual Docker network, +database image, unique database alias and absence of published ports, then clones +the recovered PostgreSQL database and copies its content. All commits and crash +tests run against that disposable clone. Real child-process exits at intent, +SQL, file and commit boundaries reconcile correctly. All 13 tables and 82 files +match the expected state after every case; the successful correction changes +only the selected SQL date and index. Replay refuses. Before/after SQL digests and +file hashes prove the retained recovery pair is unchanged. The disposable SQL +clone is removed. Evidence: `source-date-journal-real-pair-2026-10-30.json`. +This uses explicit source overlays on the retained candidate image, not a newly +released application image. + +To repeat against a retained isolated restore, provide private paths locally: + +```sh +uv run python scripts/rehearse-source-date-journal.py \ + --restore-receipt /private/restore-receipt.json \ + --plan /private/date-backfill-plan.json \ + --candidate-image '' \ + --output /tmp/date-journal-real-pair.json +``` + Remaining wrapper requirements: bind exact reviewed source/plan/target and fresh backup/readback hashes; verify the external writer fence and full live SQL/content baseline; supervise the privileged action; refuse outstanding journal state; diff --git a/scripts/rehearse-source-date-journal.py b/scripts/rehearse-source-date-journal.py new file mode 100644 index 0000000..77e0f4c --- /dev/null +++ b/scripts/rehearse-source-date-journal.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Prove date-journal crash recovery on a disposable clone of a recovered pair. + +No production endpoint is accepted. The retained recovery database/files are +observed before/after; SQL commits occur only in a fresh temporary database clone. +Private page/SQL values never appear in the public result. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import runpy +import shutil +import subprocess +import tempfile +import uuid +from pathlib import Path + +# Share the existing reviewed private SQL/file observation helpers, without +# invoking the old rehearsal's main or its transaction/file mutation section. +PREVIOUS = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py"))) +COMMON = PREVIOUS["CHECKS"].split("before_files = files()", 1)[0] +SNAPSHOT = COMMON.replace("patched = prepare_index_date_patch(plan, original)", "") + ''' +print(json.dumps({"tables_sha256": hashlib.sha256(json.dumps( + digest(database()), sort_keys=True).encode()).hexdigest(), "files": files()})) +''' +CHECKS = COMMON + r''' +import os, subprocess, sys +from coulomb_social.apps.spaces.date_backfill_journal import ( + apply_date_patch, reconcile_date_patch) + +before_files, before = files(), database() +prior = source_timestamp(plan["observed_sql_created_at"]) +desired = source_timestamp(plan["source_created_at"]) +assert prior != desired # this exact cohort needs a SQL date correction +selected = [r for r in before["spaces_space"] if str(r["id"]) == selection["space_id"]] +assert len(selected) == 1 and selected[0]["created_at"] == prior +expected = copy.deepcopy(before) +next(r for r in expected["spaces_space"] + if str(r["id"]) == selection["space_id"])["created_at"] = desired +expected_files = dict(before_files) +expected_files[str(index.relative_to(root))] = hashlib.sha256(patched).hexdigest() + +child = """ +import django, json, os +from pathlib import Path +django.setup() +from coulomb_social.apps.spaces.date_backfill_journal import apply_date_patch +apply_date_patch(Path('/rehearsal-content'), json.loads(Path('/review/plan.json').read_bytes()), + Path(os.environ['JOURNAL']), backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'], + checkpoint=lambda phase: os._exit(73) if phase == os.environ['CRASH_PHASE'] else None) +""" +phases = [] +for phase in ["intent", "sql", "file", "commit"]: + journal = Path("/journal") / (phase + ".json") + result = subprocess.run([sys.executable, "-c", child], capture_output=True, timeout=45, + env={**os.environ, "JOURNAL": str(journal), "CRASH_PHASE": phase}) + assert result.returncode == 73 + assert json.loads(journal.read_bytes())["state"] == "pending" + verdict = reconcile_date_patch(root, journal) + committed = phase == "commit" + assert verdict == ("completed" if committed else "rolled_back") + assert digest(database()) == digest(expected if committed else before) + assert files() == (expected_files if committed else before_files) + phases.append({"phase": phase, "child_exit": 73, "reconciled": verdict, + "all_tables_and_files_match_expected": True}) + # Explicitly reverse only the disposable clone for the next scenario. + if committed: + assert Space.objects.filter(pk=selection["space_id"], created_at=desired).update( + created_at=prior) == 1 + index.write_bytes(original) + assert digest(database()) == digest(before) and files() == before_files + +final = Path("/journal/success.json") +apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256']) +assert digest(database()) == digest(expected) and files() == expected_files +assert json.loads(final.read_bytes())["state"] == "completed" +try: + apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256']) +except (ValueError, FileExistsError): + pass +else: + raise AssertionError("old plan replay unexpectedly succeeded") +assert digest(database()) == digest(expected) and files() == expected_files +print(json.dumps({"status": "recovered_pair_journal_verified", "phases": phases, + "successful_commit_verified": True, "replay_refused": True, + "only_selected_sql_date_and_index_changed": True, + "all_tables_compared": len(before), "all_files_compared": len(before_files), + "grants_and_local_edits_preserved": True})) +''' + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--restore-receipt", type=Path, required=True) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--candidate-image", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists() or args.output.is_symlink(): + raise ValueError("Fresh output required") + raw_receipt = args.restore_receipt.read_bytes() + state = json.loads(raw_receipt) + if (state["status"] != "restored_verified" + or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"]) + or state["network"] != state["database_container"] + "-net"): + raise ValueError("Isolated recovery receipt required") + run, hashes = PREVIOUS["run"], PREVIOUS["hashes"] + network = json.loads(run(["docker", "network", "inspect", state["network"]]))[0] + db = json.loads(run(["docker", "inspect", state["database_container"]]))[0] + if (network["Internal"] is not True or not db["State"]["Running"] + or set(db["NetworkSettings"]["Networks"]) != {state["network"]} + or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"] + or any(db["NetworkSettings"]["Ports"].values()) + or db["Image"] != state["postgres_image_id"]): + raise ValueError("Recovery isolation mismatch") + # The alias used by the app must identify only the inspected database. + for identity in network["Containers"]: + if identity == db["Id"]: + continue + other = json.loads(run(["docker", "inspect", identity]))[0] + if "recovery-db" in (other["NetworkSettings"]["Networks"][state["network"]].get( + "Aliases") or []): + raise ValueError("Ambiguous recovery database alias") + image = run(["docker", "image", "inspect", args.candidate_image, + "--format", "{{.Id}}"]).decode().strip() + if not re.fullmatch(r"sha256:[a-f0-9]{64}", image): + raise ValueError("Locally retained image required") + root = Path(state["restored_directory"]) / "content" + if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to( + Path("/tmp")): + raise ValueError("Real temporary recovery directory required") + original_hashes = hashes(root) + repo = Path(__file__).resolve().parents[1] + modules = {name: (repo / "coulomb_social/apps/spaces" / name).read_bytes() + for name in ["date_backfill.py", "date_backfill_journal.py"]} + plan_bytes = args.plan.read_bytes() + clone = "csoc_date_" + uuid.uuid4().hex + with tempfile.TemporaryDirectory(prefix="csoc-date-journal-rehearsal-") as temporary: + work = Path(temporary) + shutil.copytree(root, work / "content") + (work / "plan.json").write_bytes(plan_bytes) + (work / "journal").mkdir(mode=0o700) + for name, data in modules.items(): + (work / name).write_bytes(data) + + def application(database, code): + name = "csoc-date-journal-" + uuid.uuid4().hex + command = ["docker", "run", "--rm", "--name", name, + "--network", state["network"], "--read-only", "--tmpfs", "/tmp", + "--cap-drop=ALL", "--security-opt=no-new-privileges", + "--user", f"{os.getuid()}:{os.getgid()}", + "--entrypoint", "/app/.venv/bin/python", + "-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base", + "-e", f"DATABASE_URL=postgres://postgres@recovery-db/{database}", + "-e", "CONTENT_ROOT=/rehearsal-content", "-e", "FORGEJO_BASE_URL=", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-e", "BACKUP_RECEIPT_SHA256=" + sha(raw_receipt), + "-v", str(work / "content") + ":/rehearsal-content", + "-v", str(work / "plan.json") + ":/review/plan.json:ro", + "-v", str(work / "journal") + ":/journal"] + for member in modules: + command += ["-v", str(work / member) + + ":/app/coulomb_social/apps/spaces/" + member + ":ro"] + try: + result = subprocess.run(command + [image, "-c", code], capture_output=True, + timeout=240) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, + timeout=30, check=False) + if result.returncode: + fd, path = tempfile.mkstemp(prefix="csoc-journal-private-diagnostics-") + with os.fdopen(fd, "wb") as stream: + stream.write(result.stdout + result.stderr) + print(json.dumps({"private_diagnostics": path})) + raise ValueError("Private application check failed") + rows = [json.loads(line) for line in result.stdout.splitlines() + if line.startswith(b"{")] + if len(rows) != 1: + raise ValueError("Single result required") + return rows[0] + + before = application("coulomb_social_restore", SNAPSHOT) + # A PostgreSQL template clone keeps every original row/sequence while + # isolating all subsequent commits from the retained recovery database. + run(["docker", "exec", state["database_container"], "createdb", "-U", "postgres", + "-T", "coulomb_social_restore", clone]) + try: + verdict = application(clone, CHECKS) + after = application("coulomb_social_restore", SNAPSHOT) + # SNAPSHOT reads the disposable content copy; the source DB itself + # must be identical. Original files are checked separately below. + if before["tables_sha256"] != after["tables_sha256"]: + raise ValueError("Original recovery SQL changed") + if hashes(root) != original_hashes: + raise ValueError("Original recovery files changed") + finally: + run(["docker", "exec", state["database_container"], "dropdb", "-U", "postgres", + "--force", clone]) + proof = { + "schema": "coulomb.recovered-date-journal-rehearsal.v1", **verdict, + "candidate_image_id": image, "source_overlay_used": True, + "modules_sha256": {k: sha(v) for k, v in modules.items()}, + "runner_sha256": sha(Path(__file__).read_bytes()), + "shared_runner_sha256": sha(Path(__file__).with_name( + "rehearse-source-date-backfill.py").read_bytes()), + "private_plan_sha256": sha(plan_bytes), "restore_receipt_sha256": sha(raw_receipt), + "network_internal": True, "published_ports": 0, + "original_recovered_sql_and_files_unchanged": True, + "disposable_sql_clone_removed": True, + "production_access": False, "production_writes": False, + "supervised_live_executor_prepared": False, + } + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(proof, stream, indent=2) + stream.write("\n") + print(json.dumps(proof)) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, KeyError, subprocess.SubprocessError): + raise SystemExit("Journal rehearsal refused; inspect private inputs/diagnostics locally.") diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index fbcd2ae..fa6dd99 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -176,6 +176,18 @@ source-date-journal-rehearsal-2026-10-30.json. The supervised wrapper, verified writer fence, fresh backup binding, recovered-real-pair rehearsal and productive postflight remain required; this primitive does not establish those prerequisites. +Recovered-pair follow-through: scripts/rehearse-source-date-journal.py now verifies +the native internal/no-published-port database boundary and uses a fresh disposable +PostgreSQL template clone plus copied files. All four real process deaths reconcile +correctly on the exact private accepted-cohort plan. Every case compares all 13 +tables and 82 files; normal commit changes only the selected SQL date/index and +replay refuses. The retained recovery SQL and files remain identical and the +disposable clone is removed. Proof: +docs/release/source-date-journal-real-pair-2026-10-30.json. Source overlays remain +explicit; no new image, productive date correction or admission is inferred. +The journal's real-pair rehearsal is complete; the external supervised wrapper, +fresh verified backup/fence and productive postflight are still required. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.