Rehearse date-journal process crashes against the recovered application pair

Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3

docs/release/source-date-journal-real-pair-2026-10-30.json

@@ -0,0 +1,53 @@
+{
+  "schema": "coulomb.recovered-date-journal-rehearsal.v1",
+  "status": "recovered_pair_journal_verified",
+  "phases": [
+    {
+      "phase": "intent",
+      "child_exit": 73,
+      "reconciled": "rolled_back",
+      "all_tables_and_files_match_expected": true
+    },
+    {
+      "phase": "sql",
+      "child_exit": 73,
+      "reconciled": "rolled_back",
+      "all_tables_and_files_match_expected": true
+    },
+    {
+      "phase": "file",
+      "child_exit": 73,
+      "reconciled": "rolled_back",
+      "all_tables_and_files_match_expected": true
+    },
+    {
+      "phase": "commit",
+      "child_exit": 73,
+      "reconciled": "completed",
+      "all_tables_and_files_match_expected": true
+    }
+  ],
+  "successful_commit_verified": true,
+  "replay_refused": true,
+  "only_selected_sql_date_and_index_changed": true,
+  "all_tables_compared": 13,
+  "all_files_compared": 82,
+  "grants_and_local_edits_preserved": true,
+  "candidate_image_id": "sha256:512819463a1902b43a615beee14be7bf97e6a05e016385082b9380692ad1a8cd",
+  "source_overlay_used": true,
+  "modules_sha256": {
+    "date_backfill.py": "84ce5d3369389e1af9ab999aad3804456fd2f66e654e290496be6930cd4885e4",
+    "date_backfill_journal.py": "f3eea06fb9f444d183c06f72ec95dd05e2be576cc8d40f93c935eea819f6d1be"
+  },
+  "runner_sha256": "0bc6b21adadc1ed2a10b136495e76fd89d1af23134dda51bcb9fe0a53bd8e6a2",
+  "shared_runner_sha256": "812fc4b6325ea1b4c9c3698ad1175fe5a95d77ce5c1dc1631174f46e93bbb956",
+  "private_plan_sha256": "44369d5abbf92d3e6d52e909cdc1e1c3a90b8817be7027c3a43846fbfc381ef8",
+  "restore_receipt_sha256": "5f745cdae0a34c4a98f0e4429ceea49475b4e5cd9d2f8f09dea6be000e200258",
+  "network_internal": true,
+  "published_ports": 0,
+  "original_recovered_sql_and_files_unchanged": true,
+  "disposable_sql_clone_removed": true,
+  "production_access": false,
+  "production_writes": false,
+  "supervised_live_executor_prepared": false
+}

docs/release/source-date-journal-review-2026-10-30.md

@@ -42,6 +42,29 @@ data, a temporary Docker container and a loopback-only port; it did not connect
 to any production/recovered database or provider. The full `make check` gate is
 recorded in the companion evidence JSON.
 
+The retained independently recovered pair now also passes the journal rehearsal.
+`scripts/rehearse-source-date-journal.py` verifies the actual Docker network,
+database image, unique database alias and absence of published ports, then clones
+the recovered PostgreSQL database and copies its content. All commits and crash
+tests run against that disposable clone. Real child-process exits at intent,
+SQL, file and commit boundaries reconcile correctly. All 13 tables and 82 files
+match the expected state after every case; the successful correction changes
+only the selected SQL date and index. Replay refuses. Before/after SQL digests and
+file hashes prove the retained recovery pair is unchanged. The disposable SQL
+clone is removed. Evidence: `source-date-journal-real-pair-2026-10-30.json`.
+This uses explicit source overlays on the retained candidate image, not a newly
+released application image.
+
+To repeat against a retained isolated restore, provide private paths locally:
+
+```sh
+uv run python scripts/rehearse-source-date-journal.py \
+  --restore-receipt /private/restore-receipt.json \
+  --plan /private/date-backfill-plan.json \
+  --candidate-image '<locally retained image digest>' \
+  --output /tmp/date-journal-real-pair.json
+```
+
 Remaining wrapper requirements: bind exact reviewed source/plan/target and fresh
 backup/readback hashes; verify the external writer fence and full live SQL/content
 baseline; supervise the privileged action; refuse outstanding journal state;

scripts/rehearse-source-date-journal.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Prove date-journal crash recovery on a disposable clone of a recovered pair.
+
+No production endpoint is accepted. The retained recovery database/files are
+observed before/after; SQL commits occur only in a fresh temporary database clone.
+Private page/SQL values never appear in the public result.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import re
+import runpy
+import shutil
+import subprocess
+import tempfile
+import uuid
+from pathlib import Path
+
+# Share the existing reviewed private SQL/file observation helpers, without
+# invoking the old rehearsal's main or its transaction/file mutation section.
+PREVIOUS = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py")))
+COMMON = PREVIOUS["CHECKS"].split("before_files = files()", 1)[0]
+SNAPSHOT = COMMON.replace("patched = prepare_index_date_patch(plan, original)", "") + '''
+print(json.dumps({"tables_sha256": hashlib.sha256(json.dumps(
+    digest(database()), sort_keys=True).encode()).hexdigest(), "files": files()}))
+'''
+CHECKS = COMMON + r'''
+import os, subprocess, sys
+from coulomb_social.apps.spaces.date_backfill_journal import (
+    apply_date_patch, reconcile_date_patch)
+
+before_files, before = files(), database()
+prior = source_timestamp(plan["observed_sql_created_at"])
+desired = source_timestamp(plan["source_created_at"])
+assert prior != desired  # this exact cohort needs a SQL date correction
+selected = [r for r in before["spaces_space"] if str(r["id"]) == selection["space_id"]]
+assert len(selected) == 1 and selected[0]["created_at"] == prior
+expected = copy.deepcopy(before)
+next(r for r in expected["spaces_space"]
+     if str(r["id"]) == selection["space_id"])["created_at"] = desired
+expected_files = dict(before_files)
+expected_files[str(index.relative_to(root))] = hashlib.sha256(patched).hexdigest()
+
+child = """
+import django, json, os
+from pathlib import Path
+django.setup()
+from coulomb_social.apps.spaces.date_backfill_journal import apply_date_patch
+apply_date_patch(Path('/rehearsal-content'), json.loads(Path('/review/plan.json').read_bytes()),
+    Path(os.environ['JOURNAL']), backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'],
+    checkpoint=lambda phase: os._exit(73) if phase == os.environ['CRASH_PHASE'] else None)
+"""
+phases = []
+for phase in ["intent", "sql", "file", "commit"]:
+    journal = Path("/journal") / (phase + ".json")
+    result = subprocess.run([sys.executable, "-c", child], capture_output=True, timeout=45,
+        env={**os.environ, "JOURNAL": str(journal), "CRASH_PHASE": phase})
+    assert result.returncode == 73
+    assert json.loads(journal.read_bytes())["state"] == "pending"
+    verdict = reconcile_date_patch(root, journal)
+    committed = phase == "commit"
+    assert verdict == ("completed" if committed else "rolled_back")
+    assert digest(database()) == digest(expected if committed else before)
+    assert files() == (expected_files if committed else before_files)
+    phases.append({"phase": phase, "child_exit": 73, "reconciled": verdict,
+                   "all_tables_and_files_match_expected": True})
+    # Explicitly reverse only the disposable clone for the next scenario.
+    if committed:
+        assert Space.objects.filter(pk=selection["space_id"], created_at=desired).update(
+            created_at=prior) == 1
+        index.write_bytes(original)
+    assert digest(database()) == digest(before) and files() == before_files
+
+final = Path("/journal/success.json")
+apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
+assert digest(database()) == digest(expected) and files() == expected_files
+assert json.loads(final.read_bytes())["state"] == "completed"
+try:
+    apply_date_patch(root, plan, final, backup_sha256=os.environ['BACKUP_RECEIPT_SHA256'])
+except (ValueError, FileExistsError):
+    pass
+else:
+    raise AssertionError("old plan replay unexpectedly succeeded")
+assert digest(database()) == digest(expected) and files() == expected_files
+print(json.dumps({"status": "recovered_pair_journal_verified", "phases": phases,
+    "successful_commit_verified": True, "replay_refused": True,
+    "only_selected_sql_date_and_index_changed": True,
+    "all_tables_compared": len(before), "all_files_compared": len(before_files),
+    "grants_and_local_edits_preserved": True}))
+'''
+
+
+def sha(data):
+    return hashlib.sha256(data).hexdigest()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--restore-receipt", type=Path, required=True)
+    parser.add_argument("--plan", type=Path, required=True)
+    parser.add_argument("--candidate-image", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+    if args.output.exists() or args.output.is_symlink():
+        raise ValueError("Fresh output required")
+    raw_receipt = args.restore_receipt.read_bytes()
+    state = json.loads(raw_receipt)
+    if (state["status"] != "restored_verified"
+            or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"])
+            or state["network"] != state["database_container"] + "-net"):
+        raise ValueError("Isolated recovery receipt required")
+    run, hashes = PREVIOUS["run"], PREVIOUS["hashes"]
+    network = json.loads(run(["docker", "network", "inspect", state["network"]]))[0]
+    db = json.loads(run(["docker", "inspect", state["database_container"]]))[0]
+    if (network["Internal"] is not True or not db["State"]["Running"]
+            or set(db["NetworkSettings"]["Networks"]) != {state["network"]}
+            or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"]
+            or any(db["NetworkSettings"]["Ports"].values())
+            or db["Image"] != state["postgres_image_id"]):
+        raise ValueError("Recovery isolation mismatch")
+    # The alias used by the app must identify only the inspected database.
+    for identity in network["Containers"]:
+        if identity == db["Id"]:
+            continue
+        other = json.loads(run(["docker", "inspect", identity]))[0]
+        if "recovery-db" in (other["NetworkSettings"]["Networks"][state["network"]].get(
+                "Aliases") or []):
+            raise ValueError("Ambiguous recovery database alias")
+    image = run(["docker", "image", "inspect", args.candidate_image,
+                 "--format", "{{.Id}}"]).decode().strip()
+    if not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
+        raise ValueError("Locally retained image required")
+    root = Path(state["restored_directory"]) / "content"
+    if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to(
+            Path("/tmp")):
+        raise ValueError("Real temporary recovery directory required")
+    original_hashes = hashes(root)
+    repo = Path(__file__).resolve().parents[1]
+    modules = {name: (repo / "coulomb_social/apps/spaces" / name).read_bytes()
+               for name in ["date_backfill.py", "date_backfill_journal.py"]}
+    plan_bytes = args.plan.read_bytes()
+    clone = "csoc_date_" + uuid.uuid4().hex
+    with tempfile.TemporaryDirectory(prefix="csoc-date-journal-rehearsal-") as temporary:
+        work = Path(temporary)
+        shutil.copytree(root, work / "content")
+        (work / "plan.json").write_bytes(plan_bytes)
+        (work / "journal").mkdir(mode=0o700)
+        for name, data in modules.items():
+            (work / name).write_bytes(data)
+
+        def application(database, code):
+            name = "csoc-date-journal-" + uuid.uuid4().hex
+            command = ["docker", "run", "--rm", "--name", name,
+                       "--network", state["network"], "--read-only", "--tmpfs", "/tmp",
+                       "--cap-drop=ALL", "--security-opt=no-new-privileges",
+                       "--user", f"{os.getuid()}:{os.getgid()}",
+                       "--entrypoint", "/app/.venv/bin/python",
+                       "-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base",
+                       "-e", f"DATABASE_URL=postgres://postgres@recovery-db/{database}",
+                       "-e", "CONTENT_ROOT=/rehearsal-content", "-e", "FORGEJO_BASE_URL=",
+                       "-e", "PYTHONDONTWRITEBYTECODE=1",
+                       "-e", "BACKUP_RECEIPT_SHA256=" + sha(raw_receipt),
+                       "-v", str(work / "content") + ":/rehearsal-content",
+                       "-v", str(work / "plan.json") + ":/review/plan.json:ro",
+                       "-v", str(work / "journal") + ":/journal"]
+            for member in modules:
+                command += ["-v", str(work / member)
+                            + ":/app/coulomb_social/apps/spaces/" + member + ":ro"]
+            try:
+                result = subprocess.run(command + [image, "-c", code], capture_output=True,
+                                        timeout=240)
+            finally:
+                subprocess.run(["docker", "rm", "-f", name], capture_output=True,
+                               timeout=30, check=False)
+            if result.returncode:
+                fd, path = tempfile.mkstemp(prefix="csoc-journal-private-diagnostics-")
+                with os.fdopen(fd, "wb") as stream:
+                    stream.write(result.stdout + result.stderr)
+                print(json.dumps({"private_diagnostics": path}))
+                raise ValueError("Private application check failed")
+            rows = [json.loads(line) for line in result.stdout.splitlines()
+                    if line.startswith(b"{")]
+            if len(rows) != 1:
+                raise ValueError("Single result required")
+            return rows[0]
+
+        before = application("coulomb_social_restore", SNAPSHOT)
+        # A PostgreSQL template clone keeps every original row/sequence while
+        # isolating all subsequent commits from the retained recovery database.
+        run(["docker", "exec", state["database_container"], "createdb", "-U", "postgres",
+             "-T", "coulomb_social_restore", clone])
+        try:
+            verdict = application(clone, CHECKS)
+            after = application("coulomb_social_restore", SNAPSHOT)
+            # SNAPSHOT reads the disposable content copy; the source DB itself
+            # must be identical. Original files are checked separately below.
+            if before["tables_sha256"] != after["tables_sha256"]:
+                raise ValueError("Original recovery SQL changed")
+            if hashes(root) != original_hashes:
+                raise ValueError("Original recovery files changed")
+        finally:
+            run(["docker", "exec", state["database_container"], "dropdb", "-U", "postgres",
+                 "--force", clone])
+    proof = {
+        "schema": "coulomb.recovered-date-journal-rehearsal.v1", **verdict,
+        "candidate_image_id": image, "source_overlay_used": True,
+        "modules_sha256": {k: sha(v) for k, v in modules.items()},
+        "runner_sha256": sha(Path(__file__).read_bytes()),
+        "shared_runner_sha256": sha(Path(__file__).with_name(
+            "rehearse-source-date-backfill.py").read_bytes()),
+        "private_plan_sha256": sha(plan_bytes), "restore_receipt_sha256": sha(raw_receipt),
+        "network_internal": True, "published_ports": 0,
+        "original_recovered_sql_and_files_unchanged": True,
+        "disposable_sql_clone_removed": True,
+        "production_access": False, "production_writes": False,
+        "supervised_live_executor_prepared": False,
+    }
+    fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(fd, "w") as stream:
+        json.dump(proof, stream, indent=2)
+        stream.write("\n")
+    print(json.dumps(proof))
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (ValueError, OSError, KeyError, subprocess.SubprocessError):
+        raise SystemExit("Journal rehearsal refused; inspect private inputs/diagnostics locally.")

workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md

@@ -176,6 +176,18 @@ source-date-journal-rehearsal-2026-10-30.json. The supervised wrapper, verified
 writer fence, fresh backup binding, recovered-real-pair rehearsal and productive
 postflight remain required; this primitive does not establish those prerequisites.
 
+Recovered-pair follow-through: scripts/rehearse-source-date-journal.py now verifies
+the native internal/no-published-port database boundary and uses a fresh disposable
+PostgreSQL template clone plus copied files. All four real process deaths reconcile
+correctly on the exact private accepted-cohort plan. Every case compares all 13
+tables and 82 files; normal commit changes only the selected SQL date/index and
+replay refuses. The retained recovery SQL and files remain identical and the
+disposable clone is removed. Proof:
+docs/release/source-date-journal-real-pair-2026-10-30.json. Source overlays remain
+explicit; no new image, productive date correction or admission is inferred.
+The journal's real-pair rehearsal is complete; the external supervised wrapper,
+fresh verified backup/fence and productive postflight are still required.
+
 ## Completion
 
 Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.