From 80fe387dcc303c864ff9081997ff7b64ba8b9b8a Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 31 Oct 2026 18:07:34 +0100 Subject: [PATCH] Complete matching uncertain import checkpoints with retained evidence Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a192e9-c28b-7533-bf25-e2341364c22a --- .../apps/spaces/import_inspection.py | 36 ++-- .../apps/spaces/import_reconciliation.py | 72 +++++++ .../commands/reconcile_content_import.py | 42 ++++ .../corpus-import-inspection-2026-10-31.md | 5 + ...corpus-import-reconciliation-2026-10-31.md | 96 +++++++++ ...rt-reconciliation-postgres-2026-10-31.json | 31 +++ scripts/rehearse-import-inspection.py | 6 +- tests/test_import_reconciliation.py | 194 ++++++++++++++++++ ...-WP-0014-cohort-transfer-bubble-offramp.md | 17 ++ 9 files changed, 483 insertions(+), 16 deletions(-) create mode 100644 coulomb_social/apps/spaces/import_reconciliation.py create mode 100644 coulomb_social/apps/spaces/management/commands/reconcile_content_import.py create mode 100644 docs/release/corpus-import-reconciliation-2026-10-31.md create mode 100644 docs/release/corpus-import-reconciliation-postgres-2026-10-31.json create mode 100644 tests/test_import_reconciliation.py diff --git a/coulomb_social/apps/spaces/import_inspection.py b/coulomb_social/apps/spaces/import_inspection.py index 02b1ad4..dc2e8ef 100644 --- a/coulomb_social/apps/spaces/import_inspection.py +++ b/coulomb_social/apps/spaces/import_inspection.py @@ -129,7 +129,8 @@ def observe_unit(receipts, index, unit): return result -def inspect(receipts): +@contextmanager +def inspection_session(receipts): if connection.in_atomic_block or not connection.get_autocommit(): raise ValueError("Inspection requires an independent read transaction") private_path(receipts.path) @@ -158,18 +159,23 @@ def inspect(receipts): ) ): raise ValueError("Source changed during inspection") - staging = list((pageops.content_root() / "spaces").glob(".import-*")) - result = { - "schema": "coulomb.corpus-import-inspection.v1", - "binding": receipts.binding, - "checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(), - "spaces": first, - "unattributed_staging_entries": len(staging), - "read_only": True, - "external_writer_fence_verified": False, - "checkpoint_completed": False, - "retry_authorized": False, - "data_restored": False, - } - result["inspection_sha256"] = digest(result) + staging = list((pageops.content_root() / "spaces").glob(".import-*")) + result = { + "schema": "coulomb.corpus-import-inspection.v1", + "binding": receipts.binding, + "checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(), + "spaces": first, + "unattributed_staging_entries": len(staging), + "read_only": True, + "external_writer_fence_verified": False, + "checkpoint_completed": False, + "retry_authorized": False, + "data_restored": False, + } + result["inspection_sha256"] = digest(result) + yield result, data + + +def inspect(receipts): + with inspection_session(receipts) as (result, _): return result diff --git a/coulomb_social/apps/spaces/import_reconciliation.py b/coulomb_social/apps/spaces/import_reconciliation.py new file mode 100644 index 0000000..f8546df --- /dev/null +++ b/coulomb_social/apps/spaces/import_reconciliation.py @@ -0,0 +1,72 @@ +"""Explicit checkpoint completion for an exactly observed, already committed import. + +No SQL/content repair or automatic retry. The deployment owner still supplies +admission, a paired backup and an external writer fence before invoking this. +""" + +import hashlib +import re + +from . import pageops +from .import_inspection import inspection_session, observe_unit +from .import_receipts import digest, sync_tree, tree_digest +from .models import Space + + +def complete_matching_commit(receipts, *, index, expected_inspection): + if (type(index) is not int or not 0 <= index < len(receipts.plan) + or not isinstance(expected_inspection, str) + or not re.fullmatch(r"[a-f0-9]{64}", expected_inspection)): + raise ValueError("An exact inspection digest and valid unit index are required") + with inspection_session(receipts) as (inspection, data): + if inspection["inspection_sha256"] != expected_inspection: + raise ValueError("Inspection changed; retain evidence and review again") + observed = inspection["spaces"][index] + if observed["classification"] != "reserved_destination_matches_source": + raise ValueError("Only an exact reserved committed import can be completed") + if inspection["unattributed_staging_entries"]: + raise ValueError("Unattributed staging requires owner reconciliation first") + tree, _, slug, _ = receipts.plan[index] + prior_unit = data["spaces"][index] + row = Space.objects.filter(pk=prior_unit["space_id"]).values().get() + if digest([row]) != observed["space_sha256"]: + raise ValueError("Observed SQL identity changed") + # The original importer can die before fsync or its completed checkpoint. + # Make the matching published tree durable before acknowledging it. + sync_tree(pageops.space_dir(slug)) + if (observe_unit(receipts, index, prior_unit) != observed + or tree_digest(tree) != receipts.source_hashes[index] + or hashlib.sha256(receipts.path.read_bytes()).hexdigest() + != inspection["checkpoint_sha256"]): + raise ValueError("State changed before checkpoint completion") + data["spaces"][index] = { + "state": "completed", + "space_id": prior_unit["space_id"], + "files_sha256": observed["files_sha256"], + "space_sha256": digest(row), + "grants_sha256": observed["grants_sha256"], + "pages": len(list((tree / "pages").glob("*.md"))), + "assets": sum(p.is_file() for p in (tree / "assets").rglob("*")), + "reconciliation": { + "schema": "coulomb.corpus-matching-commit-reconciliation.v1", + "prior_unit": prior_unit, + "inspection": inspection, + }, + } + # Existing atomic replacement + fsync keeps the prior started checkpoint + # or the completed one with its evidence. Lost output requires readback; + # replay of the old inspection cannot acknowledge a different state. + receipts.save(data) + return { + "schema": "coulomb.corpus-matching-commit-return.v1", + "status": "matching_commit_checkpoint_completed", + "index": index, + "inspection_sha256": expected_inspection, + "checkpoint_sha256": hashlib.sha256(receipts.path.read_bytes()).hexdigest(), + "checkpoint_completed": True, + "sql_modified": False, + "content_modified": False, + "data_restored": False, + "external_writer_fence_verified": False, + "remaining_import_authorized": False, + } diff --git a/coulomb_social/apps/spaces/management/commands/reconcile_content_import.py b/coulomb_social/apps/spaces/management/commands/reconcile_content_import.py new file mode 100644 index 0000000..c17efcc --- /dev/null +++ b/coulomb_social/apps/spaces/management/commands/reconcile_content_import.py @@ -0,0 +1,42 @@ +"""Record one matching committed import; never restore or delete application data.""" + +import json +from pathlib import Path + +from django.core.management.base import BaseCommand, CommandError + +from coulomb_social.apps.spaces.import_receipts import CorpusReceipts +from coulomb_social.apps.spaces.import_reconciliation import complete_matching_commit +from coulomb_social.apps.spaces.management.commands.import_content_corpus import ( + Command as ImportCommand, +) + + +class Command(BaseCommand): + help = "Complete one exact matching import checkpoint after owner reconciliation review" + + def add_arguments(self, parser): + parser.add_argument("manifest") + parser.add_argument("--receipt-dir", required=True) + parser.add_argument("--unit-index", type=int, required=True) + parser.add_argument("--expect-inspection", required=True) + parser.add_argument("--complete-matching-commit", action="store_true", required=True) + + def handle(self, *args, **options): + if not options["complete_matching_commit"]: + raise CommandError("Explicit matching-commit completion is required") + try: + manifest = Path(options["manifest"]) + plan = ImportCommand().preflight(manifest, replace=False, check_existing=False) + receipts = CorpusReceipts(options["receipt_dir"], manifest, plan) + result = complete_matching_commit( + receipts, index=options["unit_index"], + expected_inspection=options["expect_inspection"], + ) + except Exception as exc: + # Failure can follow durable checkpoint replacement. Never advise replay + # or disclose source paths, identities or SQL values in the error. + raise CommandError( + "Reconciliation stopped; inspect the retained checkpoint before further action" + ) from exc + self.stdout.write(json.dumps(result, sort_keys=True)) diff --git a/docs/release/corpus-import-inspection-2026-10-31.md b/docs/release/corpus-import-inspection-2026-10-31.md index 2a2c471..dc5c1f1 100644 --- a/docs/release/corpus-import-inspection-2026-10-31.md +++ b/docs/release/corpus-import-inspection-2026-10-31.md @@ -57,6 +57,11 @@ authorized and no data was restored. The inspector does not convert `started` to uncertain state with its retained backup, writer fence and reviewed operation. Do not remove the checkpoint or use replacement import to bypass uncertainty. +The subsequent [matching-commit procedure](corpus-import-reconciliation-2026-10-31.md) +adds a separate, explicit checkpoint operation for one exactly matching reserved +commit. Inspection itself remains read-only. Partial, changed, unbound and +unattributed-staging states still require owner reconciliation. + ## Verification and release boundary Fifteen new cases cover matching committed state without receipt, partial stores, diff --git a/docs/release/corpus-import-reconciliation-2026-10-31.md b/docs/release/corpus-import-reconciliation-2026-10-31.md new file mode 100644 index 0000000..af176f1 --- /dev/null +++ b/docs/release/corpus-import-reconciliation-2026-10-31.md @@ -0,0 +1,96 @@ +# Completing a matching uncertain import checkpoint + +CSOC-WP-0014-T07 now has an explicit checkpoint-completion operation for an +already committed import whose reserved identity and complete source-derived +state match. This is source preparation, not a production reconciliation or +admission of any cohort. The existing native executor and frozen release packets +are unchanged. + +## Owner procedure + +Before any native use, the deployment owner must identify the exact original +attempt, stop and independently verify application and external writers, retain +the original checkpoint and a coherent paired recovery point, and confirm the +source/tenant/identity/grant mapping was admitted. Neither of the commands below +verifies an external writer fence or provider backup. The current production +image does not contain these new commands; owner integration and release remain +required. + +1. Inspect the original manifest and existing private receipt directory using + `inspect_content_corpus`. Keep the result privately with the operation evidence. +2. Review the selected zero-based unit index, reserved identity, source binding, + complete file/grant/metadata comparison and the original operation's outcome. + Only `reserved_destination_matches_source`, with no unattributed `.import-*` + entries, is eligible for the operation below. A match describes current state; + it is not proof that no intervening edit ever occurred. +3. Supply the exact inspection SHA-256 and explicit completion action: + + ```sh + python manage.py reconcile_content_import /private/reviewed/manifest.json \ + --receipt-dir /private/import-attempt \ + --unit-index 0 \ + --expect-inspection \ + --complete-matching-commit + ``` + + The command reacquires the existing receipt/content locks and a consistent + read-only SQL transaction, recomputes the whole inspection, and refuses a + changed digest. It fsyncs the matching destination, rechecks the selected unit, + source and checkpoint bytes, then atomically replaces only the checkpoint. + The completed unit retains its original reserved identity, ordinary importer + fingerprints/counts, prior started unit and full hash/count inspection evidence. + SQL rows, grants and content bytes are not modified. +4. Independently read the checkpoint and inspect again. A lost response or failure + may occur after replacement; do not infer failure to complete from the absence + of terminal output. The previous inspection cannot be replayed against an + already completed checkpoint. An interrupted temporary checkpoint file is + retained as evidence and is not automatically deleted or adopted. +5. If continuation is separately admitted, the existing receipted importer skips + the completed unit and preserves later edits and grants. Completion of this + checkpoint does not authorize importing the remaining corpus. + +## Other states remain explicit owner work + +| Observed state | Required disposition | +| --- | --- | +| Reserved identity differs from source | Preserve later edits/grants; reconcile against the original operation and paired point. This command never overwrites them. | +| SQL only, files only, or neither | Retain the uncertain checkpoint and inspect private staging, transaction outcome and the paired recovery point. No automatic reset to pending, retry, restore or deletion is implemented here. | +| Foreign identity or legacy unbound started receipt | Resolve ownership and identity from independent evidence. Do not adopt a row because its slug matches. | +| Unattributed staging entries | Establish their exact attempt/ownership and disposition separately. Matching published bytes do not authorize cleaning up those entries. | +| Pending or already completed unit | No reconciliation mutation is permitted. Completed identities remain historical boundaries and may have valid later edits. | + +The helper uses cooperating file locks and PostgreSQL repeatable-read/read-only +transactions. These are not an external writer fence. The return explicitly +reports that limit, no SQL/content changes, no restore, and no authorization to +continue imports. The owner must verify native prerequisites before execution; +passing synthetic tests does not supply them. + +## Validation + +The focused suite covers exact completion and evidence retention, unchanged SQL/ +content, preservation of later edits on resumed import, stale evidence, changed +source/mapping/checkpoint, partial or foreign stores, legacy receipts, unattributed +staging, invalid selection, sync failure and drift during fsync. Real child-process +exits immediately before and after checkpoint replacement leave either the prior +started checkpoint or the completed checkpoint with its reconciliation evidence. + +Full `make check`: **648 passed, 69 existing warnings, zero failures/errors/skips +in 150.38s**, including required browser journeys, rerun after the interrupted +session. Nineteen new reconciliation +cases pass; the focused corpus checkpoint/inspection/reconciliation selection +passes all 52 cases. Log: `/tmp/csoc-import-reconciliation-check-20261031.log`. + +The isolated PostgreSQL 16.14 rehearsal passes with 243 actual read-only and +repeatable-read transaction checks. It uses synthetic data, explicit source and +locked dependency overlays, an internal Docker network and no published ports. +Its disposable containers, volumes and network were removed. Source-bound receipt: +`corpus-import-reconciliation-postgres-2026-10-31.json`. This is not packaged +candidate or production proof. + +After resuming, all 12 source hashes in that PostgreSQL receipt were independently +checked against the current files and matched. State Hub consistency passed and +the T07 description was independently read back with the reconciliation evidence. + +T07 remains in progress for the other reconciliation states, supervised owner +integration, source/grant admission, native release/import/postflight and +productive verdicts. diff --git a/docs/release/corpus-import-reconciliation-postgres-2026-10-31.json b/docs/release/corpus-import-reconciliation-postgres-2026-10-31.json new file mode 100644 index 0000000..a5362b5 --- /dev/null +++ b/docs/release/corpus-import-reconciliation-postgres-2026-10-31.json @@ -0,0 +1,31 @@ +{ + "schema": "coulomb.corpus-inspection-rehearsal.v1", + "proof": { + "status": "passed", + "database_vendor": "postgresql", + "server_version": 160014, + "inspection_transaction_mode_checks": 243, + "source_overlay": true, + "locked_development_dependencies": true, + "production_data_used": false + }, + "app_dependency_image": "sha256:522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed", + "postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "internal_network": true, + "published_ports": false, + "disposable_resources_removed": true, + "source_sha256": { + "scripts/rehearse-import-inspection.py": "34a95a57df6b93132a45c70f95b56b8ee9ef5be8677efa6f27f48c41bdc46f17", + "uv.lock": "e8ebb6fc4b5abd1ac80f1c9012f1983642d0528a8a213cfcc394261606fad887", + "coulomb_social/apps/spaces/import_tree.py": "f232bde0d1741668ff28a6cb24306b7f67d8bf9fa3838507a8db4f4e80361784", + "coulomb_social/apps/spaces/import_receipts.py": "670a5eb48613df47bf8fd0cf7d9306f7d2e992d46dff69680e6e7998f64f9cb2", + "coulomb_social/apps/spaces/import_inspection.py": "908a62bd31e0a6cfab3cc94f45c671ff4f1ec403035434790a29adfc5275f5ab", + "coulomb_social/apps/spaces/import_reconciliation.py": "75abc50ec1c6e15a12534eb39edc5ad3fa4128c37a354d4cea2bf6c37f745771", + "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py": "b083e39aa766c8ba268788fae98507780604d40832e119db1cd9f7c10f1aa212", + "coulomb_social/apps/spaces/management/commands/reconcile_content_import.py": "2e25d94ab3a9659e38a3ea4c8b6c2e87c710216ab5834a248fccdd00da9fdbb9", + "tests/test_import_corpus.py": "5a0670c408740de86ce601cd9d2a0a12cf0aaa5afda416867605180c568a1c5f", + "tests/test_import_receipts.py": "134e8901991dc493333b7543bbd6ff1f1e7b7ca3d5cf9d7849d7c45212e20aab", + "tests/test_import_inspection.py": "890e783eec30dd1b9e93ad7c3ab3173b57174a096938520865eaaf420bc605fb", + "tests/test_import_reconciliation.py": "89d7161715a2c298c30c6ae38f8d9647688e2ea261d2913a8a9b8969202ac298" + } +} diff --git a/scripts/rehearse-import-inspection.py b/scripts/rehearse-import-inspection.py index e0647bd..98e4d9e 100644 --- a/scripts/rehearse-import-inspection.py +++ b/scripts/rehearse-import-inspection.py @@ -39,7 +39,8 @@ def observe(*args): import_inspection.observe_unit=observe import pytest code=pytest.main(['-q','-p','no:cacheprovider','/app/tests/test_import_receipts.py', - '/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py']) + '/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py', + '/app/tests/test_import_reconciliation.py']) assert code==0 and checks print(json.dumps({'status':'passed','database_vendor':'postgresql','server_version':version, 'inspection_transaction_mode_checks':len(checks),'source_overlay':True, @@ -124,13 +125,16 @@ def main(): ROOT / "coulomb_social/apps/spaces/import_tree.py", ROOT / "coulomb_social/apps/spaces/import_receipts.py", ROOT / "coulomb_social/apps/spaces/import_inspection.py", + ROOT / "coulomb_social/apps/spaces/import_reconciliation.py", ROOT / "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py", + ROOT / "coulomb_social/apps/spaces/management/commands/reconcile_content_import.py", *( ROOT / "tests" / name for name in ( "test_import_corpus.py", "test_import_receipts.py", "test_import_inspection.py", + "test_import_reconciliation.py", ) ), ] diff --git a/tests/test_import_reconciliation.py b/tests/test_import_reconciliation.py new file mode 100644 index 0000000..6a158b2 --- /dev/null +++ b/tests/test_import_reconciliation.py @@ -0,0 +1,194 @@ +"""Exact checkpoint reconciliation preserves stores and survives process death.""" + +import io +import json +import os +import subprocess +import sys + +import pytest +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import CommandError + +from coulomb_social.apps.spaces import import_reconciliation, pageops +from coulomb_social.apps.spaces.models import Space, SpaceMembership +from tests import test_import_corpus +from tests.test_import_inspection import inspect, uncertain + +pytestmark = pytest.mark.django_db(transaction=True) +corpus = test_import_corpus.corpus + + +def complete(manifest, directory, inspection, index=0): + output = io.StringIO() + call_command( + "reconcile_content_import", str(manifest), receipt_dir=str(directory), + unit_index=index, expect_inspection=inspection["inspection_sha256"], + complete_matching_commit=True, stdout=output, + ) + return json.loads(output.getvalue()) + + +def stores(): + return ( + list(Space.objects.order_by("pk").values()), + list(SpaceMembership.objects.order_by("pk").values()), + {str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()}, + ) + + +def test_matching_commit_completes_only_checkpoint_then_resume_preserves_later_edits( + corpus, tmp_path, monkeypatch +): + manifest, _, member = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + before = stores() + prior = json.loads((directory / "checkpoint.json").read_bytes()) + report = inspect(manifest, directory) + result = complete(manifest, directory, report) + checkpoint = json.loads((directory / "checkpoint.json").read_bytes()) + assert stores() == before + assert result["checkpoint_completed"] and not result["remaining_import_authorized"] + assert not result["external_writer_fence_verified"] + assert not result["sql_modified"] and not result["content_modified"] + assert checkpoint["spaces"][1] == prior["spaces"][1] + reconciled = checkpoint["spaces"][0] + assert reconciled["reconciliation"]["prior_unit"] == prior["spaces"][0] + assert reconciled["reconciliation"]["inspection"] == report + assert (directory / "checkpoint.json").stat().st_mode & 0o777 == 0o600 + saved = (directory / "checkpoint.json").read_bytes() + with pytest.raises(CommandError): + complete(manifest, directory, report) + assert (directory / "checkpoint.json").read_bytes() == saved + page = pageops.space_dir("one") / "pages/index.md" + page.write_text("later private edit") + Space.objects.filter(slug="one").update(title="later title") + SpaceMembership.objects.filter(space__slug="one", member=member).update(role="viewer") + output = io.StringIO() + call_command("import_content_corpus", str(manifest), receipt_dir=str(directory), stdout=output) + assert "Imported 1 space(s); skipped 1" in output.getvalue() + assert page.read_text() == "later private edit" + assert Space.objects.get(slug="one").title == "later title" + assert SpaceMembership.objects.get(space__slug="one").role == "viewer" + + +@pytest.mark.parametrize("change", [ + "body", "grant", "metadata", "source", "manifest", "checkpoint", "foreign", + "legacy", "no_sql", "no_files", "staging", "wrong_digest", "pending", "bad_index", +]) +def test_changed_or_nonmatching_return_never_modifies_stores_or_checkpoint( + corpus, tmp_path, monkeypatch, change +): + manifest, entries, member = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + report = inspect(manifest, directory) + checkpoint = directory / "checkpoint.json" + if change == "body": + (pageops.space_dir("one") / "pages/index.md").write_text("later content") + elif change == "grant": + SpaceMembership.objects.filter(member=member).update(role="viewer") + elif change == "metadata": + Space.objects.filter(slug="one").update(description="later metadata") + elif change == "source": + (manifest.parent / entries[0]["tree"] / "pages/index.md").write_text("source drift") + elif change == "manifest": + manifest.write_text(manifest.read_text() + "\n") + elif change in {"checkpoint", "legacy"}: + data = json.loads(checkpoint.read_bytes()) + if change == "legacy": + data["spaces"][0].pop("space_id") + checkpoint.write_text(json.dumps(data, indent=2)) + elif change in {"no_sql", "foreign"}: + Space.objects.get(slug="one").delete() + if change == "foreign": + Space.objects.create(tenant_id="tenant:a", slug="one", title="foreign") + elif change == "no_files": + import shutil + + shutil.rmtree(pageops.space_dir("one")) + elif change == "staging": + (pageops.content_root() / "spaces/.import-unattributed").mkdir() + elif change == "wrong_digest": + report["inspection_sha256"] = "0" * 64 + # Test both stale inspection and fresh inspection of nonmatching state. + if change in {"body", "grant", "metadata", "legacy", "foreign", "no_sql", "no_files", + "staging"}: + report = inspect(manifest, directory) + before = stores() + saved = checkpoint.read_bytes() + index = 1 if change == "pending" else -1 if change == "bad_index" else 0 + with pytest.raises(CommandError): + complete(manifest, directory, report, index=index) + assert stores() == before and checkpoint.read_bytes() == saved + + +@pytest.mark.parametrize("failure", ["fsync", "drift"]) +def test_failed_sync_or_late_drift_leaves_uncertain_checkpoint( + corpus, tmp_path, monkeypatch, failure +): + manifest, _, _ = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + report = inspect(manifest, directory) + saved = (directory / "checkpoint.json").read_bytes() + + def sync(tree): + if failure == "fsync": + raise OSError("synthetic disk failure") + (tree / "pages/index.md").write_text("external writer") + + monkeypatch.setattr(import_reconciliation, "sync_tree", sync) + with pytest.raises(CommandError): + complete(manifest, directory, report) + assert (directory / "checkpoint.json").read_bytes() == saved + + +@pytest.mark.parametrize("phase", ["before_replace", "after_replace"]) +def test_process_exit_retains_started_or_completed_checkpoint_with_evidence( + corpus, tmp_path, monkeypatch, phase +): + manifest, _, _ = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + report = inspect(manifest, directory) + before = stores() + code = ''' +import json,os,sys +from django.conf import settings +settings.DATABASES['default']=json.loads(sys.argv[1]) +settings.CONTENT_ROOT=sys.argv[2] +import django +django.setup() +from django.core.management import call_command +from coulomb_social.apps.spaces import import_receipts +original=import_receipts.os.replace +def replace(*args): + if sys.argv[6]=='before_replace':os._exit(74) + original(*args) + os._exit(74) +import_receipts.os.replace=replace +call_command('reconcile_content_import',sys.argv[3],receipt_dir=sys.argv[4], + unit_index=0,expect_inspection=sys.argv[5],complete_matching_commit=True) +''' + result = subprocess.run( + [sys.executable, "-c", code, json.dumps(settings.DATABASES["default"], default=str), + str(pageops.content_root()), str(manifest), str(directory), + report["inspection_sha256"], phase], + env={**os.environ, "DJANGO_SETTINGS_MODULE": "coulomb_social.settings.test"}, + capture_output=True, timeout=30, + ) + assert result.returncode == 74, result.stderr.decode() + assert stores() == before + unit = json.loads((directory / "checkpoint.json").read_bytes())["spaces"][0] + if phase == "before_replace": + assert unit["state"] == "started" + # A fresh explicit review can complete; no importer automatically retries. + complete(manifest, directory, inspect(manifest, directory)) + else: + assert unit["state"] == "completed" + assert unit["reconciliation"]["inspection"] == report + with pytest.raises(CommandError): + complete(manifest, directory, report) diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 27d3363..07d0ad5 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -407,6 +407,23 @@ T07 remains progress: concrete state reconciliation, supervised owner integratio source/grant admission, native release/import/postflight and productive verdicts are still required. No live cohort or approved executor was changed. +2026-10-31 matching-commit reconciliation: added an explicit operation that +completes one uncertain checkpoint only when its reserved identity, source files, +metadata and mapped grants match the exact reviewed inspection. It reacquires +existing locks, uses a read-only SQL snapshot, makes destination files durable, +rechecks state and atomically retains the original started unit and inspection +with the completed checkpoint. It changes no SQL/content and refuses partial, +foreign, changed, unbound or unattributed-staging states. Real process exits on +both sides of replacement retain inspectable evidence; resumed imports preserve +later edits/grants. Full make check passes 648 tests with required browsers and +zero failures/errors/skips. Isolated PostgreSQL 16.14 also passes, with 243 +read-only/repeatable-read transaction checks and disposable-resource cleanup. +Review: [matching-commit reconciliation](../docs/release/corpus-import-reconciliation-2026-10-31.md). +This advances preparation within T07, which remains progress. Other uncertain +states, owner integration and verified external fencing/paired backup, cohort +admission, native release/import and productive verdicts remain open. No live +checkpoint, source cohort, grant or frozen execution packet was changed. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.