Complete matching uncertain import checkpoints with retained evidence

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a192e9-c28b-7533-bf25-e2341364c22a
This commit is contained in:
tegwick 2026-10-31 18:07:34 +01:00
parent b800daedbc
commit 80fe387dcc
9 changed files with 483 additions and 16 deletions

View file

@ -129,7 +129,8 @@ def observe_unit(receipts, index, unit):
return result
def inspect(receipts):
@contextmanager
def inspection_session(receipts):
if connection.in_atomic_block or not connection.get_autocommit():
raise ValueError("Inspection requires an independent read transaction")
private_path(receipts.path)
@ -158,18 +159,23 @@ def inspect(receipts):
)
):
raise ValueError("Source changed during inspection")
staging = list((pageops.content_root() / "spaces").glob(".import-*"))
result = {
"schema": "coulomb.corpus-import-inspection.v1",
"binding": receipts.binding,
"checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(),
"spaces": first,
"unattributed_staging_entries": len(staging),
"read_only": True,
"external_writer_fence_verified": False,
"checkpoint_completed": False,
"retry_authorized": False,
"data_restored": False,
}
result["inspection_sha256"] = digest(result)
staging = list((pageops.content_root() / "spaces").glob(".import-*"))
result = {
"schema": "coulomb.corpus-import-inspection.v1",
"binding": receipts.binding,
"checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(),
"spaces": first,
"unattributed_staging_entries": len(staging),
"read_only": True,
"external_writer_fence_verified": False,
"checkpoint_completed": False,
"retry_authorized": False,
"data_restored": False,
}
result["inspection_sha256"] = digest(result)
yield result, data
def inspect(receipts):
with inspection_session(receipts) as (result, _):
return result

View file

@ -0,0 +1,72 @@
"""Explicit checkpoint completion for an exactly observed, already committed import.
No SQL/content repair or automatic retry. The deployment owner still supplies
admission, a paired backup and an external writer fence before invoking this.
"""
import hashlib
import re
from . import pageops
from .import_inspection import inspection_session, observe_unit
from .import_receipts import digest, sync_tree, tree_digest
from .models import Space
def complete_matching_commit(receipts, *, index, expected_inspection):
if (type(index) is not int or not 0 <= index < len(receipts.plan)
or not isinstance(expected_inspection, str)
or not re.fullmatch(r"[a-f0-9]{64}", expected_inspection)):
raise ValueError("An exact inspection digest and valid unit index are required")
with inspection_session(receipts) as (inspection, data):
if inspection["inspection_sha256"] != expected_inspection:
raise ValueError("Inspection changed; retain evidence and review again")
observed = inspection["spaces"][index]
if observed["classification"] != "reserved_destination_matches_source":
raise ValueError("Only an exact reserved committed import can be completed")
if inspection["unattributed_staging_entries"]:
raise ValueError("Unattributed staging requires owner reconciliation first")
tree, _, slug, _ = receipts.plan[index]
prior_unit = data["spaces"][index]
row = Space.objects.filter(pk=prior_unit["space_id"]).values().get()
if digest([row]) != observed["space_sha256"]:
raise ValueError("Observed SQL identity changed")
# The original importer can die before fsync or its completed checkpoint.
# Make the matching published tree durable before acknowledging it.
sync_tree(pageops.space_dir(slug))
if (observe_unit(receipts, index, prior_unit) != observed
or tree_digest(tree) != receipts.source_hashes[index]
or hashlib.sha256(receipts.path.read_bytes()).hexdigest()
!= inspection["checkpoint_sha256"]):
raise ValueError("State changed before checkpoint completion")
data["spaces"][index] = {
"state": "completed",
"space_id": prior_unit["space_id"],
"files_sha256": observed["files_sha256"],
"space_sha256": digest(row),
"grants_sha256": observed["grants_sha256"],
"pages": len(list((tree / "pages").glob("*.md"))),
"assets": sum(p.is_file() for p in (tree / "assets").rglob("*")),
"reconciliation": {
"schema": "coulomb.corpus-matching-commit-reconciliation.v1",
"prior_unit": prior_unit,
"inspection": inspection,
},
}
# Existing atomic replacement + fsync keeps the prior started checkpoint
# or the completed one with its evidence. Lost output requires readback;
# replay of the old inspection cannot acknowledge a different state.
receipts.save(data)
return {
"schema": "coulomb.corpus-matching-commit-return.v1",
"status": "matching_commit_checkpoint_completed",
"index": index,
"inspection_sha256": expected_inspection,
"checkpoint_sha256": hashlib.sha256(receipts.path.read_bytes()).hexdigest(),
"checkpoint_completed": True,
"sql_modified": False,
"content_modified": False,
"data_restored": False,
"external_writer_fence_verified": False,
"remaining_import_authorized": False,
}

View file

@ -0,0 +1,42 @@
"""Record one matching committed import; never restore or delete application data."""
import json
from pathlib import Path
from django.core.management.base import BaseCommand, CommandError
from coulomb_social.apps.spaces.import_receipts import CorpusReceipts
from coulomb_social.apps.spaces.import_reconciliation import complete_matching_commit
from coulomb_social.apps.spaces.management.commands.import_content_corpus import (
Command as ImportCommand,
)
class Command(BaseCommand):
help = "Complete one exact matching import checkpoint after owner reconciliation review"
def add_arguments(self, parser):
parser.add_argument("manifest")
parser.add_argument("--receipt-dir", required=True)
parser.add_argument("--unit-index", type=int, required=True)
parser.add_argument("--expect-inspection", required=True)
parser.add_argument("--complete-matching-commit", action="store_true", required=True)
def handle(self, *args, **options):
if not options["complete_matching_commit"]:
raise CommandError("Explicit matching-commit completion is required")
try:
manifest = Path(options["manifest"])
plan = ImportCommand().preflight(manifest, replace=False, check_existing=False)
receipts = CorpusReceipts(options["receipt_dir"], manifest, plan)
result = complete_matching_commit(
receipts, index=options["unit_index"],
expected_inspection=options["expect_inspection"],
)
except Exception as exc:
# Failure can follow durable checkpoint replacement. Never advise replay
# or disclose source paths, identities or SQL values in the error.
raise CommandError(
"Reconciliation stopped; inspect the retained checkpoint before further action"
) from exc
self.stdout.write(json.dumps(result, sort_keys=True))

View file

@ -57,6 +57,11 @@ authorized and no data was restored. The inspector does not convert `started` to
uncertain state with its retained backup, writer fence and reviewed operation.
Do not remove the checkpoint or use replacement import to bypass uncertainty.
The subsequent [matching-commit procedure](corpus-import-reconciliation-2026-10-31.md)
adds a separate, explicit checkpoint operation for one exactly matching reserved
commit. Inspection itself remains read-only. Partial, changed, unbound and
unattributed-staging states still require owner reconciliation.
## Verification and release boundary
Fifteen new cases cover matching committed state without receipt, partial stores,

View file

@ -0,0 +1,96 @@
# Completing a matching uncertain import checkpoint
CSOC-WP-0014-T07 now has an explicit checkpoint-completion operation for an
already committed import whose reserved identity and complete source-derived
state match. This is source preparation, not a production reconciliation or
admission of any cohort. The existing native executor and frozen release packets
are unchanged.
## Owner procedure
Before any native use, the deployment owner must identify the exact original
attempt, stop and independently verify application and external writers, retain
the original checkpoint and a coherent paired recovery point, and confirm the
source/tenant/identity/grant mapping was admitted. Neither of the commands below
verifies an external writer fence or provider backup. The current production
image does not contain these new commands; owner integration and release remain
required.
1. Inspect the original manifest and existing private receipt directory using
`inspect_content_corpus`. Keep the result privately with the operation evidence.
2. Review the selected zero-based unit index, reserved identity, source binding,
complete file/grant/metadata comparison and the original operation's outcome.
Only `reserved_destination_matches_source`, with no unattributed `.import-*`
entries, is eligible for the operation below. A match describes current state;
it is not proof that no intervening edit ever occurred.
3. Supply the exact inspection SHA-256 and explicit completion action:
```sh
python manage.py reconcile_content_import /private/reviewed/manifest.json \
--receipt-dir /private/import-attempt \
--unit-index 0 \
--expect-inspection <inspection_sha256> \
--complete-matching-commit
```
The command reacquires the existing receipt/content locks and a consistent
read-only SQL transaction, recomputes the whole inspection, and refuses a
changed digest. It fsyncs the matching destination, rechecks the selected unit,
source and checkpoint bytes, then atomically replaces only the checkpoint.
The completed unit retains its original reserved identity, ordinary importer
fingerprints/counts, prior started unit and full hash/count inspection evidence.
SQL rows, grants and content bytes are not modified.
4. Independently read the checkpoint and inspect again. A lost response or failure
may occur after replacement; do not infer failure to complete from the absence
of terminal output. The previous inspection cannot be replayed against an
already completed checkpoint. An interrupted temporary checkpoint file is
retained as evidence and is not automatically deleted or adopted.
5. If continuation is separately admitted, the existing receipted importer skips
the completed unit and preserves later edits and grants. Completion of this
checkpoint does not authorize importing the remaining corpus.
## Other states remain explicit owner work
| Observed state | Required disposition |
| --- | --- |
| Reserved identity differs from source | Preserve later edits/grants; reconcile against the original operation and paired point. This command never overwrites them. |
| SQL only, files only, or neither | Retain the uncertain checkpoint and inspect private staging, transaction outcome and the paired recovery point. No automatic reset to pending, retry, restore or deletion is implemented here. |
| Foreign identity or legacy unbound started receipt | Resolve ownership and identity from independent evidence. Do not adopt a row because its slug matches. |
| Unattributed staging entries | Establish their exact attempt/ownership and disposition separately. Matching published bytes do not authorize cleaning up those entries. |
| Pending or already completed unit | No reconciliation mutation is permitted. Completed identities remain historical boundaries and may have valid later edits. |
The helper uses cooperating file locks and PostgreSQL repeatable-read/read-only
transactions. These are not an external writer fence. The return explicitly
reports that limit, no SQL/content changes, no restore, and no authorization to
continue imports. The owner must verify native prerequisites before execution;
passing synthetic tests does not supply them.
## Validation
The focused suite covers exact completion and evidence retention, unchanged SQL/
content, preservation of later edits on resumed import, stale evidence, changed
source/mapping/checkpoint, partial or foreign stores, legacy receipts, unattributed
staging, invalid selection, sync failure and drift during fsync. Real child-process
exits immediately before and after checkpoint replacement leave either the prior
started checkpoint or the completed checkpoint with its reconciliation evidence.
Full `make check`: **648 passed, 69 existing warnings, zero failures/errors/skips
in 150.38s**, including required browser journeys, rerun after the interrupted
session. Nineteen new reconciliation
cases pass; the focused corpus checkpoint/inspection/reconciliation selection
passes all 52 cases. Log: `/tmp/csoc-import-reconciliation-check-20261031.log`.
The isolated PostgreSQL 16.14 rehearsal passes with 243 actual read-only and
repeatable-read transaction checks. It uses synthetic data, explicit source and
locked dependency overlays, an internal Docker network and no published ports.
Its disposable containers, volumes and network were removed. Source-bound receipt:
`corpus-import-reconciliation-postgres-2026-10-31.json`. This is not packaged
candidate or production proof.
After resuming, all 12 source hashes in that PostgreSQL receipt were independently
checked against the current files and matched. State Hub consistency passed and
the T07 description was independently read back with the reconciliation evidence.
T07 remains in progress for the other reconciliation states, supervised owner
integration, source/grant admission, native release/import/postflight and
productive verdicts.

View file

@ -0,0 +1,31 @@
{
"schema": "coulomb.corpus-inspection-rehearsal.v1",
"proof": {
"status": "passed",
"database_vendor": "postgresql",
"server_version": 160014,
"inspection_transaction_mode_checks": 243,
"source_overlay": true,
"locked_development_dependencies": true,
"production_data_used": false
},
"app_dependency_image": "sha256:522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed",
"postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70",
"internal_network": true,
"published_ports": false,
"disposable_resources_removed": true,
"source_sha256": {
"scripts/rehearse-import-inspection.py": "34a95a57df6b93132a45c70f95b56b8ee9ef5be8677efa6f27f48c41bdc46f17",
"uv.lock": "e8ebb6fc4b5abd1ac80f1c9012f1983642d0528a8a213cfcc394261606fad887",
"coulomb_social/apps/spaces/import_tree.py": "f232bde0d1741668ff28a6cb24306b7f67d8bf9fa3838507a8db4f4e80361784",
"coulomb_social/apps/spaces/import_receipts.py": "670a5eb48613df47bf8fd0cf7d9306f7d2e992d46dff69680e6e7998f64f9cb2",
"coulomb_social/apps/spaces/import_inspection.py": "908a62bd31e0a6cfab3cc94f45c671ff4f1ec403035434790a29adfc5275f5ab",
"coulomb_social/apps/spaces/import_reconciliation.py": "75abc50ec1c6e15a12534eb39edc5ad3fa4128c37a354d4cea2bf6c37f745771",
"coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py": "b083e39aa766c8ba268788fae98507780604d40832e119db1cd9f7c10f1aa212",
"coulomb_social/apps/spaces/management/commands/reconcile_content_import.py": "2e25d94ab3a9659e38a3ea4c8b6c2e87c710216ab5834a248fccdd00da9fdbb9",
"tests/test_import_corpus.py": "5a0670c408740de86ce601cd9d2a0a12cf0aaa5afda416867605180c568a1c5f",
"tests/test_import_receipts.py": "134e8901991dc493333b7543bbd6ff1f1e7b7ca3d5cf9d7849d7c45212e20aab",
"tests/test_import_inspection.py": "890e783eec30dd1b9e93ad7c3ab3173b57174a096938520865eaaf420bc605fb",
"tests/test_import_reconciliation.py": "89d7161715a2c298c30c6ae38f8d9647688e2ea261d2913a8a9b8969202ac298"
}
}

View file

@ -39,7 +39,8 @@ def observe(*args):
import_inspection.observe_unit=observe
import pytest
code=pytest.main(['-q','-p','no:cacheprovider','/app/tests/test_import_receipts.py',
'/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py'])
'/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py',
'/app/tests/test_import_reconciliation.py'])
assert code==0 and checks
print(json.dumps({'status':'passed','database_vendor':'postgresql','server_version':version,
'inspection_transaction_mode_checks':len(checks),'source_overlay':True,
@ -124,13 +125,16 @@ def main():
ROOT / "coulomb_social/apps/spaces/import_tree.py",
ROOT / "coulomb_social/apps/spaces/import_receipts.py",
ROOT / "coulomb_social/apps/spaces/import_inspection.py",
ROOT / "coulomb_social/apps/spaces/import_reconciliation.py",
ROOT / "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py",
ROOT / "coulomb_social/apps/spaces/management/commands/reconcile_content_import.py",
*(
ROOT / "tests" / name
for name in (
"test_import_corpus.py",
"test_import_receipts.py",
"test_import_inspection.py",
"test_import_reconciliation.py",
)
),
]

View file

@ -0,0 +1,194 @@
"""Exact checkpoint reconciliation preserves stores and survives process death."""
import io
import json
import os
import subprocess
import sys
import pytest
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from coulomb_social.apps.spaces import import_reconciliation, pageops
from coulomb_social.apps.spaces.models import Space, SpaceMembership
from tests import test_import_corpus
from tests.test_import_inspection import inspect, uncertain
pytestmark = pytest.mark.django_db(transaction=True)
corpus = test_import_corpus.corpus
def complete(manifest, directory, inspection, index=0):
output = io.StringIO()
call_command(
"reconcile_content_import", str(manifest), receipt_dir=str(directory),
unit_index=index, expect_inspection=inspection["inspection_sha256"],
complete_matching_commit=True, stdout=output,
)
return json.loads(output.getvalue())
def stores():
return (
list(Space.objects.order_by("pk").values()),
list(SpaceMembership.objects.order_by("pk").values()),
{str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()},
)
def test_matching_commit_completes_only_checkpoint_then_resume_preserves_later_edits(
corpus, tmp_path, monkeypatch
):
manifest, _, member = corpus
directory = tmp_path / "receipts"
uncertain(manifest, directory, monkeypatch)
before = stores()
prior = json.loads((directory / "checkpoint.json").read_bytes())
report = inspect(manifest, directory)
result = complete(manifest, directory, report)
checkpoint = json.loads((directory / "checkpoint.json").read_bytes())
assert stores() == before
assert result["checkpoint_completed"] and not result["remaining_import_authorized"]
assert not result["external_writer_fence_verified"]
assert not result["sql_modified"] and not result["content_modified"]
assert checkpoint["spaces"][1] == prior["spaces"][1]
reconciled = checkpoint["spaces"][0]
assert reconciled["reconciliation"]["prior_unit"] == prior["spaces"][0]
assert reconciled["reconciliation"]["inspection"] == report
assert (directory / "checkpoint.json").stat().st_mode & 0o777 == 0o600
saved = (directory / "checkpoint.json").read_bytes()
with pytest.raises(CommandError):
complete(manifest, directory, report)
assert (directory / "checkpoint.json").read_bytes() == saved
page = pageops.space_dir("one") / "pages/index.md"
page.write_text("later private edit")
Space.objects.filter(slug="one").update(title="later title")
SpaceMembership.objects.filter(space__slug="one", member=member).update(role="viewer")
output = io.StringIO()
call_command("import_content_corpus", str(manifest), receipt_dir=str(directory), stdout=output)
assert "Imported 1 space(s); skipped 1" in output.getvalue()
assert page.read_text() == "later private edit"
assert Space.objects.get(slug="one").title == "later title"
assert SpaceMembership.objects.get(space__slug="one").role == "viewer"
@pytest.mark.parametrize("change", [
"body", "grant", "metadata", "source", "manifest", "checkpoint", "foreign",
"legacy", "no_sql", "no_files", "staging", "wrong_digest", "pending", "bad_index",
])
def test_changed_or_nonmatching_return_never_modifies_stores_or_checkpoint(
corpus, tmp_path, monkeypatch, change
):
manifest, entries, member = corpus
directory = tmp_path / "receipts"
uncertain(manifest, directory, monkeypatch)
report = inspect(manifest, directory)
checkpoint = directory / "checkpoint.json"
if change == "body":
(pageops.space_dir("one") / "pages/index.md").write_text("later content")
elif change == "grant":
SpaceMembership.objects.filter(member=member).update(role="viewer")
elif change == "metadata":
Space.objects.filter(slug="one").update(description="later metadata")
elif change == "source":
(manifest.parent / entries[0]["tree"] / "pages/index.md").write_text("source drift")
elif change == "manifest":
manifest.write_text(manifest.read_text() + "\n")
elif change in {"checkpoint", "legacy"}:
data = json.loads(checkpoint.read_bytes())
if change == "legacy":
data["spaces"][0].pop("space_id")
checkpoint.write_text(json.dumps(data, indent=2))
elif change in {"no_sql", "foreign"}:
Space.objects.get(slug="one").delete()
if change == "foreign":
Space.objects.create(tenant_id="tenant:a", slug="one", title="foreign")
elif change == "no_files":
import shutil
shutil.rmtree(pageops.space_dir("one"))
elif change == "staging":
(pageops.content_root() / "spaces/.import-unattributed").mkdir()
elif change == "wrong_digest":
report["inspection_sha256"] = "0" * 64
# Test both stale inspection and fresh inspection of nonmatching state.
if change in {"body", "grant", "metadata", "legacy", "foreign", "no_sql", "no_files",
"staging"}:
report = inspect(manifest, directory)
before = stores()
saved = checkpoint.read_bytes()
index = 1 if change == "pending" else -1 if change == "bad_index" else 0
with pytest.raises(CommandError):
complete(manifest, directory, report, index=index)
assert stores() == before and checkpoint.read_bytes() == saved
@pytest.mark.parametrize("failure", ["fsync", "drift"])
def test_failed_sync_or_late_drift_leaves_uncertain_checkpoint(
corpus, tmp_path, monkeypatch, failure
):
manifest, _, _ = corpus
directory = tmp_path / "receipts"
uncertain(manifest, directory, monkeypatch)
report = inspect(manifest, directory)
saved = (directory / "checkpoint.json").read_bytes()
def sync(tree):
if failure == "fsync":
raise OSError("synthetic disk failure")
(tree / "pages/index.md").write_text("external writer")
monkeypatch.setattr(import_reconciliation, "sync_tree", sync)
with pytest.raises(CommandError):
complete(manifest, directory, report)
assert (directory / "checkpoint.json").read_bytes() == saved
@pytest.mark.parametrize("phase", ["before_replace", "after_replace"])
def test_process_exit_retains_started_or_completed_checkpoint_with_evidence(
corpus, tmp_path, monkeypatch, phase
):
manifest, _, _ = corpus
directory = tmp_path / "receipts"
uncertain(manifest, directory, monkeypatch)
report = inspect(manifest, directory)
before = stores()
code = '''
import json,os,sys
from django.conf import settings
settings.DATABASES['default']=json.loads(sys.argv[1])
settings.CONTENT_ROOT=sys.argv[2]
import django
django.setup()
from django.core.management import call_command
from coulomb_social.apps.spaces import import_receipts
original=import_receipts.os.replace
def replace(*args):
if sys.argv[6]=='before_replace':os._exit(74)
original(*args)
os._exit(74)
import_receipts.os.replace=replace
call_command('reconcile_content_import',sys.argv[3],receipt_dir=sys.argv[4],
unit_index=0,expect_inspection=sys.argv[5],complete_matching_commit=True)
'''
result = subprocess.run(
[sys.executable, "-c", code, json.dumps(settings.DATABASES["default"], default=str),
str(pageops.content_root()), str(manifest), str(directory),
report["inspection_sha256"], phase],
env={**os.environ, "DJANGO_SETTINGS_MODULE": "coulomb_social.settings.test"},
capture_output=True, timeout=30,
)
assert result.returncode == 74, result.stderr.decode()
assert stores() == before
unit = json.loads((directory / "checkpoint.json").read_bytes())["spaces"][0]
if phase == "before_replace":
assert unit["state"] == "started"
# A fresh explicit review can complete; no importer automatically retries.
complete(manifest, directory, inspect(manifest, directory))
else:
assert unit["state"] == "completed"
assert unit["reconciliation"]["inspection"] == report
with pytest.raises(CommandError):
complete(manifest, directory, report)

View file

@ -407,6 +407,23 @@ T07 remains progress: concrete state reconciliation, supervised owner integratio
source/grant admission, native release/import/postflight and productive verdicts
are still required. No live cohort or approved executor was changed.
2026-10-31 matching-commit reconciliation: added an explicit operation that
completes one uncertain checkpoint only when its reserved identity, source files,
metadata and mapped grants match the exact reviewed inspection. It reacquires
existing locks, uses a read-only SQL snapshot, makes destination files durable,
rechecks state and atomically retains the original started unit and inspection
with the completed checkpoint. It changes no SQL/content and refuses partial,
foreign, changed, unbound or unattributed-staging states. Real process exits on
both sides of replacement retain inspectable evidence; resumed imports preserve
later edits/grants. Full make check passes 648 tests with required browsers and
zero failures/errors/skips. Isolated PostgreSQL 16.14 also passes, with 243
read-only/repeatable-read transaction checks and disposable-resource cleanup.
Review: [matching-commit reconciliation](../docs/release/corpus-import-reconciliation-2026-10-31.md).
This advances preparation within T07, which remains progress. Other uncertain
states, owner integration and verified external fencing/paired backup, cohort
admission, native release/import and productive verdicts remain open. No live
checkpoint, source cohort, grant or frozen execution packet was changed.
## Completion
Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.