Complete matching uncertain import checkpoints with retained evidence
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a192e9-c28b-7533-bf25-e2341364c22a
This commit is contained in:
parent
b800daedbc
commit
80fe387dcc
9 changed files with 483 additions and 16 deletions
|
|
@ -129,7 +129,8 @@ def observe_unit(receipts, index, unit):
|
|||
return result
|
||||
|
||||
|
||||
def inspect(receipts):
|
||||
@contextmanager
|
||||
def inspection_session(receipts):
|
||||
if connection.in_atomic_block or not connection.get_autocommit():
|
||||
raise ValueError("Inspection requires an independent read transaction")
|
||||
private_path(receipts.path)
|
||||
|
|
@ -158,18 +159,23 @@ def inspect(receipts):
|
|||
)
|
||||
):
|
||||
raise ValueError("Source changed during inspection")
|
||||
staging = list((pageops.content_root() / "spaces").glob(".import-*"))
|
||||
result = {
|
||||
"schema": "coulomb.corpus-import-inspection.v1",
|
||||
"binding": receipts.binding,
|
||||
"checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(),
|
||||
"spaces": first,
|
||||
"unattributed_staging_entries": len(staging),
|
||||
"read_only": True,
|
||||
"external_writer_fence_verified": False,
|
||||
"checkpoint_completed": False,
|
||||
"retry_authorized": False,
|
||||
"data_restored": False,
|
||||
}
|
||||
result["inspection_sha256"] = digest(result)
|
||||
staging = list((pageops.content_root() / "spaces").glob(".import-*"))
|
||||
result = {
|
||||
"schema": "coulomb.corpus-import-inspection.v1",
|
||||
"binding": receipts.binding,
|
||||
"checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(),
|
||||
"spaces": first,
|
||||
"unattributed_staging_entries": len(staging),
|
||||
"read_only": True,
|
||||
"external_writer_fence_verified": False,
|
||||
"checkpoint_completed": False,
|
||||
"retry_authorized": False,
|
||||
"data_restored": False,
|
||||
}
|
||||
result["inspection_sha256"] = digest(result)
|
||||
yield result, data
|
||||
|
||||
|
||||
def inspect(receipts):
|
||||
with inspection_session(receipts) as (result, _):
|
||||
return result
|
||||
|
|
|
|||
72
coulomb_social/apps/spaces/import_reconciliation.py
Normal file
72
coulomb_social/apps/spaces/import_reconciliation.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
"""Explicit checkpoint completion for an exactly observed, already committed import.
|
||||
|
||||
No SQL/content repair or automatic retry. The deployment owner still supplies
|
||||
admission, a paired backup and an external writer fence before invoking this.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from . import pageops
|
||||
from .import_inspection import inspection_session, observe_unit
|
||||
from .import_receipts import digest, sync_tree, tree_digest
|
||||
from .models import Space
|
||||
|
||||
|
||||
def complete_matching_commit(receipts, *, index, expected_inspection):
|
||||
if (type(index) is not int or not 0 <= index < len(receipts.plan)
|
||||
or not isinstance(expected_inspection, str)
|
||||
or not re.fullmatch(r"[a-f0-9]{64}", expected_inspection)):
|
||||
raise ValueError("An exact inspection digest and valid unit index are required")
|
||||
with inspection_session(receipts) as (inspection, data):
|
||||
if inspection["inspection_sha256"] != expected_inspection:
|
||||
raise ValueError("Inspection changed; retain evidence and review again")
|
||||
observed = inspection["spaces"][index]
|
||||
if observed["classification"] != "reserved_destination_matches_source":
|
||||
raise ValueError("Only an exact reserved committed import can be completed")
|
||||
if inspection["unattributed_staging_entries"]:
|
||||
raise ValueError("Unattributed staging requires owner reconciliation first")
|
||||
tree, _, slug, _ = receipts.plan[index]
|
||||
prior_unit = data["spaces"][index]
|
||||
row = Space.objects.filter(pk=prior_unit["space_id"]).values().get()
|
||||
if digest([row]) != observed["space_sha256"]:
|
||||
raise ValueError("Observed SQL identity changed")
|
||||
# The original importer can die before fsync or its completed checkpoint.
|
||||
# Make the matching published tree durable before acknowledging it.
|
||||
sync_tree(pageops.space_dir(slug))
|
||||
if (observe_unit(receipts, index, prior_unit) != observed
|
||||
or tree_digest(tree) != receipts.source_hashes[index]
|
||||
or hashlib.sha256(receipts.path.read_bytes()).hexdigest()
|
||||
!= inspection["checkpoint_sha256"]):
|
||||
raise ValueError("State changed before checkpoint completion")
|
||||
data["spaces"][index] = {
|
||||
"state": "completed",
|
||||
"space_id": prior_unit["space_id"],
|
||||
"files_sha256": observed["files_sha256"],
|
||||
"space_sha256": digest(row),
|
||||
"grants_sha256": observed["grants_sha256"],
|
||||
"pages": len(list((tree / "pages").glob("*.md"))),
|
||||
"assets": sum(p.is_file() for p in (tree / "assets").rglob("*")),
|
||||
"reconciliation": {
|
||||
"schema": "coulomb.corpus-matching-commit-reconciliation.v1",
|
||||
"prior_unit": prior_unit,
|
||||
"inspection": inspection,
|
||||
},
|
||||
}
|
||||
# Existing atomic replacement + fsync keeps the prior started checkpoint
|
||||
# or the completed one with its evidence. Lost output requires readback;
|
||||
# replay of the old inspection cannot acknowledge a different state.
|
||||
receipts.save(data)
|
||||
return {
|
||||
"schema": "coulomb.corpus-matching-commit-return.v1",
|
||||
"status": "matching_commit_checkpoint_completed",
|
||||
"index": index,
|
||||
"inspection_sha256": expected_inspection,
|
||||
"checkpoint_sha256": hashlib.sha256(receipts.path.read_bytes()).hexdigest(),
|
||||
"checkpoint_completed": True,
|
||||
"sql_modified": False,
|
||||
"content_modified": False,
|
||||
"data_restored": False,
|
||||
"external_writer_fence_verified": False,
|
||||
"remaining_import_authorized": False,
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
"""Record one matching committed import; never restore or delete application data."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from coulomb_social.apps.spaces.import_receipts import CorpusReceipts
|
||||
from coulomb_social.apps.spaces.import_reconciliation import complete_matching_commit
|
||||
from coulomb_social.apps.spaces.management.commands.import_content_corpus import (
|
||||
Command as ImportCommand,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Complete one exact matching import checkpoint after owner reconciliation review"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("manifest")
|
||||
parser.add_argument("--receipt-dir", required=True)
|
||||
parser.add_argument("--unit-index", type=int, required=True)
|
||||
parser.add_argument("--expect-inspection", required=True)
|
||||
parser.add_argument("--complete-matching-commit", action="store_true", required=True)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
if not options["complete_matching_commit"]:
|
||||
raise CommandError("Explicit matching-commit completion is required")
|
||||
try:
|
||||
manifest = Path(options["manifest"])
|
||||
plan = ImportCommand().preflight(manifest, replace=False, check_existing=False)
|
||||
receipts = CorpusReceipts(options["receipt_dir"], manifest, plan)
|
||||
result = complete_matching_commit(
|
||||
receipts, index=options["unit_index"],
|
||||
expected_inspection=options["expect_inspection"],
|
||||
)
|
||||
except Exception as exc:
|
||||
# Failure can follow durable checkpoint replacement. Never advise replay
|
||||
# or disclose source paths, identities or SQL values in the error.
|
||||
raise CommandError(
|
||||
"Reconciliation stopped; inspect the retained checkpoint before further action"
|
||||
) from exc
|
||||
self.stdout.write(json.dumps(result, sort_keys=True))
|
||||
|
|
@ -57,6 +57,11 @@ authorized and no data was restored. The inspector does not convert `started` to
|
|||
uncertain state with its retained backup, writer fence and reviewed operation.
|
||||
Do not remove the checkpoint or use replacement import to bypass uncertainty.
|
||||
|
||||
The subsequent [matching-commit procedure](corpus-import-reconciliation-2026-10-31.md)
|
||||
adds a separate, explicit checkpoint operation for one exactly matching reserved
|
||||
commit. Inspection itself remains read-only. Partial, changed, unbound and
|
||||
unattributed-staging states still require owner reconciliation.
|
||||
|
||||
## Verification and release boundary
|
||||
|
||||
Fifteen new cases cover matching committed state without receipt, partial stores,
|
||||
|
|
|
|||
96
docs/release/corpus-import-reconciliation-2026-10-31.md
Normal file
96
docs/release/corpus-import-reconciliation-2026-10-31.md
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
# Completing a matching uncertain import checkpoint
|
||||
|
||||
CSOC-WP-0014-T07 now has an explicit checkpoint-completion operation for an
|
||||
already committed import whose reserved identity and complete source-derived
|
||||
state match. This is source preparation, not a production reconciliation or
|
||||
admission of any cohort. The existing native executor and frozen release packets
|
||||
are unchanged.
|
||||
|
||||
## Owner procedure
|
||||
|
||||
Before any native use, the deployment owner must identify the exact original
|
||||
attempt, stop and independently verify application and external writers, retain
|
||||
the original checkpoint and a coherent paired recovery point, and confirm the
|
||||
source/tenant/identity/grant mapping was admitted. Neither of the commands below
|
||||
verifies an external writer fence or provider backup. The current production
|
||||
image does not contain these new commands; owner integration and release remain
|
||||
required.
|
||||
|
||||
1. Inspect the original manifest and existing private receipt directory using
|
||||
`inspect_content_corpus`. Keep the result privately with the operation evidence.
|
||||
2. Review the selected zero-based unit index, reserved identity, source binding,
|
||||
complete file/grant/metadata comparison and the original operation's outcome.
|
||||
Only `reserved_destination_matches_source`, with no unattributed `.import-*`
|
||||
entries, is eligible for the operation below. A match describes current state;
|
||||
it is not proof that no intervening edit ever occurred.
|
||||
3. Supply the exact inspection SHA-256 and explicit completion action:
|
||||
|
||||
```sh
|
||||
python manage.py reconcile_content_import /private/reviewed/manifest.json \
|
||||
--receipt-dir /private/import-attempt \
|
||||
--unit-index 0 \
|
||||
--expect-inspection <inspection_sha256> \
|
||||
--complete-matching-commit
|
||||
```
|
||||
|
||||
The command reacquires the existing receipt/content locks and a consistent
|
||||
read-only SQL transaction, recomputes the whole inspection, and refuses a
|
||||
changed digest. It fsyncs the matching destination, rechecks the selected unit,
|
||||
source and checkpoint bytes, then atomically replaces only the checkpoint.
|
||||
The completed unit retains its original reserved identity, ordinary importer
|
||||
fingerprints/counts, prior started unit and full hash/count inspection evidence.
|
||||
SQL rows, grants and content bytes are not modified.
|
||||
4. Independently read the checkpoint and inspect again. A lost response or failure
|
||||
may occur after replacement; do not infer failure to complete from the absence
|
||||
of terminal output. The previous inspection cannot be replayed against an
|
||||
already completed checkpoint. An interrupted temporary checkpoint file is
|
||||
retained as evidence and is not automatically deleted or adopted.
|
||||
5. If continuation is separately admitted, the existing receipted importer skips
|
||||
the completed unit and preserves later edits and grants. Completion of this
|
||||
checkpoint does not authorize importing the remaining corpus.
|
||||
|
||||
## Other states remain explicit owner work
|
||||
|
||||
| Observed state | Required disposition |
|
||||
| --- | --- |
|
||||
| Reserved identity differs from source | Preserve later edits/grants; reconcile against the original operation and paired point. This command never overwrites them. |
|
||||
| SQL only, files only, or neither | Retain the uncertain checkpoint and inspect private staging, transaction outcome and the paired recovery point. No automatic reset to pending, retry, restore or deletion is implemented here. |
|
||||
| Foreign identity or legacy unbound started receipt | Resolve ownership and identity from independent evidence. Do not adopt a row because its slug matches. |
|
||||
| Unattributed staging entries | Establish their exact attempt/ownership and disposition separately. Matching published bytes do not authorize cleaning up those entries. |
|
||||
| Pending or already completed unit | No reconciliation mutation is permitted. Completed identities remain historical boundaries and may have valid later edits. |
|
||||
|
||||
The helper uses cooperating file locks and PostgreSQL repeatable-read/read-only
|
||||
transactions. These are not an external writer fence. The return explicitly
|
||||
reports that limit, no SQL/content changes, no restore, and no authorization to
|
||||
continue imports. The owner must verify native prerequisites before execution;
|
||||
passing synthetic tests does not supply them.
|
||||
|
||||
## Validation
|
||||
|
||||
The focused suite covers exact completion and evidence retention, unchanged SQL/
|
||||
content, preservation of later edits on resumed import, stale evidence, changed
|
||||
source/mapping/checkpoint, partial or foreign stores, legacy receipts, unattributed
|
||||
staging, invalid selection, sync failure and drift during fsync. Real child-process
|
||||
exits immediately before and after checkpoint replacement leave either the prior
|
||||
started checkpoint or the completed checkpoint with its reconciliation evidence.
|
||||
|
||||
Full `make check`: **648 passed, 69 existing warnings, zero failures/errors/skips
|
||||
in 150.38s**, including required browser journeys, rerun after the interrupted
|
||||
session. Nineteen new reconciliation
|
||||
cases pass; the focused corpus checkpoint/inspection/reconciliation selection
|
||||
passes all 52 cases. Log: `/tmp/csoc-import-reconciliation-check-20261031.log`.
|
||||
|
||||
The isolated PostgreSQL 16.14 rehearsal passes with 243 actual read-only and
|
||||
repeatable-read transaction checks. It uses synthetic data, explicit source and
|
||||
locked dependency overlays, an internal Docker network and no published ports.
|
||||
Its disposable containers, volumes and network were removed. Source-bound receipt:
|
||||
`corpus-import-reconciliation-postgres-2026-10-31.json`. This is not packaged
|
||||
candidate or production proof.
|
||||
|
||||
After resuming, all 12 source hashes in that PostgreSQL receipt were independently
|
||||
checked against the current files and matched. State Hub consistency passed and
|
||||
the T07 description was independently read back with the reconciliation evidence.
|
||||
|
||||
T07 remains in progress for the other reconciliation states, supervised owner
|
||||
integration, source/grant admission, native release/import/postflight and
|
||||
productive verdicts.
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"schema": "coulomb.corpus-inspection-rehearsal.v1",
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"database_vendor": "postgresql",
|
||||
"server_version": 160014,
|
||||
"inspection_transaction_mode_checks": 243,
|
||||
"source_overlay": true,
|
||||
"locked_development_dependencies": true,
|
||||
"production_data_used": false
|
||||
},
|
||||
"app_dependency_image": "sha256:522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed",
|
||||
"postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70",
|
||||
"internal_network": true,
|
||||
"published_ports": false,
|
||||
"disposable_resources_removed": true,
|
||||
"source_sha256": {
|
||||
"scripts/rehearse-import-inspection.py": "34a95a57df6b93132a45c70f95b56b8ee9ef5be8677efa6f27f48c41bdc46f17",
|
||||
"uv.lock": "e8ebb6fc4b5abd1ac80f1c9012f1983642d0528a8a213cfcc394261606fad887",
|
||||
"coulomb_social/apps/spaces/import_tree.py": "f232bde0d1741668ff28a6cb24306b7f67d8bf9fa3838507a8db4f4e80361784",
|
||||
"coulomb_social/apps/spaces/import_receipts.py": "670a5eb48613df47bf8fd0cf7d9306f7d2e992d46dff69680e6e7998f64f9cb2",
|
||||
"coulomb_social/apps/spaces/import_inspection.py": "908a62bd31e0a6cfab3cc94f45c671ff4f1ec403035434790a29adfc5275f5ab",
|
||||
"coulomb_social/apps/spaces/import_reconciliation.py": "75abc50ec1c6e15a12534eb39edc5ad3fa4128c37a354d4cea2bf6c37f745771",
|
||||
"coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py": "b083e39aa766c8ba268788fae98507780604d40832e119db1cd9f7c10f1aa212",
|
||||
"coulomb_social/apps/spaces/management/commands/reconcile_content_import.py": "2e25d94ab3a9659e38a3ea4c8b6c2e87c710216ab5834a248fccdd00da9fdbb9",
|
||||
"tests/test_import_corpus.py": "5a0670c408740de86ce601cd9d2a0a12cf0aaa5afda416867605180c568a1c5f",
|
||||
"tests/test_import_receipts.py": "134e8901991dc493333b7543bbd6ff1f1e7b7ca3d5cf9d7849d7c45212e20aab",
|
||||
"tests/test_import_inspection.py": "890e783eec30dd1b9e93ad7c3ab3173b57174a096938520865eaaf420bc605fb",
|
||||
"tests/test_import_reconciliation.py": "89d7161715a2c298c30c6ae38f8d9647688e2ea261d2913a8a9b8969202ac298"
|
||||
}
|
||||
}
|
||||
|
|
@ -39,7 +39,8 @@ def observe(*args):
|
|||
import_inspection.observe_unit=observe
|
||||
import pytest
|
||||
code=pytest.main(['-q','-p','no:cacheprovider','/app/tests/test_import_receipts.py',
|
||||
'/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py'])
|
||||
'/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py',
|
||||
'/app/tests/test_import_reconciliation.py'])
|
||||
assert code==0 and checks
|
||||
print(json.dumps({'status':'passed','database_vendor':'postgresql','server_version':version,
|
||||
'inspection_transaction_mode_checks':len(checks),'source_overlay':True,
|
||||
|
|
@ -124,13 +125,16 @@ def main():
|
|||
ROOT / "coulomb_social/apps/spaces/import_tree.py",
|
||||
ROOT / "coulomb_social/apps/spaces/import_receipts.py",
|
||||
ROOT / "coulomb_social/apps/spaces/import_inspection.py",
|
||||
ROOT / "coulomb_social/apps/spaces/import_reconciliation.py",
|
||||
ROOT / "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py",
|
||||
ROOT / "coulomb_social/apps/spaces/management/commands/reconcile_content_import.py",
|
||||
*(
|
||||
ROOT / "tests" / name
|
||||
for name in (
|
||||
"test_import_corpus.py",
|
||||
"test_import_receipts.py",
|
||||
"test_import_inspection.py",
|
||||
"test_import_reconciliation.py",
|
||||
)
|
||||
),
|
||||
]
|
||||
|
|
|
|||
194
tests/test_import_reconciliation.py
Normal file
194
tests/test_import_reconciliation.py
Normal file
|
|
@ -0,0 +1,194 @@
|
|||
"""Exact checkpoint reconciliation preserves stores and survives process death."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
|
||||
from coulomb_social.apps.spaces import import_reconciliation, pageops
|
||||
from coulomb_social.apps.spaces.models import Space, SpaceMembership
|
||||
from tests import test_import_corpus
|
||||
from tests.test_import_inspection import inspect, uncertain
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
corpus = test_import_corpus.corpus
|
||||
|
||||
|
||||
def complete(manifest, directory, inspection, index=0):
|
||||
output = io.StringIO()
|
||||
call_command(
|
||||
"reconcile_content_import", str(manifest), receipt_dir=str(directory),
|
||||
unit_index=index, expect_inspection=inspection["inspection_sha256"],
|
||||
complete_matching_commit=True, stdout=output,
|
||||
)
|
||||
return json.loads(output.getvalue())
|
||||
|
||||
|
||||
def stores():
|
||||
return (
|
||||
list(Space.objects.order_by("pk").values()),
|
||||
list(SpaceMembership.objects.order_by("pk").values()),
|
||||
{str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()},
|
||||
)
|
||||
|
||||
|
||||
def test_matching_commit_completes_only_checkpoint_then_resume_preserves_later_edits(
|
||||
corpus, tmp_path, monkeypatch
|
||||
):
|
||||
manifest, _, member = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
before = stores()
|
||||
prior = json.loads((directory / "checkpoint.json").read_bytes())
|
||||
report = inspect(manifest, directory)
|
||||
result = complete(manifest, directory, report)
|
||||
checkpoint = json.loads((directory / "checkpoint.json").read_bytes())
|
||||
assert stores() == before
|
||||
assert result["checkpoint_completed"] and not result["remaining_import_authorized"]
|
||||
assert not result["external_writer_fence_verified"]
|
||||
assert not result["sql_modified"] and not result["content_modified"]
|
||||
assert checkpoint["spaces"][1] == prior["spaces"][1]
|
||||
reconciled = checkpoint["spaces"][0]
|
||||
assert reconciled["reconciliation"]["prior_unit"] == prior["spaces"][0]
|
||||
assert reconciled["reconciliation"]["inspection"] == report
|
||||
assert (directory / "checkpoint.json").stat().st_mode & 0o777 == 0o600
|
||||
saved = (directory / "checkpoint.json").read_bytes()
|
||||
with pytest.raises(CommandError):
|
||||
complete(manifest, directory, report)
|
||||
assert (directory / "checkpoint.json").read_bytes() == saved
|
||||
page = pageops.space_dir("one") / "pages/index.md"
|
||||
page.write_text("later private edit")
|
||||
Space.objects.filter(slug="one").update(title="later title")
|
||||
SpaceMembership.objects.filter(space__slug="one", member=member).update(role="viewer")
|
||||
output = io.StringIO()
|
||||
call_command("import_content_corpus", str(manifest), receipt_dir=str(directory), stdout=output)
|
||||
assert "Imported 1 space(s); skipped 1" in output.getvalue()
|
||||
assert page.read_text() == "later private edit"
|
||||
assert Space.objects.get(slug="one").title == "later title"
|
||||
assert SpaceMembership.objects.get(space__slug="one").role == "viewer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", [
|
||||
"body", "grant", "metadata", "source", "manifest", "checkpoint", "foreign",
|
||||
"legacy", "no_sql", "no_files", "staging", "wrong_digest", "pending", "bad_index",
|
||||
])
|
||||
def test_changed_or_nonmatching_return_never_modifies_stores_or_checkpoint(
|
||||
corpus, tmp_path, monkeypatch, change
|
||||
):
|
||||
manifest, entries, member = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
report = inspect(manifest, directory)
|
||||
checkpoint = directory / "checkpoint.json"
|
||||
if change == "body":
|
||||
(pageops.space_dir("one") / "pages/index.md").write_text("later content")
|
||||
elif change == "grant":
|
||||
SpaceMembership.objects.filter(member=member).update(role="viewer")
|
||||
elif change == "metadata":
|
||||
Space.objects.filter(slug="one").update(description="later metadata")
|
||||
elif change == "source":
|
||||
(manifest.parent / entries[0]["tree"] / "pages/index.md").write_text("source drift")
|
||||
elif change == "manifest":
|
||||
manifest.write_text(manifest.read_text() + "\n")
|
||||
elif change in {"checkpoint", "legacy"}:
|
||||
data = json.loads(checkpoint.read_bytes())
|
||||
if change == "legacy":
|
||||
data["spaces"][0].pop("space_id")
|
||||
checkpoint.write_text(json.dumps(data, indent=2))
|
||||
elif change in {"no_sql", "foreign"}:
|
||||
Space.objects.get(slug="one").delete()
|
||||
if change == "foreign":
|
||||
Space.objects.create(tenant_id="tenant:a", slug="one", title="foreign")
|
||||
elif change == "no_files":
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(pageops.space_dir("one"))
|
||||
elif change == "staging":
|
||||
(pageops.content_root() / "spaces/.import-unattributed").mkdir()
|
||||
elif change == "wrong_digest":
|
||||
report["inspection_sha256"] = "0" * 64
|
||||
# Test both stale inspection and fresh inspection of nonmatching state.
|
||||
if change in {"body", "grant", "metadata", "legacy", "foreign", "no_sql", "no_files",
|
||||
"staging"}:
|
||||
report = inspect(manifest, directory)
|
||||
before = stores()
|
||||
saved = checkpoint.read_bytes()
|
||||
index = 1 if change == "pending" else -1 if change == "bad_index" else 0
|
||||
with pytest.raises(CommandError):
|
||||
complete(manifest, directory, report, index=index)
|
||||
assert stores() == before and checkpoint.read_bytes() == saved
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", ["fsync", "drift"])
|
||||
def test_failed_sync_or_late_drift_leaves_uncertain_checkpoint(
|
||||
corpus, tmp_path, monkeypatch, failure
|
||||
):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
report = inspect(manifest, directory)
|
||||
saved = (directory / "checkpoint.json").read_bytes()
|
||||
|
||||
def sync(tree):
|
||||
if failure == "fsync":
|
||||
raise OSError("synthetic disk failure")
|
||||
(tree / "pages/index.md").write_text("external writer")
|
||||
|
||||
monkeypatch.setattr(import_reconciliation, "sync_tree", sync)
|
||||
with pytest.raises(CommandError):
|
||||
complete(manifest, directory, report)
|
||||
assert (directory / "checkpoint.json").read_bytes() == saved
|
||||
|
||||
|
||||
@pytest.mark.parametrize("phase", ["before_replace", "after_replace"])
|
||||
def test_process_exit_retains_started_or_completed_checkpoint_with_evidence(
|
||||
corpus, tmp_path, monkeypatch, phase
|
||||
):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
report = inspect(manifest, directory)
|
||||
before = stores()
|
||||
code = '''
|
||||
import json,os,sys
|
||||
from django.conf import settings
|
||||
settings.DATABASES['default']=json.loads(sys.argv[1])
|
||||
settings.CONTENT_ROOT=sys.argv[2]
|
||||
import django
|
||||
django.setup()
|
||||
from django.core.management import call_command
|
||||
from coulomb_social.apps.spaces import import_receipts
|
||||
original=import_receipts.os.replace
|
||||
def replace(*args):
|
||||
if sys.argv[6]=='before_replace':os._exit(74)
|
||||
original(*args)
|
||||
os._exit(74)
|
||||
import_receipts.os.replace=replace
|
||||
call_command('reconcile_content_import',sys.argv[3],receipt_dir=sys.argv[4],
|
||||
unit_index=0,expect_inspection=sys.argv[5],complete_matching_commit=True)
|
||||
'''
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-c", code, json.dumps(settings.DATABASES["default"], default=str),
|
||||
str(pageops.content_root()), str(manifest), str(directory),
|
||||
report["inspection_sha256"], phase],
|
||||
env={**os.environ, "DJANGO_SETTINGS_MODULE": "coulomb_social.settings.test"},
|
||||
capture_output=True, timeout=30,
|
||||
)
|
||||
assert result.returncode == 74, result.stderr.decode()
|
||||
assert stores() == before
|
||||
unit = json.loads((directory / "checkpoint.json").read_bytes())["spaces"][0]
|
||||
if phase == "before_replace":
|
||||
assert unit["state"] == "started"
|
||||
# A fresh explicit review can complete; no importer automatically retries.
|
||||
complete(manifest, directory, inspect(manifest, directory))
|
||||
else:
|
||||
assert unit["state"] == "completed"
|
||||
assert unit["reconciliation"]["inspection"] == report
|
||||
with pytest.raises(CommandError):
|
||||
complete(manifest, directory, report)
|
||||
|
|
@ -407,6 +407,23 @@ T07 remains progress: concrete state reconciliation, supervised owner integratio
|
|||
source/grant admission, native release/import/postflight and productive verdicts
|
||||
are still required. No live cohort or approved executor was changed.
|
||||
|
||||
2026-10-31 matching-commit reconciliation: added an explicit operation that
|
||||
completes one uncertain checkpoint only when its reserved identity, source files,
|
||||
metadata and mapped grants match the exact reviewed inspection. It reacquires
|
||||
existing locks, uses a read-only SQL snapshot, makes destination files durable,
|
||||
rechecks state and atomically retains the original started unit and inspection
|
||||
with the completed checkpoint. It changes no SQL/content and refuses partial,
|
||||
foreign, changed, unbound or unattributed-staging states. Real process exits on
|
||||
both sides of replacement retain inspectable evidence; resumed imports preserve
|
||||
later edits/grants. Full make check passes 648 tests with required browsers and
|
||||
zero failures/errors/skips. Isolated PostgreSQL 16.14 also passes, with 243
|
||||
read-only/repeatable-read transaction checks and disposable-resource cleanup.
|
||||
Review: [matching-commit reconciliation](../docs/release/corpus-import-reconciliation-2026-10-31.md).
|
||||
This advances preparation within T07, which remains progress. Other uncertain
|
||||
states, owner integration and verified external fencing/paired backup, cohort
|
||||
admission, native release/import and productive verdicts remain open. No live
|
||||
checkpoint, source cohort, grant or frozen execution packet was changed.
|
||||
|
||||
## Completion
|
||||
|
||||
Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue