From e75d3b0f6a090ee23d040d9b9509ce2066819699 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 31 Oct 2026 17:31:38 +0100 Subject: [PATCH] Bind import attempts to exclusive space identities and add inspection Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3 --- .../apps/spaces/import_inspection.py | 175 ++++++++++++++ coulomb_social/apps/spaces/import_receipts.py | 8 +- coulomb_social/apps/spaces/import_tree.py | 32 ++- .../commands/inspect_content_corpus.py | 34 +++ .../corpus-import-inspection-2026-10-31.md | 85 +++++++ ...import-inspection-postgres-2026-10-31.json | 28 +++ .../corpus-import-receipts-2026-10-31.md | 7 + scripts/rehearse-import-inspection.py | 158 +++++++++++++ tests/test_import_inspection.py | 217 ++++++++++++++++++ tests/test_import_receipts.py | 9 + ...-WP-0014-cohort-transfer-bubble-offramp.md | 17 ++ 11 files changed, 760 insertions(+), 10 deletions(-) create mode 100644 coulomb_social/apps/spaces/import_inspection.py create mode 100644 coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py create mode 100644 docs/release/corpus-import-inspection-2026-10-31.md create mode 100644 docs/release/corpus-import-inspection-postgres-2026-10-31.json create mode 100644 scripts/rehearse-import-inspection.py create mode 100644 tests/test_import_inspection.py diff --git a/coulomb_social/apps/spaces/import_inspection.py b/coulomb_social/apps/spaces/import_inspection.py new file mode 100644 index 0000000..02b1ad4 --- /dev/null +++ b/coulomb_social/apps/spaces/import_inspection.py @@ -0,0 +1,175 @@ +"""Read-only classification of uncertain corpus imports; never repairs a store.""" + +from __future__ import annotations + +import fcntl +import hashlib +import os +import uuid +from contextlib import contextmanager + +from django.db import connection, transaction + +from . import pageops +from .import_receipts import digest, private_path, tree_digest +from .import_tree import _metadata, _source_dates +from .models import Space, SpaceMembership + + +@contextmanager +def existing_lock(path): + private_path(path) + # Inspection never creates a lock, journal, content root or receipt directory. + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise ValueError("Import or content writer is active; inspect after it stops") from exc + yield + finally: + os.close(fd) + + +def expected_metadata(tree, tenant, slug, grants): + title, description, visual = _metadata(tree, slug) + values = { + "tenant_id": tenant, + "slug": slug, + "title": title, + "description": description, + "visual": visual, + "is_active": True, + "content_root": "pages", + "default_branch": "main", + "forgejo_owner": "", + "forgejo_repo": "", + "created_by_id": next(m.pk for m, role in grants if role == SpaceMembership.Role.OWNER), + } + index = tree / "pages/index.md" + if index.is_file(): + meta, _ = pageops.parse_frontmatter(index.read_text(encoding="utf-8")) + values.update(_source_dates(meta)) + return values + + +def observe_unit(receipts, index, unit): + tree, tenant, slug, grants = receipts.plan[index] + rows = list(Space.objects.filter(slug=slug).values()) + destination = pageops.space_dir(slug) + if destination.is_symlink() or any(p.is_symlink() for p in destination.parents): + raise ValueError("Destination must not contain symlinks") + files_hash = tree_digest(destination) if destination.is_dir() else None + grant_rows = sorted( + (str(member), role) + for member, role in SpaceMembership.objects.filter(space__slug=slug).values_list( + "member_id", "role" + ) + ) + result = { + "index": index, + "checkpoint_state": unit.get("state"), + "sql_rows": len(rows), + "published_tree_present": destination.exists(), + "files_sha256": files_hash, + "space_sha256": digest(rows), + "grants_sha256": digest(grant_rows), + } + state = unit.get("state") + if state == "completed": + result["classification"] = ( + "completed_identity_present" + if len(rows) == 1 + and str(rows[0]["id"]) == unit.get("space_id") + and rows[0]["tenant_id"] == tenant + else "completed_identity_changed" + ) + return result + if state not in {"pending", "started"}: + raise ValueError("Invalid checkpoint state") + if state == "pending": + result["classification"] = ( + "pending_destination_occupied" if rows or destination.exists() else "pending" + ) + return result + try: + reserved = uuid.UUID(unit["space_id"]) + except (KeyError, ValueError, TypeError, AttributeError): + result["classification"] = "started_identity_unbound" + return result + if not rows and not destination.exists(): + result["classification"] = "no_sql_or_published_tree" + return result + if not rows: + result["classification"] = "files_without_sql" + return result + if len(rows) != 1 or rows[0]["id"] != reserved or rows[0]["tenant_id"] != tenant: + result["classification"] = "foreign_destination_identity" + return result + if not destination.exists(): + result["classification"] = "sql_without_files" + return result + wanted = expected_metadata(tree, tenant, slug, grants) + metadata_match = all(rows[0][key] == value for key, value in wanted.items()) + grants_match = grant_rows == sorted((str(m.pk), role) for m, role in grants) + # The importer copies root Markdown pages and the assets subtree. Extra + # source files, or a synthesized index, cannot be called byte-exact here. + files_match = files_hash == receipts.source_hashes[index] + result.update( + metadata_matches=metadata_match, + grants_match=grants_match, + source_files_match=files_match, + original_space_date_checked="created_at" in wanted, + ) + result["classification"] = ( + "reserved_destination_matches_source" + if metadata_match and grants_match and files_match + else "reserved_destination_differs" + ) + return result + + +def inspect(receipts): + if connection.in_atomic_block or not connection.get_autocommit(): + raise ValueError("Inspection requires an independent read transaction") + private_path(receipts.path) + if not receipts.path.is_file(): + raise ValueError("An existing checkpoint is required for inspection") + with ( + existing_lock(receipts.directory / ".lock"), + existing_lock(pageops.content_root() / ".writer.lock"), + ): + checkpoint = receipts.path.read_bytes() + data = receipts.read() + if any(not isinstance(unit, dict) for unit in data["spaces"]): + raise ValueError("Invalid space checkpoint") + with transaction.atomic(): + if connection.vendor == "postgresql": + with connection.cursor() as cursor: + cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + first = [observe_unit(receipts, i, unit) for i, unit in enumerate(data["spaces"])] + second = [observe_unit(receipts, i, unit) for i, unit in enumerate(data["spaces"])] + if first != second or checkpoint != receipts.path.read_bytes(): + raise ValueError("Destination or checkpoint changed during inspection") + if any( + tree_digest(tree) != checksum + for (tree, _, _, _), checksum in zip( + receipts.plan, receipts.source_hashes, strict=True + ) + ): + raise ValueError("Source changed during inspection") + staging = list((pageops.content_root() / "spaces").glob(".import-*")) + result = { + "schema": "coulomb.corpus-import-inspection.v1", + "binding": receipts.binding, + "checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(), + "spaces": first, + "unattributed_staging_entries": len(staging), + "read_only": True, + "external_writer_fence_verified": False, + "checkpoint_completed": False, + "retry_authorized": False, + "data_restored": False, + } + result["inspection_sha256"] = digest(result) + return result diff --git a/coulomb_social/apps/spaces/import_receipts.py b/coulomb_social/apps/spaces/import_receipts.py index 57979ce..707623c 100644 --- a/coulomb_social/apps/spaces/import_receipts.py +++ b/coulomb_social/apps/spaces/import_receipts.py @@ -205,13 +205,17 @@ class CorpusReceipts: raise ValueError("Source changed after preflight") # Persist uncertainty BEFORE the first import write. Any failure, # including a lost commit reply or receipt fsync, stays uncertain. - data["spaces"][index] = {"state": "started"} + space_id = uuid.uuid4() + data["spaces"][index] = {"state": "started", "space_id": str(space_id)} self.save(data) report = import_space_tree(tree, tenant_id=tenant, slug=slug, - memberships=grants, replace=False) + memberships=grants, replace=False, + new_space_id=space_id) if tree_digest(tree) != self.source_hashes[index]: raise ValueError("Source changed during import; reconciliation required") space = Space.objects.get(tenant_id=tenant, slug=slug) + if space.pk != space_id: + raise ValueError("Imported space identity changed; reconciliation required") observed_grants = sorted((str(m), role) for m, role in SpaceMembership.objects.filter(space=space).values_list("member_id", "role")) if observed_grants != sorted((str(m.pk), role) for m, role in grants): diff --git a/coulomb_social/apps/spaces/import_tree.py b/coulomb_social/apps/spaces/import_tree.py index 6da1f1f..767427f 100644 --- a/coulomb_social/apps/spaces/import_tree.py +++ b/coulomb_social/apps/spaces/import_tree.py @@ -4,6 +4,7 @@ from __future__ import annotations import shutil import tempfile +import uuid from dataclasses import dataclass, field from pathlib import Path @@ -132,7 +133,13 @@ def import_space_tree( tree: Path, *, tenant_id: str, slug: str | None = None, member: Member | None = None, title: str | None = None, replace: bool = False, memberships: list[tuple[Member, str]] | None = None, + new_space_id: uuid.UUID | None = None, ) -> ImportReport: + if new_space_id is not None: + if replace or not isinstance(new_space_id, uuid.UUID): + raise pageops.PageOpsError("Reserved import identity requires a new UUID space") + if Space.objects.filter(pk=new_space_id).exists(): + raise pageops.PageOpsError("Reserved import identity already exists") # Invalid proposals remain write-free, including lock-file creation. validate_space_tree(tree, tenant_id=tenant_id, slug=slug, member=member, replace=replace, title=title) @@ -140,7 +147,8 @@ def import_space_tree( if mapped.tenant_id != tenant_id or role not in SpaceMembership.Role.values: raise pageops.PageOpsError("Invalid membership tenant or role") return _import_space_tree(tree, tenant_id=tenant_id, slug=slug, member=member, - title=title, replace=replace, memberships=memberships) + title=title, replace=replace, memberships=memberships, + new_space_id=new_space_id) @pageops.content_writer @@ -148,6 +156,7 @@ def _import_space_tree( tree: Path, *, tenant_id: str, slug: str | None = None, member: Member | None = None, title: str | None = None, replace: bool = False, memberships: list[tuple[Member, str]] | None = None, + new_space_id: uuid.UUID | None = None, ) -> ImportReport: """Stage content, then publish with DB writes; restore on ordinary errors. @@ -194,15 +203,22 @@ def _import_space_tree( try: with transaction.atomic(): try: - space, created = Space.objects.update_or_create( - tenant_id=tenant_id, slug=space_slug, - defaults={"title": display_title, "description": abstractor, - "visual": visual, "is_active": True, - "content_root": "pages", "default_branch": "main"}, - ) + defaults = {"title": display_title, "description": abstractor, + "visual": visual, "is_active": True, + "content_root": "pages", "default_branch": "main"} + if new_space_id is not None: + # A receipted attempt owns only its pre-recorded UUID. + # Never update a row created concurrently at this slug. + space = Space.objects.create(id=new_space_id, tenant_id=tenant_id, + slug=space_slug, **defaults) + created = True + else: + space, created = Space.objects.update_or_create( + tenant_id=tenant_id, slug=space_slug, defaults=defaults) except IntegrityError as exc: raise pageops.PageOpsError( - "Content slug belongs to another tenant" + "Reserved import identity or destination already exists" + if new_space_id is not None else "Content slug belongs to another tenant" ) from exc report.space_created = created report.space_updated = not created diff --git a/coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py b/coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py new file mode 100644 index 0000000..8059119 --- /dev/null +++ b/coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py @@ -0,0 +1,34 @@ +"""Inspect uncertain receipt-backed imports without changing content or checkpoints.""" + +import json +from pathlib import Path + +from django.core.management.base import BaseCommand, CommandError + +from coulomb_social.apps.spaces import pageops +from coulomb_social.apps.spaces.import_inspection import inspect +from coulomb_social.apps.spaces.import_receipts import CorpusReceipts +from coulomb_social.apps.spaces.management.commands.import_content_corpus import ( + Command as ImportCommand, +) + + +class Command(BaseCommand): + help = "Classify an existing corpus checkpoint; never repair or authorize a retry" + + def add_arguments(self, parser): + parser.add_argument("manifest") + parser.add_argument("--receipt-dir", required=True) + + def handle(self, *args, **options): + try: + manifest = Path(options["manifest"]) + plan = ImportCommand().preflight(manifest, replace=False, check_existing=False) + receipts = CorpusReceipts(options["receipt_dir"], manifest, plan) + result = inspect(receipts) + except (ValueError, KeyError, TypeError, OSError, pageops.PageOpsError) as exc: + # Do not disclose source paths, identities or row contents in diagnostics. + raise CommandError( + "Corpus inspection refused; retain the checkpoint and inspect privately" + ) from exc + self.stdout.write(json.dumps(result, sort_keys=True)) diff --git a/docs/release/corpus-import-inspection-2026-10-31.md b/docs/release/corpus-import-inspection-2026-10-31.md new file mode 100644 index 0000000..2a2c471 --- /dev/null +++ b/docs/release/corpus-import-inspection-2026-10-31.md @@ -0,0 +1,85 @@ +# Inspecting uncertain corpus imports + +CSOC-WP-0014-T07 preparation. The receipt-backed importer now reserves the new +Space UUID in its durable `started` checkpoint before import writes. Its SQL +operation creates that exact UUID exclusively. A competing row at the slug or +reserved UUID refuses instead of being updated or receiving imported grants. +The committed return must still have the reserved identity before completion. +Ordinary imports without receipt mode retain their existing behavior. + +This closes an identity gap in the earlier checkpoint implementation: a started +receipt previously named no intended SQL identity, and the underlying generic +importer could update an existing same-tenant row when its content tree was absent. +Legacy completed checkpoints remain usable. Legacy started checkpoints remain +uncertain and are reported as identity-unbound; no identity is invented for them. + +## Read-only inspection + +The new management command is: + +```sh +python manage.py inspect_content_corpus /private/reviewed/manifest.json \ + --receipt-dir /private/import-attempt +``` + +It requires the original manifest/source/mapping/destination binding and an +existing checkpoint and lock files. It does not create missing directories or +locks. It takes the existing importer/content locks without invoking content +recovery, then reads SQL in one transaction. PostgreSQL uses `REPEATABLE READ, +READ ONLY`. It compares destination and checkpoint bookends and rechecks source +hashes before reporting. These cooperating locks are **not** proof of an external +writer fence; the report explicitly records that limit. + +Output contains unit indexes, classifications, counts and hashes, without source +bodies, identity names or SQL rows. It binds the checkpoint bytes and observed SQL, +grants and files. Classifications distinguish: + +- A reserved identity whose current source-derived metadata, mapped grants and + complete source file hashes match. +- A reserved identity with differences, including later content, metadata, grant + or original-date changes. +- Foreign/replacement identities, files without SQL, or SQL without files. +- No SQL row or published tree, which does not establish absence of staging. +- Older started receipts without a reserved identity. +- Pending destinations and the continued presence of completed identities. + +Original space dates are compared when supplied by the source index; absence is +reported. Insertion/update timestamps that cannot be derived from source are +fingerprinted, not fabricated. A matching result does not prove that no intervening +edit ever occurred. Synthesized indexes or omitted extra source files cannot be +called byte-exact by the whole-tree comparison. Completed units may have later +edits; their original receipt remains historical evidence, not an overwrite target. +Unattributed `.import-*` entries are counted, never assigned to an attempt or removed. + +Every result explicitly says that no checkpoint was completed, no retry was +authorized and no data was restored. The inspector does not convert `started` to +`completed` or `pending`. The deployment owner must still reconcile the specific +uncertain state with its retained backup, writer fence and reviewed operation. +Do not remove the checkpoint or use replacement import to bypass uncertainty. + +## Verification and release boundary + +Fifteen new cases cover matching committed state without receipt, partial stores, +legacy/unrelated identities, later edits/grants, original dates, active locks, +missing paths, changing observations and occupied-slug refusal. The existing real +process-exit tests now inspect both the pre-write and committed-without-receipt +states while proving unchanged checkpoints. A row inserted after corpus preflight +is retained without imported grants or destination files. + +Full `make check`: **629 passed, 69 existing warnings, zero failures/errors/skips +in 71.11s**, including the required browser journeys. Ruff and whitespace checks +pass. Log: `/tmp/csoc-import-inspection-check-20261031.log`. + +`scripts/rehearse-import-inspection.py` also runs the corpus, receipt and inspection +tests against disposable PostgreSQL 16.14 on an internal Docker network with no +published ports. It passed 60 checks of actual inspection transactions' read-only +and repeatable-read modes. Current source and locked development dependencies are explicit +read-only overlays on the cached application image; this is not packaged-candidate +or production proof. Its containers, anonymous test volumes and network were +removed. Source-bound receipt: +`corpus-import-inspection-postgres-2026-10-31.json`. + +No live cohort, grant, deployment, release pin or previously reviewed executor +changed. T07 remains in progress: source/identity/grant reconciliation, a concrete +owner procedure for uncertain imports, supervised executor integration, release, +fresh backup/fencing, native postflight and productive acceptance remain required. diff --git a/docs/release/corpus-import-inspection-postgres-2026-10-31.json b/docs/release/corpus-import-inspection-postgres-2026-10-31.json new file mode 100644 index 0000000..6c7a1c1 --- /dev/null +++ b/docs/release/corpus-import-inspection-postgres-2026-10-31.json @@ -0,0 +1,28 @@ +{ + "schema": "coulomb.corpus-inspection-rehearsal.v1", + "proof": { + "status": "passed", + "database_vendor": "postgresql", + "server_version": 160014, + "inspection_transaction_mode_checks": 60, + "source_overlay": true, + "locked_development_dependencies": true, + "production_data_used": false + }, + "app_dependency_image": "sha256:522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed", + "postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "internal_network": true, + "published_ports": false, + "disposable_resources_removed": true, + "source_sha256": { + "scripts/rehearse-import-inspection.py": "2f560e25f00b08cdbcf1a27440c02fc74c57d255429b0496e3f4e25b7eb90468", + "uv.lock": "e8ebb6fc4b5abd1ac80f1c9012f1983642d0528a8a213cfcc394261606fad887", + "coulomb_social/apps/spaces/import_tree.py": "f232bde0d1741668ff28a6cb24306b7f67d8bf9fa3838507a8db4f4e80361784", + "coulomb_social/apps/spaces/import_receipts.py": "670a5eb48613df47bf8fd0cf7d9306f7d2e992d46dff69680e6e7998f64f9cb2", + "coulomb_social/apps/spaces/import_inspection.py": "627f7c006f6aedf337cd146b1052028afe8342ce820283cdfb7963266d9e754d", + "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py": "b083e39aa766c8ba268788fae98507780604d40832e119db1cd9f7c10f1aa212", + "tests/test_import_corpus.py": "5a0670c408740de86ce601cd9d2a0a12cf0aaa5afda416867605180c568a1c5f", + "tests/test_import_receipts.py": "134e8901991dc493333b7543bbd6ff1f1e7b7ca3d5cf9d7849d7c45212e20aab", + "tests/test_import_inspection.py": "890e783eec30dd1b9e93ad7c3ab3173b57174a096938520865eaaf420bc605fb" + } +} diff --git a/docs/release/corpus-import-receipts-2026-10-31.md b/docs/release/corpus-import-receipts-2026-10-31.md index 10ca015..737a829 100644 --- a/docs/release/corpus-import-receipts-2026-10-31.md +++ b/docs/release/corpus-import-receipts-2026-10-31.md @@ -26,6 +26,8 @@ is preflighted before import writes. Dry-run does not create content or receipts an atomic, fsynced completed checkpoint containing the stable Space ID, content, metadata and grant hashes and page/asset counts. It refuses an enclosing database transaction, so a completed receipt cannot precede an outer rollback. + The later inspection follow-through also reserves the new Space UUID before + writes and uses exclusive SQL creation; see the linked review below. - A retry checks the binding and skips completed spaces with the same stable SQL identity. Later body, title and grant edits remain intact. A deleted or replaced identity requires reconciliation. Historical completion hashes describe the @@ -56,6 +58,11 @@ helper supplies no generic automatic repair or permission to restore production. Source fidelity, original-date verification, assets/links and productive transfer verdicts remain the existing workplan requirements. +Read-only classification is now available through `inspect_content_corpus`. +It distinguishes exact reserved identities from partial, changed, foreign and +legacy identity-unbound outcomes without editing a checkpoint or authorizing a +retry. See [inspection review](corpus-import-inspection-2026-10-31.md). + ## Verification The new synthetic suite exercises real process termination before import, after diff --git a/scripts/rehearse-import-inspection.py b/scripts/rehearse-import-inspection.py new file mode 100644 index 0000000..e0647bd --- /dev/null +++ b/scripts/rehearse-import-inspection.py @@ -0,0 +1,158 @@ +"""Run synthetic corpus/receipt tests on disposable isolated PostgreSQL.""" + +import hashlib +import json +import subprocess +import sysconfig +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +IMAGE = ( + "forgejo.coulomb.social/coulomb/coulomb-social@sha256:" + "522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed" +) +CODE = """import json,os,time +os.environ['DJANGO_SETTINGS_MODULE']='coulomb_social.settings.test' +from django.conf import settings +settings.DATABASES={'default':{'ENGINE':'django.db.backends.postgresql', + 'HOST':'inspection-db','NAME':'inspection','USER':'postgres','PORT':'5432'}} +import django +django.setup() +from django.db import connection +for attempt in range(30): + try: + connection.ensure_connection();break + except Exception:time.sleep(1) +else:raise RuntimeError('disposable_database_not_ready') +version=connection.pg_version +connection.close() +from coulomb_social.apps.spaces import import_inspection +original=import_inspection.observe_unit +checks=[] +def observe(*args): + with connection.cursor() as cursor: + cursor.execute('SHOW transaction_read_only');assert cursor.fetchone()[0]=='on' + cursor.execute('SHOW transaction_isolation');assert cursor.fetchone()[0]=='repeatable read' + checks.append(True) + return original(*args) +import_inspection.observe_unit=observe +import pytest +code=pytest.main(['-q','-p','no:cacheprovider','/app/tests/test_import_receipts.py', + '/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py']) +assert code==0 and checks +print(json.dumps({'status':'passed','database_vendor':'postgresql','server_version':version, + 'inspection_transaction_mode_checks':len(checks),'source_overlay':True, + 'locked_development_dependencies':True,'production_data_used':False})) +""" + + +def docker(*args): + return subprocess.run( + ["docker", *args], capture_output=True, text=True, check=True, timeout=180 + ).stdout.strip() + + +def main(): + app = docker("image", "inspect", IMAGE, "--format", "{{.Id}}") + pg = docker("image", "inspect", "postgres:16", "--format", "{{.Id}}") + network = database = runner = None + name = "csoc-inspection-" + uuid.uuid4().hex[:12] + try: + network = docker("network", "create", "--internal", name) + database = docker( + "create", + "--name", + name + "-db", + "--network", + network, + "--network-alias", + "inspection-db", + "-e", + "POSTGRES_HOST_AUTH_METHOD=trust", + "-e", + "POSTGRES_DB=inspection", + pg, + ) + docker("start", database) + runner = docker( + "create", + "--name", + name + "-app", + "--network", + network, + "--read-only", + "--tmpfs", + "/tmp", + "--entrypoint", + "/app/.venv/bin/python", + "-e", + "PYTHONDONTWRITEBYTECODE=1", + "-e", + "PYTHONPATH=/test-deps", + "-v", + str(ROOT / "coulomb_social") + ":/app/coulomb_social:ro", + "-v", + str(ROOT / "tests") + ":/app/tests:ro", + "-v", + sysconfig.get_path("purelib") + ":/test-deps:ro", + app, + "-c", + CODE, + ) + for identity in (database, runner): + metadata = json.loads(docker("inspect", identity))[0] + if metadata["HostConfig"].get("PortBindings"): + raise RuntimeError("published_ports_refused") + if not json.loads(docker("network", "inspect", network))[0]["Internal"]: + raise RuntimeError("internal_network_required") + result = docker("start", "--attach", runner) + if docker("inspect", runner, "--format", "{{.State.ExitCode}}") != "0": + raise RuntimeError("rehearsal_failed") + proof = json.loads(result.splitlines()[-1]) + finally: + # Delete only IDs returned by this invocation, including their anonymous + # test volumes. Retained recovery containers and networks are unrelated. + for identity in (runner, database): + if identity: + docker("rm", "--force", "--volumes", identity) + if network: + docker("network", "rm", network) + files = [ + Path(__file__), + ROOT / "uv.lock", + ROOT / "coulomb_social/apps/spaces/import_tree.py", + ROOT / "coulomb_social/apps/spaces/import_receipts.py", + ROOT / "coulomb_social/apps/spaces/import_inspection.py", + ROOT / "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py", + *( + ROOT / "tests" / name + for name in ( + "test_import_corpus.py", + "test_import_receipts.py", + "test_import_inspection.py", + ) + ), + ] + print( + json.dumps( + { + "schema": "coulomb.corpus-inspection-rehearsal.v1", + "proof": proof, + "app_dependency_image": app, + "postgres_image": pg, + "internal_network": True, + "published_ports": False, + "disposable_resources_removed": True, + "source_sha256": { + str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in files + }, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_import_inspection.py b/tests/test_import_inspection.py new file mode 100644 index 0000000..259e02f --- /dev/null +++ b/tests/test_import_inspection.py @@ -0,0 +1,217 @@ +"""Uncertain imports retain identity and can be inspected without repair writes.""" + +import fcntl +import io +import json +import os +import uuid +from datetime import UTC + +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError + +from coulomb_social.apps.spaces import import_inspection, import_receipts, pageops +from coulomb_social.apps.spaces.import_tree import import_space_tree +from coulomb_social.apps.spaces.models import Space, SpaceMembership +from tests import test_import_corpus + +pytestmark = pytest.mark.django_db(transaction=True) +corpus = test_import_corpus.corpus + + +def inspect(manifest, directory): + output = io.StringIO() + call_command("inspect_content_corpus", str(manifest), receipt_dir=str(directory), stdout=output) + return json.loads(output.getvalue()) + + +def uncertain(manifest, directory, monkeypatch): + original = import_receipts.CorpusReceipts.save + + def lose_receipt(self, data): + if data["spaces"][0]["state"] == "completed": + raise OSError("synthetic lost receipt") + original(self, data) + + with monkeypatch.context() as patch: + patch.setattr(import_receipts.CorpusReceipts, "save", lose_receipt) + with pytest.raises(CommandError): + call_command( + "import_content_corpus", + str(manifest), + receipt_dir=str(directory), + stdout=io.StringIO(), + ) + + +def test_committed_without_receipt_is_classified_without_writes(corpus, tmp_path, monkeypatch): + manifest, _, _ = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + checkpoint = (directory / "checkpoint.json").read_bytes() + rows = list(Space.objects.values()) + grants = list(SpaceMembership.objects.values()) + files = {str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()} + result = inspect(manifest, directory) + assert result["spaces"][0]["classification"] == "reserved_destination_matches_source" + assert result["spaces"][1]["classification"] == "pending" + assert ( + result["read_only"] + and not result["retry_authorized"] + and not result["checkpoint_completed"] + ) + assert not result["external_writer_fence_verified"] + assert (directory / "checkpoint.json").read_bytes() == checkpoint + assert list(Space.objects.values()) == rows and list(SpaceMembership.objects.values()) == grants + assert { + str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file() + } == files + assert result == inspect(manifest, directory) + assert str(Space.objects.get().pk) == json.loads(checkpoint)["spaces"][0]["space_id"] + + +@pytest.mark.parametrize( + "change", ["body", "metadata", "grant", "identity", "legacy", "no_files", "no_sql"] +) +def test_inspection_distinguishes_later_changes_partial_and_unbound_states( + corpus, tmp_path, monkeypatch, change +): + manifest, _, member = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + path = pageops.space_dir("one") / "pages/index.md" + if change == "body": + path.write_text("later local edit") + if change == "metadata": + Space.objects.filter(slug="one").update(title="later title") + if change == "grant": + SpaceMembership.objects.filter(member=member).update(role="viewer") + if change == "identity": + Space.objects.get(slug="one").delete() + Space.objects.create(slug="one", tenant_id="tenant:a", title="one") + if change == "legacy": + checkpoint = directory / "checkpoint.json" + data = json.loads(checkpoint.read_bytes()) + data["spaces"][0].pop("space_id") + checkpoint.write_text(json.dumps(data)) + if change == "no_files": + import shutil + + shutil.rmtree(pageops.space_dir("one")) + if change == "no_sql": + Space.objects.get(slug="one").delete() + before = (directory / "checkpoint.json").read_bytes() + result = inspect(manifest, directory) + expected = { + "identity": "foreign_destination_identity", + "legacy": "started_identity_unbound", + "no_files": "sql_without_files", + "no_sql": "files_without_sql", + }.get(change, "reserved_destination_differs") + assert result["spaces"][0]["classification"] == expected + assert (directory / "checkpoint.json").read_bytes() == before + + +def test_reserved_id_creation_refuses_occupied_slug_without_updating_row(corpus): + manifest, entries, member = corpus + source = manifest.parent / entries[0]["tree"] + original = Space.objects.create + reserved = uuid.uuid4() + # Insert outside the import transaction, at the entry to the importer. It + # must not adopt/update the already occupied slug even with no content tree. + original(slug="one", tenant_id="tenant:a", title="concurrent owner row") + before = list(Space.objects.values()) + with pytest.raises(pageops.PageOpsError): + import_space_tree( + source, tenant_id="tenant:a", memberships=[(member, "owner")], new_space_id=reserved + ) + assert list(Space.objects.values()) == before and not SpaceMembership.objects.exists() + assert not pageops.space_dir("one").exists() + + +def test_row_created_after_corpus_preflight_is_not_adopted(corpus, tmp_path, monkeypatch): + manifest, _, _ = corpus + original = import_receipts.import_space_tree + + def race(*args, **kwargs): + Space.objects.create(slug="one", tenant_id="tenant:a", title="concurrent row") + return original(*args, **kwargs) + + monkeypatch.setattr(import_receipts, "import_space_tree", race) + directory = tmp_path / "receipts" + with pytest.raises(CommandError): + call_command( + "import_content_corpus", str(manifest), receipt_dir=str(directory), stdout=io.StringIO() + ) + assert Space.objects.get().title == "concurrent row" and not SpaceMembership.objects.exists() + assert not pageops.space_dir("one").exists() + assert ( + inspect(manifest, directory)["spaces"][0]["classification"] + == "foreign_destination_identity" + ) + + +def test_source_original_date_is_compared(corpus, tmp_path, monkeypatch): + from datetime import datetime + + manifest, entries, _ = corpus + source = manifest.parent / entries[0]["tree"] / "pages/index.md" + source.write_text( + "---\ntitle: Original\nsource_created_at: '2020-01-02T03:04:05+00:00'\n---\nBody\n" + ) + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + result = inspect(manifest, directory) + assert result["spaces"][0]["original_space_date_checked"] + assert result["spaces"][0]["classification"] == "reserved_destination_matches_source" + Space.objects.filter(slug="one").update(created_at=datetime(2021, 1, 1, tzinfo=UTC)) + assert ( + inspect(manifest, directory)["spaces"][0]["classification"] + == "reserved_destination_differs" + ) + + +@pytest.mark.parametrize("lock_name", ["receipt", "content"]) +def test_inspection_refuses_active_writers(corpus, tmp_path, monkeypatch, lock_name): + manifest, _, _ = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + path = ( + directory / ".lock" if lock_name == "receipt" else pageops.content_root() / ".writer.lock" + ) + fd = os.open(path, os.O_RDONLY) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + with pytest.raises(CommandError): + inspect(manifest, directory) + finally: + os.close(fd) + + +def test_no_inspection_directory_or_locks_are_created(corpus, tmp_path): + manifest, _, _ = corpus + directory = tmp_path / "absent-receipts" + with pytest.raises(CommandError): + inspect(manifest, directory) + assert not directory.exists() and not pageops.content_root().exists() + + +def test_inspection_refuses_drift_during_comparison(corpus, tmp_path, monkeypatch): + manifest, _, _ = corpus + directory = tmp_path / "receipts" + uncertain(manifest, directory, monkeypatch) + original = import_inspection.observe_unit + calls = 0 + + def observe(*args): + nonlocal calls + result = original(*args) + calls += 1 + if calls == 2: + (pageops.space_dir("one") / "pages/index.md").write_text("external writer") + return result + + monkeypatch.setattr(import_inspection, "observe_unit", observe) + with pytest.raises(CommandError): + inspect(manifest, directory) diff --git a/tests/test_import_receipts.py b/tests/test_import_receipts.py index fc262f8..3fc3814 100644 --- a/tests/test_import_receipts.py +++ b/tests/test_import_receipts.py @@ -100,6 +100,15 @@ call_command("import_content_corpus", sys.argv[3], receipt_dir=sys.argv[4]) assert checkpoint.read_bytes() == before assert Space.objects.count() == count assert count == (1 if phase == "committed" else 0) + output = io.StringIO() + call_command("inspect_content_corpus", str(manifest), receipt_dir=str(private), + stdout=output) + report = json.loads(output.getvalue()) + assert report["spaces"][0]["classification"] == ( + "reserved_destination_matches_source" if phase == "committed" + else "no_sql_or_published_tree" + ) + assert checkpoint.read_bytes() == before and not report["retry_authorized"] @pytest.mark.parametrize("drift", ["source", "manifest", "identity", "content_root"]) diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 192183d..27d3363 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -390,6 +390,23 @@ deployment, per-cohort postflight and productive acceptance remain open. No furt source cohort was imported into production, and the existing reviewed executor or release packets were not changed. +2026-10-31 uncertain-import follow-through: receipt mode now persists a reserved +Space UUID before writes and creates it exclusively, preventing an existing or +concurrently created same-slug SQL row from being updated/adopted. The new +`inspect_content_corpus` command classifies matching reserved commits, partial +stores, later edits, foreign identities and legacy unbound receipts without +changing content, SQL or checkpoints. It compares original source dates when +present, uses existing locks and a consistent read transaction, and explicitly +does not authorize retry or claim an external writer fence. Fifteen new cases and +the extended real process-exit inspection pass. Full make check:629 passes with +required browsers and no skips. Disposable PostgreSQL16.14 corpus/receipt/inspection +tests also pass with60 actual read-only/repeatable-read mode checks; all disposable +resources were removed. Review: +[uncertain-import inspection](../docs/release/corpus-import-inspection-2026-10-31.md). +T07 remains progress: concrete state reconciliation, supervised owner integration, +source/grant admission, native release/import/postflight and productive verdicts +are still required. No live cohort or approved executor was changed. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.