Bind import attempts to exclusive space identities and add inspection
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 7m14s
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 7m14s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3
This commit is contained in:
parent
0cc45be42c
commit
e75d3b0f6a
11 changed files with 760 additions and 10 deletions
175
coulomb_social/apps/spaces/import_inspection.py
Normal file
175
coulomb_social/apps/spaces/import_inspection.py
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
"""Read-only classification of uncertain corpus imports; never repairs a store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
|
||||
from django.db import connection, transaction
|
||||
|
||||
from . import pageops
|
||||
from .import_receipts import digest, private_path, tree_digest
|
||||
from .import_tree import _metadata, _source_dates
|
||||
from .models import Space, SpaceMembership
|
||||
|
||||
|
||||
@contextmanager
|
||||
def existing_lock(path):
|
||||
private_path(path)
|
||||
# Inspection never creates a lock, journal, content root or receipt directory.
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
try:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError as exc:
|
||||
raise ValueError("Import or content writer is active; inspect after it stops") from exc
|
||||
yield
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def expected_metadata(tree, tenant, slug, grants):
|
||||
title, description, visual = _metadata(tree, slug)
|
||||
values = {
|
||||
"tenant_id": tenant,
|
||||
"slug": slug,
|
||||
"title": title,
|
||||
"description": description,
|
||||
"visual": visual,
|
||||
"is_active": True,
|
||||
"content_root": "pages",
|
||||
"default_branch": "main",
|
||||
"forgejo_owner": "",
|
||||
"forgejo_repo": "",
|
||||
"created_by_id": next(m.pk for m, role in grants if role == SpaceMembership.Role.OWNER),
|
||||
}
|
||||
index = tree / "pages/index.md"
|
||||
if index.is_file():
|
||||
meta, _ = pageops.parse_frontmatter(index.read_text(encoding="utf-8"))
|
||||
values.update(_source_dates(meta))
|
||||
return values
|
||||
|
||||
|
||||
def observe_unit(receipts, index, unit):
|
||||
tree, tenant, slug, grants = receipts.plan[index]
|
||||
rows = list(Space.objects.filter(slug=slug).values())
|
||||
destination = pageops.space_dir(slug)
|
||||
if destination.is_symlink() or any(p.is_symlink() for p in destination.parents):
|
||||
raise ValueError("Destination must not contain symlinks")
|
||||
files_hash = tree_digest(destination) if destination.is_dir() else None
|
||||
grant_rows = sorted(
|
||||
(str(member), role)
|
||||
for member, role in SpaceMembership.objects.filter(space__slug=slug).values_list(
|
||||
"member_id", "role"
|
||||
)
|
||||
)
|
||||
result = {
|
||||
"index": index,
|
||||
"checkpoint_state": unit.get("state"),
|
||||
"sql_rows": len(rows),
|
||||
"published_tree_present": destination.exists(),
|
||||
"files_sha256": files_hash,
|
||||
"space_sha256": digest(rows),
|
||||
"grants_sha256": digest(grant_rows),
|
||||
}
|
||||
state = unit.get("state")
|
||||
if state == "completed":
|
||||
result["classification"] = (
|
||||
"completed_identity_present"
|
||||
if len(rows) == 1
|
||||
and str(rows[0]["id"]) == unit.get("space_id")
|
||||
and rows[0]["tenant_id"] == tenant
|
||||
else "completed_identity_changed"
|
||||
)
|
||||
return result
|
||||
if state not in {"pending", "started"}:
|
||||
raise ValueError("Invalid checkpoint state")
|
||||
if state == "pending":
|
||||
result["classification"] = (
|
||||
"pending_destination_occupied" if rows or destination.exists() else "pending"
|
||||
)
|
||||
return result
|
||||
try:
|
||||
reserved = uuid.UUID(unit["space_id"])
|
||||
except (KeyError, ValueError, TypeError, AttributeError):
|
||||
result["classification"] = "started_identity_unbound"
|
||||
return result
|
||||
if not rows and not destination.exists():
|
||||
result["classification"] = "no_sql_or_published_tree"
|
||||
return result
|
||||
if not rows:
|
||||
result["classification"] = "files_without_sql"
|
||||
return result
|
||||
if len(rows) != 1 or rows[0]["id"] != reserved or rows[0]["tenant_id"] != tenant:
|
||||
result["classification"] = "foreign_destination_identity"
|
||||
return result
|
||||
if not destination.exists():
|
||||
result["classification"] = "sql_without_files"
|
||||
return result
|
||||
wanted = expected_metadata(tree, tenant, slug, grants)
|
||||
metadata_match = all(rows[0][key] == value for key, value in wanted.items())
|
||||
grants_match = grant_rows == sorted((str(m.pk), role) for m, role in grants)
|
||||
# The importer copies root Markdown pages and the assets subtree. Extra
|
||||
# source files, or a synthesized index, cannot be called byte-exact here.
|
||||
files_match = files_hash == receipts.source_hashes[index]
|
||||
result.update(
|
||||
metadata_matches=metadata_match,
|
||||
grants_match=grants_match,
|
||||
source_files_match=files_match,
|
||||
original_space_date_checked="created_at" in wanted,
|
||||
)
|
||||
result["classification"] = (
|
||||
"reserved_destination_matches_source"
|
||||
if metadata_match and grants_match and files_match
|
||||
else "reserved_destination_differs"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def inspect(receipts):
|
||||
if connection.in_atomic_block or not connection.get_autocommit():
|
||||
raise ValueError("Inspection requires an independent read transaction")
|
||||
private_path(receipts.path)
|
||||
if not receipts.path.is_file():
|
||||
raise ValueError("An existing checkpoint is required for inspection")
|
||||
with (
|
||||
existing_lock(receipts.directory / ".lock"),
|
||||
existing_lock(pageops.content_root() / ".writer.lock"),
|
||||
):
|
||||
checkpoint = receipts.path.read_bytes()
|
||||
data = receipts.read()
|
||||
if any(not isinstance(unit, dict) for unit in data["spaces"]):
|
||||
raise ValueError("Invalid space checkpoint")
|
||||
with transaction.atomic():
|
||||
if connection.vendor == "postgresql":
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")
|
||||
first = [observe_unit(receipts, i, unit) for i, unit in enumerate(data["spaces"])]
|
||||
second = [observe_unit(receipts, i, unit) for i, unit in enumerate(data["spaces"])]
|
||||
if first != second or checkpoint != receipts.path.read_bytes():
|
||||
raise ValueError("Destination or checkpoint changed during inspection")
|
||||
if any(
|
||||
tree_digest(tree) != checksum
|
||||
for (tree, _, _, _), checksum in zip(
|
||||
receipts.plan, receipts.source_hashes, strict=True
|
||||
)
|
||||
):
|
||||
raise ValueError("Source changed during inspection")
|
||||
staging = list((pageops.content_root() / "spaces").glob(".import-*"))
|
||||
result = {
|
||||
"schema": "coulomb.corpus-import-inspection.v1",
|
||||
"binding": receipts.binding,
|
||||
"checkpoint_sha256": hashlib.sha256(checkpoint).hexdigest(),
|
||||
"spaces": first,
|
||||
"unattributed_staging_entries": len(staging),
|
||||
"read_only": True,
|
||||
"external_writer_fence_verified": False,
|
||||
"checkpoint_completed": False,
|
||||
"retry_authorized": False,
|
||||
"data_restored": False,
|
||||
}
|
||||
result["inspection_sha256"] = digest(result)
|
||||
return result
|
||||
|
|
@ -205,13 +205,17 @@ class CorpusReceipts:
|
|||
raise ValueError("Source changed after preflight")
|
||||
# Persist uncertainty BEFORE the first import write. Any failure,
|
||||
# including a lost commit reply or receipt fsync, stays uncertain.
|
||||
data["spaces"][index] = {"state": "started"}
|
||||
space_id = uuid.uuid4()
|
||||
data["spaces"][index] = {"state": "started", "space_id": str(space_id)}
|
||||
self.save(data)
|
||||
report = import_space_tree(tree, tenant_id=tenant, slug=slug,
|
||||
memberships=grants, replace=False)
|
||||
memberships=grants, replace=False,
|
||||
new_space_id=space_id)
|
||||
if tree_digest(tree) != self.source_hashes[index]:
|
||||
raise ValueError("Source changed during import; reconciliation required")
|
||||
space = Space.objects.get(tenant_id=tenant, slug=slug)
|
||||
if space.pk != space_id:
|
||||
raise ValueError("Imported space identity changed; reconciliation required")
|
||||
observed_grants = sorted((str(m), role) for m, role in
|
||||
SpaceMembership.objects.filter(space=space).values_list("member_id", "role"))
|
||||
if observed_grants != sorted((str(m.pk), role) for m, role in grants):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import shutil
|
||||
import tempfile
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -132,7 +133,13 @@ def import_space_tree(
|
|||
tree: Path, *, tenant_id: str, slug: str | None = None,
|
||||
member: Member | None = None, title: str | None = None,
|
||||
replace: bool = False, memberships: list[tuple[Member, str]] | None = None,
|
||||
new_space_id: uuid.UUID | None = None,
|
||||
) -> ImportReport:
|
||||
if new_space_id is not None:
|
||||
if replace or not isinstance(new_space_id, uuid.UUID):
|
||||
raise pageops.PageOpsError("Reserved import identity requires a new UUID space")
|
||||
if Space.objects.filter(pk=new_space_id).exists():
|
||||
raise pageops.PageOpsError("Reserved import identity already exists")
|
||||
# Invalid proposals remain write-free, including lock-file creation.
|
||||
validate_space_tree(tree, tenant_id=tenant_id, slug=slug, member=member,
|
||||
replace=replace, title=title)
|
||||
|
|
@ -140,7 +147,8 @@ def import_space_tree(
|
|||
if mapped.tenant_id != tenant_id or role not in SpaceMembership.Role.values:
|
||||
raise pageops.PageOpsError("Invalid membership tenant or role")
|
||||
return _import_space_tree(tree, tenant_id=tenant_id, slug=slug, member=member,
|
||||
title=title, replace=replace, memberships=memberships)
|
||||
title=title, replace=replace, memberships=memberships,
|
||||
new_space_id=new_space_id)
|
||||
|
||||
|
||||
@pageops.content_writer
|
||||
|
|
@ -148,6 +156,7 @@ def _import_space_tree(
|
|||
tree: Path, *, tenant_id: str, slug: str | None = None,
|
||||
member: Member | None = None, title: str | None = None,
|
||||
replace: bool = False, memberships: list[tuple[Member, str]] | None = None,
|
||||
new_space_id: uuid.UUID | None = None,
|
||||
) -> ImportReport:
|
||||
"""Stage content, then publish with DB writes; restore on ordinary errors.
|
||||
|
||||
|
|
@ -194,15 +203,22 @@ def _import_space_tree(
|
|||
try:
|
||||
with transaction.atomic():
|
||||
try:
|
||||
space, created = Space.objects.update_or_create(
|
||||
tenant_id=tenant_id, slug=space_slug,
|
||||
defaults = {"title": display_title, "description": abstractor,
|
||||
"visual": visual, "is_active": True,
|
||||
"content_root": "pages", "default_branch": "main"},
|
||||
)
|
||||
"content_root": "pages", "default_branch": "main"}
|
||||
if new_space_id is not None:
|
||||
# A receipted attempt owns only its pre-recorded UUID.
|
||||
# Never update a row created concurrently at this slug.
|
||||
space = Space.objects.create(id=new_space_id, tenant_id=tenant_id,
|
||||
slug=space_slug, **defaults)
|
||||
created = True
|
||||
else:
|
||||
space, created = Space.objects.update_or_create(
|
||||
tenant_id=tenant_id, slug=space_slug, defaults=defaults)
|
||||
except IntegrityError as exc:
|
||||
raise pageops.PageOpsError(
|
||||
"Content slug belongs to another tenant"
|
||||
"Reserved import identity or destination already exists"
|
||||
if new_space_id is not None else "Content slug belongs to another tenant"
|
||||
) from exc
|
||||
report.space_created = created
|
||||
report.space_updated = not created
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
"""Inspect uncertain receipt-backed imports without changing content or checkpoints."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from coulomb_social.apps.spaces import pageops
|
||||
from coulomb_social.apps.spaces.import_inspection import inspect
|
||||
from coulomb_social.apps.spaces.import_receipts import CorpusReceipts
|
||||
from coulomb_social.apps.spaces.management.commands.import_content_corpus import (
|
||||
Command as ImportCommand,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Classify an existing corpus checkpoint; never repair or authorize a retry"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("manifest")
|
||||
parser.add_argument("--receipt-dir", required=True)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
try:
|
||||
manifest = Path(options["manifest"])
|
||||
plan = ImportCommand().preflight(manifest, replace=False, check_existing=False)
|
||||
receipts = CorpusReceipts(options["receipt_dir"], manifest, plan)
|
||||
result = inspect(receipts)
|
||||
except (ValueError, KeyError, TypeError, OSError, pageops.PageOpsError) as exc:
|
||||
# Do not disclose source paths, identities or row contents in diagnostics.
|
||||
raise CommandError(
|
||||
"Corpus inspection refused; retain the checkpoint and inspect privately"
|
||||
) from exc
|
||||
self.stdout.write(json.dumps(result, sort_keys=True))
|
||||
85
docs/release/corpus-import-inspection-2026-10-31.md
Normal file
85
docs/release/corpus-import-inspection-2026-10-31.md
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# Inspecting uncertain corpus imports
|
||||
|
||||
CSOC-WP-0014-T07 preparation. The receipt-backed importer now reserves the new
|
||||
Space UUID in its durable `started` checkpoint before import writes. Its SQL
|
||||
operation creates that exact UUID exclusively. A competing row at the slug or
|
||||
reserved UUID refuses instead of being updated or receiving imported grants.
|
||||
The committed return must still have the reserved identity before completion.
|
||||
Ordinary imports without receipt mode retain their existing behavior.
|
||||
|
||||
This closes an identity gap in the earlier checkpoint implementation: a started
|
||||
receipt previously named no intended SQL identity, and the underlying generic
|
||||
importer could update an existing same-tenant row when its content tree was absent.
|
||||
Legacy completed checkpoints remain usable. Legacy started checkpoints remain
|
||||
uncertain and are reported as identity-unbound; no identity is invented for them.
|
||||
|
||||
## Read-only inspection
|
||||
|
||||
The new management command is:
|
||||
|
||||
```sh
|
||||
python manage.py inspect_content_corpus /private/reviewed/manifest.json \
|
||||
--receipt-dir /private/import-attempt
|
||||
```
|
||||
|
||||
It requires the original manifest/source/mapping/destination binding and an
|
||||
existing checkpoint and lock files. It does not create missing directories or
|
||||
locks. It takes the existing importer/content locks without invoking content
|
||||
recovery, then reads SQL in one transaction. PostgreSQL uses `REPEATABLE READ,
|
||||
READ ONLY`. It compares destination and checkpoint bookends and rechecks source
|
||||
hashes before reporting. These cooperating locks are **not** proof of an external
|
||||
writer fence; the report explicitly records that limit.
|
||||
|
||||
Output contains unit indexes, classifications, counts and hashes, without source
|
||||
bodies, identity names or SQL rows. It binds the checkpoint bytes and observed SQL,
|
||||
grants and files. Classifications distinguish:
|
||||
|
||||
- A reserved identity whose current source-derived metadata, mapped grants and
|
||||
complete source file hashes match.
|
||||
- A reserved identity with differences, including later content, metadata, grant
|
||||
or original-date changes.
|
||||
- Foreign/replacement identities, files without SQL, or SQL without files.
|
||||
- No SQL row or published tree, which does not establish absence of staging.
|
||||
- Older started receipts without a reserved identity.
|
||||
- Pending destinations and the continued presence of completed identities.
|
||||
|
||||
Original space dates are compared when supplied by the source index; absence is
|
||||
reported. Insertion/update timestamps that cannot be derived from source are
|
||||
fingerprinted, not fabricated. A matching result does not prove that no intervening
|
||||
edit ever occurred. Synthesized indexes or omitted extra source files cannot be
|
||||
called byte-exact by the whole-tree comparison. Completed units may have later
|
||||
edits; their original receipt remains historical evidence, not an overwrite target.
|
||||
Unattributed `.import-*` entries are counted, never assigned to an attempt or removed.
|
||||
|
||||
Every result explicitly says that no checkpoint was completed, no retry was
|
||||
authorized and no data was restored. The inspector does not convert `started` to
|
||||
`completed` or `pending`. The deployment owner must still reconcile the specific
|
||||
uncertain state with its retained backup, writer fence and reviewed operation.
|
||||
Do not remove the checkpoint or use replacement import to bypass uncertainty.
|
||||
|
||||
## Verification and release boundary
|
||||
|
||||
Fifteen new cases cover matching committed state without receipt, partial stores,
|
||||
legacy/unrelated identities, later edits/grants, original dates, active locks,
|
||||
missing paths, changing observations and occupied-slug refusal. The existing real
|
||||
process-exit tests now inspect both the pre-write and committed-without-receipt
|
||||
states while proving unchanged checkpoints. A row inserted after corpus preflight
|
||||
is retained without imported grants or destination files.
|
||||
|
||||
Full `make check`: **629 passed, 69 existing warnings, zero failures/errors/skips
|
||||
in 71.11s**, including the required browser journeys. Ruff and whitespace checks
|
||||
pass. Log: `/tmp/csoc-import-inspection-check-20261031.log`.
|
||||
|
||||
`scripts/rehearse-import-inspection.py` also runs the corpus, receipt and inspection
|
||||
tests against disposable PostgreSQL 16.14 on an internal Docker network with no
|
||||
published ports. It passed 60 checks of actual inspection transactions' read-only
|
||||
and repeatable-read modes. Current source and locked development dependencies are explicit
|
||||
read-only overlays on the cached application image; this is not packaged-candidate
|
||||
or production proof. Its containers, anonymous test volumes and network were
|
||||
removed. Source-bound receipt:
|
||||
`corpus-import-inspection-postgres-2026-10-31.json`.
|
||||
|
||||
No live cohort, grant, deployment, release pin or previously reviewed executor
|
||||
changed. T07 remains in progress: source/identity/grant reconciliation, a concrete
|
||||
owner procedure for uncertain imports, supervised executor integration, release,
|
||||
fresh backup/fencing, native postflight and productive acceptance remain required.
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"schema": "coulomb.corpus-inspection-rehearsal.v1",
|
||||
"proof": {
|
||||
"status": "passed",
|
||||
"database_vendor": "postgresql",
|
||||
"server_version": 160014,
|
||||
"inspection_transaction_mode_checks": 60,
|
||||
"source_overlay": true,
|
||||
"locked_development_dependencies": true,
|
||||
"production_data_used": false
|
||||
},
|
||||
"app_dependency_image": "sha256:522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed",
|
||||
"postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70",
|
||||
"internal_network": true,
|
||||
"published_ports": false,
|
||||
"disposable_resources_removed": true,
|
||||
"source_sha256": {
|
||||
"scripts/rehearse-import-inspection.py": "2f560e25f00b08cdbcf1a27440c02fc74c57d255429b0496e3f4e25b7eb90468",
|
||||
"uv.lock": "e8ebb6fc4b5abd1ac80f1c9012f1983642d0528a8a213cfcc394261606fad887",
|
||||
"coulomb_social/apps/spaces/import_tree.py": "f232bde0d1741668ff28a6cb24306b7f67d8bf9fa3838507a8db4f4e80361784",
|
||||
"coulomb_social/apps/spaces/import_receipts.py": "670a5eb48613df47bf8fd0cf7d9306f7d2e992d46dff69680e6e7998f64f9cb2",
|
||||
"coulomb_social/apps/spaces/import_inspection.py": "627f7c006f6aedf337cd146b1052028afe8342ce820283cdfb7963266d9e754d",
|
||||
"coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py": "b083e39aa766c8ba268788fae98507780604d40832e119db1cd9f7c10f1aa212",
|
||||
"tests/test_import_corpus.py": "5a0670c408740de86ce601cd9d2a0a12cf0aaa5afda416867605180c568a1c5f",
|
||||
"tests/test_import_receipts.py": "134e8901991dc493333b7543bbd6ff1f1e7b7ca3d5cf9d7849d7c45212e20aab",
|
||||
"tests/test_import_inspection.py": "890e783eec30dd1b9e93ad7c3ab3173b57174a096938520865eaaf420bc605fb"
|
||||
}
|
||||
}
|
||||
|
|
@ -26,6 +26,8 @@ is preflighted before import writes. Dry-run does not create content or receipts
|
|||
an atomic, fsynced completed checkpoint containing the stable Space ID, content,
|
||||
metadata and grant hashes and page/asset counts. It refuses an enclosing database
|
||||
transaction, so a completed receipt cannot precede an outer rollback.
|
||||
The later inspection follow-through also reserves the new Space UUID before
|
||||
writes and uses exclusive SQL creation; see the linked review below.
|
||||
- A retry checks the binding and skips completed spaces with the same stable SQL
|
||||
identity. Later body, title and grant edits remain intact. A deleted or replaced
|
||||
identity requires reconciliation. Historical completion hashes describe the
|
||||
|
|
@ -56,6 +58,11 @@ helper supplies no generic automatic repair or permission to restore production.
|
|||
Source fidelity, original-date verification, assets/links and productive transfer
|
||||
verdicts remain the existing workplan requirements.
|
||||
|
||||
Read-only classification is now available through `inspect_content_corpus`.
|
||||
It distinguishes exact reserved identities from partial, changed, foreign and
|
||||
legacy identity-unbound outcomes without editing a checkpoint or authorizing a
|
||||
retry. See [inspection review](corpus-import-inspection-2026-10-31.md).
|
||||
|
||||
## Verification
|
||||
|
||||
The new synthetic suite exercises real process termination before import, after
|
||||
|
|
|
|||
158
scripts/rehearse-import-inspection.py
Normal file
158
scripts/rehearse-import-inspection.py
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
"""Run synthetic corpus/receipt tests on disposable isolated PostgreSQL."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sysconfig
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
IMAGE = (
|
||||
"forgejo.coulomb.social/coulomb/coulomb-social@sha256:"
|
||||
"522bcd7fce92436765a60c1e4afa2598f3cc40913352153ff1c4319a99ca92ed"
|
||||
)
|
||||
CODE = """import json,os,time
|
||||
os.environ['DJANGO_SETTINGS_MODULE']='coulomb_social.settings.test'
|
||||
from django.conf import settings
|
||||
settings.DATABASES={'default':{'ENGINE':'django.db.backends.postgresql',
|
||||
'HOST':'inspection-db','NAME':'inspection','USER':'postgres','PORT':'5432'}}
|
||||
import django
|
||||
django.setup()
|
||||
from django.db import connection
|
||||
for attempt in range(30):
|
||||
try:
|
||||
connection.ensure_connection();break
|
||||
except Exception:time.sleep(1)
|
||||
else:raise RuntimeError('disposable_database_not_ready')
|
||||
version=connection.pg_version
|
||||
connection.close()
|
||||
from coulomb_social.apps.spaces import import_inspection
|
||||
original=import_inspection.observe_unit
|
||||
checks=[]
|
||||
def observe(*args):
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute('SHOW transaction_read_only');assert cursor.fetchone()[0]=='on'
|
||||
cursor.execute('SHOW transaction_isolation');assert cursor.fetchone()[0]=='repeatable read'
|
||||
checks.append(True)
|
||||
return original(*args)
|
||||
import_inspection.observe_unit=observe
|
||||
import pytest
|
||||
code=pytest.main(['-q','-p','no:cacheprovider','/app/tests/test_import_receipts.py',
|
||||
'/app/tests/test_import_inspection.py','/app/tests/test_import_corpus.py'])
|
||||
assert code==0 and checks
|
||||
print(json.dumps({'status':'passed','database_vendor':'postgresql','server_version':version,
|
||||
'inspection_transaction_mode_checks':len(checks),'source_overlay':True,
|
||||
'locked_development_dependencies':True,'production_data_used':False}))
|
||||
"""
|
||||
|
||||
|
||||
def docker(*args):
|
||||
return subprocess.run(
|
||||
["docker", *args], capture_output=True, text=True, check=True, timeout=180
|
||||
).stdout.strip()
|
||||
|
||||
|
||||
def main():
|
||||
app = docker("image", "inspect", IMAGE, "--format", "{{.Id}}")
|
||||
pg = docker("image", "inspect", "postgres:16", "--format", "{{.Id}}")
|
||||
network = database = runner = None
|
||||
name = "csoc-inspection-" + uuid.uuid4().hex[:12]
|
||||
try:
|
||||
network = docker("network", "create", "--internal", name)
|
||||
database = docker(
|
||||
"create",
|
||||
"--name",
|
||||
name + "-db",
|
||||
"--network",
|
||||
network,
|
||||
"--network-alias",
|
||||
"inspection-db",
|
||||
"-e",
|
||||
"POSTGRES_HOST_AUTH_METHOD=trust",
|
||||
"-e",
|
||||
"POSTGRES_DB=inspection",
|
||||
pg,
|
||||
)
|
||||
docker("start", database)
|
||||
runner = docker(
|
||||
"create",
|
||||
"--name",
|
||||
name + "-app",
|
||||
"--network",
|
||||
network,
|
||||
"--read-only",
|
||||
"--tmpfs",
|
||||
"/tmp",
|
||||
"--entrypoint",
|
||||
"/app/.venv/bin/python",
|
||||
"-e",
|
||||
"PYTHONDONTWRITEBYTECODE=1",
|
||||
"-e",
|
||||
"PYTHONPATH=/test-deps",
|
||||
"-v",
|
||||
str(ROOT / "coulomb_social") + ":/app/coulomb_social:ro",
|
||||
"-v",
|
||||
str(ROOT / "tests") + ":/app/tests:ro",
|
||||
"-v",
|
||||
sysconfig.get_path("purelib") + ":/test-deps:ro",
|
||||
app,
|
||||
"-c",
|
||||
CODE,
|
||||
)
|
||||
for identity in (database, runner):
|
||||
metadata = json.loads(docker("inspect", identity))[0]
|
||||
if metadata["HostConfig"].get("PortBindings"):
|
||||
raise RuntimeError("published_ports_refused")
|
||||
if not json.loads(docker("network", "inspect", network))[0]["Internal"]:
|
||||
raise RuntimeError("internal_network_required")
|
||||
result = docker("start", "--attach", runner)
|
||||
if docker("inspect", runner, "--format", "{{.State.ExitCode}}") != "0":
|
||||
raise RuntimeError("rehearsal_failed")
|
||||
proof = json.loads(result.splitlines()[-1])
|
||||
finally:
|
||||
# Delete only IDs returned by this invocation, including their anonymous
|
||||
# test volumes. Retained recovery containers and networks are unrelated.
|
||||
for identity in (runner, database):
|
||||
if identity:
|
||||
docker("rm", "--force", "--volumes", identity)
|
||||
if network:
|
||||
docker("network", "rm", network)
|
||||
files = [
|
||||
Path(__file__),
|
||||
ROOT / "uv.lock",
|
||||
ROOT / "coulomb_social/apps/spaces/import_tree.py",
|
||||
ROOT / "coulomb_social/apps/spaces/import_receipts.py",
|
||||
ROOT / "coulomb_social/apps/spaces/import_inspection.py",
|
||||
ROOT / "coulomb_social/apps/spaces/management/commands/inspect_content_corpus.py",
|
||||
*(
|
||||
ROOT / "tests" / name
|
||||
for name in (
|
||||
"test_import_corpus.py",
|
||||
"test_import_receipts.py",
|
||||
"test_import_inspection.py",
|
||||
)
|
||||
),
|
||||
]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"schema": "coulomb.corpus-inspection-rehearsal.v1",
|
||||
"proof": proof,
|
||||
"app_dependency_image": app,
|
||||
"postgres_image": pg,
|
||||
"internal_network": True,
|
||||
"published_ports": False,
|
||||
"disposable_resources_removed": True,
|
||||
"source_sha256": {
|
||||
str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
for p in files
|
||||
},
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
217
tests/test_import_inspection.py
Normal file
217
tests/test_import_inspection.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
"""Uncertain imports retain identity and can be inspected without repair writes."""
|
||||
|
||||
import fcntl
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from datetime import UTC
|
||||
|
||||
import pytest
|
||||
from django.core.management import call_command
|
||||
from django.core.management.base import CommandError
|
||||
|
||||
from coulomb_social.apps.spaces import import_inspection, import_receipts, pageops
|
||||
from coulomb_social.apps.spaces.import_tree import import_space_tree
|
||||
from coulomb_social.apps.spaces.models import Space, SpaceMembership
|
||||
from tests import test_import_corpus
|
||||
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
corpus = test_import_corpus.corpus
|
||||
|
||||
|
||||
def inspect(manifest, directory):
|
||||
output = io.StringIO()
|
||||
call_command("inspect_content_corpus", str(manifest), receipt_dir=str(directory), stdout=output)
|
||||
return json.loads(output.getvalue())
|
||||
|
||||
|
||||
def uncertain(manifest, directory, monkeypatch):
|
||||
original = import_receipts.CorpusReceipts.save
|
||||
|
||||
def lose_receipt(self, data):
|
||||
if data["spaces"][0]["state"] == "completed":
|
||||
raise OSError("synthetic lost receipt")
|
||||
original(self, data)
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(import_receipts.CorpusReceipts, "save", lose_receipt)
|
||||
with pytest.raises(CommandError):
|
||||
call_command(
|
||||
"import_content_corpus",
|
||||
str(manifest),
|
||||
receipt_dir=str(directory),
|
||||
stdout=io.StringIO(),
|
||||
)
|
||||
|
||||
|
||||
def test_committed_without_receipt_is_classified_without_writes(corpus, tmp_path, monkeypatch):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
checkpoint = (directory / "checkpoint.json").read_bytes()
|
||||
rows = list(Space.objects.values())
|
||||
grants = list(SpaceMembership.objects.values())
|
||||
files = {str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()}
|
||||
result = inspect(manifest, directory)
|
||||
assert result["spaces"][0]["classification"] == "reserved_destination_matches_source"
|
||||
assert result["spaces"][1]["classification"] == "pending"
|
||||
assert (
|
||||
result["read_only"]
|
||||
and not result["retry_authorized"]
|
||||
and not result["checkpoint_completed"]
|
||||
)
|
||||
assert not result["external_writer_fence_verified"]
|
||||
assert (directory / "checkpoint.json").read_bytes() == checkpoint
|
||||
assert list(Space.objects.values()) == rows and list(SpaceMembership.objects.values()) == grants
|
||||
assert {
|
||||
str(p): p.read_bytes() for p in pageops.content_root().rglob("*") if p.is_file()
|
||||
} == files
|
||||
assert result == inspect(manifest, directory)
|
||||
assert str(Space.objects.get().pk) == json.loads(checkpoint)["spaces"][0]["space_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change", ["body", "metadata", "grant", "identity", "legacy", "no_files", "no_sql"]
|
||||
)
|
||||
def test_inspection_distinguishes_later_changes_partial_and_unbound_states(
|
||||
corpus, tmp_path, monkeypatch, change
|
||||
):
|
||||
manifest, _, member = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
path = pageops.space_dir("one") / "pages/index.md"
|
||||
if change == "body":
|
||||
path.write_text("later local edit")
|
||||
if change == "metadata":
|
||||
Space.objects.filter(slug="one").update(title="later title")
|
||||
if change == "grant":
|
||||
SpaceMembership.objects.filter(member=member).update(role="viewer")
|
||||
if change == "identity":
|
||||
Space.objects.get(slug="one").delete()
|
||||
Space.objects.create(slug="one", tenant_id="tenant:a", title="one")
|
||||
if change == "legacy":
|
||||
checkpoint = directory / "checkpoint.json"
|
||||
data = json.loads(checkpoint.read_bytes())
|
||||
data["spaces"][0].pop("space_id")
|
||||
checkpoint.write_text(json.dumps(data))
|
||||
if change == "no_files":
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(pageops.space_dir("one"))
|
||||
if change == "no_sql":
|
||||
Space.objects.get(slug="one").delete()
|
||||
before = (directory / "checkpoint.json").read_bytes()
|
||||
result = inspect(manifest, directory)
|
||||
expected = {
|
||||
"identity": "foreign_destination_identity",
|
||||
"legacy": "started_identity_unbound",
|
||||
"no_files": "sql_without_files",
|
||||
"no_sql": "files_without_sql",
|
||||
}.get(change, "reserved_destination_differs")
|
||||
assert result["spaces"][0]["classification"] == expected
|
||||
assert (directory / "checkpoint.json").read_bytes() == before
|
||||
|
||||
|
||||
def test_reserved_id_creation_refuses_occupied_slug_without_updating_row(corpus):
|
||||
manifest, entries, member = corpus
|
||||
source = manifest.parent / entries[0]["tree"]
|
||||
original = Space.objects.create
|
||||
reserved = uuid.uuid4()
|
||||
# Insert outside the import transaction, at the entry to the importer. It
|
||||
# must not adopt/update the already occupied slug even with no content tree.
|
||||
original(slug="one", tenant_id="tenant:a", title="concurrent owner row")
|
||||
before = list(Space.objects.values())
|
||||
with pytest.raises(pageops.PageOpsError):
|
||||
import_space_tree(
|
||||
source, tenant_id="tenant:a", memberships=[(member, "owner")], new_space_id=reserved
|
||||
)
|
||||
assert list(Space.objects.values()) == before and not SpaceMembership.objects.exists()
|
||||
assert not pageops.space_dir("one").exists()
|
||||
|
||||
|
||||
def test_row_created_after_corpus_preflight_is_not_adopted(corpus, tmp_path, monkeypatch):
|
||||
manifest, _, _ = corpus
|
||||
original = import_receipts.import_space_tree
|
||||
|
||||
def race(*args, **kwargs):
|
||||
Space.objects.create(slug="one", tenant_id="tenant:a", title="concurrent row")
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(import_receipts, "import_space_tree", race)
|
||||
directory = tmp_path / "receipts"
|
||||
with pytest.raises(CommandError):
|
||||
call_command(
|
||||
"import_content_corpus", str(manifest), receipt_dir=str(directory), stdout=io.StringIO()
|
||||
)
|
||||
assert Space.objects.get().title == "concurrent row" and not SpaceMembership.objects.exists()
|
||||
assert not pageops.space_dir("one").exists()
|
||||
assert (
|
||||
inspect(manifest, directory)["spaces"][0]["classification"]
|
||||
== "foreign_destination_identity"
|
||||
)
|
||||
|
||||
|
||||
def test_source_original_date_is_compared(corpus, tmp_path, monkeypatch):
|
||||
from datetime import datetime
|
||||
|
||||
manifest, entries, _ = corpus
|
||||
source = manifest.parent / entries[0]["tree"] / "pages/index.md"
|
||||
source.write_text(
|
||||
"---\ntitle: Original\nsource_created_at: '2020-01-02T03:04:05+00:00'\n---\nBody\n"
|
||||
)
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
result = inspect(manifest, directory)
|
||||
assert result["spaces"][0]["original_space_date_checked"]
|
||||
assert result["spaces"][0]["classification"] == "reserved_destination_matches_source"
|
||||
Space.objects.filter(slug="one").update(created_at=datetime(2021, 1, 1, tzinfo=UTC))
|
||||
assert (
|
||||
inspect(manifest, directory)["spaces"][0]["classification"]
|
||||
== "reserved_destination_differs"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lock_name", ["receipt", "content"])
|
||||
def test_inspection_refuses_active_writers(corpus, tmp_path, monkeypatch, lock_name):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
path = (
|
||||
directory / ".lock" if lock_name == "receipt" else pageops.content_root() / ".writer.lock"
|
||||
)
|
||||
fd = os.open(path, os.O_RDONLY)
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
with pytest.raises(CommandError):
|
||||
inspect(manifest, directory)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def test_no_inspection_directory_or_locks_are_created(corpus, tmp_path):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "absent-receipts"
|
||||
with pytest.raises(CommandError):
|
||||
inspect(manifest, directory)
|
||||
assert not directory.exists() and not pageops.content_root().exists()
|
||||
|
||||
|
||||
def test_inspection_refuses_drift_during_comparison(corpus, tmp_path, monkeypatch):
|
||||
manifest, _, _ = corpus
|
||||
directory = tmp_path / "receipts"
|
||||
uncertain(manifest, directory, monkeypatch)
|
||||
original = import_inspection.observe_unit
|
||||
calls = 0
|
||||
|
||||
def observe(*args):
|
||||
nonlocal calls
|
||||
result = original(*args)
|
||||
calls += 1
|
||||
if calls == 2:
|
||||
(pageops.space_dir("one") / "pages/index.md").write_text("external writer")
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(import_inspection, "observe_unit", observe)
|
||||
with pytest.raises(CommandError):
|
||||
inspect(manifest, directory)
|
||||
|
|
@ -100,6 +100,15 @@ call_command("import_content_corpus", sys.argv[3], receipt_dir=sys.argv[4])
|
|||
assert checkpoint.read_bytes() == before
|
||||
assert Space.objects.count() == count
|
||||
assert count == (1 if phase == "committed" else 0)
|
||||
output = io.StringIO()
|
||||
call_command("inspect_content_corpus", str(manifest), receipt_dir=str(private),
|
||||
stdout=output)
|
||||
report = json.loads(output.getvalue())
|
||||
assert report["spaces"][0]["classification"] == (
|
||||
"reserved_destination_matches_source" if phase == "committed"
|
||||
else "no_sql_or_published_tree"
|
||||
)
|
||||
assert checkpoint.read_bytes() == before and not report["retry_authorized"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drift", ["source", "manifest", "identity", "content_root"])
|
||||
|
|
|
|||
|
|
@ -390,6 +390,23 @@ deployment, per-cohort postflight and productive acceptance remain open. No furt
|
|||
source cohort was imported into production, and the existing reviewed executor
|
||||
or release packets were not changed.
|
||||
|
||||
2026-10-31 uncertain-import follow-through: receipt mode now persists a reserved
|
||||
Space UUID before writes and creates it exclusively, preventing an existing or
|
||||
concurrently created same-slug SQL row from being updated/adopted. The new
|
||||
`inspect_content_corpus` command classifies matching reserved commits, partial
|
||||
stores, later edits, foreign identities and legacy unbound receipts without
|
||||
changing content, SQL or checkpoints. It compares original source dates when
|
||||
present, uses existing locks and a consistent read transaction, and explicitly
|
||||
does not authorize retry or claim an external writer fence. Fifteen new cases and
|
||||
the extended real process-exit inspection pass. Full make check:629 passes with
|
||||
required browsers and no skips. Disposable PostgreSQL16.14 corpus/receipt/inspection
|
||||
tests also pass with60 actual read-only/repeatable-read mode checks; all disposable
|
||||
resources were removed. Review:
|
||||
[uncertain-import inspection](../docs/release/corpus-import-inspection-2026-10-31.md).
|
||||
T07 remains progress: concrete state reconciliation, supervised owner integration,
|
||||
source/grant admission, native release/import/postflight and productive verdicts
|
||||
are still required. No live cohort or approved executor was changed.
|
||||
|
||||
## Completion
|
||||
|
||||
Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue