From 789e69902df3773e2b0955ebd19175cdd16ff806 Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 27 Oct 2026 23:51:27 +0100 Subject: [PATCH] Harden original-date migration and verify Coulomb recovery admission Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a17f16-d99c-7ea3-95f0-8212897fba6b --- WORK-RECORDS.md | 12 +- coulomb_social/apps/spaces/bubble_transfer.py | 157 ++++++++++++++++ coulomb_social/apps/spaces/import_tree.py | 35 +++- docs/migration/corpus-import.md | 100 ++++++++++ docs/release/acceptance-audit-2026-10-27.md | 175 ++++++++++++++++++ .../release/backup-boundaries-2026-10-27.json | 32 ++++ docs/release/backup-delivery-2026-10-27.json | 18 ++ ...up-version-delete-boundary-2026-10-27.json | 9 + ...p-wrong-namespace-boundary-2026-10-27.json | 13 ++ ...ble-capture-reconciliation-2026-10-27.json | 32 ++++ ...service-contract-conformance-2026-10-27.md | 28 +++ .../storage-candidate-c11300a-2026-10-27.json | 49 +++++ ...torage-migration-rehearsal-2026-10-27.json | 22 +++ docs/usecases/communication-regressions.json | 29 ++- docs/usecases/platform-communication.md | 23 ++- docs/usecases/sharing-portability-review.md | 20 ++ scripts/bubble-corpus-utils.mjs | 103 ++++++++++- scripts/capture-bubble-references.mjs | 104 +++++++++++ scripts/check-communication.py | 17 +- scripts/export-bubble-corpus.mjs | 136 ++++++++------ scripts/plan-bubble-transfer.py | 53 ++++++ scripts/rehearse-storage-migration.py | 168 +++++++++++++++++ scripts/tests/bubble-corpus-utils.test.mjs | 49 ++++- tests/test_bubble_transfer.py | 151 +++++++++++++++ tests/test_communication_gate.py | 30 +++ tests/test_import_tree.py | 10 +- tests/test_storage_migration_lifecycle.py | 62 +++++++ ...0012-durable-recovery-service-readiness.md | 31 +++- .../CSOC-WP-0013-sharing-content-hardening.md | 19 +- ...-WP-0014-cohort-transfer-bubble-offramp.md | 63 ++++++- ...P-0015-communication-runtime-acceptance.md | 21 ++- 31 files changed, 1685 insertions(+), 86 deletions(-) create mode 100644 coulomb_social/apps/spaces/bubble_transfer.py create mode 100644 docs/release/acceptance-audit-2026-10-27.md create mode 100644 docs/release/backup-boundaries-2026-10-27.json create mode 100644 docs/release/backup-delivery-2026-10-27.json create mode 100644 docs/release/backup-version-delete-boundary-2026-10-27.json create mode 100644 docs/release/backup-wrong-namespace-boundary-2026-10-27.json create mode 100644 docs/release/bubble-capture-reconciliation-2026-10-27.json create mode 100644 docs/release/service-contract-conformance-2026-10-27.md create mode 100644 docs/release/storage-candidate-c11300a-2026-10-27.json create mode 100644 docs/release/storage-migration-rehearsal-2026-10-27.json create mode 100644 docs/usecases/sharing-portability-review.md create mode 100644 scripts/capture-bubble-references.mjs create mode 100644 scripts/plan-bubble-transfer.py create mode 100644 scripts/rehearse-storage-migration.py create mode 100644 tests/test_bubble_transfer.py create mode 100644 tests/test_storage_migration_lifecycle.py diff --git a/WORK-RECORDS.md b/WORK-RECORDS.md index 232fd82..2bfb909 100644 --- a/WORK-RECORDS.md +++ b/WORK-RECORDS.md @@ -24,7 +24,7 @@ | workplan | CSOC-WP-0014 | active | — | workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md | | workplan | CSOC-WP-0015 | active | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | | workplan | CSOC-WP-0016 | finished | — | workplans/CSOC-WP-0016-mfa-setup-link.md | -| workplan | CSOC-WP-0017 | ready | — | workplans/CSOC-WP-0017-wave2-resource-right-sizing.md | +| workplan | CSOC-WP-0017 | finished | — | workplans/CSOC-WP-0017-wave2-resource-right-sizing.md | | task | CSOC-WP-0001-T01 | done | — | workplans/CSOC-WP-0001-bubble-io-exit-assessment.md | | task | CSOC-WP-0001-T02 | done | — | workplans/CSOC-WP-0001-bubble-io-exit-assessment.md | | task | CSOC-WP-0001-T03 | done | — | workplans/CSOC-WP-0001-bubble-io-exit-assessment.md | @@ -86,21 +86,21 @@ | task | CSOC-WP-0011-T01 | todo | — | workplans/CSOC-WP-0011-ui-framework-architecture.md | | task | CSOC-WP-0011-T02 | todo | — | workplans/CSOC-WP-0011-ui-framework-architecture.md | | task | CSOC-WP-0011-T03 | todo | — | workplans/CSOC-WP-0011-ui-framework-architecture.md | -| task | CSOC-WP-0012-T01 | wait | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | +| task | CSOC-WP-0012-T01 | progress | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | | task | CSOC-WP-0012-T02 | todo | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | | task | CSOC-WP-0012-T03 | todo | — | workplans/CSOC-WP-0012-durable-recovery-service-readiness.md | | task | CSOC-WP-0013-T01 | progress | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | | task | CSOC-WP-0013-T02 | progress | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | | task | CSOC-WP-0013-T03 | todo | — | workplans/CSOC-WP-0013-sharing-content-hardening.md | -| task | CSOC-WP-0014-T01 | todo | — | workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md | +| task | CSOC-WP-0014-T01 | progress | — | workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md | | task | CSOC-WP-0014-T02 | todo | — | workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md | | task | CSOC-WP-0014-T03 | wait | — | workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md | -| task | CSOC-WP-0015-T01 | wait | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | +| task | CSOC-WP-0015-T01 | done | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | | task | CSOC-WP-0015-T02 | todo | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | -| task | CSOC-WP-0015-T03 | wait | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | +| task | CSOC-WP-0015-T03 | done | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | | task | CSOC-WP-0015-T04 | progress | — | workplans/CSOC-WP-0015-communication-runtime-acceptance.md | | task | CSOC-WP-0016-T01 | done | — | workplans/CSOC-WP-0016-mfa-setup-link.md | -| task | CSOC-WP-0017-T01 | wait | — | workplans/CSOC-WP-0017-wave2-resource-right-sizing.md | +| task | CSOC-WP-0017-T01 | done | — | workplans/CSOC-WP-0017-wave2-resource-right-sizing.md | | intake | CSOC-IN-0001 | closed | blue | docs/intakes/csoc-residuals.md | | intake | CSOC-IN-0002 | closed | green | docs/intakes/csoc-residuals.md | | intake | CSOC-IN-0003 | open | blue | docs/intakes/csoc-residuals.md | diff --git a/coulomb_social/apps/spaces/bubble_transfer.py b/coulomb_social/apps/spaces/bubble_transfer.py new file mode 100644 index 0000000..b064573 --- /dev/null +++ b/coulomb_social/apps/spaces/bubble_transfer.py @@ -0,0 +1,157 @@ +"""Deterministic, offline Bubble transfer inventory; never grants access or imports.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime, timedelta + + +def source_timestamp(value) -> datetime: + """Bubble numeric dates are epoch milliseconds, not filesystem/import times.""" + if isinstance(value, bool): + raise ValueError("Invalid source timestamp") + if isinstance(value, str) and re.fullmatch(r"\d{13}", value): + value = int(value) + if isinstance(value, int): + result = datetime(1970, 1, 1, tzinfo=UTC) + timedelta(milliseconds=value) + elif isinstance(value, str): + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + if result.tzinfo is None: + raise ValueError("Source timestamp requires a timezone") + else: + raise ValueError("Missing or invalid source timestamp") + return result.astimezone(UTC) + + +def normalize_ref(value) -> str: + if not isinstance(value, str) or not value: + raise ValueError("Invalid Bubble reference") + if "LOOKUP__" in value: + match = re.search(r"[0-9]+x[0-9]+", value.rsplit("LOOKUP__", 1)[-1]) + if not match: + raise ValueError("Invalid Bubble lookup reference") + return match[0] + return value + + +def _records(records): + if not isinstance(records, dict): + raise ValueError("Expected a source-id keyed capture") + result = {} + for key, record in records.items(): + if not isinstance(record, dict): + raise ValueError("Invalid source record") + identity = normalize_ref(key) + if identity in result or normalize_ref(record.get("_id", key)) != identity: + raise ValueError("Duplicate or mismatched source identity") + result[identity] = record + return result + + +def transfer_plan(spaces, articles, *, inventory_spaces=None) -> dict: + """Rank every space newest first; expose incomplete dates/chunk closure. + + Output contains private IDs, but no bodies or user identities. Captured + timestamps are preserved verbatim alongside normalized UTC equivalents. + Unknown dates sort last and block transfer preparation. A complete captured + graph is still not proof that the live source has no additional records. + """ + spaces, articles = _records(spaces), _records(articles) + fresh_space_ids = set(spaces) + inventory = _records(inventory_spaces) if inventory_spaces is not None else {} + # Retain the entire known inventory in the queue. An old record may guide + # ordering/closure, but can never certify that the space was refreshed. + spaces = {**inventory, **spaces} + entries = [] + orphan_articles = [] + assigned = {identity: set() for identity in spaces} + for identity, record in articles.items(): + owner = record.get("space_custom_space") + if owner and normalize_ref(owner) in spaces: + assigned[normalize_ref(owner)].add(identity) + else: + orphan_articles.append(identity) + + def dates(record, gaps, identity): + result = {} + for source, target in [("Created Date", "created_at"), ("Modified Date", "modified_at")]: + raw = record.get(source) + result[target + "_raw"] = raw + try: + result[target] = source_timestamp(raw).isoformat() + except (ValueError, OverflowError, OSError): + result[target] = None + gaps.add((identity, "missing_or_invalid_" + target)) + return result + + for identity, space in spaces.items(): + gaps = set() + fresh = identity in fresh_space_ids + if not fresh: + gaps.add((identity, "space_not_refreshed")) + date_fields = dates(space, gaps, identity) + pending = sorted(assigned[identity]) + root_id = space.get("_chunk_custom_article") + root_id = normalize_ref(root_id) if root_id else None + if root_id: + if root_id not in articles: + gaps.add((root_id, "missing_space_root_record")) + else: + pending.append(root_id) + included = set() + chunks = set() + while pending: + article_id = pending.pop() + if article_id in included: + continue + included.add(article_id) + article = articles.get(article_id) + if article is None: + gaps.add((article_id, "missing_chunk_record")) + continue + owner = article.get("space_custom_space") + if owner and normalize_ref(owner) != identity: + gaps.add((article_id, "chunk_belongs_to_other_space")) + refs = article.get("chunks_list_custom_article", []) + if not isinstance(refs, list): + gaps.add((article_id, "invalid_chunk_list")) + continue + for ref in refs: + child = normalize_ref(ref) + chunks.add(child) + pending.append(child) + pages = [] + for article_id in sorted(included): + article = articles.get(article_id) + if article is None: + continue + if not isinstance(article.get("content_text"), str): + gaps.add((article_id, "missing_body")) + parent = article.get("context_custom_article") + parent = normalize_ref(parent) if parent else None + if parent and parent not in articles: + gaps.add((article_id, "missing_parent_record")) + pages.append({ + "id": article_id, "is_chunk": article_id in chunks, "parent_id": parent, + **dates(article, gaps, article_id), + }) + entries.append({ + "id": identity, **date_fields, "root_article_id": root_id, "articles": pages, + "source_record_fresh": fresh, + "captured_graph_complete": not gaps, + "gaps": [{"id": key, "reason": reason} for key, reason in sorted(gaps)], + }) + entries.sort(key=lambda item: ( + item["created_at"] is None, + -source_timestamp(item["created_at"]).timestamp() if item["created_at"] else 0, + item["id"], + )) + referenced = {article["id"] for entry in entries for article in entry["articles"]} + return { + "schema": "coulomb.bubble-transfer-plan.v1", "order": "space_created_at_desc_id_asc", + "spaces": entries, + "unassigned_article_ids": sorted(set(orphan_articles) - referenced), + "inventory_only_space_ids": sorted(set(inventory) - fresh_space_ids), + "live_source_completeness_verified": False, "memberships_authorized": False, + "production_imported": False, + } diff --git a/coulomb_social/apps/spaces/import_tree.py b/coulomb_social/apps/spaces/import_tree.py index f133610..1b64011 100644 --- a/coulomb_social/apps/spaces/import_tree.py +++ b/coulomb_social/apps/spaces/import_tree.py @@ -12,6 +12,7 @@ from django.db import IntegrityError, transaction from coulomb_social.apps.members.models import Member from . import pageops +from .bubble_transfer import source_timestamp from .models import Space, SpaceMembership @@ -32,6 +33,17 @@ def normalize_import_slug(value: str) -> str: return pageops.normalize_slug(value) +def _source_dates(meta: dict) -> dict: + dates = {} + for source, target in [("source_created_at", "created_at")]: + if source in meta: + try: + dates[target] = source_timestamp(meta[source]) + except (ValueError, OverflowError, OSError) as exc: + raise pageops.PageOpsError("Invalid source creation timestamp") from exc + return dates + + def _metadata(tree: Path, slug: str, title: str | None = None) -> tuple[str, str, str]: index = tree / "pages" / "index.md" meta, _ = ( @@ -76,10 +88,25 @@ def validate_space_tree( pages = sorted((tree / "pages").glob("*.md")) if not pages: raise pageops.PageOpsError("Source contains no markdown pages") + page_metadata = [] for path in pages: if not path.is_file() or pageops.normalize_slug(path.stem) != path.stem: raise pageops.PageOpsError("Source page filenames must be normalized slugs") - path.read_text(encoding="utf-8") + meta, _ = pageops.parse_frontmatter(path.read_text(encoding="utf-8")) + _source_dates(meta) + page_metadata.append(meta) + bubble_tree = any( + meta.get("bubble_type") in {"space", "article"} + or str(meta.get("id", "")).startswith(("bubble:", "bubble-space:")) + for meta in page_metadata + ) + if bubble_tree and ( + not (tree / "pages" / "index.md").is_file() + or any("source_created_at" not in meta for meta in page_metadata) + ): + raise pageops.PageOpsError( + "Bubble imports require original creation dates for the space and every page/chunk" + ) _metadata(tree, space_slug, title) root = pageops.content_root() dest = pageops.space_dir(space_slug) @@ -183,6 +210,12 @@ def _import_space_tree( if owner and space.created_by_id is None: space.created_by = owner space.save(update_fields=["created_by", "updated_at"]) + if index.is_file(): + meta, _ = pageops.parse_frontmatter(index.read_text(encoding="utf-8")) + dates = _source_dates(meta) + if dates: + # auto_now_add records insertion time unless explicitly restored. + Space.objects.filter(pk=space.pk).update(**dates) if dest.exists(): dest.rename(backup) staging.rename(dest) diff --git a/docs/migration/corpus-import.md b/docs/migration/corpus-import.md index 78a27dd..49dd3fc 100644 --- a/docs/migration/corpus-import.md +++ b/docs/migration/corpus-import.md @@ -5,6 +5,106 @@ Bubble, provision accounts or verify the completeness of a Bubble export. Use private local trees and manifests outside the checkout. Never commit member mappings, corpus content, DB dumps or backup archives. +## All-space order and original dates — 2026-10-27 + +The user specifies eventual transfer of **all** coulomb.social spaces, using +deterministic tooling and newest space creation first (LIFO guidance). Preserve +creation dates of spaces, pages and every nested chunk. Source modification +date, export time and filesystem mtime must not determine this order. + +Plan an existing private capture without contacting Bubble or an inference service: + +```bash +uv run python scripts/plan-bubble-transfer.py /private/capture --output /private/new-plan.json +``` + +Input is the exporter's source-ID-keyed `spaces.json` and `articles.json`. +The new output is mode 0600 and refuses overwrite; its parent directory should +be private. It contains private IDs and normalized/raw dates, no article bodies +or identities. Input hashes bind it to the exact capture. Every space appears, +sorted by creation descending with a stable ID tie-breaker. Missing dates sort +last and are flagged. Recursively follow `chunks_list_custom_article`, including +children without their own space reference; cycles do not duplicate records. +Report missing children/parents, foreign-space chunks, missing bodies/dates and +unassigned captured articles. Never fabricate dates from Bubble IDs or import time. + +`captured_graph_complete` means only that those checks pass for captured records. +It does not prove pagination/source completeness, asset/link fidelity, original +access rules, approved grants or productive acceptance. Fresh source capture, +all-tree conversion and per-space asset/link reconciliation remain required. +The August retained inventory has 71 spaces/109 articles, with 11 incomplete +captured graphs: 16 missing chunk references, 29 missing parent references and +one space missing both creation/modification dates. All remain in the plan. + +Prepared Markdown uses `source_created_at` for each space index/page/chunk. +The importer accepts timezone-aware ISO values or Bubble epoch milliseconds, +rejects invalid present dates before writes, copies page/chunk metadata unchanged, +and restores the space's SQL `created_at` (including repeat import). Normal edits +retain page/chunk source metadata. Bubble-marked trees require original creation +dates for the space index and every page/chunk before any writes; the importer refuses to synthesize an index +with an unknown original space date. Generic hand-authored, non-Bubble trees +can still omit source metadata. +The capture exporter now includes original space dates in its inventory/index, +alongside the existing original article dates. + +When refreshing an older inventory, add +`--inventory-spaces /private/old-capture/spaces.json` to the planning command. +It keeps every known space queued, including absent records, explicitly marked +`source_record_fresh: false` and `space_not_refreshed`. Retained timestamps may +guide ordering; they never certify current source completeness or transfer readiness. + +After renewing the private Bubble session with `scripts/capture-auth-state.sh`, +the read-only capture can discover every visible space root instead of using +the historical fixed space-name list: + +```bash +COULOMB_SOCIAL_ALL_SPACES=1 COULOMB_SOCIAL_CORPUS_DIR=/private/new-capture \ + node scripts/export-bubble-corpus.mjs +``` + +This mode never seeds old responses, ranks discovered roots by their captured +space creation date, retains unknown dates last and records each visited/missing/ +failed root in the private index. It still uses bounded UI scrolling and does +not certify pagination/source completeness. The source JSON capture covers all +visited roots; the legacy exporter still produces only one sample tree. Run the +planner on the JSON and reconcile missing references before all-tree conversion; +do not interpret that one sample tree as migration of the entire capture. + +Optionally set `COULOMB_SOCIAL_ORDER_SPACES_FILE=/private/old-capture/spaces.json` +to rank by retained original creation dates before every space's fresh metadata +has been fetched. That file is used for ordering only, with its hash recorded; +older articles/bodies are never seeded in all-space mode. Private source JSON +and `capture-state.json` checkpoints preserve partial successful capture. Three +consecutive navigation failures stop the pass with explicit private step diagnostics. +Bubble's hidden root anchors identify the correct card; capture clicks its visible +title rather than the hidden anchor. A visited card is not pagination proof. + +Resume a partial capture through explicit missing references, without another full +space walk or using older article bodies: + +```bash +node scripts/capture-bubble-references.mjs --capture /private/checkpoint \ + --inventory-spaces /private/old-capture/spaces.json --output /private/new-reference-pass +``` + +This follows only source-referenced article/root addresses, including missing +chunk/parent records, absent bodies/dates and unrefreshed space roots. Input hashes, +attempts and unresolved references remain in private checkpoints. Output must be +new and outside the repository. Response reads are bounded; a completed pass is +not a completeness claim. Plan its resulting source files again. The October27 +pass captured 70 fresh spaces/162 articles; all 71 known spaces remain represented, +with 67 captured graphs passing and the remaining source/ownership gaps explicit. + +Packaged SQL migration rehearsal (internal disposable PostgreSQL, no production): + +```bash +uv run python scripts/rehearse-storage-migration.py \ + --app-image forgejo.coulomb.social/coulomb/coulomb-social:c11300a --packaged-source +``` + +Packaged mode uses the image's application and Django; only test tools are copied +into a read-only mount. Its test database/container/network are disposable. + ## Manifest Version 1 JSON. Tree paths are relative to the manifest directory or absolute. diff --git a/docs/release/acceptance-audit-2026-10-27.md b/docs/release/acceptance-audit-2026-10-27.md new file mode 100644 index 0000000..d0ec31b --- /dev/null +++ b/docs/release/acceptance-audit-2026-10-27.md @@ -0,0 +1,175 @@ +# Workplan acceptance audit — 2026-10-27 + +WP-0012–0015 remain active. UC-008 weekly-call acceptance and UC-009 personal +reset acceptance retain their original scope. Neither authorizes a new cohort +import, a mailbox scan, a provider email, or Bubble retirement. + +## Verified returns + +`make check`: Ruff and 219 tests pass, including mandatory real-browser journeys. +`make check-communication EMAIL_CONNECT_ROOT=/home/worsch/email-connect`: 103 tests +pass, zero skipped, no missing required functions. Its private snapshot now includes +four explicitly declared recovery helpers. Source/test/helper hashes are retained in +`var/test-results/communication.json`; no owner configuration, credentials or ledger +is copied. Recovery covers all six tables and accepted/unknown deliveries, +consumed/cancelled handles, suppression and persisted caller quota continuity. + +The real migration executor passes on both the normal SQLite test database and +disposable PostgreSQL 16.14. Legacy collisions refuse before application of 0003, +without modifying either conflicting row or memberships. Reversing 0003 preserves +rows, grants and tenant uniqueness; reapplying restores global uniqueness. +Reproduce with `uv run python scripts/rehearse-storage-migration.py`. +[Hash-bound PostgreSQL proof](storage-migration-rehearsal-2026-10-27.json). +This does not prove a production migration or a packaged candidate release. + +Read-only SSH checks on railiance01 found: + +- App Ready 1/1 at `sha256:9a646d8e01a61ba5316d3c2eb0273064e17f2d0300ad07af766ccedda65ecb18`. + Latest owning rollout is Helm revision 17 (WP-0017); revision 16 is its + unchanged product-image provenance. Spaces migrations 0001/0002 are applied; + global slug constraint is absent. Two spaces, zero duplicate-slug groups, + 82 content files and zero pending recovery journals; registration URL configured. +- `/etc/coulomb-social-backup/primary.env` is absent. CCR-2026-0036 is applied + for the reader, but its three delivery admission flags remain false. The + record explicitly says node delivery is unadmitted. RPF-WP-0053-T01/T02 + still own execution and delivery admission; do not substitute the email key. +- Scoped email receiver Ready 1/1 at `sha256:5d79fd1fec15d3a279383820f170d263de3a55f5308df01a0a34732ebaea6d3d`; + generation 66 observed. Its caller policy ConfigMap is + `email-connect-callers-user-engine-276257849bf2`, with caller projection present. +- User-engine Ready 1/1 at `sha256:1e6547c40c93c49f01438edcf76c20f561ccebc110220bae020723505aa56c70`; + generation 77 observed. Caller projection and exact `tenant:coulomb`, + `tenant:*-family`, `tenant:family-*` bindings to client `coulomb-social` remain. + This is a newer caller image than the October 9 rollout, not a fresh mail receipt. +- Both ledger timers active. Backup service last succeeded 04:20:49 UTC today; + freshness service last succeeded 20:00:00 UTC. Last capture 04:19:47 UTC, + primary version `1793074846197328`, ciphertext SHA256 + `53f0b616876da23a34d8a3fe2d7376128fa41bff381798c11a625991db052bc3`. + Verified independent mirror version `1793074847672146`; versioned receipt + `1793074848495349`. No credentials or message bodies were read. + +## Requirement disposition + +| Task | Evidence and remaining requirement | +| --- | --- | +| WP-0012-T01 | Reader/provider pair preparation exists, node delivery absent; execution admission, independent application receipt/key custody, capacity/retention and temporary app-pair disposition still required. | +| WP-0012-T02 | Capture/retrieve/decrypt/restore tooling exists; no first admitted offsite Coulomb point. Run exact version retrieval and content/membership/denial rehearsal after admission. | +| WP-0012-T03 | Prepared units/runbook and historical local restore exist; recurring Coulomb backup, received failure/stale signal, measured complete recovery and service conformance remain unproven. CSOC-IN-0004 live CRUD/transfer smoke remains open. | +| WP-0013-T01 | Constraint and refusal/concurrency tests plus actual forward/reverse PostgreSQL proof pass. Guarded image release, fresh pre-release pair, production migration and postflight still required. | +| WP-0013-T02 | Retention/recovery implementation and tests pass. Representative original custom Markdown/raw HTML/body assets and productive fidelity verdict missing; no live purge/recovery was run. | +| WP-0013-T03 | Sharing/export/exit UI implemented and browser-tested; separate user UC-003/004/005/006 verdicts missing. | +| WP-0014-T01 | 71-space/109-article historical inventory remains inventory. User now specifies all spaces, newest creation first. Deterministic planner and date-preserving import implemented; fresh private source, identity/grants, complete chunks/assets and retained workflow comparison remain. | +| WP-0014-T02 | Catalogue preserves stable IDs and confidence. Original/rebuild comparison and user disposition required for remaining journeys and later candidates. | +| WP-0014-T03 | Existing cutover decision `186be722-e2c7-4109-a1e6-d120fe8fca5b` remains open; needs WP-0012/0013 returns and explicit cohort/apex decision. | +| WP-0015-T01 | Complete: EMAIL-WP-0008 admitted/activated the scoped original caller, template/operation/binding quotas and separate token custody; current runtime retains those bindings. Productive per-flow verdict remains T04. | +| WP-0015-T02 | Platform sender, named support/reply owner, authorized inbound access, DNS alignment and correlated bounce/suppression/retention evidence remain missing. | +| WP-0015-T03 | Complete: EMAIL-WP-0007 independent six-table recovery/copy custody and recurring operation accepted October 10; current timers and independently verified archive mirror confirmed. See references below. | +| WP-0015-T04 | Loopback double covers app browser mechanics; actual owner-spanning issuer/directory/mail-sink/app journey and productive UC-010/011 verdicts remain missing. UC-012 still needs a nominated product caller/template. | + +Owner returns, inspected in their repositories: + +- `email-connect/docs/scoped-caller-live-2026-10-09.json` and EMAIL-WP-0008: + admitted policy/custody, coordinated rollout, retired bearer refusal, no-send + tenant/payload/burst/suppression denials and preserved accepted deliveries. +- `email-connect/docs/ledger-scaleway-custody-2026-10-10.json`, + `deploy/backup/operating-acceptance-2026-10-10.json` and EMAIL-WP-0007: + provider-retrieved nonexpiring kit and escrow, separately bound recovery reader, + native worker/read/write/delete denials and network-disabled image-bound restore. + Live empty transactional states are covered synthetically, not claimed observed. + Daily mirrored archives expire after 30 days; protected kit has no expiry. + Platform-root owns weekly retention/capacity/kit refresh and failure reception. + Historical private pre-release copies remain retained pending explicit owner + disposition; no expiry or destruction is inferred. Production restore requires + caller/challenge fencing and post-capture reconciliation; 60-minute RTO is a target. + +## Concrete next returns + +1. Platform owner must admit the existing Coulomb node executor/delivery contract + in CCR-2026-0036 and RPF-WP-0053, with native provider boundary proof. Use its + pair-aware delivery helper, exact CCR hash and pinned KV version; deliver only + the Coulomb pair to root:root 0600 `primary.env` beneath a 0700 parent. No + bucket/IAM/escrow administration belongs in this worker. Existing founder + schedule approval (daily 03:15 UTC; RPO 24h, stale 26h, RTO target 60min) is + already recorded in RPF-WP-0053; do not ask for that schedule again. +2. Retain independent version/hash receipts and key/image recovery custody, bind + lifecycle/headroom, run/retrieve/rehearse the first point, then activate and + prove monitored backup/freshness units. A local paired capture does not meet this. +3. Release the storage/content hardening only with fresh paired rollback evidence + and the PostgreSQL migration proof above. Restore production only under a + separate incident decision accounting for writes after capture. +4. User specifies eventual transfer of all spaces, newest-created first, with + original space/page/chunk creation dates preserved. Use the deterministic + planner in docs/migration/corpus-import.md and refresh the expired Bubble + session; record each comparison/verdict using the existing + [review walkthrough](../usecases/first-transfer-review.md). Bulk productive import still requires complete source/chunk/asset reconciliation + and explicit grants. Prepared UC-003–006 steps are in + docs/usecases/sharing-portability-review.md. +5. Complete real owner-spanning communication browser tests and appoint the + platform sender/reply/bounce operator. No live mail or mailbox scan was performed. + + +## Subsequent approved node delivery — 2026-10-27 + +The earlier absent/unadmitted file observations above were pre-admission checks. +User decision 9557e2b8-2b37-5b94-a5e6-e5fe08d0351e now approves the Coulomb-only +node executor and delivery. Native boundary checks pass and version-2 own-pair +delivery is complete, hash-bound to CCR 1f624cd21f73bcbbb5920c6bd8d8dcf9ccee2e8389a06560644b35947c333925. +See backup-boundaries-2026-10-27.json and backup-delivery-2026-10-27.json. +No completed object, first paired capture, independent escrow/retrieval or timer +activation is implied. Those remain WP-0012-T01/T02/T03 returns. + + +Platform admission source sync succeeded at commit +4583b22fd697b4212ebff46750930fdb06a61adc (primary railiance01, +expected commit equals derived commit, applied). The original admission source +commit is 98b1190; Repo Manager's required identifier assignment produced the +synced follow-up commit. Normal hooks, including the pinned secret scanner, +passed after clarifying a public source-file SHA256 field name. + +Installed paired/snapshot helpers passed the current-image server dry-run, +with no app pause or archive upload. Node /var/lib free space observed +122,820,976,640 bytes of 206,900,281,344 total. This observation is not a +retention/independent custody acceptance. + + +## Packaged candidate and fresh source reconciliation + +Published image `c11300a`, digest +`sha256:fb579adefa6e3e0c16fedeac134b2e05f4ed6216807cb25995fbc0da826c1167`, +passes the packaged PostgreSQL migration lifecycle (image source and Django, +only test tools injected), migration-drift check and server dry-run. Three-resource +Helm comparison against revision17 changes only the deployment image. +[Candidate proof](storage-candidate-c11300a-2026-10-27.json). Not deployed; +fresh paired rollback/recovery evidence remains required before the guarded release. + +The all-visible-space pass checkpointed 70 visits but hit its parent deadline +before a final index. Those records were retained, not reclassified as complete. +The new reference continuation follows only source-referenced UI addresses, hashes +its input files and bounds each response read. It completed 34 visits without a +response timeout, yielding 70 fresh spaces/162 articles. The planner combines +fresh records with the entire retained inventory, marking absent spaces +`space_not_refreshed`, never certifying old metadata as fresh. It retains all 71 +known spaces: 67 captured graphs pass; one inventory-only space, two absent body +fields, two missing/invalid creation and modification dates (including the stale +space), one cross-space chunk relation and one unassigned article remain visible. +No graph pass certifies assets, privacy, live source completeness or acceptance. +[Non-secret reconciliation](bubble-capture-reconciliation-2026-10-27.json). +Private source bodies, IDs, account mapping and checkpoints remain outside Git. + + +Bubble-marked imports now require an original creation date on the space index +and every page/chunk, before any writes, including refusal to synthesize an index +with an unknown original space date. Updated synthetic rollback fixtures preserve +date metadata while testing real rollback. Full gate219 passes; Node capture8 +passes. Candidate c11300a predates this strict guard and must be refreshed before +its production release; it remains an interim packaged SQL proof, not deployment +approval. An additional native DeleteObjectVersion probe returns403 on a unique +never-published key, touching no retained object. Receipt: +backup-version-delete-boundary-2026-10-27.json. + +Wrong-namespace login using the same admitted ServiceAccount name also returns403, +without reading sibling credentials or any data. Receipt: +backup-wrong-namespace-boundary-2026-10-27.json. Whole-capture field-completeness +counts are separate from per-space graph counts: 70 fresh space dates and 160 +article creation dates are present/valid; 159 article records have a text body, +and two captured article records contain only an ID. Those stubs are not proof +of deletion, permission, an empty body or an original creation date. diff --git a/docs/release/backup-boundaries-2026-10-27.json b/docs/release/backup-boundaries-2026-10-27.json new file mode 100644 index 0000000..5f11361 --- /dev/null +++ b/docs/release/backup-boundaries-2026-10-27.json @@ -0,0 +1,32 @@ +{ + "decision_id": "9557e2b8-2b37-5b94-a5e6-e5fe08d0351e", + "proof": { + "completed_objects_created": false, + "credential_values_printed": false, + "kv_version": 2, + "negative_login_denials": { + "wrong_audience": 403, + "wrong_serviceaccount": 403 + }, + "own_multipart_create_abort": true, + "positive_reader_and_revoke": true, + "provider_denials": { + "bucket_policy": 403, + "delete": 403, + "iam_list": 403, + "native_prefix": 403, + "sibling_prefix": 403 + }, + "reader_denials": { + "admin": 403, + "list": 403, + "metadata": 403, + "sibling": 403, + "write_cas0_existing": 403 + }, + "schema": "coulomb.backup-boundary-proof.v1", + "status": "verified" + }, + "verifier_sha256": "071285b9ecf33a1b80ff6af14f23ccf0a76cad1b30a9f9b34d27262ef1e636de", + "delivery_source_sha256": "afefce5b3de43539d4c4d02a4b92da395326eb2f3b410b5baf4586afeb05b107" +} diff --git a/docs/release/backup-delivery-2026-10-27.json b/docs/release/backup-delivery-2026-10-27.json new file mode 100644 index 0000000..236fa9e --- /dev/null +++ b/docs/release/backup-delivery-2026-10-27.json @@ -0,0 +1,18 @@ +{ + "application": "coulomb-social", + "ccr": "CCR-2026-0036", + "ccr_sha256": "1f624cd21f73bcbbb5920c6bd8d8dcf9ccee2e8389a06560644b35947c333925", + "cluster_uid": "a553c742-0115-43d4-99a4-a5ca56fe0786", + "delivered_at": "2026-10-27T21:18:21.952636+00:00", + "directory_mode": "0700", + "kv_path": "platform/workloads/coulomb-social/backup-primary", + "kv_version": 2, + "mode": "0600", + "outcome": "created", + "owner": "root:root", + "readback_equal": true, + "schema": "platform.application-backup-credential-delivery.v1", + "status": "delivered", + "target": "/etc/coulomb-social-backup/primary.env", + "token_revoked": true +} diff --git a/docs/release/backup-version-delete-boundary-2026-10-27.json b/docs/release/backup-version-delete-boundary-2026-10-27.json new file mode 100644 index 0000000..9b069e1 --- /dev/null +++ b/docs/release/backup-version-delete-boundary-2026-10-27.json @@ -0,0 +1,9 @@ +{ + "schema": "coulomb.backup-version-delete-boundary.v1", + "status": "denied", + "http_status": 403, + "action": "DeleteObjectVersion", + "probe_key_never_published": true, + "retained_objects_touched": false, + "credential_values_printed": false +} diff --git a/docs/release/backup-wrong-namespace-boundary-2026-10-27.json b/docs/release/backup-wrong-namespace-boundary-2026-10-27.json new file mode 100644 index 0000000..4df975c --- /dev/null +++ b/docs/release/backup-wrong-namespace-boundary-2026-10-27.json @@ -0,0 +1,13 @@ +{ + "schema": "coulomb.backup-wrong-namespace-boundary.v1", + "status": "denied", + "http_status": 403, + "target_role_application": "coulomb-social", + "same_service_account_name": true, + "wrong_namespace": true, + "data_reads": 0, + "sibling_credentials_read": false, + "credential_values_printed": false, + "probe_identity_source": "Fresh TokenRequest for the same pre-existing ServiceAccount name in another namespace; used only for negative login to the Coulomb role", + "sibling_backup_secret_values_read": false +} diff --git a/docs/release/bubble-capture-reconciliation-2026-10-27.json b/docs/release/bubble-capture-reconciliation-2026-10-27.json new file mode 100644 index 0000000..b47bc3c --- /dev/null +++ b/docs/release/bubble-capture-reconciliation-2026-10-27.json @@ -0,0 +1,32 @@ +{ + "date": "2026-10-27", + "known_spaces": 71, + "fresh_spaces": 70, + "captured_articles": 162, + "captured_graph_complete": 67, + "inventory_only_spaces": 1, + "unassigned_articles": 1, + "gap_counts": { + "space_not_refreshed": 1, + "missing_body": 2, + "chunk_belongs_to_other_space": 1, + "missing_or_invalid_created_at": 2, + "missing_or_invalid_modified_at": 2 + }, + "source_sha256": { + "spaces": "ae20b0cfd1705f486c0b67ef48ac84b396f953cbcef09f9a8f07d07a66f1bf71", + "articles": "629edc87200902b9b1f552d26fe940c9d1aee535e6e4a12037114e5f54087bc1", + "retained_spaces": "92038249ec2e4eed4c8d61ffcdbe030232cebc52e59ba005baee506a16930550" + }, + "reference_visits": 34, + "reference_pass_completed": true, + "response_timeouts": 0, + "live_source_completeness_verified": false, + "production_imported": false, + "record_completeness": { + "fresh_spaces_with_valid_creation_date": 69, + "article_records_with_valid_creation_date": 160, + "article_records_with_text_body": 159, + "article_records_containing_only_id": 2 + } +} diff --git a/docs/release/service-contract-conformance-2026-10-27.md b/docs/release/service-contract-conformance-2026-10-27.md new file mode 100644 index 0000000..3e7e466 --- /dev/null +++ b/docs/release/service-contract-conformance-2026-10-27.md @@ -0,0 +1,28 @@ +# Business-app service contract evidence — 2026-10-27 + +Assessment against the accepted `the-custodian/canon/standards/business-app-service-contract_v0.1.md` +§§1–6. **Conformance is not yet established.** Each row separates implemented +evidence from the returns required to finish WP-0012/0014. + +| Section | Current evidence | Remaining proof | +| --- | --- | --- | +| 1: shape/tenancy | Standalone repo, `coulomb_social_db`, dedicated deployment/values in railiance-apps. Space/Member tenant keys and explicit membership joins; tests deny foreign tenant/member access. Slug-keyed content has global collision refusal in source. | Deploy and verify the SQL slug constraint. Reconcile all source grants and prove complete tenant export/consolidation without ambiguous paths; global uniqueness alone does not certify a multi-client migration. | +| 2: identity | OIDC/PKCE behind `apps/identity`; app-local users have unusable passwords. Accepted ADR-0001 specializes the local-account baseline. Historical Case B and personal recovery are recorded in docs/identity/smoke.md and the October 6 receipts. | Current registration Case B and source-cohort identity/grant return. Link the governing State Hub decision for the ADR specialization before claiming §8.3 deviation acceptance; the ADR alone does not establish that link. | +| 3: build/release | Forgejo immutable product digest `9a646d8e…`, source `dd83c90`, values/release and smoke in the revision16 receipt; resource-only revision17 in WP-0017. Release-unit declaration in registry/release-units/coulomb-social.yaml. | Guarded storage/content release and matching source/image/values/smoke receipt. Old contract §3.2 names Gitea; current delivery uses Forgejo. Resolve the fleet successor provenance rather than silently treating that as identical. | +| 4: operations | Deployment Ready, health endpoint/runbooks, capture and isolated restore tooling. Actual historical paired restore exists. | Current ops-hub client/environment/endpoint registration evidence; admitted recurring *paired* backup, independent receipt/key/image recovery and measured complete recovery; failed/stale signal reception; post-release stage-1 CRUD/copy/transfer smoke and CSOC-IN-0004 reconciliation. Database-only backup does not cover CONTENT_ROOT. | +| 5: cost | Attribution key `resource:tenant:coulomb:coulomb-social`, dated workload/demand/labor records in docs/resource-evidence; WP-0017 records current requests 15m/176Mi. | Demonstrate the key's client × app × instance join in current finance ingestion and per-instance infrastructure cost report. Planning labor rates and CPU requests are not booked costs. | +| 6: protection | Platform instance isolation, explicit read/write denial, sanitized Markdown, private images, account-local JSON export and successor-aware local exit. No credentials copied into these records. | Full client-dataset export, instance teardown plus backup expiry contract (§6.2), and schema-derived personal-data inventory. Personal export is narrower than complete client offboarding. Productive sharing/portability verdicts and retained-data disposition remain WP-0013. | + +The August service-objectives record is historical. Its RPO ≤60min/RTO ≤4h and +Forgejo content durability assumptions predate the persistent CONTENT_ROOT +design. RPF-WP-0053 records the founder-approved paired schedule at 03:15 UTC, +RPO target 24h, stale threshold 26h and attended RTO target 60min. This supersedes +the old *planning assumption*, but no such Coulomb schedule is active yet. +Availability 99.5%, health p99 500ms and page latency objectives remain planning +targets, with no current measurement window proving attainment. Preserve dated +observations; do not rewrite August measurements as current facts. + +WP-0012 owns the operating/service returns above. WP-0013 owns privacy/content +verdicts; WP-0014 owns all-space reconciliation and the separate cohort/apex +cutover decision. Existing canonical cross-owner records remain in their owners; +this assessment creates no credential grant or service acceptance. diff --git a/docs/release/storage-candidate-c11300a-2026-10-27.json b/docs/release/storage-candidate-c11300a-2026-10-27.json new file mode 100644 index 0000000..1da91e0 --- /dev/null +++ b/docs/release/storage-candidate-c11300a-2026-10-27.json @@ -0,0 +1,49 @@ +{ + "packaged_app_image": "sha256:fb579adefa6e3e0c16fedeac134b2e05f4ed6216807cb25995fbc0da826c1167", + "postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "proof": { + "status": "passed", + "database_vendor": "postgresql", + "server_version": 160014, + "forward_collision_refusal": true, + "refusal_preserves_rows_and_grants": true, + "rollback_preserves_rows_grants_and_tenant_constraint": true, + "reapply_enforces_global_constraint": true, + "production_access": false + }, + "source_sha256": { + "tests/test_storage_migration_lifecycle.py": "34df20aafd86b723c9192b3d45941ab3bb78bb2837267f2c3f7b056072bb5e7b", + "coulomb_social/apps/spaces/migrations/0003_space_slug_global_uniq.py": "e567cf8fd5f3b3761e5336a3f0209908c1fb00beac10009d2eeda6c91a9bbdad", + "uv.lock": "06a1bfff320560086dc0210e01e2504e07cf18cc4ae8646e6980c2f472e29935" + }, + "date": "2026-10-27", + "source_commit": "c11300a17cf202a8f38d2b1f2ddd000f1ea26740", + "registry_image": "forgejo.coulomb.social/coulomb/coulomb-social@sha256:fb579adefa6e3e0c16fedeac134b2e05f4ed6216807cb25995fbc0da826c1167", + "published_tag": "c11300a", + "published": true, + "packaged_makemigrations_check": "No changes detected", + "server_dry_run": "passed", + "production_deployed": false, + "candidate_render_sha256": "120f88734f01be2d668c6b56bf79d171f2e08a8959b6f4e8d302d1c70cff248a", + "baseline_helm_revision": 17, + "packaged_source_and_django": true, + "injected_test_tools": [ + "pytest", + "_pytest", + "pluggy", + "iniconfig", + "pygments", + "py.py" + ], + "reproduce": "uv run python scripts/rehearse-storage-migration.py --app-image forgejo.coulomb.social/coulomb/coulomb-social:c11300a --packaged-source", + "manifest_comparison": { + "resource_count": 3, + "changes": [ + "Deployment app image" + ], + "other_changes": false + }, + "rehearsal_script_sha256": "799f009af4e7b3c163bc72c1cf474bd1d5871636f38e1a69c7967c72eeba67f3", + "production_rollout_ready": false, + "requires_rebuild_for_strict_date_guard": true +} diff --git a/docs/release/storage-migration-rehearsal-2026-10-27.json b/docs/release/storage-migration-rehearsal-2026-10-27.json new file mode 100644 index 0000000..c8a596f --- /dev/null +++ b/docs/release/storage-migration-rehearsal-2026-10-27.json @@ -0,0 +1,22 @@ +{ + "app_dependency_image": "sha256:9a646d8e01a61ba5316d3c2eb0273064e17f2d0300ad07af766ccedda65ecb18", + "postgres_image": "sha256:ff23cdce56cac62ada6f66013e1a50864c0abbe79d132d40b6e05bd80f378a70", + "proof": { + "status": "passed", + "database_vendor": "postgresql", + "server_version": 160014, + "forward_collision_refusal": true, + "refusal_preserves_rows_and_grants": true, + "rollback_preserves_rows_grants_and_tenant_constraint": true, + "reapply_enforces_global_constraint": true, + "production_access": false + }, + "source_sha256": { + "tests/test_storage_migration_lifecycle.py": "34df20aafd86b723c9192b3d45941ab3bb78bb2837267f2c3f7b056072bb5e7b", + "coulomb_social/apps/spaces/migrations/0003_space_slug_global_uniq.py": "e567cf8fd5f3b3761e5336a3f0209908c1fb00beac10009d2eeda6c91a9bbdad", + "uv.lock": "06a1bfff320560086dc0210e01e2504e07cf18cc4ae8646e6980c2f472e29935" + }, + "date": "2026-10-27", + "command": "uv run python scripts/rehearse-storage-migration.py", + "scope": "Synthetic PostgreSQL migration lifecycle; current source and read-only locked test dependencies over cached app image; not a packaged release or production migration" +} diff --git a/docs/usecases/communication-regressions.json b/docs/usecases/communication-regressions.json index d30718e..4be86ee 100644 --- a/docs/usecases/communication-regressions.json +++ b/docs/usecases/communication-regressions.json @@ -38,18 +38,31 @@ "test_callers::test_definitive_failure_retains_retry_policy_after_restart", "test_callers::test_legacy_invitation_adoption_preserves_accepted_receipt", "test_callers::test_historical_failed_reservation_is_conservatively_terminal", - "test_callers::test_invalid_policy_refuses_startup_before_database_mutation" + "test_callers::test_invalid_policy_refuses_startup_before_database_mutation", + "test_ledger_backup::test_sqlite_backup_includes_committed_wal_without_source_checkpoint", + "test_ledger_backup::test_real_encryption_recovery_preserves_challenge_and_delivery_behavior", + "test_ledger_backup::test_packet_refuses_all_unsafe_members_before_extract", + "test_ledger_backup::test_capture_resumes_unchanged_receiver_after_failures", + "test_ledger_backup::test_backup_success_requires_readback_and_preserves_one_pending_capture", + "test_ledger_backup::test_invalid_capture_cannot_be_sealed", + "test_ledger_backup::test_helper_has_no_secret_or_token_mount_and_requires_no_egress", + "test_ledger_backup::test_scoped_ledger_rehearsal_preserves_ownership_and_budgets" ], "remaining_acceptance": [ - "CSOC-UC-009 isolated issuer reset, inbox and app browser journey", - "CSOC-UC-010/011 isolated registration and invitation browser journeys", - "Owner runtime admission of independent caller policies, credentials and limits", - "Platform sender and independent ledger backup custody admission", - "Named inbound/reply/bounce operations, alignment and retention", - "Per-flow productive user acceptance" + "CSOC-UC-009/010/011 real owner-spanning isolated issuer/directory/mailbox/app browser journeys (current app browser uses doubles)", + "Platform sender identity and named inbound/reply/bounce operations, DNS alignment and retention", + "Per-flow productive user acceptance; UC-009 personal reset acceptance remains recorded", + "Temporary pre-release copies remain private pending explicit owner disposition; no automatic deletion" ], "test_files": [ "tests/test_transactional.py", - "tests/test_callers.py" + "tests/test_callers.py", + "tests/test_ledger_backup.py" + ], + "helper_files": [ + "scripts/ledger_archive.py", + "scripts/capture-ledger.py", + "scripts/backup-ledger.py", + "scripts/rehearse-ledger.py" ] } diff --git a/docs/usecases/platform-communication.md b/docs/usecases/platform-communication.md index affaafc..f11e94a 100644 --- a/docs/usecases/platform-communication.md +++ b/docs/usecases/platform-communication.md @@ -38,7 +38,7 @@ live sends are part of the gate. | Invitation auth/template/recipient/key/suppression | email-connect `tests/test_transactional.py`: `test_rejects_auth_recipient_template_and_key`, `test_suppression_blocks_send` | Separate caller identities, caller/template/tenant scope, request limits; no authority from payload `source` | | Invitation retry / uncertain outcome | Owner tests `test_sends_fixed_template_once`, `test_resend_is_separate_event_and_sends_again`, `test_lost_post_send_record_never_resends_after_restart`, `test_definitive_failure_retries_but_changed_payload_is_rejected`, `test_disconnect_after_smtp_acceptance_never_releases_reservation` | Per-caller event namespace, bounded retry schedule and operator handling of unknown outcomes | | Verification privacy and replay | Owner tests `test_registration_verification_is_digest_only_and_single_use`, `test_registration_cancellation_is_single_use_and_prevents_verification`, `test_mailbox_evidence_is_not_authorization`, plus retry/restart/concurrency functions bound below | Expired-handle and binding denials through an isolated owner-spanning browser journey | -| Provider failure / diagnostic redaction | Owner tests `test_provider_timeout_temporary_and_permanent_are_redacted`, `test_status_authenticates_without_sending_and_redacts_errors` | No tokens, recipient/body or provider details in diagnostic projections; caller-scoped status; provider acceptance and inbox receipt shown separately | +| Provider failure / diagnostic redaction | Owner tests `test_provider_timeout_temporary_and_permanent_are_redacted`, `test_status_authenticates_without_sending_and_redacts_errors` | Diagnostic projections omit bearer values, recipient addresses, message bodies and provider details; caller-scoped status; provider acceptance and inbox receipt shown separately | | Reply/bounce operations | Offline scanner fixtures exist in email-connect | Named monitored inbox, authorized inbound access, correlated bounce → suppression, bounded retention and support escalation | Historical owner baseline (2026-10-07): 27 transactional tests pass at source @@ -93,3 +93,24 @@ and receiver deployment remain evidenced. Its unfinished T04/T05/T06 scopes are superseded by WP-0015-T01/T03, WP-0015-T02/T03 and WP-0015-T04 respectively. EMAIL-WP-0008-T03, EMAIL-WP-0007 and RPF-WP-0053 retain canonical owner/custody work. WP-0010 references above are historical provenance, not active gates. + + +## Verified owner returns — 2026-10-27 + +EMAIL-WP-0008 completed scoped caller custody/activation on October 9; +EMAIL-WP-0007 completed independent ledger escrow/kit/reader, mirrored daily +archives and monitored schedules on October 10. Current receiver policy and +user-engine bindings remain deployed/Ready. Today's backup and freshness +checks succeeded and its independent versioned mirror is verified. +WP-0015-T01/T03 are done; earlier source-only statements above are historical. + +The offline gate now declares four recovery Python helpers and the synthetic +ledger-backup test module as well as caller/transactional tests. It copies/hashes +only those helpers, source and tests and passes **103 tests, zero skips**, +including real ephemeral age encryption/decryption, all-six-table recovery and +scoped quota continuity. No production configuration/ledger/custody is copied. + +Actual owner-spanning browser integration, platform sender/inbound/reply/bounce +operations and productive UC-010/011 verdicts remain WP-0015-T02/T04. +UC-009 personal recovery acceptance is preserved; UC-012 remains deferred. +Details: ../release/acceptance-audit-2026-10-27.md. diff --git a/docs/usecases/sharing-portability-review.md b/docs/usecases/sharing-portability-review.md new file mode 100644 index 0000000..8e97c2a --- /dev/null +++ b/docs/usecases/sharing-portability-review.md @@ -0,0 +1,20 @@ +# Prepared review — UC-003 through UC-006 + +Review on `https://app.coulomb.social`, using a disposable page in an intended +space and a second destination space. Use an explicitly chosen reader and an +ungranted same-tenant member; keep their identities and real notes private. +Current product image is the accepted weekly-call image; storage/content +hardening from October 11 is not yet deployed. Record the actual release/date +with each verdict. Automated checks do not supply these verdicts. + +| Journey | Review steps | Expected result | +| --- | --- | --- | +| UC-003: authoring | Create a disposable page with title, abstract, visual and body. Save/reopen; edit and reopen again. Open two edit tabs, save in one, then submit the older tab. | Content remains recognizable. Stale submission preserves your text and asks you to reopen/merge. Verify representative custom Markdown/raw HTML/assets separately; sanitization differences need an explicit verdict. | +| UC-004: copy/move | Copy that page to the second space. Edit the copy; reopen the source. Move another disposable page, then follow its old address as both an authorized reader and an outsider. | Copy is independent. Moved content/assets remain intact; old address still requires destination permission. No overwrite or access grant follows from moving. | +| UC-005: sharing | As owner, use Share space to grant the chosen reader access. Reader opens page and visual, then tries edit/copy/move directly. Ungranted member tries those same page/image URLs. Revoke the reader and retry. | Reader can read but cannot modify. Outsider receives no private content. Revoked reader loses page and image access. Review which existing grants/creator access remain. | +| UC-006: portability | Open Your app data/export. Inspect the private account-local JSON export and leaving/ownership explanation. Use a disposable app-local member for any actual exit test. | Export has the intended local identity/grant scope. Shared content remains with a named successor; sole owner must resolve succession first. App-local exit leaves platform identity under its own owner. Do not delete the productive owner to review the screen. | + +For each journey reply `accepted`, `gap` or `deferred`, with a short explanation. +An observed gap becomes a live task before that journey is accepted. Weekly-call +acceptance remains UC-008 only. A source-date check for newly migrated pages and +all their chunks is also required under the user's October 27 migration instruction. diff --git a/scripts/bubble-corpus-utils.mjs b/scripts/bubble-corpus-utils.mjs index db8ac1b..b6402c8 100644 --- a/scripts/bubble-corpus-utils.mjs +++ b/scripts/bubble-corpus-utils.mjs @@ -1,8 +1,19 @@ /** Pure capture guards; fixtures must never contain real cookies or notes. */ export function normalizeBubbleRef(value) { if (typeof value !== 'string') return value; - if (!value.startsWith('LOOKUP__')) return value; - return value.match(/[0-9]+x[0-9]+/)?.[0] || value; + if (!value.includes('LOOKUP__')) return value; + return value.split('LOOKUP__').at(-1).match(/[0-9]+x[0-9]+/)?.[0] || value; +} + +export async function boundedCaptureBody(response, milliseconds = 8000) { + let timer; + try { + return await Promise.race([ + response.text().then(text => ({status:'read', text})), + new Promise(resolve => { timer = setTimeout(() => resolve({status:'timeout'}), milliseconds); }), + ]); + } catch { return {status:'failed'}; } + finally { clearTimeout(timer); } } export function assertSessionFresh(state, now = Date.now() / 1000) { @@ -28,3 +39,91 @@ export function selectTargetArticles(articles, rootId) { } return [spaceId, selected]; } + +/** UI root links map through captured root articles to *space* creation dates. */ +export function rankSpaceLinks(links, spaces, articles) { + const unique = new Map(); + for (const link of links) { + const path = new URL(link.href, 'https://source.invalid').pathname; + const match = path.match(/^\/pg_chunk\/([A-Za-z0-9_-]+)$/); + if (!match) continue; + const rootId = normalizeBubbleRef(match[1]); + const spaceId = normalizeBubbleRef(articles[rootId]?.space_custom_space); + const created = spaces[spaceId]?.['Created Date']; + const createdMs = Number.isSafeInteger(created) ? created : null; + // One deterministic entry per source root, even when the UI repeats cards. + const entry = { href: path, rootId, spaceId: spaceId || null, createdMs, + label: String(articles[rootId]?.nomer_text || link.label || '').trim() }; + if (!unique.has(rootId) || entry.label < unique.get(rootId).label) unique.set(rootId, entry); + } + return [...unique.values()].sort((a, b) => + (b.createdMs ?? -Infinity) - (a.createdMs ?? -Infinity) || + (a.rootId < b.rootId ? -1 : a.rootId > b.rootId ? 1 : 0)); +} + +/** Follow only explicit source references or records missing their body/date. */ +export function pendingCaptureReferences(spaces, articles, inventory = {}) { + const needed = new Set(); + const add = value => { + if (typeof value !== 'string') return; + const id = normalizeBubbleRef(value); + if (/^[0-9]+x[0-9]+$/.test(id)) needed.add(id); + }; + for (const [id, space] of Object.entries({...inventory, ...spaces})) { + const root = normalizeBubbleRef(space._chunk_custom_article || inventory[id]?._chunk_custom_article); + if (root && (!articles[root] || !spaces[id] || spaces[id]['Created Date'] == null)) add(root); + } + for (const [id, article] of Object.entries(articles)) { + if (typeof article.content_text !== 'string' || article['Created Date'] == null) add(id); + for (const ref of Array.isArray(article.chunks_list_custom_article) + ? article.chunks_list_custom_article : []) { + if (!articles[normalizeBubbleRef(ref)]) add(ref); + } + const parent = normalizeBubbleRef(article.context_custom_article); + if (parent && !articles[parent]) add(parent); + } + return [...needed].sort(); +} + +function ingest(obj, spaces, articles) { + if (Array.isArray(obj)) { + for (const x of obj) ingest(x, spaces, articles); + return; + } + if (!obj || typeof obj !== "object") return; + const t = obj._type; + const src = obj._source && typeof obj._source === "object" ? obj._source : null; + const id = obj._id || (src && src._id); + if (t === "custom.space" && src && id) { + spaces[normalizeBubbleRef(id)] = mergeCapturedRecord(spaces[normalizeBubbleRef(id)], src, id); + } + if (t === "custom.article" && src && id) { + const a = mergeCapturedRecord(articles[normalizeBubbleRef(id)], src, id); + if (a.space_custom_space) a.space_custom_space = normalizeBubbleRef(a.space_custom_space); + articles[normalizeBubbleRef(id)] = a; + } + if (Array.isArray(obj.responses)) { + for (const r of obj.responses) ingest(r, spaces, articles); + } + if (obj.hits && typeof obj.hits === "object") { + for (const h of obj.hits.hits || []) ingest(h, spaces, articles); + } + for (const v of Object.values(obj)) { + if (v && typeof v === "object") ingest(v, spaces, articles); + } +} +export function ingestCapturedBody(text, spaces, articles) { + try { + ingest(JSON.parse(text), spaces, articles); + return; + } catch { + /* try NDJSON */ + } + for (const line of text.split("\n")) { + try { + ingest(JSON.parse(line), spaces, articles); + } catch { + /* ignore */ + } + } +} diff --git a/scripts/capture-bubble-references.mjs b/scripts/capture-bubble-references.mjs new file mode 100644 index 0000000..c09b292 --- /dev/null +++ b/scripts/capture-bubble-references.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +/** Continue a private checkpoint through explicit UI references, without a full recrawl. */ +import { chromium } from 'playwright'; +import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, resolve, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createHash } from 'node:crypto'; +import { parseArgs } from 'node:util'; +import { assertSessionFresh, ingestCapturedBody, boundedCaptureBody, + pendingCaptureReferences } from './bubble-corpus-utils.mjs'; + +const {values} = parseArgs({options: { + capture: {type:'string'}, output: {type:'string'}, + 'inventory-spaces': {type:'string'}, 'max-visits': {type:'string', default:'200'}, +}}); +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const BASE = 'https://coulomb.social'; +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +let browser; +try { + if (!values.capture || !values.output) throw new Error('private_input_output_required'); + const out = resolve(values.output); + if (out === ROOT || out.startsWith(ROOT + '/')) throw new Error('output_must_be_outside_repo'); + const limit = Number(values['max-visits']); + if (!Number.isInteger(limit) || limit < 1 || limit > 2000) throw new Error('invalid_limit'); + const spacesBytes = readFileSync(join(values.capture, 'spaces.json')); + const articlesBytes = readFileSync(join(values.capture, 'articles.json')); + const spaces = JSON.parse(spacesBytes), articles = JSON.parse(articlesBytes); + const inventoryBytes = values['inventory-spaces'] ? readFileSync(values['inventory-spaces']) : null; + const inventory = inventoryBytes ? JSON.parse(inventoryBytes) : {}; + const state = process.env.COULOMB_SOCIAL_STORAGE_STATE || + join(homedir(), '.config/coulomb-social/auth/storage-state.json'); + assertSessionFresh(JSON.parse(readFileSync(state, 'utf8'))); + mkdirSync(out, {mode:0o700}); // Refuse overwrite of any earlier checkpoint. + const attempts = []; + let responseTimeouts = 0; + function write(name, value) { + const temporary = join(out, name + '.partial'); + writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', {mode:0o600}); + renameSync(temporary, join(out, name)); + } + function checkpoint(status = 'capturing_references') { + write('spaces.json', spaces); write('articles.json', articles); + write('capture-state.json', { + status, space_count:Object.keys(spaces).length, article_count:Object.keys(articles).length, + attempts, response_timeouts:responseTimeouts, + remaining_references:pendingCaptureReferences(spaces, articles, inventory), + base_spaces_sha256:createHash('sha256').update(spacesBytes).digest('hex'), + base_articles_sha256:createHash('sha256').update(articlesBytes).digest('hex'), + inventory_sha256:inventoryBytes ? createHash('sha256').update(inventoryBytes).digest('hex') : null, + live_source_completeness_verified:false, production_imported:false, + }); + } + checkpoint(); + browser = await chromium.launch({headless:true}); + const context = await browser.newContext({storageState:state}); + const page = await context.newPage(); + const pending = new Set(); + const observer = response => { + if (!response.url().includes('elasticsearch')) return; + const operation = (async () => { + try { + const result = await boundedCaptureBody(response); + if (result.status === 'timeout') responseTimeouts++; + const text = result.text; + if (typeof text === 'string' && text.length < 5_000_000) { + ingestCapturedBody(text, spaces, articles); checkpoint(); + } + } catch { /* A cancelled/denied body leaves its reference unresolved. */ } + })(); + pending.add(operation); operation.finally(() => pending.delete(operation)); + }; + page.on('response', observer); + const attempted = new Set(); + while (attempted.size < limit) { + const next = pendingCaptureReferences(spaces, articles, inventory) + .find(id => !attempted.has(id)); + if (!next) break; + attempted.add(next); + let status = 'visited'; + try { + await page.goto(BASE + '/pg_chunk/' + next, + {waitUntil:'domcontentloaded', timeout:20000}); + await delay(2000); + } catch { status = 'navigation_failed'; } + await Promise.allSettled([...pending]); // Each response read has its own bound. + attempts.push({id:next,status}); checkpoint(); + console.log(JSON.stringify({visits:attempted.size, spaces:Object.keys(spaces).length, + articles:Object.keys(articles).length})); + } + page.off('response', observer); + await Promise.allSettled([...pending]); + await browser.close(); browser = null; + checkpoint('completed_reference_pass'); + console.log(JSON.stringify({status:'completed_reference_pass', visits:attempts.length, + spaces:Object.keys(spaces).length, articles:Object.keys(articles).length, + remaining_references:pendingCaptureReferences(spaces, articles, inventory).length, + live_source_completeness_verified:false, production_imported:false})); +} catch { + if (browser) await browser.close(); + console.error('Reference capture refused or stopped; private checkpoints retained'); + process.exitCode = 1; +} diff --git a/scripts/check-communication.py b/scripts/check-communication.py index f44c121..89a0e6a 100644 --- a/scripts/check-communication.py +++ b/scripts/check-communication.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Run exact owner retry/privacy regressions on a private, network-denied snapshot. -Only source and declared synthetic test modules are copied. The owner's configuration, -environment, caches and runtime databases are neither read nor mutated. +Only source, declared Python helpers and synthetic test modules are copied. +The owner's configuration, environment, caches and runtime databases are untouched. """ import argparse @@ -51,11 +51,20 @@ def check(owner, receipt): ): raise ValueError("Synthetic tests must be explicit test modules") tests = [owner / name for name in test_files] + helper_files = contract.get("helper_files", []) + if any( + not isinstance(name, str) or Path(name).parent != Path("scripts") + or Path(name).suffix != ".py" + for name in helper_files + ): + raise ValueError("Owner helpers must be explicit Python files under scripts") + helpers = [owner / name for name in helper_files] if not source.is_dir() or source.is_symlink() or any( - not test.is_file() or test.is_symlink() for test in tests + not path.is_file() or path.is_symlink() or path.parent.is_symlink() + for path in tests + helpers ): raise ValueError("Owner source and synthetic tests must be regular local files") - files = sorted(source.rglob("*.py")) + tests + files = sorted(source.rglob("*.py")) + tests + helpers if any(p.is_symlink() for p in files): raise ValueError("Owner Python sources must not be symlinks") # Snapshot exact bytes first. Hash receipts bind the code actually executed, diff --git a/scripts/export-bubble-corpus.mjs b/scripts/export-bubble-corpus.mjs index 74a2aa8..fc613be 100755 --- a/scripts/export-bubble-corpus.mjs +++ b/scripts/export-bubble-corpus.mjs @@ -8,7 +8,8 @@ */ import { chromium } from "playwright"; import { assertSessionFresh, normalizeBubbleRef, mergeCapturedRecord, - selectTargetArticles } from "./bubble-corpus-utils.mjs"; + selectTargetArticles, rankSpaceLinks, ingestCapturedBody as parseBody, + boundedCaptureBody } from "./bubble-corpus-utils.mjs"; import { mkdirSync, writeFileSync, @@ -19,6 +20,7 @@ import { createWriteStream, } from "node:fs"; import { homedir } from "node:os"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import https from "node:https"; import http from "node:http"; @@ -28,6 +30,10 @@ const STATE = join(homedir(), ".config", "coulomb-social", "auth", "storage-state.json"); const BASE = process.env.COULOMB_SOCIAL_URL || "https://coulomb.social"; const TARGET_LABEL = process.env.COULOMB_SOCIAL_TARGET_SPACE_LABEL || ""; +const ALL_SPACES = process.env.COULOMB_SOCIAL_ALL_SPACES === "1"; +const ORDER_SPACES_FILE = process.env.COULOMB_SOCIAL_ORDER_SPACES_FILE || ""; +const orderSource = ORDER_SPACES_FILE ? readFileSync(ORDER_SPACES_FILE) : null; +const orderSpaces = orderSource ? JSON.parse(orderSource.toString("utf8")) : {}; const TARGET_ROUTE = process.env.COULOMB_SOCIAL_CAPTURE_ROUTE || "/vw_pages"; const SEED_NET = process.env.COULOMB_SOCIAL_SEED_NETWORK || @@ -79,48 +85,6 @@ function slugify(s) { } function normId(id) { return normalizeBubbleRef(id); } -function ingest(obj, spaces, articles) { - if (Array.isArray(obj)) { - for (const x of obj) ingest(x, spaces, articles); - return; - } - if (!obj || typeof obj !== "object") return; - const t = obj._type; - const src = obj._source && typeof obj._source === "object" ? obj._source : null; - const id = obj._id || (src && src._id); - if (t === "custom.space" && src && id) { - spaces[normId(id)] = mergeCapturedRecord(spaces[normId(id)], src, id); - } - if (t === "custom.article" && src && id) { - const a = mergeCapturedRecord(articles[normId(id)], src, id); - if (a.space_custom_space) a.space_custom_space = normId(a.space_custom_space); - articles[normId(id)] = a; - } - if (Array.isArray(obj.responses)) { - for (const r of obj.responses) ingest(r, spaces, articles); - } - if (obj.hits && typeof obj.hits === "object") { - for (const h of obj.hits.hits || []) ingest(h, spaces, articles); - } - for (const v of Object.values(obj)) { - if (v && typeof v === "object") ingest(v, spaces, articles); - } -} -function parseBody(text, spaces, articles) { - try { - ingest(JSON.parse(text), spaces, articles); - return; - } catch { - /* try NDJSON */ - } - for (const line of text.split("\n")) { - try { - ingest(JSON.parse(line), spaces, articles); - } catch { - /* ignore */ - } - } -} function download(url, dest) { if (!url || typeof url !== "string") return Promise.resolve(null); let u = url; @@ -155,6 +119,7 @@ function download(url, dest) { } resolve(null); }); + req.setTimeout(20000, () => req.destroy()); }); } @@ -172,7 +137,7 @@ try { const spaces = {}; const articles = {}; -if (!TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET)) { +if (!ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET)) { for (const name of readdirSync(SEED_NET)) { if (!name.endsWith(".body.txt")) continue; parseBody(readFileSync(join(SEED_NET, name), "utf8"), spaces, articles); @@ -181,6 +146,19 @@ if (!TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync } ensure(OUT); +const captureAttempts = []; +let responseTimeouts = 0; +let responseFailures = 0; +function checkpoint(status = "capturing") { + write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2)); + write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2)); + write(join(OUT, "capture-state.json"), JSON.stringify({ + status, all_visible_spaces_requested: ALL_SPACES, + response_timeouts: responseTimeouts, response_failures: responseFailures, + capture_attempts: captureAttempts, space_count: Object.keys(spaces).length, + article_count: Object.keys(articles).length, live_source_completeness_verified: false, + }, null, 2)); +} const browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ storageState: STATE }); const page = await context.newPage(); @@ -189,8 +167,14 @@ page.on("response", (response) => { pendingResponses.push((async () => { try { if (!response.url().includes("elasticsearch")) return; - const text = await response.text(); - if (text && text.length < 5_000_000) parseBody(text, spaces, articles); + const result = await boundedCaptureBody(response); + if (result.status === "timeout") responseTimeouts++; + if (result.status === "failed") responseFailures++; + const text = result.text; + if (text && text.length < 5_000_000) { + parseBody(text, spaces, articles); + checkpoint(); + } } catch { /* ignore */ } @@ -256,25 +240,61 @@ if (TARGET_LABEL) { process.exit(3); } } else { -for (const name of SPACE_CLICKS) { +let targets = SPACE_CLICKS.map(label => ({ label })); +if (ALL_SPACES) { + await loadSpacesList(); + await Promise.allSettled(pendingResponses); + const links = await page.locator('a[href*="/pg_chunk/"]').evaluateAll(elements => + elements.map(el => ({ href: el.href, label: el.textContent }))); + targets = rankSpaceLinks(links, { ...orderSpaces, ...spaces }, articles); + checkpoint(); + if (!targets.length) { + await browser.close(); + console.error("No space roots found; no capture published."); + process.exit(5); + } +} +for (const target of targets) { + const name = target.label; + let capturePhase = "space_list"; try { await loadSpacesList(); - const loc = page.getByText(name, { exact: true }).first(); - if ((await loc.count()) === 0) { - console.log("skip", name); + // Bubble's open-in-new-tab anchor is hidden until hover. Select the visible + // space title within the card containing that exact source-root address. + const loc = target.rootId + ? page.locator('.group-item:visible').filter({ + has: page.locator(`a[href$="/pg_chunk/${target.rootId}"]`), + }).first().getByText(name, { exact: true }).filter({ visible: true }).first() + : page.getByText(name, { exact: true }).first(); + if ((await loc.count()) !== 1) { + captureAttempts.push({ ...target, status: "root_not_visible" }); + checkpoint(); continue; } + capturePhase = "root_click"; await loc.click({ timeout: 5000 }); await page.waitForTimeout(2500); + capturePhase = "pages_navigation"; await page.goto(`${BASE}/vw_pages`, { waitUntil: "domcontentloaded", timeout: 60000 }); await page.waitForTimeout(3000); + capturePhase = "pages_scroll"; for (let i = 0; i < 4; i++) { await page.mouse.wheel(0, 1800); await page.waitForTimeout(400); } - console.log("ok", name, "articles", Object.keys(articles).length); + captureAttempts.push({ ...target, status: "visited" }); + checkpoint(); + console.log("visited", captureAttempts.length, "articles", Object.keys(articles).length); } catch (e) { - console.log("fail", name, "capture_step_failed"); + captureAttempts.push({ ...target, status: "capture_step_failed", phase: capturePhase, + error_type: e.name }); + checkpoint(); + console.log("capture_step_failed", captureAttempts.length); + if (ALL_SPACES && captureAttempts.slice(-3).every(attempt => + attempt.status === "capture_step_failed") && captureAttempts.length >= 3) { + console.log("stopped_after_repeated_failures"); + break; + } } } } @@ -318,9 +338,14 @@ const lab = TARGET_LABEL ? { ...spaceLabel(sampleId), slug: slugify(TARGET_LABEL const index = { exported_at: new Date().toISOString(), + all_visible_spaces_requested: ALL_SPACES, + ordering_inventory_sha256: orderSource ? createHash("sha256").update(orderSource).digest("hex") : null, + ordering_inventory_used_for_dates_only: Boolean(orderSource), + capture_attempts: captureAttempts, + live_source_completeness_verified: false, target_requested: TARGET_LABEL || null, source_route: TARGET_LABEL ? TARGET_ROUTE : null, - old_seed_used: !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET), + old_seed_used: !ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET), space_count: Object.keys(spaces).length, article_count: Object.keys(articles).length, sample_tree: sampleId ? `trees/${lab.slug}` : null, @@ -331,6 +356,8 @@ const index = { slug: L.slug, bubble_slug: L.rawSlug, title_guess: L.title, + source_created_at: spaces[id]["Created Date"] ?? null, + source_modified_at: spaces[id]["Modified Date"] ?? null, article_count: (bySpace[id] || []).length, }; }), @@ -397,6 +424,8 @@ abstractor: ${JSON.stringify(TARGET_LABEL visual: ${JSON.stringify(targetVisual)} space: ${JSON.stringify(lab.slug)} bubble_type: space +source_created_at: ${JSON.stringify(spaces[sampleId]?.["Created Date"] ?? null)} +source_modified_at: ${JSON.stringify(spaces[sampleId]?.["Modified Date"] ?? null)} --- ${TARGET_LABEL ? (articles[targetRootId]?.content_text || `# ${lab.title}`) : `# ${lab.title}`} @@ -415,3 +444,4 @@ Do not commit — may contain private content. ); console.log("OUT", OUT); console.log("tree", treeRoot, "pages", sampleArts.length, "visuals", visuals); +checkpoint("completed_capture_pass"); diff --git a/scripts/plan-bubble-transfer.py b/scripts/plan-bubble-transfer.py new file mode 100644 index 0000000..f245595 --- /dev/null +++ b/scripts/plan-bubble-transfer.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Plan all captured spaces newest first, offline; private output, no import.""" + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from coulomb_social.apps.spaces.bubble_transfer import transfer_plan # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("capture", type=Path, help="Private spaces.json/articles.json directory") + parser.add_argument("--output", type=Path, required=True, help="New private JSON file") + parser.add_argument("--inventory-spaces", type=Path, + help="Retained spaces.json; keep absent spaces queued as not refreshed") + args = parser.parse_args() + try: + sources = {name: (args.capture / (name + ".json")).read_bytes() + for name in ("spaces", "articles")} + inventory = None + if args.inventory_spaces: + sources["retained_spaces"] = args.inventory_spaces.read_bytes() + inventory = json.loads(sources["retained_spaces"]) + plan = transfer_plan(json.loads(sources["spaces"]), json.loads(sources["articles"]), + inventory_spaces=inventory) + plan["source_sha256"] = {name: hashlib.sha256(data).hexdigest() + for name, data in sources.items()} + # Never print private IDs, names, notes or source errors to the terminal. + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(plan, stream, indent=2) + stream.write("\n") + print(json.dumps({ + "spaces": len(plan["spaces"]), + "captured_graph_complete": sum(s["captured_graph_complete"] for s in plan["spaces"]), + "unassigned_articles": len(plan["unassigned_article_ids"]), + "inventory_only_spaces": len(plan["inventory_only_space_ids"]), + "live_source_completeness_verified": False, "production_imported": False, + })) + except (ValueError, OSError, TypeError): + print("Transfer plan refused; inspect the private capture/output locally", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rehearse-storage-migration.py b/scripts/rehearse-storage-migration.py new file mode 100644 index 0000000..182d9b4 --- /dev/null +++ b/scripts/rehearse-storage-migration.py @@ -0,0 +1,168 @@ +"""Rehearse migration 0003 on disposable internal-network PostgreSQL. + +Uses cached images and read-only source/test dependencies; no published ports, +production credentials, live data, or production migration. Requires Docker, +postgres:16, the deployed dd83c90 app image and the locked local dev environment. +""" + +import argparse +import hashlib +import json +import shutil +import subprocess +import sysconfig +import tempfile +import uuid +from pathlib import Path + +name = "csoc-migration-" + uuid.uuid4().hex[:10] +root = Path(__file__).resolve().parents[1] +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument( + "--app-image", + default="forgejo.coulomb.social/coulomb/coulomb-social:dd83c90", +) +parser.add_argument( + "--packaged-source", + action="store_true", + help="Use image source/dependencies, injecting only test tools", +) +args = parser.parse_args() + + +def run(*args): + return subprocess.run(args, capture_output=True, text=True, check=True).stdout.strip() + + +app = run( + "docker", + "image", + "inspect", + args.app_image, + "--format", + "{{.Id}}", +) +pg = run("docker", "image", "inspect", "postgres:16", "--format", "{{.Id}}") +code = """import time, json, django +django.setup() +from django.db import connection +for i in range(30): + try: + connection.ensure_connection() + break + except Exception: + time.sleep(1) +else: + raise RuntimeError('isolated_postgres_not_ready') +from django.core.management import call_command +call_command('migrate', verbosity=0) +from tests import test_storage_migration_lifecycle as lifecycle +assert connection.vendor == 'postgresql' +lifecycle.test_forward_refusal_rollback_and_reapply_preserve_rows() +print(json.dumps({ + 'status': 'passed', 'database_vendor': connection.vendor, + 'server_version': connection.pg_version, 'forward_collision_refusal': True, + 'refusal_preserves_rows_and_grants': True, + 'rollback_preserves_rows_grants_and_tenant_constraint': True, + 'reapply_enforces_global_constraint': True, 'production_access': False, +})) +""" +tool_directory = None +source_mounts = ["-v", str(root / "coulomb_social") + ":/app/coulomb_social:ro"] +dependency_directory = sysconfig.get_path("purelib") +test_tools = ["pytest", "_pytest", "pluggy", "iniconfig", "pygments", "py.py"] +if args.packaged_source: + source_mounts = [] + tool_directory = Path(tempfile.mkdtemp(prefix="csoc-migration-test-tools-")) + for package in test_tools: + source = Path(dependency_directory) / package + if source.is_dir(): + shutil.copytree(source, tool_directory / package) + else: + shutil.copyfile(source, tool_directory / package) + # Only public test-tool sources are copied. The image runs as UID10001. + tool_directory.chmod(0o755) + dependency_directory = str(tool_directory) + code = code.replace( + "django.setup()", + "assert django.__file__.startswith('/app/.venv/'), 'packaged_django_required'\n" + "django.setup()", + ) +try: + run("docker", "network", "create", "--internal", name) + run( + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "--network", + name, + "--network-alias", + "storage-db", + "-e", + "POSTGRES_HOST_AUTH_METHOD=trust", + "-e", + "POSTGRES_DB=storage_rehearsal", + pg, + ) + result = run( + "docker", + "run", + "--rm", + "--network", + name, + "--read-only", + "--tmpfs", + "/tmp", + "--entrypoint", + "/app/.venv/bin/python", + "-e", + "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base", + "-e", + "DATABASE_URL=postgres://postgres@storage-db/storage_rehearsal", + "-e", + "PYTHONDONTWRITEBYTECODE=1", + *source_mounts, + "-v", + str(root / "tests") + ":/app/tests:ro", + "-v", + dependency_directory + ":/test-deps:ro", + "-e", + "PYTHONPATH=/test-deps", + app, + "-c", + code, + ) + print( + json.dumps( + { + "app_dependency_image": app, + "packaged_source_and_django": args.packaged_source, + "injected_test_tools": test_tools + if args.packaged_source + else "locked dev environment", + "postgres_image": pg, + "proof": json.loads(result.splitlines()[-1]), + "source_sha256": { + str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in [ + root / "tests/test_storage_migration_lifecycle.py", + root + / "coulomb_social/apps/spaces/migrations/0003_space_slug_global_uniq.py", + root / "uv.lock", + ] + }, + }, + indent=2, + ) + ) +except subprocess.CalledProcessError as e: + print(e.stdout, e.stderr) + raise +finally: + subprocess.run(["docker", "rm", "-fv", name], capture_output=True) + subprocess.run(["docker", "network", "rm", name], capture_output=True) + if tool_directory is not None: + shutil.rmtree(tool_directory) diff --git a/scripts/tests/bubble-corpus-utils.test.mjs b/scripts/tests/bubble-corpus-utils.test.mjs index d671cbf..464b40d 100644 --- a/scripts/tests/bubble-corpus-utils.test.mjs +++ b/scripts/tests/bubble-corpus-utils.test.mjs @@ -1,12 +1,14 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { normalizeBubbleRef, assertSessionFresh, mergeCapturedRecord, - selectTargetArticles } from '../bubble-corpus-utils.mjs'; + selectTargetArticles, rankSpaceLinks, pendingCaptureReferences, + ingestCapturedBody, boundedCaptureBody } from '../bubble-corpus-utils.mjs'; test('typed lookup references normalize to the same space identifier', () => { const id = '1700000000000x1234567890'; assert.equal(normalizeBubbleRef(`LOOKUP__custom.space__${id}`), id); assert.equal(normalizeBubbleRef(`LOOKUP__${id}`), id); + assert.equal(normalizeBubbleRef(`1234567890__LOOKUP__${id}`), id); assert.equal(normalizeBubbleRef(id), id); }); @@ -36,3 +38,48 @@ test('explicit target selection never substitutes the largest space', () => { delete records.call; assert.throws(() => selectTargetArticles(records,'root'), /target_protocols_not_captured/); }); + +test('all-space capture ranks by space creation, keeps unknowns and deduplicates roots', () => { + const links = ['old', 'new', 'unknown', 'new'].map(id => + ({ href: `/pg_chunk/${id}`, label: id })); + const spaces = {a:{'Created Date':1,'Modified Date':99},b:{'Created Date':2}}; + const articles = {old:{space_custom_space:'a','Created Date':100}, + new:{space_custom_space:'b','Created Date':0,nomer_text:'New space title'}}; + const ranked = rankSpaceLinks(links, spaces, articles); + assert.deepEqual(ranked.map(r => r.rootId), ['new', 'old', 'unknown']); + assert.equal(ranked[2].createdMs, null); + assert.equal(ranked[0].label, 'New space title'); + assert.deepEqual(ranked, rankSpaceLinks([...links].reverse(), spaces, articles)); +}); + +test('reference continuation follows missing chunks, parents, bodies and unrefreshed space roots', () => { + const root='1700000000000x1', child='1700000000000x2', parent='1700000000000x3'; + const inventory={space:{_chunk_custom_article:root}}; + const articles={[root]:{content_text:'Synthetic', 'Created Date':1, + chunks_list_custom_article:[`12__LOOKUP__${child}`], context_custom_article:parent}}; + assert.deepEqual(pendingCaptureReferences({},articles,inventory),[root,child,parent]); + articles[child]={content_text:'Child','Created Date':2}; + articles[parent]={content_text:'Parent','Created Date':3}; + assert.deepEqual(pendingCaptureReferences({space:{'Created Date':1}},articles,inventory),[]); + delete articles[child].content_text; + assert.deepEqual(pendingCaptureReferences({space:{'Created Date':1}},articles,inventory),[child]); +}); + +test('shared response ingestion normalizes prefixed references and preserves already captured body', () => { + const spaces={},articles={}; + const id='1700000000000x1', space='1700000000000x2'; + ingestCapturedBody(JSON.stringify({_type:'custom.article',_id:id, + _source:{content_text:'Synthetic only',space_custom_space:`12__LOOKUP__${space}`}}),spaces,articles); + ingestCapturedBody(JSON.stringify({_type:'custom.article',_id:id, + _source:{nomer_text:'Synthetic title'}}),spaces,articles); + assert.equal(articles[id].space_custom_space,space); + assert.equal(articles[id].content_text,'Synthetic only'); +}); + +test('unfinished source responses cannot stall the capture drain or leak diagnostics', async () => { + assert.deepEqual(await boundedCaptureBody({text:()=>Promise.resolve('synthetic')}), + {status:'read',text:'synthetic'}); + assert.deepEqual(await boundedCaptureBody({text:()=>new Promise(()=>{})},1),{status:'timeout'}); + assert.deepEqual(await boundedCaptureBody({text:()=>Promise.reject(new Error('synthetic detail'))}), + {status:'failed'}); +}); diff --git a/tests/test_bubble_transfer.py b/tests/test_bubble_transfer.py new file mode 100644 index 0000000..ec9fef8 --- /dev/null +++ b/tests/test_bubble_transfer.py @@ -0,0 +1,151 @@ +"""Synthetic source identities, timestamps and nested chunks only.""" + +from datetime import UTC, datetime + +import pytest + +from coulomb_social.apps.spaces.bubble_transfer import ( + normalize_ref, + source_timestamp, + transfer_plan, +) +from coulomb_social.apps.spaces.import_tree import import_space_tree +from coulomb_social.apps.spaces.models import Space +from coulomb_social.apps.spaces.services import put_space_page +from tests.test_import_tree import _write_sample_tree +from tests.test_space_permissions import make_member + + +def record(created, **fields): + return {"Created Date": created, "Modified Date": created, **fields} + + +def test_numeric_prefixed_lookup_references_use_the_record_id_after_lookup(): + identity = "1750000000000x123456789012345678" + assert normalize_ref("1234567890__LOOKUP__" + identity) == identity + + +def test_lifo_is_deterministic_and_uses_creation_not_modification(): + spaces = {"old": record(1700000000000), "b": record(1750000000123), + "a": record(1750000000123), "unknown": {}} + spaces["old"]["Modified Date"] = 1800000000000 + plan = transfer_plan(spaces, {}) + assert [s["id"] for s in plan["spaces"]] == ["a", "b", "old", "unknown"] + assert plan == transfer_plan(dict(reversed(list(spaces.items()))), {}) + assert plan["spaces"][0]["created_at_raw"] == 1750000000123 + assert plan["spaces"][0]["created_at"] == "2025-06-15T15:06:40.123000+00:00" + assert not plan["spaces"][-1]["captured_graph_complete"] + assert not plan["production_imported"] + + +def test_recursive_chunks_preserve_every_date_and_report_missing_records(): + spaces = {"space": record(1750000000000)} + articles = { + "page": record(1750000001000, space_custom_space="space", content_text="Page", + chunks_list_custom_article=["child"]), + "child": record(1750000002000, content_text="Nested", context_custom_article="page", + chunks_list_custom_article=["grandchild"]), + "grandchild": record(1750000003000, content_text="Leaf", + chunks_list_custom_article=["page", "missing"]), + } + plan = transfer_plan(spaces, articles) + entry = plan["spaces"][0] + assert len(entry["articles"]) == 3 + assert {p["created_at_raw"] for p in entry["articles"]} == { + 1750000001000, 1750000002000, 1750000003000, + } + assert {g["reason"] for g in entry["gaps"]} == {"missing_chunk_record"} + assert not plan["unassigned_article_ids"] + del articles["grandchild"]["Created Date"] + assert "missing_or_invalid_created_at" in { + g["reason"] for g in transfer_plan(spaces, articles)["spaces"][0]["gaps"] + } + + +def test_space_root_and_its_chunks_are_included_without_article_space_fields(): + spaces = {"space": record(1750000000000, _chunk_custom_article="root")} + articles = {"root": record(1750000001000, content_text="Root", + chunks_list_custom_article=["child"]), + "child": record(1750000002000, content_text="Child")} + plan = transfer_plan(spaces, articles) + assert plan["spaces"][0]["captured_graph_complete"] + assert plan["spaces"][0]["root_article_id"] == "root" + assert len(plan["spaces"][0]["articles"]) == 2 + assert not plan["unassigned_article_ids"] + del articles["root"] + assert not transfer_plan(spaces, articles)["spaces"][0]["captured_graph_complete"] + + +def test_fresh_capture_does_not_silently_drop_retained_inventory_or_certify_stale_dates(): + inventory = {"missing": record(1750000000000), "fresh": record(1700000000000)} + plan = transfer_plan({"fresh": record(1700000000000)}, {}, inventory_spaces=inventory) + assert [s["id"] for s in plan["spaces"]] == ["missing", "fresh"] + assert plan["inventory_only_space_ids"] == ["missing"] + assert plan["spaces"][0]["source_record_fresh"] is False + assert plan["spaces"][0]["gaps"] == [{"id": "missing", "reason": "space_not_refreshed"}] + assert not plan["spaces"][0]["captured_graph_complete"] + assert plan["spaces"][1]["captured_graph_complete"] + + +@pytest.mark.parametrize("value", [None, True, "2026-10-27T12:00:00", "invalid"]) +def test_missing_or_timezone_ambiguous_source_dates_are_not_fabricated(value): + with pytest.raises(ValueError): + source_timestamp(value) + + +@pytest.mark.django_db +def test_import_preserves_creation_date_in_db_pages_and_chunks_after_edit(tmp_path, settings): + settings.CONTENT_ROOT = str(tmp_path / "content") + tree = _write_sample_tree(tmp_path / "export") + owner = make_member("date-owner") + for filename, timestamp in [("index", "2020-01-02T03:04:05.123Z"), + ("notes", "2021-02-03T04:05:06.456Z"), + ("chunk", "2022-03-04T05:06:07.789Z")]: + (tree / "pages" / (filename + ".md")).write_text( + f'---\ntitle: "{filename}"\nsource_created_at: "{timestamp}"\n' + 'parent: "bubble:page"\n---\n\nOriginal\n' + ) + import_space_tree(tree, tenant_id=owner.tenant_id, member=owner) + space = Space.objects.get(slug="sample-space") + assert space.created_at == datetime(2020, 1, 2, 3, 4, 5, 123000, tzinfo=UTC) + from coulomb_social.apps.spaces import pageops + for slug in ["notes", "chunk"]: + before = pageops.read_page(space.slug, slug) + put_space_page(owner, space, slug, title=slug, body="Edited") + assert pageops.read_page(space.slug, slug).meta["source_created_at"] == ( + before.meta["source_created_at"] + ) + import_space_tree(tree, tenant_id=owner.tenant_id, member=owner, replace=True) + space.refresh_from_db() + assert space.created_at.year == 2020 + + +@pytest.mark.django_db +def test_invalid_import_date_refuses_before_content_or_database_writes(tmp_path, settings): + from coulomb_social.apps.spaces import pageops + settings.CONTENT_ROOT = str(tmp_path / "content") + tree = _write_sample_tree(tmp_path / "export") + with (tree / "pages/index.md").open("w") as stream: + stream.write('---\nsource_created_at: "unknown"\n---\nBody\n') + with pytest.raises(pageops.PageOpsError, match="source creation timestamp"): + import_space_tree(tree, tenant_id="tenant:a") + assert not Space.objects.exists() + assert not pageops.content_root().exists() + + +@pytest.mark.django_db +@pytest.mark.parametrize("missing", ["index", "notes", "index-file"]) +def test_bubble_import_requires_every_original_date_before_any_writes(tmp_path, settings, missing): + from coulomb_social.apps.spaces import pageops + settings.CONTENT_ROOT = str(tmp_path / "content") + tree = _write_sample_tree(tmp_path / "export") + path = tree / "pages" / ("index.md" if missing == "index-file" else missing + ".md") + if missing == "index-file": + path.unlink() + else: + path.write_text("\n".join(line for line in path.read_text().splitlines() + if not line.startswith("source_created_at:")) + "\n") + with pytest.raises(pageops.PageOpsError, match="original creation dates"): + import_space_tree(tree, tenant_id="tenant:a") + assert not Space.objects.exists() + assert not pageops.content_root().exists() diff --git a/tests/test_communication_gate.py b/tests/test_communication_gate.py index 510c520..4535a27 100644 --- a/tests/test_communication_gate.py +++ b/tests/test_communication_gate.py @@ -73,3 +73,33 @@ def test_communication_gate_binds_each_required_function_to_its_module(gate): assert evidence["missing_required_tests"] == ["test_other::test_contract"] other.write_text("def test_contract():\n pass\n") assert module.check(owner, receipt)["passed"] + + +def test_communication_gate_snapshots_and_hashes_explicit_recovery_helper(gate): + module, owner, receipt = gate + contract = json.loads(module.CONTRACT.read_text()) + contract["helper_files"] = ["scripts/recovery.py"] + module.CONTRACT.write_text(json.dumps(contract)) + (owner / "scripts").mkdir() + helper = owner / "scripts/recovery.py" + helper.write_text("PROOF = 'recovered'\n") + (owner / "tests/test_contract.py").write_text( + "import runpy\nfrom pathlib import Path\ndef test_contract():\n" + " helper = Path(__file__).parents[1] / 'scripts/recovery.py'\n" + " assert runpy.run_path(str(helper))['PROOF'] == 'recovered'\n" + ) + evidence = module.check(owner, receipt) + assert evidence["passed"] + assert "scripts/recovery.py" in evidence["source_sha256"] + helper.write_text("PROOF = 'lost'\n") + assert not module.check(owner, receipt)["passed"] + + +@pytest.mark.parametrize("name", ["../recovery.py", "scripts/../../recovery.py", "scripts/key.env"]) +def test_communication_gate_refuses_helpers_outside_python_allowlist(gate, name): + module, owner, receipt = gate + contract = json.loads(module.CONTRACT.read_text()) + contract["helper_files"] = [name] + module.CONTRACT.write_text(json.dumps(contract)) + with pytest.raises(ValueError, match="explicit Python files"): + module.check(owner, receipt) diff --git a/tests/test_import_tree.py b/tests/test_import_tree.py index e831997..f656c13 100644 --- a/tests/test_import_tree.py +++ b/tests/test_import_tree.py @@ -32,6 +32,7 @@ def _write_sample_tree(root: Path) -> Path: 'visual: ""\n' 'space: "sample-space"\n' "bubble_type: space\n" + 'source_created_at: "2001-01-02T03:04:05Z"\n' "---\n\n" "# Sample Space\n\nLanding.\n", encoding="utf-8", @@ -43,6 +44,7 @@ def _write_sample_tree(root: Path) -> Path: 'visual: "assets/notes-visual.png"\n' 'space: "sample-space"\n' "bubble_type: article\n" + 'source_created_at: "2001-01-03T04:05:06Z"\n' "---\n\n" "# Notes\n\nBody text.\n", encoding="utf-8", @@ -162,7 +164,9 @@ def test_import_rolls_back_content_on_publish_error(content_root, tmp_path, monk import_space_tree(tree, tenant_id="tenant:a") path = pageops.page_file("sample-space", "index") original = path.read_bytes() - (tree / "pages" / "index.md").write_text('# Changed\n') + meta, _ = pageops.parse_frontmatter((tree / "pages" / "index.md").read_text()) + meta["title"] = "Changed" + (tree / "pages" / "index.md").write_text(pageops.dump_frontmatter(meta, '# Changed\n')) real_rename = Path.rename def fail_publish(self, target): @@ -200,7 +204,9 @@ def test_import_restores_tree_when_database_commit_fails(content_root, tmp_path, import_space_tree(tree, tenant_id="tenant:a") path = pageops.page_file("sample-space", "index") original = path.read_bytes() - (tree / "pages" / "index.md").write_text('# Changed\n') + meta, _ = pageops.parse_frontmatter((tree / "pages" / "index.md").read_text()) + meta["title"] = "Changed" + (tree / "pages" / "index.md").write_text(pageops.dump_frontmatter(meta, '# Changed\n')) def fail_commit(): raise DatabaseError("simulated database commit failure") diff --git a/tests/test_storage_migration_lifecycle.py b/tests/test_storage_migration_lifecycle.py new file mode 100644 index 0000000..7a29bfc --- /dev/null +++ b/tests/test_storage_migration_lifecycle.py @@ -0,0 +1,62 @@ +"""Exercise the real forward/reverse migration, including a refused legacy DB.""" + +import pytest +from django.db import IntegrityError, connection, transaction +from django.db.migrations.executor import MigrationExecutor + +from coulomb_social.apps.spaces.models import Space, SpaceMembership +from tests.test_space_permissions import make_member + +OLD = ("spaces", "0002_space_visual") +NEW = ("spaces", "0003_space_slug_global_uniq") + + +def migrate(target): + executor = MigrationExecutor(connection) + executor.migrate([target]) + return executor + + +@pytest.mark.django_db(transaction=True) +def test_forward_refusal_rollback_and_reapply_preserve_rows(): + owner = make_member("migration-owner") + space = Space.objects.create( + tenant_id=owner.tenant_id, slug="retained", title="Retained", created_by=owner, + ) + SpaceMembership.objects.create(space=space, member=owner, role="owner") + before_spaces = list(Space.objects.order_by("pk").values()) + before_grants = list(SpaceMembership.objects.order_by("pk").values()) + duplicate = None + try: + migrate(OLD) + assert list(Space.objects.order_by("pk").values()) == before_spaces + assert list(SpaceMembership.objects.order_by("pk").values()) == before_grants + # The tenant constraint must survive rollback of the global constraint. + with pytest.raises(IntegrityError), transaction.atomic(): + Space.objects.create(tenant_id=space.tenant_id, slug=space.slug, title="Denied") + duplicate = Space.objects.create( + tenant_id="tenant:legacy", slug=space.slug, title="Legacy collision", is_active=False, + ) + collided = list(Space.objects.order_by("pk").values()) + with pytest.raises(RuntimeError, match="does not rename, merge, or delete"): + migrate(NEW) + assert list(Space.objects.order_by("pk").values()) == collided + assert list(SpaceMembership.objects.order_by("pk").values()) == before_grants + assert NEW not in MigrationExecutor(connection).loader.applied_migrations + # Reconciliation here removes only the test's extra synthetic row. + duplicate.delete() + duplicate = None + migrate(NEW) + with pytest.raises(IntegrityError), transaction.atomic(): + Space.objects.create(tenant_id="tenant:other", slug=space.slug, title="Denied") + assert list(Space.objects.order_by("pk").values()) == before_spaces + assert list(SpaceMembership.objects.order_by("pk").values()) == before_grants + assert NEW in MigrationExecutor(connection).loader.applied_migrations + migrate(OLD) + migrate(NEW) + assert list(Space.objects.order_by("pk").values()) == before_spaces + assert list(SpaceMembership.objects.order_by("pk").values()) == before_grants + finally: + if duplicate is not None: + duplicate.delete() + migrate(NEW) diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 73dcaa4..647ea1b 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-08" +updated: "2026-10-27" origin: residual origin_ref: CSOC-WP-0008 state_hub_workstream_id: "0cd1f513-f26e-5707-b646-8b4bbe6369fc" @@ -20,13 +20,30 @@ Residual acceptance from CSOC-WP-0008, split at the user's request. Existing imp ```task id: CSOC-WP-0012-T01 -status: wait +status: progress priority: high +needs_human: false state_hub_task_id: "dc470337-585f-517d-b2a5-92e89a7dbec8" ``` Use existing RPF-WP-0053-T02/T03/T06 credential-pair/prefix/escrow admission. Own the coulomb-social consumer acceptance, not another credential provisioning lane. Record capacity, retained independent retrieval receipts, RPO/RTO and expiry of temporary production-pair copies; no secrets in records. +2026-10-27: User approved the scoped node executor/delivery (decision +9557e2b8-2b37-5b94-a5e6-e5fe08d0351e). Native own-reader/version/revocation, +five reader denials, wrong-account/audience denials, own multipart create/abort, +and sibling/native-prefix/delete/bucket-policy/IAM denials pass. The exact +hash-pinned helper delivered only this application's version-2 pair to root:root +0600 under a 0700 parent, with equal readback and revoked token. No completed +object or timer created. Proofs: docs/release/backup-boundaries-2026-10-27.json +and backup-delivery-2026-10-27.json. Independent recovery/receipt custody, +retention/capacity and first paired recovery remain open. + +2026-10-27 supplement: native DeleteObjectVersion also denies403 on a unique +never-published probe key; no retained object touched. Wrong-namespace login +with the same ServiceAccount name also denies403, no data or sibling credential +read. Receipts: docs/release/backup-version-delete-boundary-2026-10-27.json and +backup-wrong-namespace-boundary-2026-10-27.json. + ## T02 — Activate and prove the first encrypted paired recovery point ```task @@ -51,6 +68,16 @@ Enable reviewed backup/freshness units only after the exact first-run return. Ve with docs/identity/smoke.md before closing that intake. Current revision16 and persistent CONTENT_ROOT are already deployed. Keep apex/Bubble retirement in WP-0014. +2026-10-27: Read-only node check confirms the Coulomb primary credential file +is absent; current reader CCR retains false node/provider/boundary admission +flags. No backup activation performed. Added requirement-by-requirement service +contract §§1–6 assessment in docs/release/service-contract-conformance-2026-10-27.md; +conformance remains unproven, including full-client offboarding versus personal +export, current finance join and ops registration. The older August recovery +assumptions are historical; founder-approved paired schedule already exists in +RPF-WP-0053 and does not need repeat schedule approval. Current findings and +next returns: docs/release/acceptance-audit-2026-10-27.md. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records. diff --git a/workplans/CSOC-WP-0013-sharing-content-hardening.md b/workplans/CSOC-WP-0013-sharing-content-hardening.md index 54a3461..b9c0c69 100644 --- a/workplans/CSOC-WP-0013-sharing-content-hardening.md +++ b/workplans/CSOC-WP-0013-sharing-content-hardening.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-11" +updated: "2026-10-27" origin: residual origin_ref: CSOC-WP-0008 state_hub_workstream_id: "70370f4e-7cba-5b0a-853b-fedcd2bd49a5" @@ -36,6 +36,19 @@ Tests cover parallel inserts, a missed import pre-check, an admin race, inactive slug reservation, migration refusal, and the legacy fail-closed reads. The live database has not been migrated. +2026-10-27: Added a real MigrationExecutor forward/refusal/reverse/reapply +regression and rehearsed it on disposable PostgreSQL 16.14. Conflicting rows and +grants stay unchanged; rollback retains tenant uniqueness. Current production has +zero duplicate slug groups and still only migrations 0001/0002. Guarded release +remains open. Proof: docs/release/storage-migration-rehearsal-2026-10-27.json. + +2026-10-27 candidate: c11300a published at digest fb579adefa6e3e0c16fedeac134b2e05f4ed6216807cb25995fbc0da826c1167. +Its actual packaged Django/source pass the PostgreSQL lifecycle proof, with only +pytest tools injected. Migration drift check and server dry-run pass; Helm render +comparison changes only the app image across three resources. Live rollout still +requires a fresh paired rollback point/rehearsal. Proof: +docs/release/storage-candidate-c11300a-2026-10-27.json. + ## T02 — Complete content fidelity and revision lifecycle ```task @@ -70,6 +83,10 @@ state_hub_task_id: "8edd7661-17b0-5aa0-b177-6170aa159623" Review owner Share space/revocation, reader/outsider denial, account-local export, successor ownership and local exit semantics with the user. Keep platform identity outside app deletion. Record UC-003/004/005/006 verdicts independently; weekly-call acceptance is UC-008 only. +2026-10-27: User requested prepared review steps rather than supplying verdicts. +Prepared exact UC-003–006 steps in docs/usecases/sharing-portability-review.md; +no acceptance inferred. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records. diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index cd3c7d1..1163e5f 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-08" +updated: "2026-10-27" origin: residual origin_ref: CSOC-WP-0008 state_hub_workstream_id: "db6b85dc-f2f0-5283-ac52-3a6af28eb5b5" @@ -20,13 +20,46 @@ Residual acceptance from CSOC-WP-0008, split at the user's request. Existing imp ```task id: CSOC-WP-0014-T01 -status: todo +status: progress priority: high state_hub_task_id: "0002882d-f156-5a5d-b524-d04793f4684d" ``` Treat 71-space/109-article capture as inventory, not bulk-import authorization. Reconcile fresh/private source, identity and explicit grants, retained/discontinued workflows and assets/links. ReichelAg weekly-call cohort is already live and accepted. Record exact next nominated cohort and non-secret lossiness evidence. +2026-10-27: User specifies all coulomb.social spaces as the eventual migration +scope, using deterministic tooling to minimize token cost and LIFO guidance +(newest space creation first). Preserve original creation dates of spaces, +pages and every chunk. Decision: 4aab0481-8597-5e47-bc8c-46e8d3b524ed. +This supersedes the need to nominate just one next space; +productive workflow/cohort review and host cutover remain separate gates. + +Implemented offline source-hash-bound planner for every captured space, deterministic +creation-date ordering/ties and recursive chunk closure with missing-date/body/ +reference gaps. Retained capture: 71 spaces, 109 articles; 60 captured graphs +complete, 11 with gaps, no unassigned captured articles. This old capture does not +prove current source completeness. Import restores source space creation date in +SQL and retains page/chunk source date metadata through edit/reimport. Missing or +invalid dates refuse before writes. User renewed the Bubble session. Fresh capture exposed numeric-prefixed LOOKUP +references; both normalizers now handle them. All-visible-space discovery now +uses exact-root cards and visible titles, fresh-source checkpoints and step +diagnostics; the live pass is progressing without navigation failures. Prior +metadata is used only to guide LIFO ordering, never to seed older article bodies. +Next: complete fresh capture/reconciliation, deterministic all-tree +conversion, asset/link reconciliation and restartable per-space receipts before +productive import. No bulk import performed. + +2026-10-27 follow-through: the all-visible pass checkpointed 70 visits but timed +out before its final index; records were preserved as partial. Bounded targeted +reference continuation completed 34 visits, yielding 70 fresh spaces/162 articles. +All 71 known spaces remain queued (67 captured graphs pass), with one stale space, +body/date gaps, cross-space chunk relation and one unassigned article explicitly +unresolved. No old body seeding, date fabrication or productive import. Native +response read bounds/checkpoints now prevent the observed drain stall. Evidence: +docs/release/bubble-capture-reconciliation-2026-10-27.json. Next: deterministic +all-tree preparation, asset/link reconciliation, source exceptions and explicit +tenant/identity/grant mapping before stepwise LIFO cohort transfer. + ## T02 — Review remaining journeys and record transfer decisions ```task @@ -44,6 +77,8 @@ Maintain docs/usecases/catalog.md confidence and stable UC IDs. Privately compar id: CSOC-WP-0014-T03 status: wait priority: high +needs_human: true +intervention_note: "Existing cutover decision 186be722-e2c7-4109-a1e6-d120fe8fca5b remains open, after WP-0012 recovery/service and WP-0013 privacy/content returns." state_hub_task_id: "599b4b6a-829d-5fe5-b52d-ff419e548fa8" ``` @@ -51,6 +86,30 @@ Depends on WP-0012 service/recovery and WP-0013 privacy/content returns. Link se for the cutover return, not resolved by closing WP-0008. Retain parallel hosts until an explicit cohort/apex decision; Bubble retirement requires all retained use cases transferred or explicitly discontinued. +## T04 — Preserve original creation dates and reconcile the accepted cohort + +```task +id: CSOC-WP-0014-T04 +status: progress +priority: high +state_hub_task_id: "3018caa3-0f7b-5cc4-85e9-4dc4f41be9f7" +``` + +User requirement from 2026-10-27 (decision +4aab0481-8597-5e47-bc8c-46e8d3b524ed): preserve creation dates of spaces, +pages and every chunk. Import now restores SQL Space.created_at and preserves +page/chunk source timestamps, requiring them throughout Bubble trees before +writes. Tests include missing index/date refusal, timezone-aware/epoch dates, +edit/reimport preservation and rollback. Do not invent dates from IDs or mtime. + +Still required: deterministic tree conversion/date manifest and per-source-ID +comparison on rehearsal/target; reconcile source date exceptions. Backfill the +accepted cohort's missing original space date without replacing its content or +losing post-transfer edits, copies or aliases. Source provenance on copied local +pages is not authority to overwrite those copies during a bulk reimport. Refresh +the interim c11300a candidate for the stricter date guard before production use. +Do not close on source-level tests alone. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records. diff --git a/workplans/CSOC-WP-0015-communication-runtime-acceptance.md b/workplans/CSOC-WP-0015-communication-runtime-acceptance.md index 762aa03..cdaeea9 100644 --- a/workplans/CSOC-WP-0015-communication-runtime-acceptance.md +++ b/workplans/CSOC-WP-0015-communication-runtime-acceptance.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-11" +updated: "2026-10-27" origin: residual origin_ref: CSOC-WP-0010 state_hub_workstream_id: "38208f72-c285-56fa-be44-de9b59a4e031" @@ -20,13 +20,19 @@ Residual acceptance from CSOC-WP-0010, split at the user's request. Existing imp ```task id: CSOC-WP-0015-T01 -status: wait +status: done priority: high state_hub_task_id: "e87ea643-7bee-5427-b4f5-b32f20dc2f29" ``` Receiver source is implemented and image deployment recorded in EMAIL-WP-0008. EMAIL-WP-0008-T03 owns actual caller/binding/template/operation/quotas, separate token custody/delivery and original-caller transition. Product acceptance verifies the admitted integration; avoid duplicating receiver implementation or granting arbitrary app notifications. +2026-10-27: EMAIL-WP-0008 completed scoped caller custody and coordinated activation +on October 9. Current receiver policy/projection and user-engine tenant/client +bindings are present, both deployments Ready. Owner no-send denial/replay receipts +and the expanded 103-test isolated gate verify the admitted integration. Productive +flow verdicts remain T04. See docs/release/acceptance-audit-2026-10-27.md. + ## T02 — Establish platform sender and monitored reply/bounce ownership ```task @@ -42,13 +48,22 @@ Review replacing the receiver forgejo sender with platform identity through its ```task id: CSOC-WP-0015-T03 -status: wait +status: done priority: high state_hub_task_id: "286e60bf-76a1-5a18-bd2c-edd9d1400c41" ``` EMAIL-WP-0007-T03/T04 and RPF-WP-0053-T05/T06 own primary archive admission/atomic credential pair, first encrypted capture/retrieval/rehearsal and monitoring. Require image/schema-bound six-table recovery, restored accepted/unknown/consumed/cancelled/suppressed/quota continuity and post-capture reconciliation; govern temporary pre-release copies. +2026-10-27: EMAIL-WP-0007 completed independent provider escrow, separately bound +recovery reader, nonexpiring image/tools/receipt kit, six-table offline recovery, +mirrored rolling archives and monitored schedules on October 10. Fresh checks show +both timers active and today's backup/freshness successful, with verified mirror. +The app gate now binds real encryption/recovery and scoped quota continuity tests; +103 owner cases pass with zero skips. Historical private copies remain retained +pending explicit owner disposition, with no inferred expiry/deletion. Production +restore still requires fencing and post-capture reconciliation. See the audit. + ## T04 — Establish owner-spanning browser journeys and productive flow verdicts ```task