diff --git a/coulomb_social/apps/spaces/bubble_bundle.py b/coulomb_social/apps/spaces/bubble_bundle.py new file mode 100644 index 0000000..590cc00 --- /dev/null +++ b/coulomb_social/apps/spaces/bubble_bundle.py @@ -0,0 +1,226 @@ +"""Offline, lossless review trees; no fetching, grants, rewriting or importing.""" + +from __future__ import annotations + +import hashlib +import json +import re +from html.parser import HTMLParser +from urllib.parse import unquote, urlsplit + +import markdown + +from .bubble_transfer import _records, normalize_ref, source_timestamp, transfer_plan + + +def _digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _slug(kind: str, identity: str) -> str: + # Independent of labels, traversal order and later record additions. + return kind + "-" + _digest(identity.encode())[:24] + + +def _page(meta: dict, body: str) -> bytes: + # Preserve the source body exactly after the frontmatter delimiter. In + # particular, avoid the product editor's intentional newline normalization. + header = "\n".join(f"{key}: {json.dumps(value, ensure_ascii=True)}" + for key, value in meta.items()) + return ("---\n" + header + "\n---\n" + body).encode() + + +def _complete_article(article): + if not isinstance(article.get("content_text"), str): + return False + try: + for field in ["Created Date", "Modified Date"]: + source_timestamp(article.get(field)) + except (ValueError, OverflowError, OSError): + return False + return True + + +class _References(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.references = set() + self.unsupported = set() + + def handle_starttag(self, tag, attrs): + for key, value in attrs: + if value is None: + continue + if key in {"href", "src", "poster", "data", "action"}: + self.references.add(("link" if key == "href" else "asset", value)) + if key in {"srcset", "style"}: + self.unsupported.add(key + "_requires_review") + if tag in {"iframe", "script", "object", "embed", "style", "form"}: + self.unsupported.add(tag + "_requires_review") + + handle_startendtag = handle_starttag + + +def _reference(kind, value, targets): + result = {"kind": kind, "source": value, "resolved": False} + try: + parsed = urlsplit(value) + host = parsed.hostname + except ValueError: + return {**result, "classification": "invalid_url"} + if (not parsed.scheme and not parsed.netloc) or ( + parsed.scheme in {"http", "https"} and host in {"coulomb.social", "www.coulomb.social"} + and parsed.username is None and parsed.password is None + ): + match = re.fullmatch(r"/pg_chunk/([^/]+)/?", unquote(parsed.path)) + if match: + identity = normalize_ref(match[1]) + candidates = targets.get(identity, []) + return {**result, "classification": "bubble_article", "target_id": identity, + "candidate_targets": candidates, + "target_status": "unique" if len(candidates) == 1 else ( + "ambiguous" if candidates else "uncaptured"), + "query": parsed.query, "fragment": parsed.fragment} + if not parsed.scheme and not parsed.netloc: + return {**result, "classification": "fragment" if value.startswith("#") else "local"} + return {**result, "classification": "bubble_route_requires_review"} + return {**result, "classification": "external" if parsed.scheme in { + "http", "https", ""} else "other_scheme_requires_review"} + + +def prepare_bundle(spaces, articles, *, inventory_spaces=None): + """Return deterministic private files and reconciliation; nothing is accepted. + + Every source string body also has an exact UTF-8 sidecar, including blocked + and unassigned records. Only graph-valid spaces receive draft review trees. + Asset/link and workflow evidence is deliberately a separate unresolved gate. + """ + plan = transfer_plan(spaces, articles, inventory_spaces=inventory_spaces) + spaces, articles = _records(spaces), _records(articles) + inventory = _records(inventory_spaces) if inventory_spaces is not None else {} + all_spaces = {**inventory, **spaces} + files = {} + entries = [] + targets = {} + slugs = {} + for entry in plan["spaces"]: + identity = entry["id"] + slug = _slug("bubble", identity) + if slug in slugs: + raise ValueError("Destination identity collision") + slugs[slug] = identity + if entry["captured_graph_complete"]: + for page in entry["articles"]: + targets.setdefault(page["id"], []).append({ + "space_slug": slug, "page_slug": _slug("article", page["id"]), + }) + for candidates in targets.values(): + candidates.sort(key=lambda item: (item["space_slug"], item["page_slug"])) + + article_details = {} + for identity, article in sorted(articles.items()): + body = article.get("content_text") + detail = {"id": identity, "body_sha256": None, "body_file": None, + "references": [], "relations": {}, "review_flags": []} + parser = _References() + if isinstance(body, str): + name = "source-bodies/" + _slug("article", identity) + ".txt" + files[name] = body.encode() + detail.update(body_file=name, body_sha256=_digest(files[name])) + parser.feed(markdown.markdown(body, extensions=["extra"])) + if article.get("logo_image"): + parser.references.add(("visual", str(article["logo_image"]))) + detail["references"] = [_reference(kind, value, targets) + for kind, value in sorted(parser.references)] + detail["review_flags"] = sorted(parser.unsupported) + for field in ["chunks_list_custom_article", "contents_list_custom_article"]: + values = article.get(field, []) + if not isinstance(values, list): + detail["review_flags"].append("invalid_" + field) + continue + detail["relations"][field] = [normalize_ref(value) for value in values] + parent = article.get("context_custom_article") + detail["relations"]["context_custom_article"] = normalize_ref(parent) if parent else None + if article.get("hasattachments_boolean"): + detail["review_flags"].append("attachment_records_require_capture") + article_details[identity] = detail + + for entry in plan["spaces"]: + identity = entry["id"] + space = all_spaces[identity] + slug = _slug("bubble", identity) + tree = "review-trees/" + slug if entry["captured_graph_complete"] else None + root = articles.get(entry["root_article_id"], {}) + title = root.get("nomer_text") or space.get("Slug") or slug + if not isinstance(title, str): + raise ValueError("Invalid source title") + blockers = {"source_completeness_review", "identity_and_explicit_grants_review", + "workflow_and_rendering_review", "existing_destination_reconciliation"} + if not tree: + blockers.add("source_graph_gaps") + if not entry["root_article_id"]: + blockers.add("space_root_requires_review") + if len(title) > 255: + blockers.add("title_exceeds_product_limit") + pages = [] + for page in entry["articles"]: + article = articles[page["id"]] + detail = article_details[page["id"]] + refs = detail["references"] + if refs: + blockers.add("asset_and_link_reconciliation") + if detail["review_flags"]: + blockers.add("unsupported_content_or_attachments") + relations = detail["relations"] + relation_ids = set(relations.get("chunks_list_custom_article", [])) | set( + relations.get("contents_list_custom_article", [])) + if relations["context_custom_article"]: + relation_ids.add(relations["context_custom_article"]) + missing = sorted(relation_ids - articles.keys()) + incomplete = sorted(ref for ref in relation_ids & articles.keys() + if not _complete_article(articles[ref])) + if missing: + blockers.add("uncaptured_relation_records") + if incomplete: + blockers.add("incomplete_relation_records") + if relations.get("contents_list_custom_article"): + blockers.add("contents_relation_semantics_review") + page_slug = _slug("article", page["id"]) + path = tree + "/pages/" + page_slug + ".md" if tree else None + if path: + files[path] = _page({ + "id": "bubble:" + page["id"], "bubble_type": "article", + "title": article.get("nomer_text") or page_slug, + "abstractor": article.get("abstractor_text") or "", "visual": "", + "space": slug, "source_created_at": page["created_at_raw"], + "source_modified_at": page["modified_at_raw"], + "archived": bool(article.get("archived_boolean")), + "parent": "bubble:" + page["parent_id"] if page["parent_id"] else "", + "source_chunks": json.dumps(relations.get("chunks_list_custom_article", [])), + "source_contents": json.dumps( + relations.get("contents_list_custom_article", [])), + }, article["content_text"]) + pages.append({**page, "file": path, "body_sha256": detail["body_sha256"], + "missing_relation_ids": missing, + "incomplete_relation_ids": incomplete, + "file_sha256": _digest(files[path]) if path else None}) + if tree: + # Keep source space birthdate distinct from its root article's birthdate. + root_slug = (_slug("article", entry["root_article_id"]) + if entry["root_article_id"] else None) + body = f"[Source root](?page={root_slug})\n" if root_slug else "" + files[tree + "/pages/index.md"] = _page({ + "id": "bubble-space:" + identity, "bubble_type": "space", "title": title, + "space": slug, "source_created_at": entry["created_at_raw"], + "source_modified_at": entry["modified_at_raw"], + "source_root": "bubble:" + entry["root_article_id"] if root_slug else "", + }, body) + entries.append({**entry, "slug": slug, "tree": tree, "articles": pages, + "blockers": sorted(blockers), "import_ready": False}) + + report = {**plan, "schema": "coulomb.bubble-review-bundle.v1", "spaces": entries, + "articles": list(article_details.values()), "body_rewriting_performed": False, + "network_requests": 0, "import_manifest_generated": False, + "asset_link_inventory_exhaustive": False, + "file_sha256": {name: _digest(data) for name, data in sorted(files.items())}} + return report, files diff --git a/docs/migration/corpus-import.md b/docs/migration/corpus-import.md index 49dd3fc..eb312cc 100644 --- a/docs/migration/corpus-import.md +++ b/docs/migration/corpus-import.md @@ -95,6 +95,47 @@ not a completeness claim. Plan its resulting source files again. The October27 pass captured 70 fresh spaces/162 articles; all 71 known spaces remain represented, with 67 captured graphs passing and the remaining source/ownership gaps explicit. +Prepare **all** captured trees and a private reconciliation bundle offline: + +```bash +uv run python scripts/prepare-bubble-bundle.py /private/checkpoint \ + --inventory-spaces /private/old-capture/spaces.json --output /private/new-review-bundle +``` + +The output directory must be new and outside the checkout. Its mode is 0700; +files are 0600. `reconciliation.json` is written last: without that completed +receipt the output is incomplete. Capture hashes and exact input snapshots bind +the bundle to the source. Every known space remains queued in LIFO order. +Only spaces whose captured graph checks pass receive `review-trees/`; blocked +spaces retain their gaps. `source-bodies/` preserves every captured text body +exactly as UTF-8, including blocked/unassigned records and leading/trailing +whitespace. Source-ID-derived slugs remain stable across label changes and +subsequent captures. These are proposed destinations; reconcile the already +accepted cohort and its local edits/copies/aliases before using any mapping. + +Each draft index preserves the **space** creation date and links to a separate +source root article. Each article/chunk preserves its own creation/modification +dates and parent/chunk/contents references; ordered lists retain duplicates. +The private report includes per-page body/file hashes and raw/UTC dates. The +root article's birthdate never substitutes for the space's birthdate. + +The bundle inventories ordinary Markdown inline/reference links and images, +HTML link/resource attributes and source Visuals, without fetching URLs or +rewriting bodies. Internal Bubble article links receive candidate destinations; +ambiguous, absent, queried and fragment targets remain for reconciliation. +External URLs, local resources, unsupported HTML/style/srcset, attachment flags +and uncaptured `contents_list_custom_article` relations remain explicit review +items. This is **not an exhaustive parser for custom Bubble conventions**, an +asset download, or proof of rendering/relationship fidelity. Source capture can +now follow missing contents relations as well as chunks/parents; their product +semantics require review before conversion. + +No import manifest or membership mapping is generated. Every space remains +`import_ready: false` until source completeness, assets/links, rendering/workflows, +existing destinations and explicit tenant/identity/grants are reconciled. Keep +all drafts private. Do not feed the whole bundle to `import_content_corpus` or +replace a live tree to backfill dates. + Packaged SQL migration rehearsal (internal disposable PostgreSQL, no production): ```bash diff --git a/docs/release/acceptance-audit-2026-10-27.md b/docs/release/acceptance-audit-2026-10-27.md index d0ec31b..6bac8d3 100644 --- a/docs/release/acceptance-audit-2026-10-27.md +++ b/docs/release/acceptance-audit-2026-10-27.md @@ -169,7 +169,33 @@ backup-version-delete-boundary-2026-10-27.json. Wrong-namespace login using the same admitted ServiceAccount name also returns403, without reading sibling credentials or any data. Receipt: backup-wrong-namespace-boundary-2026-10-27.json. Whole-capture field-completeness -counts are separate from per-space graph counts: 70 fresh space dates and 160 +counts are separate from per-space graph counts: 69 fresh space dates and 160 article creation dates are present/valid; 159 article records have a text body, and two captured article records contain only an ID. Those stubs are not proof of deletion, permission, an empty body or an original creation date. + +## Offline all-tree preparation — 2026-10-28 + +The deterministic compiler prepares all known spaces in creation-date order; +blocked spaces remain in the private queue. Captured source-body bytes, original +space/page/chunk dates and ordered parent/chunk/contents relations are preserved. +Draft space indexes link to separate source root articles rather than assigning +the root article the space's creation date. Stable source-ID-derived filenames +avoid title/order collisions. No productive manifest, identities or grants are +inferred. Asset/link/custom-rendering and existing destination reconciliation +remain explicit, with no exhaustive-parser claim or remote asset downloads. + +Following newly detected contents relations recovered additional source records +in 57 targeted visits: 244 captured articles, 241 text bodies and 242 valid article +creation dates. All 71 known spaces remain queued; 67 draft trees pass the narrow +captured-graph checks. Five capture references, the four prior space graph gaps, +contents semantics and attachment exceptions remain unresolved. One response +read timed out; it was bounded and the checkpoint/pass completed. + +An actual-capture rehearsal, with outgoing network disabled, isolated SQLite and +a synthetic owner, imported the 67 draft trees. It verified 543 prepared file +hashes, 235 included page/chunk file/body hashes and creation dates, and all 67 +SQL space creation dates without mismatch. The other six source bodies remain +in sidecars for blocked/unassigned records. No live tree, membership or database +was modified. [Counts and hash-bound proof](bubble-offline-preparation-2026-10-28.json). +The accepted-cohort date backfill and actual rendering/access review remain open. diff --git a/docs/release/bubble-offline-preparation-2026-10-28.json b/docs/release/bubble-offline-preparation-2026-10-28.json new file mode 100644 index 0000000..7bb5e9c --- /dev/null +++ b/docs/release/bubble-offline-preparation-2026-10-28.json @@ -0,0 +1,68 @@ +{ + "all_import_ready": false, + "articles_containing_only_id": 2, + "articles_with_valid_creation_date": 242, + "asset_link_inventory_exhaustive": false, + "blocker_counts": { + "asset_and_link_reconciliation": 69, + "contents_relation_semantics_review": 10, + "existing_destination_reconciliation": 71, + "identity_and_explicit_grants_review": 71, + "incomplete_relation_records": 2, + "source_completeness_review": 71, + "source_graph_gaps": 4, + "space_root_requires_review": 1, + "uncaptured_relation_records": 1, + "unsupported_content_or_attachments": 1, + "workflow_and_rendering_review": 71 + }, + "captured_articles": 244, + "date": "2026-10-28", + "draft_trees": 67, + "fresh_spaces": 70, + "fresh_spaces_with_valid_creation_date": 69, + "gap_counts": { + "chunk_belongs_to_other_space": 1, + "missing_body": 2, + "missing_or_invalid_created_at": 2, + "missing_or_invalid_modified_at": 2, + "space_not_refreshed": 1 + }, + "inventory_only_spaces": 1, + "isolated_import_rehearsal": { + "body_hash_mismatches": 0, + "bundle_sha256": "8983dd680d7684a69b7ec63f73d07a2ec153f0a74b5bf5375a87b345d9aa46dc", + "database": "isolated SQLite", + "network_disabled": true, + "page_date_mismatches": 0, + "prepared_spaces": 67, + "production_imported": false, + "real_identity_or_grants_used": false, + "runner_source_sha256": "b78704f99166e8097be00ba096ab5a1a9d95b62302b6a30c69a83aaf2077829d", + "schema": "coulomb.offline-capture-rehearsal.v1", + "space_date_mismatches": 0, + "status": "passed", + "verified_file_hashes": 543, + "verified_pages_and_chunks": 235 + }, + "known_spaces": 71, + "live_source_completeness_verified": false, + "memberships_authorized": false, + "preserved_body_sidecars": 241, + "private_bundle_report_sha256": "8983dd680d7684a69b7ec63f73d07a2ec153f0a74b5bf5375a87b345d9aa46dc", + "production_imported": false, + "reference_counts": { + "external": 259 + }, + "reference_pass_status": "completed_reference_pass", + "reference_visits": 57, + "remaining_capture_references": 5, + "response_timeouts": 1, + "schema": "coulomb.bubble-offline-preparation.v1", + "source_sha256": { + "articles": "b00f84adb5de60c724b004e3f05d64f44c8f4ca4bd0f5fef826c4e323841e87a", + "retained_spaces": "92038249ec2e4eed4c8d61ffcdbe030232cebc52e59ba005baee506a16930550", + "spaces": "ae20b0cfd1705f486c0b67ef48ac84b396f953cbcef09f9a8f07d07a66f1bf71" + }, + "unassigned_articles": 1 +} diff --git a/scripts/bubble-corpus-utils.mjs b/scripts/bubble-corpus-utils.mjs index b6402c8..3857442 100644 --- a/scripts/bubble-corpus-utils.mjs +++ b/scripts/bubble-corpus-utils.mjs @@ -75,9 +75,10 @@ export function pendingCaptureReferences(spaces, articles, inventory = {}) { } for (const [id, article] of Object.entries(articles)) { if (typeof article.content_text !== 'string' || article['Created Date'] == null) add(id); - for (const ref of Array.isArray(article.chunks_list_custom_article) - ? article.chunks_list_custom_article : []) { - if (!articles[normalizeBubbleRef(ref)]) add(ref); + for (const field of ['chunks_list_custom_article', 'contents_list_custom_article']) { + for (const ref of Array.isArray(article[field]) ? article[field] : []) { + if (!articles[normalizeBubbleRef(ref)]) add(ref); + } } const parent = normalizeBubbleRef(article.context_custom_article); if (parent && !articles[parent]) add(parent); diff --git a/scripts/prepare-bubble-bundle.py b/scripts/prepare-bubble-bundle.py new file mode 100644 index 0000000..0e3847e --- /dev/null +++ b/scripts/prepare-bubble-bundle.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Prepare private offline all-space review trees; never import or contact Bubble.""" + +import argparse +import hashlib +import json +import os +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from coulomb_social.apps.spaces.bubble_bundle import prepare_bundle # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("capture", type=Path) + parser.add_argument("--inventory-spaces", type=Path) + parser.add_argument("--output", type=Path, required=True, help="New directory outside checkout") + args = parser.parse_args() + output_created = False + try: + output = args.output.expanduser().resolve() + repo = Path(__file__).resolve().parents[1] + capture = args.capture.expanduser().resolve() + if output.is_relative_to(repo) or output.is_relative_to(capture) or ( + capture.is_relative_to(output) + ): + raise ValueError("Unsafe output location") + inputs = {name: (capture / (name + ".json")).read_bytes() + for name in ("spaces", "articles")} + if args.inventory_spaces: + if args.inventory_spaces.resolve().is_relative_to(output): + raise ValueError("Output overlaps retained inventory") + inputs["retained_spaces"] = args.inventory_spaces.read_bytes() + report, files = prepare_bundle( + json.loads(inputs["spaces"]), json.loads(inputs["articles"]), + inventory_spaces=json.loads(inputs["retained_spaces"]) + if "retained_spaces" in inputs else None, + ) + report["source_sha256"] = {name: hashlib.sha256(data).hexdigest() + for name, data in sorted(inputs.items())} + # Validate/compile entirely before creating output. Exclusive publication + # refuses overwrite; completion receipt is written last if interrupted. + output.mkdir(mode=0o700) + output_created = True + for name, data in sorted({**files, **{ + "source-capture/" + name + ".json": data for name, data in inputs.items() + }}.items()): + dest = output / name + dest.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + fd = os.open(dest, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(data) + report["status"] = "completed_offline_preparation" + fd = os.open(output / "reconciliation.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(report, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({ + "spaces": len(report["spaces"]), + "draft_trees": sum(bool(entry["tree"]) for entry in report["spaces"]), + "exact_body_sidecars": sum(a["body_file"] is not None for a in report["articles"]), + "blocker_counts": dict(sorted(Counter( + blocker for s in report["spaces"] for blocker in s["blockers"] + ).items())), "production_imported": False, + })) + except (ValueError, OSError, TypeError, KeyError): + print("Preparation refused; inspect private input/output locally. " + ( + "Output may be incomplete; no completion receipt means it is unusable." + if output_created else "No output was published." + ), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/bubble-corpus-utils.test.mjs b/scripts/tests/bubble-corpus-utils.test.mjs index 464b40d..0967e86 100644 --- a/scripts/tests/bubble-corpus-utils.test.mjs +++ b/scripts/tests/bubble-corpus-utils.test.mjs @@ -76,6 +76,15 @@ test('shared response ingestion normalizes prefixed references and preserves alr assert.equal(articles[id].content_text,'Synthetic only'); }); +test('reference continuation also retains uncaptured contents relations', () => { + const id='1700000000000x1', related='1700000000000x2'; + const articles={[id]:{content_text:'Synthetic', 'Created Date':1, + contents_list_custom_article:[`12__LOOKUP__${related}`]}}; + assert.deepEqual(pendingCaptureReferences({},articles),[related]); + articles[related]={content_text:'Related', 'Created Date':2}; + assert.deepEqual(pendingCaptureReferences({},articles),[]); +}); + test('unfinished source responses cannot stall the capture drain or leak diagnostics', async () => { assert.deepEqual(await boundedCaptureBody({text:()=>Promise.resolve('synthetic')}), {status:'read',text:'synthetic'}); diff --git a/tests/test_bubble_bundle.py b/tests/test_bubble_bundle.py new file mode 100644 index 0000000..a66fae0 --- /dev/null +++ b/tests/test_bubble_bundle.py @@ -0,0 +1,168 @@ +"""Synthetic source bodies only; offline preparation must never imply acceptance.""" + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from coulomb_social.apps.spaces.bubble_bundle import prepare_bundle +from coulomb_social.apps.spaces.import_tree import import_space_tree +from coulomb_social.apps.spaces.models import Space +from tests.test_space_permissions import make_member + + +def source(created, **kwargs): + return {"Created Date": created, "Modified Date": created, **kwargs} + + +def capture(): + return ({"s": source(1700000000123, _chunk_custom_article="r")}, { + "r": source(1700000001456, content_text="\n\n Root\r\n\n", nomer_text="Synthetic", + chunks_list_custom_article=["c"], space_custom_space="s"), + "c": source(1700000002789, content_text="Child \n", context_custom_article="r"), + }) + + +def test_bundle_deterministic_preserves_bodies_dates_and_ordered_relations(): + spaces, articles = capture() + articles["r"]["chunks_list_custom_article"] = ["c", "c"] + report, files = prepare_bundle(spaces, articles) + reversed_report, reversed_files = prepare_bundle( + dict(reversed(list(spaces.items()))), dict(reversed(list(articles.items())))) + assert (report, files) == (reversed_report, reversed_files) + entry = report["spaces"][0] + assert entry["created_at_raw"] == 1700000000123 + assert b"[Source root](?page=article-" in files[entry["tree"] + "/pages/index.md"] + for page in entry["articles"]: + raw = articles[page["id"]]["content_text"].encode() + assert files[page["file"]].endswith(b"\n---\n" + raw) + assert page["body_sha256"] == hashlib.sha256(raw).hexdigest() + assert f'source_created_at: {articles[page["id"]]["Created Date"]}'.encode() in ( + files[page["file"]]) + root = next(a for a in report["articles"] if a["id"] == "r") + assert root["relations"]["chunks_list_custom_article"] == ["c", "c"] + assert not entry["import_ready"] + assert not report["production_imported"] + + +def test_missing_dates_bodies_and_inventory_only_records_remain_queued_without_trees(): + spaces, articles = capture() + del articles["c"]["Created Date"] + articles["orphan"] = {"content_text": "Orphan original"} + report, files = prepare_bundle(spaces, articles, inventory_spaces={"old": {}}) + assert len(report["spaces"]) == 2 + assert all(s["tree"] is None for s in report["spaces"]) + assert not any(p.startswith("review-trees/") for p in files) + assert report["unassigned_article_ids"] == ["orphan"] + assert len([p for p in files if p.startswith("source-bodies/")]) == 3 + + +def test_reference_images_html_relations_and_attachments_are_unresolved_and_private(): + spaces, articles = capture() + body = ('[root](https://coulomb.social/pg_chunk/r?q=1#part)\n' + '![image][visual]\n\n[visual]: https://assets.example/synthetic.png\n\n' + '' + 'lookalike' + '') + articles["r"].update(content_text=body, contents_list_custom_article=["missing"], + logo_image="https://assets.example/logo.webp", hasattachments_boolean=True) + report, files = prepare_bundle(spaces, articles) + detail = next(a for a in report["articles"] if a["id"] == "r") + refs = detail["references"] + root = next(r for r in refs if r["classification"] == "bubble_article") + assert root["target_status"] == "unique" + assert root["query"] == "q=1" and root["fragment"] == "part" + assert {r["kind"] for r in refs} == {"asset", "visual", "link"} + assert next(r for r in refs if "attacker.example" in r["source"])["classification"] == ( + "external") + assert all(not r["resolved"] for r in refs) + entry = report["spaces"][0] + assert "uncaptured_relation_records" in entry["blockers"] + assert "attachment_records_require_capture" in detail["review_flags"] + assert "srcset_requires_review" in detail["review_flags"] + assert "iframe_requires_review" in detail["review_flags"] + assert any(data.endswith(body.encode()) for data in files.values()) + assert report["network_requests"] == 0 + + +def test_destination_names_survive_label_changes_and_additional_records(): + spaces, articles = capture() + report, _ = prepare_bundle(spaces, articles) + articles["r"]["nomer_text"] = "../../index" + articles["extra"] = source(1700000001000, space_custom_space="s", content_text="Extra") + later, files = prepare_bundle(spaces, articles) + assert report["spaces"][0]["slug"] == later["spaces"][0]["slug"] + paths = {p["id"]: p["file"] for p in later["spaces"][0]["articles"]} + assert all(p["file"] == paths[p["id"]] for p in report["spaces"][0]["articles"]) + assert all(".." not in Path(path).parts for path in files) + + +def test_shared_source_reference_never_selects_an_arbitrary_destination(): + spaces, articles = capture() + spaces["another"] = source(1700000000000, _chunk_custom_article="c") + articles["r"]["content_text"] = "[shared](/pg_chunk/c)" + report, _ = prepare_bundle(spaces, articles) + root = next(a for a in report["articles"] if a["id"] == "r") + reference = root["references"][0] + assert reference["target_status"] == "ambiguous" + assert len(reference["candidate_targets"]) == 2 + assert not reference["resolved"] + + +def test_relation_id_placeholder_does_not_count_as_a_complete_source_record(): + spaces, articles = capture() + articles["r"]["contents_list_custom_article"] = ["stub"] + articles["stub"] = {"_id": "stub"} + report, _ = prepare_bundle(spaces, articles) + entry = report["spaces"][0] + root = next(a for a in entry["articles"] if a["id"] == "r") + assert root["incomplete_relation_ids"] == ["stub"] + assert not root["missing_relation_ids"] + assert "incomplete_relation_records" in entry["blockers"] + + +@pytest.mark.django_db +def test_import_keeps_distinct_space_root_and_child_creation_dates(tmp_path, settings): + spaces, articles = capture() + report, files = prepare_bundle(spaces, articles) + for name, data in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + settings.CONTENT_ROOT = str(tmp_path / "content") + owner = make_member("synthetic-bundle-owner") + entry = report["spaces"][0] + import_space_tree(tmp_path / entry["tree"], tenant_id=owner.tenant_id, member=owner) + space = Space.objects.get(slug=entry["slug"]) + assert space.created_at.isoformat() == "2023-11-14T22:13:20.123000+00:00" + for page in entry["articles"]: + actual = Path(settings.CONTENT_ROOT) / "spaces" / entry["slug"] / "pages" / ( + Path(page["file"]).name) + assert actual.read_bytes() == files[page["file"]] + + +def test_cli_private_output_completion_hashes_no_overwrite_or_checkout_output(tmp_path): + repo = Path(__file__).resolve().parents[1] + capture_dir = tmp_path / "capture" + capture_dir.mkdir() + spaces, articles = capture() + for name, data in [("spaces", spaces), ("articles", articles)]: + (capture_dir / (name + ".json")).write_text(json.dumps(data)) + output = tmp_path / "bundle" + cmd = [sys.executable, str(repo / "scripts/prepare-bubble-bundle.py"), str(capture_dir), + "--output", str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + receipt = json.loads((output / "reconciliation.json").read_text()) + assert receipt["status"] == "completed_offline_preparation" + assert "Synthetic" not in result.stdout + assert os.stat(output).st_mode & 0o777 == 0o700 + assert all(os.stat(p).st_mode & 0o777 == 0o600 for p in output.rglob("*") if p.is_file()) + assert subprocess.run(cmd, capture_output=True).returncode == 1 + cmd[-1] = str(repo / "var/test-bundle-refusal") + assert subprocess.run(cmd, capture_output=True).returncode == 1 + assert not Path(cmd[-1]).exists() diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 1163e5f..0eae946 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -8,7 +8,7 @@ status: active owner: codex topic_slug: coulomb-social created: "2026-10-08" -updated: "2026-10-27" +updated: "2026-10-28" origin: residual origin_ref: CSOC-WP-0008 state_hub_workstream_id: "db6b85dc-f2f0-5283-ac52-3a6af28eb5b5" @@ -60,6 +60,20 @@ docs/release/bubble-capture-reconciliation-2026-10-27.json. Next: deterministic all-tree preparation, asset/link reconciliation, source exceptions and explicit tenant/identity/grant mapping before stepwise LIFO cohort transfer. +2026-10-28: Offline all-tree compiler now retains all 71 known spaces in LIFO +order, emits 67 draft review trees and preserves every captured text body exactly +in private sidecars, including blocked/unassigned records. Proposed slugs derive +from source IDs and remain stable across renamed labels/new records. Exact input +snapshots, file/body hashes and raw/UTC date manifests bind preparation to source. +No import manifest, grants, body rewriting, asset fetching or productive import. +Markdown/reference/HTML/Visual link inventory, ambiguous target candidates, +unsupported constructs, attachments and contents relations remain explicit review +gates, with no exhaustive-parser claim. Contents relations revealed 33 previously +uncaptured article references. Targeted continuation completed 57 visits, capturing +244 articles/241 bodies; five references and one response timeout remain recorded. +All four prior per-space graph exceptions persist. Non-secret counts/hashes: +docs/release/bubble-offline-preparation-2026-10-28.json. + ## T02 — Review remaining journeys and record transfer decisions ```task @@ -110,6 +124,17 @@ pages is not authority to overwrite those copies during a bulk reimport. Refresh the interim c11300a candidate for the stricter date guard before production use. Do not close on source-level tests alone. +2026-10-28: Draft indexes keep space creation dates separate from their source +root article; every article/chunk retains its own dates and ordered relations. +Actual private-capture rehearsal imported all 67 draft trees into isolated SQLite +with a synthetic owner, outgoing network disabled and no productive identities or +grants. All 235 included pages/chunks retained exact file/body hashes and original +creation dates; all 67 SQL space dates matched. Six captured text bodies outside +those trees remain in the private source sidecars. Evidence is the hash-bound +rehearsal embedded in bubble-offline-preparation-2026-10-28.json. This proves +draft conversion/import date preservation, not rendering, source completeness, +live acceptance or accepted-cohort backfill. Existing live content is untouched. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.