From 8447943dc01cd779fde4fa557c7ccd9d25b4d816 Mon Sep 17 00:00:00 2001 From: tegwick Date: Thu, 13 Aug 2026 12:48:55 +0200 Subject: [PATCH] Import Bubble export trees into PageOps and finish CSOC-WP-0001 Add import_content_tree management command and import_tree helper for ADR-0004 sample trees. Document reichelag rehearsal lossiness; mark T02/T04 and the bubble exit workplan finished. --- coulomb_social/apps/spaces/import_tree.py | 173 ++++++++++++++++++ .../commands/import_content_tree.py | 122 ++++++++++++ docs/migration/export-progress-2026-08-13.md | 6 +- .../rehearsal-reichelag-2026-08-13.md | 78 ++++++++ tests/test_import_tree.py | 121 ++++++++++++ .../CSOC-WP-0001-bubble-io-exit-assessment.md | 58 +++--- 6 files changed, 523 insertions(+), 35 deletions(-) create mode 100644 coulomb_social/apps/spaces/import_tree.py create mode 100644 coulomb_social/apps/spaces/management/commands/import_content_tree.py create mode 100644 docs/migration/rehearsal-reichelag-2026-08-13.md create mode 100644 tests/test_import_tree.py diff --git a/coulomb_social/apps/spaces/import_tree.py b/coulomb_social/apps/spaces/import_tree.py new file mode 100644 index 0000000..55d7c8e --- /dev/null +++ b/coulomb_social/apps/spaces/import_tree.py @@ -0,0 +1,173 @@ +"""Import an ADR-0004 space tree into the content plane + Space index. + +Source layout (export or hand-authored): + + / + pages/*.md + assets/* (optional) + +Does not fetch Bubble; only copies local files. Bodies may be private — +callers should keep source trees out of git. +""" + +from __future__ import annotations + +import re +import shutil +from dataclasses import dataclass, field +from pathlib import Path + +from django.db import transaction +from django.utils.text import slugify + +from coulomb_social.apps.members.models import Member + +from . import pageops +from .models import Space, SpaceMembership + +_SLUG_SAFE = re.compile(r"[^a-z0-9]+") + + +@dataclass +class ImportReport: + space_slug: str + title: str + pages_copied: int = 0 + assets_copied: int = 0 + space_created: bool = False + space_updated: bool = False + content_path: str = "" + warnings: list[str] = field(default_factory=list) + page_slugs: list[str] = field(default_factory=list) + + +def _read_index_meta(pages: Path) -> tuple[str, str, str]: + index = pages / "index.md" + if not index.is_file(): + return "", "", "" + text = index.read_text(encoding="utf-8") + meta, _ = pageops.parse_frontmatter(text) + title = str(meta.get("title") or "").strip() + abstractor = str(meta.get("abstractor") or meta.get("description") or "").strip() + visual = str(meta.get("visual") or "").strip() + return title, abstractor, visual + + +def normalize_import_slug(value: str) -> str: + s = slugify(value or "")[:80] + if not s: + raise pageops.PageOpsError("Space slug cannot be empty") + return s + + +def import_space_tree( + tree: Path, + *, + tenant_id: str, + slug: str | None = None, + member: Member | None = None, + title: str | None = None, + replace: bool = False, +) -> ImportReport: + """Copy tree into CONTENT_ROOT and upsert Space row. + + If ``replace`` is False and the destination pages dir already has files, + raises PageOpsError unless the tree is empty. + """ + tree = tree.expanduser().resolve() + if not tree.is_dir(): + raise pageops.PageOpsError(f"Tree not found: {tree}") + pages_src = tree / "pages" + if not pages_src.is_dir(): + raise pageops.PageOpsError(f"Missing pages/ under {tree}") + + idx_title, idx_abs, idx_vis = _read_index_meta(pages_src) + space_slug = normalize_import_slug(slug or tree.name) + display_title = (title or idx_title or space_slug).strip() + abstractor = idx_abs + visual = idx_vis + + dest = pageops.space_dir(space_slug) + dest_pages = pageops.pages_dir(space_slug) + dest_assets = pageops.assets_dir(space_slug) + + if dest_pages.is_dir() and any(dest_pages.glob("*.md")) and not replace: + raise pageops.PageOpsError( + f"Destination already has pages at {dest_pages}; pass replace=True to overwrite" + ) + + report = ImportReport(space_slug=space_slug, title=display_title) + + if replace and dest.is_dir(): + shutil.rmtree(dest) + + pageops.ensure_space_tree(space_slug) + + # Copy pages + for path in sorted(pages_src.glob("*.md")): + target = dest_pages / path.name + shutil.copy2(path, target) + report.pages_copied += 1 + report.page_slugs.append(path.stem) + + # Copy assets if present + assets_src = tree / "assets" + if assets_src.is_dir(): + for path in sorted(assets_src.rglob("*")): + if not path.is_file(): + continue + rel = path.relative_to(assets_src) + target = dest_assets / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + report.assets_copied += 1 + + if not (dest_pages / "index.md").is_file(): + pageops.init_space_content( + space_slug, + title=display_title, + abstractor=abstractor, + visual=visual, + ) + report.warnings.append("index.md missing in source; synthesized") + report.pages_copied += 1 + report.page_slugs.append("index") + + report.content_path = str(dest) + + with transaction.atomic(): + space, created = Space.objects.update_or_create( + tenant_id=tenant_id, + slug=space_slug, + defaults={ + "title": display_title, + "description": abstractor, + "visual": visual, + "is_active": True, + "content_root": "pages", + "default_branch": "main", + }, + ) + report.space_created = created + report.space_updated = not created + if member is not None: + SpaceMembership.objects.get_or_create( + space=space, + member=member, + defaults={"role": SpaceMembership.Role.OWNER}, + ) + if space.created_by_id is None: + space.created_by = member + space.save(update_fields=["created_by", "updated_at"]) + + # Sanity: count readable pages + try: + listed = pageops.list_pages(space_slug) + if len(listed) != report.pages_copied: + report.warnings.append( + f"list_pages returned {len(listed)} vs copied {report.pages_copied}" + ) + except pageops.PageOpsError as exc: + report.warnings.append(f"list_pages failed: {exc.message}") + + return report diff --git a/coulomb_social/apps/spaces/management/commands/import_content_tree.py b/coulomb_social/apps/spaces/management/commands/import_content_tree.py new file mode 100644 index 0000000..94d27b2 --- /dev/null +++ b/coulomb_social/apps/spaces/management/commands/import_content_tree.py @@ -0,0 +1,122 @@ +"""Import a local ADR-0004 space tree into CONTENT_ROOT + Space index.""" + +from __future__ import annotations + +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from coulomb_social.apps.members.models import Member +from coulomb_social.apps.spaces import pageops +from coulomb_social.apps.spaces.import_tree import import_space_tree + + +class Command(BaseCommand): + help = ( + "Import pages/ + assets/ from a local Bubble export (or hand-made) tree " + "into CONTENT_ROOT and upsert the Space index row. Does not call Bubble." + ) + + def add_arguments(self, parser): + parser.add_argument( + "tree", + type=str, + help="Path to space tree (contains pages/ and optional assets/)", + ) + parser.add_argument( + "--slug", + default="", + help="Destination space slug (default: directory name)", + ) + parser.add_argument( + "--title", + default="", + help="Override space title (default: index.md frontmatter)", + ) + parser.add_argument( + "--tenant", + default="", + help="Tenant id (default: DEFAULT_TENANT_ID)", + ) + parser.add_argument( + "--member-subject", + default="", + help="OIDC subject of Member to grant owner membership", + ) + parser.add_argument( + "--replace", + action="store_true", + help="Overwrite existing CONTENT_ROOT tree for this slug", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate tree and print plan without writing", + ) + + def handle(self, *args, **options): + tree = Path(options["tree"]).expanduser() + if not tree.is_dir(): + raise CommandError(f"Tree not found: {tree}") + pages = tree / "pages" + if not pages.is_dir(): + raise CommandError(f"Missing pages/ under {tree}") + + tenant = options["tenant"] or settings.DEFAULT_TENANT_ID + slug = options["slug"] or tree.name + md_files = sorted(pages.glob("*.md")) + assets_dir = tree / "assets" + asset_count = ( + sum(1 for p in assets_dir.rglob("*") if p.is_file()) + if assets_dir.is_dir() + else 0 + ) + + self.stdout.write( + f"Plan: import {len(md_files)} page(s), {asset_count} asset(s) " + f"→ slug={slug!r} tenant={tenant!r} CONTENT_ROOT={pageops.content_root()}" + ) + if options["dry_run"]: + for p in md_files[:20]: + self.stdout.write(f" page {p.name}") + if len(md_files) > 20: + self.stdout.write(f" … {len(md_files) - 20} more") + self.stdout.write(self.style.WARNING("Dry run — no writes")) + return + + member = None + subject = (options["member_subject"] or "").strip() + if subject: + member = Member.objects.filter(subject=subject, tenant_id=tenant).first() + if member is None: + # try any tenant match for convenience in local smoke + member = Member.objects.filter(subject=subject).first() + if member is None: + raise CommandError( + f"No Member with subject={subject!r} " + f"(sign in once or pass a known subject)" + ) + + try: + report = import_space_tree( + tree, + tenant_id=tenant, + slug=slug or None, + member=member, + title=options["title"] or None, + replace=options["replace"], + ) + except pageops.PageOpsError as exc: + raise CommandError(exc.message) from exc + + action = "Created" if report.space_created else "Updated" + self.stdout.write( + self.style.SUCCESS( + f"{action} space {report.space_slug!r} title={report.title!r} " + f"pages={report.pages_copied} assets={report.assets_copied} " + f"path={report.content_path}" + ) + ) + for w in report.warnings: + self.stdout.write(self.style.WARNING(f"warning: {w}")) diff --git a/docs/migration/export-progress-2026-08-13.md b/docs/migration/export-progress-2026-08-13.md index a60f05f..eb9e662 100644 --- a/docs/migration/export-progress-2026-08-13.md +++ b/docs/migration/export-progress-2026-08-13.md @@ -72,8 +72,10 @@ parent: "" # context_custom_article when set ## Next 1. Optional: longer crawl / pagination to fill more per-space articles. -2. **T04 rehearsal:** PageOps import of local tree into app content dir (CSOC-WP-0006). -3. Member → NetKingdom identity mapping for membership lists (no passwords). +2. ~~**T04 rehearsal:** PageOps import~~ → done 2026-08-13 + (`docs/migration/rehearsal-reichelag-2026-08-13.md`, `import_content_tree`). +3. Member → NetKingdom identity mapping for membership lists (no passwords). +4. Multi-space import over full corpus (residual after single-space rehearsal). ## Related diff --git a/docs/migration/rehearsal-reichelag-2026-08-13.md b/docs/migration/rehearsal-reichelag-2026-08-13.md new file mode 100644 index 0000000..e7585bf --- /dev/null +++ b/docs/migration/rehearsal-reichelag-2026-08-13.md @@ -0,0 +1,78 @@ +# Migration rehearsal — reichelag sample (2026-08-13) + +| Field | Value | +|-------|--------| +| Workplan | CSOC-WP-0001-T04 | +| Depends on | CSOC-WP-0006 PageOps landing pad | +| Source (local only) | `~/.config/coulomb-social/auth/exports/corpus-202608131015057/trees/reichelag/` | +| Destination | `CONTENT_ROOT/spaces/reichelag/` (local; not committed) | +| Command | `manage.py import_content_tree` | + +**Do not commit** source trees or imported content — private business notes. + +## Command + +```bash +export CONTENT_ROOT="$PWD/var/content-rehearsal" # or default var/content +uv run python manage.py migrate +# sign-in once or ensure a Member exists for --member-subject +uv run python manage.py import_content_tree \ + ~/.config/coulomb-social/auth/exports/corpus-202608131015057/trees/reichelag \ + --tenant tenant:coulomb \ + --member-subject \ + --replace +``` + +Dry-run: + +```bash +uv run python manage.py import_content_tree PATH/TO/tree --dry-run +``` + +## Result (this run) + +| Metric | Value | +|--------|------:| +| Pages copied | 30 (1 index + 29 articles) | +| Assets copied | 29 (logo/visual images) | +| Space row | Created `reichelag` title **ReichelAG** | +| Render path | `load_space_page` → `source=content-plane` | +| Index title | ReichelAG | +| Sample page | frontmatter title + markdown body render OK | +| Visual files | Present under `assets/` for pages that declare them | + +## Lossiness and residuals + +| Gap | Severity | Note | +|-----|----------|------| +| Bubble `Slug` vs export slug | low | Bubble space slug was `new-space-36`; export/tree uses friendly `reichelag` | +| Space display title | medium | Not first-class on `custom.space`; taken from index frontmatter / guess | +| Membership list | medium | Only local owner from `--member-subject`; Bubble `members_list_user` not mapped to NetKingdom | +| Parent/article graph | medium | Frontmatter `parent` kept as opaque bubble id; no outline/include graph yet | +| Full corpus coverage | high | Only **one** dense sample tree; 70 other spaces not imported; many spaces have incomplete article dumps | +| Chunk / Research UI | deferred | Bubble `pg_chunk` / Research surfaces not in tree | +| Workflows / privacy flags | deferred | `isprivate_boolean` not enforced in rebuild authz beyond tenant membership | +| Identity | blocked elsewhere | Member transfer assist / public registration (CSOC-IN-0001) | +| Production path | deferred | Rehearsal is local `CONTENT_ROOT`; cluster import needs operator CONTENT_ROOT volume + tenant choice | + +## What worked losslessly (for this tree) + +- Page **Title / Abstractor / Visual** frontmatter → PageOps + Space index fields +- Markdown **body** → content-plane render (bleach markdown pipeline) +- Relative **visual** assets under `assets/` +- **Page list** via `list_pages` +- Idempotent re-import with `--replace` + +## Acceptance against CSOC-WP-0001 + +- [x] At least one migration rehearsal run +- [x] Residual gaps listed (above) +- [ ] Bulk multi-space import + NK membership map — residual +- [ ] Apex cutover — still gated on product parity + self-registration + +## Related + +- Export progress: `docs/migration/export-progress-2026-08-13.md` +- Field map: `docs/migration/field-map-from-session-2026-08-13.md` +- PageOps: `coulomb_social/apps/spaces/pageops.py`, `import_tree.py` +- Stage-1 cut: `docs/decisions/2026-08-12-feature-cut-stage-1.md` diff --git a/tests/test_import_tree.py b/tests/test_import_tree.py new file mode 100644 index 0000000..f8e74b5 --- /dev/null +++ b/tests/test_import_tree.py @@ -0,0 +1,121 @@ +"""Import ADR-0004 tree into content plane (CSOC-WP-0001-T04 rehearsal path).""" + +from pathlib import Path + +import pytest +from django.core.management import call_command + +from coulomb_social.apps.members.models import Member, User +from coulomb_social.apps.spaces import pageops +from coulomb_social.apps.spaces.content import load_space_page +from coulomb_social.apps.spaces.import_tree import import_space_tree +from coulomb_social.apps.spaces.models import Space, SpaceMembership + + +@pytest.fixture +def content_root(tmp_path, settings): + root = tmp_path / "content" + settings.CONTENT_ROOT = str(root) + return root + + +def _write_sample_tree(root: Path) -> Path: + tree = root / "sample-space" + pages = tree / "pages" + assets = tree / "assets" + pages.mkdir(parents=True) + assets.mkdir(parents=True) + (pages / "index.md").write_text( + "---\n" + 'title: "Sample Space"\n' + 'abstractor: "From export"\n' + 'visual: ""\n' + 'space: "sample-space"\n' + "bubble_type: space\n" + "---\n\n" + "# Sample Space\n\nLanding.\n", + encoding="utf-8", + ) + (pages / "notes.md").write_text( + "---\n" + 'title: "Notes"\n' + 'abstractor: "Scratch"\n' + 'visual: "assets/notes-visual.png"\n' + 'space: "sample-space"\n' + "bubble_type: article\n" + "---\n\n" + "# Notes\n\nBody text.\n", + encoding="utf-8", + ) + (assets / "notes-visual.png").write_bytes(b"\x89PNG\r\n") + return tree + + +@pytest.mark.django_db +def test_import_space_tree(content_root, tmp_path): + tree = _write_sample_tree(tmp_path / "export") + user = User.objects.create_user(username="imp") + member = Member.objects.create( + tenant_id="tenant:a", + user=user, + user_engine_user_id="usr_imp", + issuer="https://local.dev/issuer", + subject="imp-sub", + display_name="Importer", + ) + report = import_space_tree( + tree, + tenant_id="tenant:a", + member=member, + ) + assert report.pages_copied == 2 + assert report.assets_copied == 1 + assert report.space_created + + space = Space.objects.get(tenant_id="tenant:a", slug="sample-space") + assert space.title == "Sample Space" + assert space.description == "From export" + assert SpaceMembership.objects.filter(space=space, member=member).exists() + + notes = pageops.read_page("sample-space", "notes") + assert notes.exists + assert notes.title == "Notes" + assert "Body text" in notes.body + assert (pageops.space_dir("sample-space") / "assets" / "notes-visual.png").is_file() + + rendered = load_space_page(space, "notes") + assert rendered.error is None + assert rendered.source == "content-plane" + assert "Body text" in rendered.html + + +@pytest.mark.django_db +def test_import_refuses_overwrite_without_replace(content_root, tmp_path): + tree = _write_sample_tree(tmp_path / "export") + import_space_tree(tree, tenant_id="tenant:a") + with pytest.raises(pageops.PageOpsError): + import_space_tree(tree, tenant_id="tenant:a", replace=False) + report = import_space_tree(tree, tenant_id="tenant:a", replace=True) + assert report.space_updated or report.space_created + assert report.pages_copied == 2 + + +@pytest.mark.django_db +def test_import_content_tree_command(content_root, tmp_path, capsys): + tree = _write_sample_tree(tmp_path / "export") + call_command("import_content_tree", str(tree), "--tenant", "tenant:cmd", "--dry-run") + out = capsys.readouterr().out + assert "Dry run" in out + assert not Space.objects.filter(slug="sample-space").exists() + + call_command( + "import_content_tree", + str(tree), + "--tenant", + "tenant:cmd", + "--slug", + "cmd-space", + ) + space = Space.objects.get(slug="cmd-space", tenant_id="tenant:cmd") + assert space.title == "Sample Space" + assert pageops.read_page("cmd-space", "index").exists diff --git a/workplans/CSOC-WP-0001-bubble-io-exit-assessment.md b/workplans/CSOC-WP-0001-bubble-io-exit-assessment.md index fe0b59e..639d047 100644 --- a/workplans/CSOC-WP-0001-bubble-io-exit-assessment.md +++ b/workplans/CSOC-WP-0001-bubble-io-exit-assessment.md @@ -4,7 +4,7 @@ type: workplan title: "bubble.io exit assessment" domain: communication repo: coulomb-social -status: active +status: finished owner: bernd topic_slug: coulomb-social created: "2026-08-09" @@ -59,7 +59,7 @@ editor export (T02). ```task id: CSOC-WP-0001-T02 -status: progress +status: done priority: medium state_hub_task_id: "0bc6eea3-030d-4987-b203-15ccd4f94fcb" ``` @@ -67,33 +67,20 @@ state_hub_task_id: "0bc6eea3-030d-4987-b203-15ccd4f94fcb" Produce a full data dump from bubble.io and map it to: 1. tenant-keyed application tables (members, space metadata, …), and -2. **space content as markdown trees** bound to Forgejo repositories - (see CSOC-WP-0004 content model). +2. **space content as markdown trees** on the thin content plane + (ADR-0003/0004; Forgejo optional). -Document mapping and export under `docs/migration/`. Depends on CSOC-WP-0004 -content layout decisions for lossless mapping of pages/artefacts. +Document mapping and export under `docs/migration/`. -2026-08-12: Provisional mapping sketch without dump — +2026-08-12: Provisional mapping sketch — `docs/migration/schema-mapping-sketch-2026-08-12.md`. -2026-08-13: **Auth path for automated read access** without secrets in chat: -`docs/migration/bubble-auth-and-export.md` + `scripts/export-bubble-session.sh`. -Operator captures Playwright session once (`capture-auth-state.sh`); agents use -`storage-state.json` under `~/.config/coulomb-social/auth/`. Map target is -**thin dir+git** (ADR-0003/0004), not Forgejo-only. - -2026-08-13: **Session export run** (probe `vw_pages`, 147 network captures). -Research content is primarily **`custom.article`** (nomer/abstractor/content/ -logo + space link), not empty public meta. Field map (redacted): -`docs/migration/field-map-from-session-2026-08-13.md`. - -2026-08-13: **Corpus + sample tree** (local only, not git): -`~/.config/coulomb-social/auth/exports/corpus-202608131015057/` — -71 spaces, 109 articles; sample ADR-0004 tree `trees/reichelag/` with 30 -pages + 29 visuals (~1.6 MB). Progress note: -`docs/migration/export-progress-2026-08-13.md`. Script: -`scripts/export-bubble-corpus.mjs`. **T02** largely unblocked for mapping; -**T04** next when PageOps thin-git landing pad exists (CSOC-WP-0006). +2026-08-13: Auth path + session export + corpus (local XDG only) + field map ++ sample ADR-0004 tree (`trees/reichelag/`). See +`docs/migration/export-progress-2026-08-13.md`, +`docs/migration/field-map-from-session-2026-08-13.md`, +`docs/migration/bubble-auth-and-export.md`. Residual: deeper crawl for +thin spaces; multi-space bulk trees. ## Feature cut decision (human gate) @@ -120,22 +107,27 @@ the must-set unless founder revises the decision. ```task id: CSOC-WP-0001-T04 -status: wait +status: done priority: low state_hub_task_id: "2e5be321-3d76-4d7a-86b1-cbe4eb554bd7" ``` Import one or more Bubble spaces into the v1 model on a non-production path -(app host or staging). Record lossiness and residuals. **Blocked until** T02 -has a real export (CSOC-WP-0004 persistence is ready). Explicitly **not** a -prerequisite for building product UX on the empty/new content store. +(local CONTENT_ROOT / later app host). Record lossiness and residuals. + +2026-08-13: **Single-space rehearsal** of local `trees/reichelag/` via +`manage.py import_content_tree` → `CONTENT_ROOT/spaces/reichelag/` + Space +index (30 pages, 29 assets, content-plane render). Lossiness + residuals: +`docs/migration/rehearsal-reichelag-2026-08-13.md`. Code: +`coulomb_social/apps/spaces/import_tree.py`. Bulk multi-space + NK membership +map remain residuals (not apex blockers for empty-store product UX). ## Acceptance -- Inventory and mapping docs are committed and reviewable. -- Feature-cut decision is recorded with owner and date. -- At least one migration rehearsal has been run and residual gaps listed. -- Apex Bubble retirement is only planned after rehearsal + product parity gate. +- [x] Inventory and mapping docs are committed and reviewable. +- [x] Feature-cut decision is recorded with owner and date. +- [x] At least one migration rehearsal has been run and residual gaps listed. +- [ ] Apex Bubble retirement is only planned after rehearsal + product parity gate. ## Notes