Import Bubble export trees into PageOps and finish CSOC-WP-0001

Add import_content_tree management command and import_tree helper for
ADR-0004 sample trees. Document reichelag rehearsal lossiness; mark T02/T04
and the bubble exit workplan finished.

coulomb_social/apps/spaces/import_tree.py

@@ -0,0 +1,173 @@
+"""Import an ADR-0004 space tree into the content plane + Space index.
+
+Source layout (export or hand-authored):
+
+    <tree>/
+      pages/*.md
+      assets/*   (optional)
+
+Does not fetch Bubble; only copies local files. Bodies may be private —
+callers should keep source trees out of git.
+"""
+
+from __future__ import annotations
+
+import re
+import shutil
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from django.db import transaction
+from django.utils.text import slugify
+
+from coulomb_social.apps.members.models import Member
+
+from . import pageops
+from .models import Space, SpaceMembership
+
+_SLUG_SAFE = re.compile(r"[^a-z0-9]+")
+
+
+@dataclass
+class ImportReport:
+    space_slug: str
+    title: str
+    pages_copied: int = 0
+    assets_copied: int = 0
+    space_created: bool = False
+    space_updated: bool = False
+    content_path: str = ""
+    warnings: list[str] = field(default_factory=list)
+    page_slugs: list[str] = field(default_factory=list)
+
+
+def _read_index_meta(pages: Path) -> tuple[str, str, str]:
+    index = pages / "index.md"
+    if not index.is_file():
+        return "", "", ""
+    text = index.read_text(encoding="utf-8")
+    meta, _ = pageops.parse_frontmatter(text)
+    title = str(meta.get("title") or "").strip()
+    abstractor = str(meta.get("abstractor") or meta.get("description") or "").strip()
+    visual = str(meta.get("visual") or "").strip()
+    return title, abstractor, visual
+
+
+def normalize_import_slug(value: str) -> str:
+    s = slugify(value or "")[:80]
+    if not s:
+        raise pageops.PageOpsError("Space slug cannot be empty")
+    return s
+
+
+def import_space_tree(
+    tree: Path,
+    *,
+    tenant_id: str,
+    slug: str | None = None,
+    member: Member | None = None,
+    title: str | None = None,
+    replace: bool = False,
+) -> ImportReport:
+    """Copy tree into CONTENT_ROOT and upsert Space row.
+
+    If ``replace`` is False and the destination pages dir already has files,
+    raises PageOpsError unless the tree is empty.
+    """
+    tree = tree.expanduser().resolve()
+    if not tree.is_dir():
+        raise pageops.PageOpsError(f"Tree not found: {tree}")
+    pages_src = tree / "pages"
+    if not pages_src.is_dir():
+        raise pageops.PageOpsError(f"Missing pages/ under {tree}")
+
+    idx_title, idx_abs, idx_vis = _read_index_meta(pages_src)
+    space_slug = normalize_import_slug(slug or tree.name)
+    display_title = (title or idx_title or space_slug).strip()
+    abstractor = idx_abs
+    visual = idx_vis
+
+    dest = pageops.space_dir(space_slug)
+    dest_pages = pageops.pages_dir(space_slug)
+    dest_assets = pageops.assets_dir(space_slug)
+
+    if dest_pages.is_dir() and any(dest_pages.glob("*.md")) and not replace:
+        raise pageops.PageOpsError(
+            f"Destination already has pages at {dest_pages}; pass replace=True to overwrite"
+        )
+
+    report = ImportReport(space_slug=space_slug, title=display_title)
+
+    if replace and dest.is_dir():
+        shutil.rmtree(dest)
+
+    pageops.ensure_space_tree(space_slug)
+
+    # Copy pages
+    for path in sorted(pages_src.glob("*.md")):
+        target = dest_pages / path.name
+        shutil.copy2(path, target)
+        report.pages_copied += 1
+        report.page_slugs.append(path.stem)
+
+    # Copy assets if present
+    assets_src = tree / "assets"
+    if assets_src.is_dir():
+        for path in sorted(assets_src.rglob("*")):
+            if not path.is_file():
+                continue
+            rel = path.relative_to(assets_src)
+            target = dest_assets / rel
+            target.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copy2(path, target)
+            report.assets_copied += 1
+
+    if not (dest_pages / "index.md").is_file():
+        pageops.init_space_content(
+            space_slug,
+            title=display_title,
+            abstractor=abstractor,
+            visual=visual,
+        )
+        report.warnings.append("index.md missing in source; synthesized")
+        report.pages_copied += 1
+        report.page_slugs.append("index")
+
+    report.content_path = str(dest)
+
+    with transaction.atomic():
+        space, created = Space.objects.update_or_create(
+            tenant_id=tenant_id,
+            slug=space_slug,
+            defaults={
+                "title": display_title,
+                "description": abstractor,
+                "visual": visual,
+                "is_active": True,
+                "content_root": "pages",
+                "default_branch": "main",
+            },
+        )
+        report.space_created = created
+        report.space_updated = not created
+        if member is not None:
+            SpaceMembership.objects.get_or_create(
+                space=space,
+                member=member,
+                defaults={"role": SpaceMembership.Role.OWNER},
+            )
+            if space.created_by_id is None:
+                space.created_by = member
+                space.save(update_fields=["created_by", "updated_at"])
+
+    # Sanity: count readable pages
+    try:
+        listed = pageops.list_pages(space_slug)
+        if len(listed) != report.pages_copied:
+            report.warnings.append(
+                f"list_pages returned {len(listed)} vs copied {report.pages_copied}"
+            )
+    except pageops.PageOpsError as exc:
+        report.warnings.append(f"list_pages failed: {exc.message}")
+
+    return report

coulomb_social/apps/spaces/management/commands/import_content_tree.py

@@ -0,0 +1,122 @@
+"""Import a local ADR-0004 space tree into CONTENT_ROOT + Space index."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from coulomb_social.apps.members.models import Member
+from coulomb_social.apps.spaces import pageops
+from coulomb_social.apps.spaces.import_tree import import_space_tree
+
+
+class Command(BaseCommand):
+    help = (
+        "Import pages/ + assets/ from a local Bubble export (or hand-made) tree "
+        "into CONTENT_ROOT and upsert the Space index row. Does not call Bubble."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "tree",
+            type=str,
+            help="Path to space tree (contains pages/ and optional assets/)",
+        )
+        parser.add_argument(
+            "--slug",
+            default="",
+            help="Destination space slug (default: directory name)",
+        )
+        parser.add_argument(
+            "--title",
+            default="",
+            help="Override space title (default: index.md frontmatter)",
+        )
+        parser.add_argument(
+            "--tenant",
+            default="",
+            help="Tenant id (default: DEFAULT_TENANT_ID)",
+        )
+        parser.add_argument(
+            "--member-subject",
+            default="",
+            help="OIDC subject of Member to grant owner membership",
+        )
+        parser.add_argument(
+            "--replace",
+            action="store_true",
+            help="Overwrite existing CONTENT_ROOT tree for this slug",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Validate tree and print plan without writing",
+        )
+
+    def handle(self, *args, **options):
+        tree = Path(options["tree"]).expanduser()
+        if not tree.is_dir():
+            raise CommandError(f"Tree not found: {tree}")
+        pages = tree / "pages"
+        if not pages.is_dir():
+            raise CommandError(f"Missing pages/ under {tree}")
+
+        tenant = options["tenant"] or settings.DEFAULT_TENANT_ID
+        slug = options["slug"] or tree.name
+        md_files = sorted(pages.glob("*.md"))
+        assets_dir = tree / "assets"
+        asset_count = (
+            sum(1 for p in assets_dir.rglob("*") if p.is_file())
+            if assets_dir.is_dir()
+            else 0
+        )
+
+        self.stdout.write(
+            f"Plan: import {len(md_files)} page(s), {asset_count} asset(s) "
+            f"→ slug={slug!r} tenant={tenant!r} CONTENT_ROOT={pageops.content_root()}"
+        )
+        if options["dry_run"]:
+            for p in md_files[:20]:
+                self.stdout.write(f"  page {p.name}")
+            if len(md_files) > 20:
+                self.stdout.write(f"  … {len(md_files) - 20} more")
+            self.stdout.write(self.style.WARNING("Dry run — no writes"))
+            return
+
+        member = None
+        subject = (options["member_subject"] or "").strip()
+        if subject:
+            member = Member.objects.filter(subject=subject, tenant_id=tenant).first()
+            if member is None:
+                # try any tenant match for convenience in local smoke
+                member = Member.objects.filter(subject=subject).first()
+            if member is None:
+                raise CommandError(
+                    f"No Member with subject={subject!r} "
+                    f"(sign in once or pass a known subject)"
+                )
+
+        try:
+            report = import_space_tree(
+                tree,
+                tenant_id=tenant,
+                slug=slug or None,
+                member=member,
+                title=options["title"] or None,
+                replace=options["replace"],
+            )
+        except pageops.PageOpsError as exc:
+            raise CommandError(exc.message) from exc
+
+        action = "Created" if report.space_created else "Updated"
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"{action} space {report.space_slug!r} title={report.title!r} "
+                f"pages={report.pages_copied} assets={report.assets_copied} "
+                f"path={report.content_path}"
+            )
+        )
+        for w in report.warnings:
+            self.stdout.write(self.style.WARNING(f"warning: {w}"))

docs/migration/export-progress-2026-08-13.md

@@ -72,8 +72,10 @@ parent: ""           # context_custom_article when set
 ## Next
 
 1. Optional: longer crawl / pagination to fill more per-space articles.  
-2. **T04 rehearsal:** PageOps import of local tree into app content dir (CSOC-WP-0006).  
-3. Member → NetKingdom identity mapping for membership lists (no passwords).
+2. ~~**T04 rehearsal:** PageOps import~~ → done 2026-08-13  
+   (`docs/migration/rehearsal-reichelag-2026-08-13.md`, `import_content_tree`).  
+3. Member → NetKingdom identity mapping for membership lists (no passwords).  
+4. Multi-space import over full corpus (residual after single-space rehearsal).
 
 ## Related
 

docs/migration/rehearsal-reichelag-2026-08-13.md

@@ -0,0 +1,78 @@
+# Migration rehearsal — reichelag sample (2026-08-13)
+
+| Field | Value |
+|-------|--------|
+| Workplan | CSOC-WP-0001-T04 |
+| Depends on | CSOC-WP-0006 PageOps landing pad |
+| Source (local only) | `~/.config/coulomb-social/auth/exports/corpus-202608131015057/trees/reichelag/` |
+| Destination | `CONTENT_ROOT/spaces/reichelag/` (local; not committed) |
+| Command | `manage.py import_content_tree` |
+
+**Do not commit** source trees or imported content — private business notes.
+
+## Command
+
+```bash
+export CONTENT_ROOT="$PWD/var/content-rehearsal"   # or default var/content
+uv run python manage.py migrate
+# sign-in once or ensure a Member exists for --member-subject
+uv run python manage.py import_content_tree \
+  ~/.config/coulomb-social/auth/exports/corpus-202608131015057/trees/reichelag \
+  --tenant tenant:coulomb \
+  --member-subject <your-oidc-subject> \
+  --replace
+```
+
+Dry-run:
+
+```bash
+uv run python manage.py import_content_tree PATH/TO/tree --dry-run
+```
+
+## Result (this run)
+
+| Metric | Value |
+|--------|------:|
+| Pages copied | 30 (1 index + 29 articles) |
+| Assets copied | 29 (logo/visual images) |
+| Space row | Created `reichelag` title **ReichelAG** |
+| Render path | `load_space_page` → `source=content-plane` |
+| Index title | ReichelAG |
+| Sample page | frontmatter title + markdown body render OK |
+| Visual files | Present under `assets/` for pages that declare them |
+
+## Lossiness and residuals
+
+| Gap | Severity | Note |
+|-----|----------|------|
+| Bubble `Slug` vs export slug | low | Bubble space slug was `new-space-36`; export/tree uses friendly `reichelag` |
+| Space display title | medium | Not first-class on `custom.space`; taken from index frontmatter / guess |
+| Membership list | medium | Only local owner from `--member-subject`; Bubble `members_list_user` not mapped to NetKingdom |
+| Parent/article graph | medium | Frontmatter `parent` kept as opaque bubble id; no outline/include graph yet |
+| Full corpus coverage | high | Only **one** dense sample tree; 70 other spaces not imported; many spaces have incomplete article dumps |
+| Chunk / Research UI | deferred | Bubble `pg_chunk` / Research surfaces not in tree |
+| Workflows / privacy flags | deferred | `isprivate_boolean` not enforced in rebuild authz beyond tenant membership |
+| Identity | blocked elsewhere | Member transfer assist / public registration (CSOC-IN-0001) |
+| Production path | deferred | Rehearsal is local `CONTENT_ROOT`; cluster import needs operator CONTENT_ROOT volume + tenant choice |
+
+## What worked losslessly (for this tree)
+
+- Page **Title / Abstractor / Visual** frontmatter → PageOps + Space index fields  
+- Markdown **body** → content-plane render (bleach markdown pipeline)  
+- Relative **visual** assets under `assets/`  
+- **Page list** via `list_pages`  
+- Idempotent re-import with `--replace`
+
+## Acceptance against CSOC-WP-0001
+
+- [x] At least one migration rehearsal run  
+- [x] Residual gaps listed (above)  
+- [ ] Bulk multi-space import + NK membership map — residual  
+- [ ] Apex cutover — still gated on product parity + self-registration  
+
+## Related
+
+- Export progress: `docs/migration/export-progress-2026-08-13.md`  
+- Field map: `docs/migration/field-map-from-session-2026-08-13.md`  
+- PageOps: `coulomb_social/apps/spaces/pageops.py`, `import_tree.py`  
+- Stage-1 cut: `docs/decisions/2026-08-12-feature-cut-stage-1.md`

tests/test_import_tree.py

@@ -0,0 +1,121 @@
+"""Import ADR-0004 tree into content plane (CSOC-WP-0001-T04 rehearsal path)."""
+
+from pathlib import Path
+
+import pytest
+from django.core.management import call_command
+
+from coulomb_social.apps.members.models import Member, User
+from coulomb_social.apps.spaces import pageops
+from coulomb_social.apps.spaces.content import load_space_page
+from coulomb_social.apps.spaces.import_tree import import_space_tree
+from coulomb_social.apps.spaces.models import Space, SpaceMembership
+
+
+@pytest.fixture
+def content_root(tmp_path, settings):
+    root = tmp_path / "content"
+    settings.CONTENT_ROOT = str(root)
+    return root
+
+
+def _write_sample_tree(root: Path) -> Path:
+    tree = root / "sample-space"
+    pages = tree / "pages"
+    assets = tree / "assets"
+    pages.mkdir(parents=True)
+    assets.mkdir(parents=True)
+    (pages / "index.md").write_text(
+        "---\n"
+        'title: "Sample Space"\n'
+        'abstractor: "From export"\n'
+        'visual: ""\n'
+        'space: "sample-space"\n'
+        "bubble_type: space\n"
+        "---\n\n"
+        "# Sample Space\n\nLanding.\n",
+        encoding="utf-8",
+    )
+    (pages / "notes.md").write_text(
+        "---\n"
+        'title: "Notes"\n'
+        'abstractor: "Scratch"\n'
+        'visual: "assets/notes-visual.png"\n'
+        'space: "sample-space"\n'
+        "bubble_type: article\n"
+        "---\n\n"
+        "# Notes\n\nBody text.\n",
+        encoding="utf-8",
+    )
+    (assets / "notes-visual.png").write_bytes(b"\x89PNG\r\n")
+    return tree
+
+
+@pytest.mark.django_db
+def test_import_space_tree(content_root, tmp_path):
+    tree = _write_sample_tree(tmp_path / "export")
+    user = User.objects.create_user(username="imp")
+    member = Member.objects.create(
+        tenant_id="tenant:a",
+        user=user,
+        user_engine_user_id="usr_imp",
+        issuer="https://local.dev/issuer",
+        subject="imp-sub",
+        display_name="Importer",
+    )
+    report = import_space_tree(
+        tree,
+        tenant_id="tenant:a",
+        member=member,
+    )
+    assert report.pages_copied == 2
+    assert report.assets_copied == 1
+    assert report.space_created
+
+    space = Space.objects.get(tenant_id="tenant:a", slug="sample-space")
+    assert space.title == "Sample Space"
+    assert space.description == "From export"
+    assert SpaceMembership.objects.filter(space=space, member=member).exists()
+
+    notes = pageops.read_page("sample-space", "notes")
+    assert notes.exists
+    assert notes.title == "Notes"
+    assert "Body text" in notes.body
+    assert (pageops.space_dir("sample-space") / "assets" / "notes-visual.png").is_file()
+
+    rendered = load_space_page(space, "notes")
+    assert rendered.error is None
+    assert rendered.source == "content-plane"
+    assert "Body text" in rendered.html
+
+
+@pytest.mark.django_db
+def test_import_refuses_overwrite_without_replace(content_root, tmp_path):
+    tree = _write_sample_tree(tmp_path / "export")
+    import_space_tree(tree, tenant_id="tenant:a")
+    with pytest.raises(pageops.PageOpsError):
+        import_space_tree(tree, tenant_id="tenant:a", replace=False)
+    report = import_space_tree(tree, tenant_id="tenant:a", replace=True)
+    assert report.space_updated or report.space_created
+    assert report.pages_copied == 2
+
+
+@pytest.mark.django_db
+def test_import_content_tree_command(content_root, tmp_path, capsys):
+    tree = _write_sample_tree(tmp_path / "export")
+    call_command("import_content_tree", str(tree), "--tenant", "tenant:cmd", "--dry-run")
+    out = capsys.readouterr().out
+    assert "Dry run" in out
+    assert not Space.objects.filter(slug="sample-space").exists()
+
+    call_command(
+        "import_content_tree",
+        str(tree),
+        "--tenant",
+        "tenant:cmd",
+        "--slug",
+        "cmd-space",
+    )
+    space = Space.objects.get(slug="cmd-space", tenant_id="tenant:cmd")
+    assert space.title == "Sample Space"
+    assert pageops.read_page("cmd-space", "index").exists

workplans/CSOC-WP-0001-bubble-io-exit-assessment.md

@@ -4,7 +4,7 @@ type: workplan
 title: "bubble.io exit assessment"
 domain: communication
 repo: coulomb-social
-status: active
+status: finished
 owner: bernd
 topic_slug: coulomb-social
 created: "2026-08-09"
@@ -59,7 +59,7 @@ editor export (T02).
 
 ```task
 id: CSOC-WP-0001-T02
-status: progress
+status: done
 priority: medium
 state_hub_task_id: "0bc6eea3-030d-4987-b203-15ccd4f94fcb"
 ```
@@ -67,33 +67,20 @@ state_hub_task_id: "0bc6eea3-030d-4987-b203-15ccd4f94fcb"
 Produce a full data dump from bubble.io and map it to:
 
 1. tenant-keyed application tables (members, space metadata, …), and  
-2. **space content as markdown trees** bound to Forgejo repositories  
-   (see CSOC-WP-0004 content model).
+2. **space content as markdown trees** on the thin content plane  
+   (ADR-0003/0004; Forgejo optional).
 
-Document mapping and export under `docs/migration/`. Depends on CSOC-WP-0004
-content layout decisions for lossless mapping of pages/artefacts.
+Document mapping and export under `docs/migration/`.
 
-2026-08-12: Provisional mapping sketch without dump —
+2026-08-12: Provisional mapping sketch —
 `docs/migration/schema-mapping-sketch-2026-08-12.md`.
 
-2026-08-13: **Auth path for automated read access** without secrets in chat:
-`docs/migration/bubble-auth-and-export.md` + `scripts/export-bubble-session.sh`.
-Operator captures Playwright session once (`capture-auth-state.sh`); agents use
-`storage-state.json` under `~/.config/coulomb-social/auth/`. Map target is
-**thin dir+git** (ADR-0003/0004), not Forgejo-only.
-
-2026-08-13: **Session export run** (probe `vw_pages`, 147 network captures).
-Research content is primarily **`custom.article`** (nomer/abstractor/content/
-logo + space link), not empty public meta. Field map (redacted):
-`docs/migration/field-map-from-session-2026-08-13.md`.
-
-2026-08-13: **Corpus + sample tree** (local only, not git):
-`~/.config/coulomb-social/auth/exports/corpus-202608131015057/` —
-71 spaces, 109 articles; sample ADR-0004 tree `trees/reichelag/` with 30
-pages + 29 visuals (~1.6 MB). Progress note:
-`docs/migration/export-progress-2026-08-13.md`. Script:
-`scripts/export-bubble-corpus.mjs`. **T02** largely unblocked for mapping;
-**T04** next when PageOps thin-git landing pad exists (CSOC-WP-0006).
+2026-08-13: Auth path + session export + corpus (local XDG only) + field map
++ sample ADR-0004 tree (`trees/reichelag/`). See
+`docs/migration/export-progress-2026-08-13.md`,
+`docs/migration/field-map-from-session-2026-08-13.md`,
+`docs/migration/bubble-auth-and-export.md`. Residual: deeper crawl for
+thin spaces; multi-space bulk trees.
 
 ## Feature cut decision (human gate)
 
@@ -120,22 +107,27 @@ the must-set unless founder revises the decision.
 
 ```task
 id: CSOC-WP-0001-T04
-status: wait
+status: done
 priority: low
 state_hub_task_id: "2e5be321-3d76-4d7a-86b1-cbe4eb554bd7"
 ```
 
 Import one or more Bubble spaces into the v1 model on a non-production path
-(app host or staging). Record lossiness and residuals. **Blocked until** T02
-has a real export (CSOC-WP-0004 persistence is ready). Explicitly **not** a
-prerequisite for building product UX on the empty/new content store.
+(local CONTENT_ROOT / later app host). Record lossiness and residuals.
+
+2026-08-13: **Single-space rehearsal** of local `trees/reichelag/` via
+`manage.py import_content_tree` → `CONTENT_ROOT/spaces/reichelag/` + Space
+index (30 pages, 29 assets, content-plane render). Lossiness + residuals:
+`docs/migration/rehearsal-reichelag-2026-08-13.md`. Code:
+`coulomb_social/apps/spaces/import_tree.py`. Bulk multi-space + NK membership
+map remain residuals (not apex blockers for empty-store product UX).
 
 ## Acceptance
 
-- Inventory and mapping docs are committed and reviewable.
-- Feature-cut decision is recorded with owner and date.
-- At least one migration rehearsal has been run and residual gaps listed.
-- Apex Bubble retirement is only planned after rehearsal + product parity gate.
+- [x] Inventory and mapping docs are committed and reviewable.
+- [x] Feature-cut decision is recorded with owner and date.
+- [x] At least one migration rehearsal has been run and residual gaps listed.
+- [ ] Apex Bubble retirement is only planned after rehearsal + product parity gate.
 
 ## Notes