coulomb-social/scripts/export-bubble-corpus.mjs

#!/usr/bin/env node
/**
 * Build a Bubble article/space corpus from a Playwright session and emit
 * one ADR-0004 markdown tree. Output stays under XDG auth exports (not git).
 *
 *   ./scripts/capture-auth-state.sh
 *   node scripts/export-bubble-corpus.mjs
 */
import { chromium } from "playwright";
import { assertSessionFresh, normalizeBubbleRef, mergeCapturedRecord,
  selectTargetArticles } from "./bubble-corpus-utils.mjs";
import {
  mkdirSync,
  writeFileSync,
  chmodSync,
  existsSync,
  readFileSync,
  readdirSync,
  createWriteStream,
} from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import https from "node:https";
import http from "node:http";

const STATE =
  process.env.COULOMB_SOCIAL_STORAGE_STATE ||
  join(homedir(), ".config", "coulomb-social", "auth", "storage-state.json");
const BASE = process.env.COULOMB_SOCIAL_URL || "https://coulomb.social";
const TARGET_LABEL = process.env.COULOMB_SOCIAL_TARGET_SPACE_LABEL || "";
const TARGET_ROUTE = process.env.COULOMB_SOCIAL_CAPTURE_ROUTE || "/vw_pages";
const SEED_NET =
  process.env.COULOMB_SOCIAL_SEED_NETWORK ||
  join(
    homedir(),
    ".config",
    "coulomb-social",
    "auth",
    "exports",
    "20260813-083747",
    "network",
  );
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 15);
const OUT =
  process.env.COULOMB_SOCIAL_CORPUS_DIR ||
  join(homedir(), ".config", "coulomb-social", "auth", "exports", `corpus-${stamp}`);

const SPACE_CLICKS = (
  process.env.COULOMB_SOCIAL_SPACE_NAMES ||
  "DeepNews,RepositoryScoping,Sandbox,CoulombConfig,IntentRoyal,DecisionGuidesLibrary,NetKingdom,TinkerBase,OpsBridge,ReichelAG,SpaceCampaign,AiFeedEditor"
).split(",").map((s) => s.trim()).filter(Boolean);

function ensure(p, mode = 0o700) {
  mkdirSync(p, { recursive: true, mode });
  try {
    chmodSync(p, mode);
  } catch {
    /* best effort */
  }
}
function write(p, data) {
  writeFileSync(p, data, { mode: 0o600 });
  try {
    chmodSync(p, 0o600);
  } catch {
    /* best effort */
  }
}
function slugify(s) {
  return (
    String(s || "untitled")
      .toLowerCase()
      .normalize("NFKD")
      .replace(/[\u0300-\u036f]/g, "")
      .replace(/[^a-z0-9]+/g, "-")
      .replace(/^-+|-+$/g, "")
      .slice(0, 80) || "untitled"
  );
}
function normId(id) { return normalizeBubbleRef(id); }

function ingest(obj, spaces, articles) {
  if (Array.isArray(obj)) {
    for (const x of obj) ingest(x, spaces, articles);
    return;
  }
  if (!obj || typeof obj !== "object") return;
  const t = obj._type;
  const src = obj._source && typeof obj._source === "object" ? obj._source : null;
  const id = obj._id || (src && src._id);
  if (t === "custom.space" && src && id) {
    spaces[normId(id)] = mergeCapturedRecord(spaces[normId(id)], src, id);
  }
  if (t === "custom.article" && src && id) {
    const a = mergeCapturedRecord(articles[normId(id)], src, id);
    if (a.space_custom_space) a.space_custom_space = normId(a.space_custom_space);
    articles[normId(id)] = a;
  }
  if (Array.isArray(obj.responses)) {
    for (const r of obj.responses) ingest(r, spaces, articles);
  }
  if (obj.hits && typeof obj.hits === "object") {
    for (const h of obj.hits.hits || []) ingest(h, spaces, articles);
  }
  for (const v of Object.values(obj)) {
    if (v && typeof v === "object") ingest(v, spaces, articles);
  }
}
function parseBody(text, spaces, articles) {
  try {
    ingest(JSON.parse(text), spaces, articles);
    return;
  } catch {
    /* try NDJSON */
  }
  for (const line of text.split("\n")) {
    try {
      ingest(JSON.parse(line), spaces, articles);
    } catch {
      /* ignore */
    }
  }
}
function download(url, dest) {
  if (!url || typeof url !== "string") return Promise.resolve(null);
  let u = url;
  if (u.startsWith("//")) u = "https:" + u;
  if (u.startsWith("/")) u = BASE.replace(/\/$/, "") + u;
  if (!u.startsWith("http")) return Promise.resolve(null);
  return new Promise((resolve) => {
    const lib = u.startsWith("https") ? https : http;
    const file = createWriteStream(dest, { mode: 0o600 });
    const req = lib.get(u, { headers: { "User-Agent": "coulomb-export/1.0" } }, (res) => {
      if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        file.close();
        download(res.headers.location, dest).then(resolve);
        return;
      }
      if (res.statusCode && res.statusCode >= 400) {
        file.close();
        resolve(null);
        return;
      }
      res.pipe(file);
      file.on("finish", () => {
        file.close();
        resolve(dest);
      });
    });
    req.on("error", () => {
      try {
        file.close();
      } catch {
        /* ignore */
      }
      resolve(null);
    });
  });
}

if (!existsSync(STATE)) {
  console.error("No session. Run ./scripts/capture-auth-state.sh first.");
  process.exit(1);
}

try {
  assertSessionFresh(JSON.parse(readFileSync(STATE, "utf8")));
} catch {
  console.error("Bubble session unavailable or expired. Run ./scripts/capture-auth-state.sh.");
  process.exit(2);
}

const spaces = {};
const articles = {};
if (!TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET)) {
  for (const name of readdirSync(SEED_NET)) {
    if (!name.endsWith(".body.txt")) continue;
    parseBody(readFileSync(join(SEED_NET, name), "utf8"), spaces, articles);
  }
  console.log("seeded", Object.keys(spaces).length, "spaces", Object.keys(articles).length, "articles");
}

ensure(OUT);
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({ storageState: STATE });
const page = await context.newPage();
const pendingResponses = [];
page.on("response", (response) => {
  pendingResponses.push((async () => {
  try {
    if (!response.url().includes("elasticsearch")) return;
    const text = await response.text();
    if (text && text.length < 5_000_000) parseBody(text, spaces, articles);
  } catch {
    /* ignore */
  }
  })());
});

async function loadSpacesList() {
  await page.goto(`${BASE}/vw_spaces`, { waitUntil: "domcontentloaded", timeout: 60000 });
  await page.waitForTimeout(3500);
  for (let i = 0; i < 6; i++) {
    await page.mouse.wheel(0, 2000);
    await page.waitForTimeout(600);
  }
}

let targetRootId = "";
let targetCaptureStep = "space_list";
if (TARGET_LABEL) {
  try {
    await loadSpacesList();
    targetCaptureStep = "space_link";
    const link = page.getByRole("link", { name: TARGET_LABEL, exact: true })
      .and(page.locator('a[href*="/pg_chunk/"]'));
    if (await link.count() !== 1) throw new Error("target_space_link_not_unique");
    const href = await link.getAttribute("href");
    if (!href || !new URL(href, BASE).pathname.startsWith("/pg_chunk/")) {
      throw new Error("target_space_root_not_found");
    }
    targetRootId = new URL(href, BASE).pathname.split("/").pop();
    targetCaptureStep = "protocol_route";
    await page.goto(`${BASE}${TARGET_ROUTE}`, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForTimeout(4000);
    targetCaptureStep = "filter_control";
    const filter = page.locator("button").filter({
      has: page.locator("svg text").filter({ hasText: /^\s*filter_list\s*$/ }),
    });
    await filter.first().click();
    targetCaptureStep = "space_filter";
    await page.waitForFunction(label =>
      [...document.querySelectorAll("select option")]
        .some(o => o.textContent.trim() === label), TARGET_LABEL, { timeout: 10000 });
    const selects = page.locator("select");
    let selected = false;
    for (let i = 0; i < await selects.count(); i++) {
      const matches = await selects.nth(i).evaluate((el, label) =>
        [...el.options].some(o => o.textContent.trim() === label), TARGET_LABEL);
      if (!matches) continue;
      await selects.nth(i).selectOption({ label: TARGET_LABEL });
      selected = true;
      break;
    }
    if (!selected) throw new Error("target_space_filter_not_found");
    targetCaptureStep = "search_control";
    await page.getByRole("button", { name: "Search", exact: true }).click();
    await page.waitForTimeout(5000);
    for (let i = 0; i < 24; i++) {
      await page.mouse.wheel(0, 1800);
      await page.waitForTimeout(500);
    }
  } catch {
    await browser.close();
    console.error(`Target capture failed at ${targetCaptureStep}; no import tree published.`);
    process.exit(3);
  }
} else {
for (const name of SPACE_CLICKS) {
  try {
    await loadSpacesList();
    const loc = page.getByText(name, { exact: true }).first();
    if ((await loc.count()) === 0) {
      console.log("skip", name);
      continue;
    }
    await loc.click({ timeout: 5000 });
    await page.waitForTimeout(2500);
    await page.goto(`${BASE}/vw_pages`, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForTimeout(3000);
    for (let i = 0; i < 4; i++) {
      await page.mouse.wheel(0, 1800);
      await page.waitForTimeout(400);
    }
    console.log("ok", name, "articles", Object.keys(articles).length);
  } catch (e) {
    console.log("fail", name, "capture_step_failed");
  }
}
}
await Promise.allSettled(pendingResponses);
await browser.close();

const bySpace = {};
for (const a of Object.values(articles)) {
  const sid = a.space_custom_space;
  if (!sid) continue;
  (bySpace[sid] ||= []).push(a);
}

function spaceLabel(sid) {
  const sp = spaces[sid] || {};
  const rawSlug = sp.Slug || "";
  const arts = bySpace[sid] || [];
  const nomers = arts.map((a) => a.nomer_text).filter(Boolean);
  let title =
    nomers.find((n) => n === "ReichelAG") ||
    nomers.find((n) => !/^\d{6}-/.test(n) && !/^new /i.test(n) && n.length < 48) ||
    rawSlug ||
    sid.slice(-12);
  return { slug: slugify(title), title, rawSlug };
}

const ranked = Object.entries(bySpace).sort((a, b) => b[1].length - a[1].length);
let sampleId, sampleArts;
if (TARGET_LABEL) {
  try {
    [sampleId, sampleArts] = selectTargetArticles(articles, targetRootId);
  } catch {
    console.error("Target protocols unavailable; no import tree published. Renew Bubble login.");
    process.exit(4);
  }
} else {
  [sampleId, sampleArts] = ranked[0] || [null, []];
}
const lab = TARGET_LABEL ? { ...spaceLabel(sampleId), slug: slugify(TARGET_LABEL),
  title: TARGET_LABEL } : spaceLabel(sampleId);

const index = {
  exported_at: new Date().toISOString(),
  target_requested: TARGET_LABEL || null,
  source_route: TARGET_LABEL ? TARGET_ROUTE : null,
  old_seed_used: !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET),
  space_count: Object.keys(spaces).length,
  article_count: Object.keys(articles).length,
  sample_tree: sampleId ? `trees/${lab.slug}` : null,
  spaces: Object.entries(spaces).map(([id]) => {
    const L = spaceLabel(id);
    return {
      id,
      slug: L.slug,
      bubble_slug: L.rawSlug,
      title_guess: L.title,
      article_count: (bySpace[id] || []).length,
    };
  }),
};
write(join(OUT, "index.json"), JSON.stringify(index, null, 2));
write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));

const treeRoot = join(OUT, "trees", lab.slug);
ensure(join(treeRoot, "pages"));
ensure(join(treeRoot, "assets"));
const used = new Set();
let visuals = 0;
let targetVisual = "";
for (const a of sampleArts) {
  let pageSlug = slugify(a.nomer_text || a._id);
  let n = 2;
  while (used.has(pageSlug)) pageSlug = `${slugify(a.nomer_text || a._id)}-${n++}`;
  used.add(pageSlug);
  let visualRel = "";
  if (a.logo_image) {
    const lower = String(a.logo_image).toLowerCase();
    const ext = lower.includes(".png")
      ? "png"
      : lower.includes(".jpg") || lower.includes(".jpeg")
        ? "jpg"
        : lower.includes(".webp")
          ? "webp"
          : "bin";
    const assetName = `${pageSlug}-visual.${ext}`;
    const dest = join(treeRoot, "assets", assetName);
    const ok = await download(a.logo_image, dest);
    if (ok && existsSync(dest)) {
      visualRel = `assets/${assetName}`;
      visuals++;
    }
  }
  if (normId(a._id) === targetRootId) targetVisual = visualRel;
  const md = `---
id: bubble:${a._id}
title: ${JSON.stringify(a.nomer_text || "")}
abstractor: ${JSON.stringify(a.abstractor_text || "")}
visual: ${JSON.stringify(visualRel)}
space: ${JSON.stringify(lab.slug)}
bubble_type: article
source_created_at: ${JSON.stringify(a["Created Date"] ?? null)}
source_modified_at: ${JSON.stringify(a["Modified Date"] ?? null)}
archived: ${Boolean(a.archived_boolean)}
parent: ${JSON.stringify(a.context_custom_article ? `bubble:${normId(String(a.context_custom_article))}` : "")}
---

${a.content_text || ""}
`;
  write(join(treeRoot, "pages", `${pageSlug}.md`), md);
}
write(
  join(treeRoot, "pages", "index.md"),
  `---
id: bubble-space:${sampleId}
title: ${JSON.stringify(lab.title)}
abstractor: ${JSON.stringify(TARGET_LABEL
  ? (articles[targetRootId]?.abstractor_text || "")
  : `Imported sample (${sampleArts.length} articles)`)}
visual: ${JSON.stringify(targetVisual)}
space: ${JSON.stringify(lab.slug)}
bubble_type: space
---

${TARGET_LABEL ? (articles[targetRootId]?.content_text || `# ${lab.title}`) : `# ${lab.title}`}
`,
);
write(
  join(OUT, "SUMMARY.md"),
  `# Corpus ${stamp}

- spaces: ${Object.keys(spaces).length}
- articles: ${Object.keys(articles).length}
- sample: trees/${lab.slug}/ (${sampleArts.length} pages, ${visuals} visuals)

Do not commit — may contain private content.
`,
);
console.log("OUT", OUT);
console.log("tree", treeRoot, "pages", sampleArts.length, "visuals", visuals);