tegwick
789e69902d
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 7m5s
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a17f16-d99c-7ea3-95f0-8212897fba6b
447 lines
16 KiB
JavaScript
Executable file
447 lines
16 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
|
/**
|
|
* Build a Bubble article/space corpus from a Playwright session and emit
|
|
* one ADR-0004 markdown tree. Output stays under XDG auth exports (not git).
|
|
*
|
|
* ./scripts/capture-auth-state.sh
|
|
* node scripts/export-bubble-corpus.mjs
|
|
*/
|
|
import { chromium } from "playwright";
|
|
import { assertSessionFresh, normalizeBubbleRef, mergeCapturedRecord,
|
|
selectTargetArticles, rankSpaceLinks, ingestCapturedBody as parseBody,
|
|
boundedCaptureBody } from "./bubble-corpus-utils.mjs";
|
|
import {
|
|
mkdirSync,
|
|
writeFileSync,
|
|
chmodSync,
|
|
existsSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
createWriteStream,
|
|
} from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { createHash } from "node:crypto";
|
|
import { join } from "node:path";
|
|
import https from "node:https";
|
|
import http from "node:http";
|
|
|
|
const STATE =
|
|
process.env.COULOMB_SOCIAL_STORAGE_STATE ||
|
|
join(homedir(), ".config", "coulomb-social", "auth", "storage-state.json");
|
|
const BASE = process.env.COULOMB_SOCIAL_URL || "https://coulomb.social";
|
|
const TARGET_LABEL = process.env.COULOMB_SOCIAL_TARGET_SPACE_LABEL || "";
|
|
const ALL_SPACES = process.env.COULOMB_SOCIAL_ALL_SPACES === "1";
|
|
const ORDER_SPACES_FILE = process.env.COULOMB_SOCIAL_ORDER_SPACES_FILE || "";
|
|
const orderSource = ORDER_SPACES_FILE ? readFileSync(ORDER_SPACES_FILE) : null;
|
|
const orderSpaces = orderSource ? JSON.parse(orderSource.toString("utf8")) : {};
|
|
const TARGET_ROUTE = process.env.COULOMB_SOCIAL_CAPTURE_ROUTE || "/vw_pages";
|
|
const SEED_NET =
|
|
process.env.COULOMB_SOCIAL_SEED_NETWORK ||
|
|
join(
|
|
homedir(),
|
|
".config",
|
|
"coulomb-social",
|
|
"auth",
|
|
"exports",
|
|
"20260813-083747",
|
|
"network",
|
|
);
|
|
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 15);
|
|
const OUT =
|
|
process.env.COULOMB_SOCIAL_CORPUS_DIR ||
|
|
join(homedir(), ".config", "coulomb-social", "auth", "exports", `corpus-${stamp}`);
|
|
|
|
const SPACE_CLICKS = (
|
|
process.env.COULOMB_SOCIAL_SPACE_NAMES ||
|
|
"DeepNews,RepositoryScoping,Sandbox,CoulombConfig,IntentRoyal,DecisionGuidesLibrary,NetKingdom,TinkerBase,OpsBridge,ReichelAG,SpaceCampaign,AiFeedEditor"
|
|
).split(",").map((s) => s.trim()).filter(Boolean);
|
|
|
|
function ensure(p, mode = 0o700) {
|
|
mkdirSync(p, { recursive: true, mode });
|
|
try {
|
|
chmodSync(p, mode);
|
|
} catch {
|
|
/* best effort */
|
|
}
|
|
}
|
|
function write(p, data) {
|
|
writeFileSync(p, data, { mode: 0o600 });
|
|
try {
|
|
chmodSync(p, 0o600);
|
|
} catch {
|
|
/* best effort */
|
|
}
|
|
}
|
|
function slugify(s) {
|
|
return (
|
|
String(s || "untitled")
|
|
.toLowerCase()
|
|
.normalize("NFKD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/[^a-z0-9]+/g, "-")
|
|
.replace(/^-+|-+$/g, "")
|
|
.slice(0, 80) || "untitled"
|
|
);
|
|
}
|
|
function normId(id) { return normalizeBubbleRef(id); }
|
|
|
|
function download(url, dest) {
|
|
if (!url || typeof url !== "string") return Promise.resolve(null);
|
|
let u = url;
|
|
if (u.startsWith("//")) u = "https:" + u;
|
|
if (u.startsWith("/")) u = BASE.replace(/\/$/, "") + u;
|
|
if (!u.startsWith("http")) return Promise.resolve(null);
|
|
return new Promise((resolve) => {
|
|
const lib = u.startsWith("https") ? https : http;
|
|
const file = createWriteStream(dest, { mode: 0o600 });
|
|
const req = lib.get(u, { headers: { "User-Agent": "coulomb-export/1.0" } }, (res) => {
|
|
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
file.close();
|
|
download(res.headers.location, dest).then(resolve);
|
|
return;
|
|
}
|
|
if (res.statusCode && res.statusCode >= 400) {
|
|
file.close();
|
|
resolve(null);
|
|
return;
|
|
}
|
|
res.pipe(file);
|
|
file.on("finish", () => {
|
|
file.close();
|
|
resolve(dest);
|
|
});
|
|
});
|
|
req.on("error", () => {
|
|
try {
|
|
file.close();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
resolve(null);
|
|
});
|
|
req.setTimeout(20000, () => req.destroy());
|
|
});
|
|
}
|
|
|
|
if (!existsSync(STATE)) {
|
|
console.error("No session. Run ./scripts/capture-auth-state.sh first.");
|
|
process.exit(1);
|
|
}
|
|
|
|
try {
|
|
assertSessionFresh(JSON.parse(readFileSync(STATE, "utf8")));
|
|
} catch {
|
|
console.error("Bubble session unavailable or expired. Run ./scripts/capture-auth-state.sh.");
|
|
process.exit(2);
|
|
}
|
|
|
|
const spaces = {};
|
|
const articles = {};
|
|
if (!ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET)) {
|
|
for (const name of readdirSync(SEED_NET)) {
|
|
if (!name.endsWith(".body.txt")) continue;
|
|
parseBody(readFileSync(join(SEED_NET, name), "utf8"), spaces, articles);
|
|
}
|
|
console.log("seeded", Object.keys(spaces).length, "spaces", Object.keys(articles).length, "articles");
|
|
}
|
|
|
|
ensure(OUT);
|
|
const captureAttempts = [];
|
|
let responseTimeouts = 0;
|
|
let responseFailures = 0;
|
|
function checkpoint(status = "capturing") {
|
|
write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
|
|
write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));
|
|
write(join(OUT, "capture-state.json"), JSON.stringify({
|
|
status, all_visible_spaces_requested: ALL_SPACES,
|
|
response_timeouts: responseTimeouts, response_failures: responseFailures,
|
|
capture_attempts: captureAttempts, space_count: Object.keys(spaces).length,
|
|
article_count: Object.keys(articles).length, live_source_completeness_verified: false,
|
|
}, null, 2));
|
|
}
|
|
const browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({ storageState: STATE });
|
|
const page = await context.newPage();
|
|
const pendingResponses = [];
|
|
page.on("response", (response) => {
|
|
pendingResponses.push((async () => {
|
|
try {
|
|
if (!response.url().includes("elasticsearch")) return;
|
|
const result = await boundedCaptureBody(response);
|
|
if (result.status === "timeout") responseTimeouts++;
|
|
if (result.status === "failed") responseFailures++;
|
|
const text = result.text;
|
|
if (text && text.length < 5_000_000) {
|
|
parseBody(text, spaces, articles);
|
|
checkpoint();
|
|
}
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
})());
|
|
});
|
|
|
|
async function loadSpacesList() {
|
|
await page.goto(`${BASE}/vw_spaces`, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
await page.waitForTimeout(3500);
|
|
for (let i = 0; i < 6; i++) {
|
|
await page.mouse.wheel(0, 2000);
|
|
await page.waitForTimeout(600);
|
|
}
|
|
}
|
|
|
|
let targetRootId = "";
|
|
let targetCaptureStep = "space_list";
|
|
if (TARGET_LABEL) {
|
|
try {
|
|
await loadSpacesList();
|
|
targetCaptureStep = "space_link";
|
|
const link = page.getByRole("link", { name: TARGET_LABEL, exact: true })
|
|
.and(page.locator('a[href*="/pg_chunk/"]'));
|
|
if (await link.count() !== 1) throw new Error("target_space_link_not_unique");
|
|
const href = await link.getAttribute("href");
|
|
if (!href || !new URL(href, BASE).pathname.startsWith("/pg_chunk/")) {
|
|
throw new Error("target_space_root_not_found");
|
|
}
|
|
targetRootId = new URL(href, BASE).pathname.split("/").pop();
|
|
targetCaptureStep = "protocol_route";
|
|
await page.goto(`${BASE}${TARGET_ROUTE}`, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
await page.waitForTimeout(4000);
|
|
targetCaptureStep = "filter_control";
|
|
const filter = page.locator("button").filter({
|
|
has: page.locator("svg text").filter({ hasText: /^\s*filter_list\s*$/ }),
|
|
});
|
|
await filter.first().click();
|
|
targetCaptureStep = "space_filter";
|
|
await page.waitForFunction(label =>
|
|
[...document.querySelectorAll("select option")]
|
|
.some(o => o.textContent.trim() === label), TARGET_LABEL, { timeout: 10000 });
|
|
const selects = page.locator("select");
|
|
let selected = false;
|
|
for (let i = 0; i < await selects.count(); i++) {
|
|
const matches = await selects.nth(i).evaluate((el, label) =>
|
|
[...el.options].some(o => o.textContent.trim() === label), TARGET_LABEL);
|
|
if (!matches) continue;
|
|
await selects.nth(i).selectOption({ label: TARGET_LABEL });
|
|
selected = true;
|
|
break;
|
|
}
|
|
if (!selected) throw new Error("target_space_filter_not_found");
|
|
targetCaptureStep = "search_control";
|
|
await page.getByRole("button", { name: "Search", exact: true }).click();
|
|
await page.waitForTimeout(5000);
|
|
for (let i = 0; i < 24; i++) {
|
|
await page.mouse.wheel(0, 1800);
|
|
await page.waitForTimeout(500);
|
|
}
|
|
} catch {
|
|
await browser.close();
|
|
console.error(`Target capture failed at ${targetCaptureStep}; no import tree published.`);
|
|
process.exit(3);
|
|
}
|
|
} else {
|
|
let targets = SPACE_CLICKS.map(label => ({ label }));
|
|
if (ALL_SPACES) {
|
|
await loadSpacesList();
|
|
await Promise.allSettled(pendingResponses);
|
|
const links = await page.locator('a[href*="/pg_chunk/"]').evaluateAll(elements =>
|
|
elements.map(el => ({ href: el.href, label: el.textContent })));
|
|
targets = rankSpaceLinks(links, { ...orderSpaces, ...spaces }, articles);
|
|
checkpoint();
|
|
if (!targets.length) {
|
|
await browser.close();
|
|
console.error("No space roots found; no capture published.");
|
|
process.exit(5);
|
|
}
|
|
}
|
|
for (const target of targets) {
|
|
const name = target.label;
|
|
let capturePhase = "space_list";
|
|
try {
|
|
await loadSpacesList();
|
|
// Bubble's open-in-new-tab anchor is hidden until hover. Select the visible
|
|
// space title within the card containing that exact source-root address.
|
|
const loc = target.rootId
|
|
? page.locator('.group-item:visible').filter({
|
|
has: page.locator(`a[href$="/pg_chunk/${target.rootId}"]`),
|
|
}).first().getByText(name, { exact: true }).filter({ visible: true }).first()
|
|
: page.getByText(name, { exact: true }).first();
|
|
if ((await loc.count()) !== 1) {
|
|
captureAttempts.push({ ...target, status: "root_not_visible" });
|
|
checkpoint();
|
|
continue;
|
|
}
|
|
capturePhase = "root_click";
|
|
await loc.click({ timeout: 5000 });
|
|
await page.waitForTimeout(2500);
|
|
capturePhase = "pages_navigation";
|
|
await page.goto(`${BASE}/vw_pages`, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
await page.waitForTimeout(3000);
|
|
capturePhase = "pages_scroll";
|
|
for (let i = 0; i < 4; i++) {
|
|
await page.mouse.wheel(0, 1800);
|
|
await page.waitForTimeout(400);
|
|
}
|
|
captureAttempts.push({ ...target, status: "visited" });
|
|
checkpoint();
|
|
console.log("visited", captureAttempts.length, "articles", Object.keys(articles).length);
|
|
} catch (e) {
|
|
captureAttempts.push({ ...target, status: "capture_step_failed", phase: capturePhase,
|
|
error_type: e.name });
|
|
checkpoint();
|
|
console.log("capture_step_failed", captureAttempts.length);
|
|
if (ALL_SPACES && captureAttempts.slice(-3).every(attempt =>
|
|
attempt.status === "capture_step_failed") && captureAttempts.length >= 3) {
|
|
console.log("stopped_after_repeated_failures");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
await Promise.allSettled(pendingResponses);
|
|
await browser.close();
|
|
|
|
const bySpace = {};
|
|
for (const a of Object.values(articles)) {
|
|
const sid = a.space_custom_space;
|
|
if (!sid) continue;
|
|
(bySpace[sid] ||= []).push(a);
|
|
}
|
|
|
|
function spaceLabel(sid) {
|
|
const sp = spaces[sid] || {};
|
|
const rawSlug = sp.Slug || "";
|
|
const arts = bySpace[sid] || [];
|
|
const nomers = arts.map((a) => a.nomer_text).filter(Boolean);
|
|
let title =
|
|
nomers.find((n) => n === "ReichelAG") ||
|
|
nomers.find((n) => !/^\d{6}-/.test(n) && !/^new /i.test(n) && n.length < 48) ||
|
|
rawSlug ||
|
|
sid.slice(-12);
|
|
return { slug: slugify(title), title, rawSlug };
|
|
}
|
|
|
|
const ranked = Object.entries(bySpace).sort((a, b) => b[1].length - a[1].length);
|
|
let sampleId, sampleArts;
|
|
if (TARGET_LABEL) {
|
|
try {
|
|
[sampleId, sampleArts] = selectTargetArticles(articles, targetRootId);
|
|
} catch {
|
|
console.error("Target protocols unavailable; no import tree published. Renew Bubble login.");
|
|
process.exit(4);
|
|
}
|
|
} else {
|
|
[sampleId, sampleArts] = ranked[0] || [null, []];
|
|
}
|
|
const lab = TARGET_LABEL ? { ...spaceLabel(sampleId), slug: slugify(TARGET_LABEL),
|
|
title: TARGET_LABEL } : spaceLabel(sampleId);
|
|
|
|
const index = {
|
|
exported_at: new Date().toISOString(),
|
|
all_visible_spaces_requested: ALL_SPACES,
|
|
ordering_inventory_sha256: orderSource ? createHash("sha256").update(orderSource).digest("hex") : null,
|
|
ordering_inventory_used_for_dates_only: Boolean(orderSource),
|
|
capture_attempts: captureAttempts,
|
|
live_source_completeness_verified: false,
|
|
target_requested: TARGET_LABEL || null,
|
|
source_route: TARGET_LABEL ? TARGET_ROUTE : null,
|
|
old_seed_used: !ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET),
|
|
space_count: Object.keys(spaces).length,
|
|
article_count: Object.keys(articles).length,
|
|
sample_tree: sampleId ? `trees/${lab.slug}` : null,
|
|
spaces: Object.entries(spaces).map(([id]) => {
|
|
const L = spaceLabel(id);
|
|
return {
|
|
id,
|
|
slug: L.slug,
|
|
bubble_slug: L.rawSlug,
|
|
title_guess: L.title,
|
|
source_created_at: spaces[id]["Created Date"] ?? null,
|
|
source_modified_at: spaces[id]["Modified Date"] ?? null,
|
|
article_count: (bySpace[id] || []).length,
|
|
};
|
|
}),
|
|
};
|
|
write(join(OUT, "index.json"), JSON.stringify(index, null, 2));
|
|
write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
|
|
write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));
|
|
|
|
const treeRoot = join(OUT, "trees", lab.slug);
|
|
ensure(join(treeRoot, "pages"));
|
|
ensure(join(treeRoot, "assets"));
|
|
const used = new Set();
|
|
let visuals = 0;
|
|
let targetVisual = "";
|
|
for (const a of sampleArts) {
|
|
let pageSlug = slugify(a.nomer_text || a._id);
|
|
let n = 2;
|
|
while (used.has(pageSlug)) pageSlug = `${slugify(a.nomer_text || a._id)}-${n++}`;
|
|
used.add(pageSlug);
|
|
let visualRel = "";
|
|
if (a.logo_image) {
|
|
const lower = String(a.logo_image).toLowerCase();
|
|
const ext = lower.includes(".png")
|
|
? "png"
|
|
: lower.includes(".jpg") || lower.includes(".jpeg")
|
|
? "jpg"
|
|
: lower.includes(".webp")
|
|
? "webp"
|
|
: "bin";
|
|
const assetName = `${pageSlug}-visual.${ext}`;
|
|
const dest = join(treeRoot, "assets", assetName);
|
|
const ok = await download(a.logo_image, dest);
|
|
if (ok && existsSync(dest)) {
|
|
visualRel = `assets/${assetName}`;
|
|
visuals++;
|
|
}
|
|
}
|
|
if (normId(a._id) === targetRootId) targetVisual = visualRel;
|
|
const md = `---
|
|
id: bubble:${a._id}
|
|
title: ${JSON.stringify(a.nomer_text || "")}
|
|
abstractor: ${JSON.stringify(a.abstractor_text || "")}
|
|
visual: ${JSON.stringify(visualRel)}
|
|
space: ${JSON.stringify(lab.slug)}
|
|
bubble_type: article
|
|
source_created_at: ${JSON.stringify(a["Created Date"] ?? null)}
|
|
source_modified_at: ${JSON.stringify(a["Modified Date"] ?? null)}
|
|
archived: ${Boolean(a.archived_boolean)}
|
|
parent: ${JSON.stringify(a.context_custom_article ? `bubble:${normId(String(a.context_custom_article))}` : "")}
|
|
---
|
|
|
|
${a.content_text || ""}
|
|
`;
|
|
write(join(treeRoot, "pages", `${pageSlug}.md`), md);
|
|
}
|
|
write(
|
|
join(treeRoot, "pages", "index.md"),
|
|
`---
|
|
id: bubble-space:${sampleId}
|
|
title: ${JSON.stringify(lab.title)}
|
|
abstractor: ${JSON.stringify(TARGET_LABEL
|
|
? (articles[targetRootId]?.abstractor_text || "")
|
|
: `Imported sample (${sampleArts.length} articles)`)}
|
|
visual: ${JSON.stringify(targetVisual)}
|
|
space: ${JSON.stringify(lab.slug)}
|
|
bubble_type: space
|
|
source_created_at: ${JSON.stringify(spaces[sampleId]?.["Created Date"] ?? null)}
|
|
source_modified_at: ${JSON.stringify(spaces[sampleId]?.["Modified Date"] ?? null)}
|
|
---
|
|
|
|
${TARGET_LABEL ? (articles[targetRootId]?.content_text || `# ${lab.title}`) : `# ${lab.title}`}
|
|
`,
|
|
);
|
|
write(
|
|
join(OUT, "SUMMARY.md"),
|
|
`# Corpus ${stamp}
|
|
|
|
- spaces: ${Object.keys(spaces).length}
|
|
- articles: ${Object.keys(articles).length}
|
|
- sample: trees/${lab.slug}/ (${sampleArts.length} pages, ${visuals} visuals)
|
|
|
|
Do not commit — may contain private content.
|
|
`,
|
|
);
|
|
console.log("OUT", OUT);
|
|
console.log("tree", treeRoot, "pages", sampleArts.length, "visuals", visuals);
|
|
checkpoint("completed_capture_pass");
|