tegwick
a0fd3cb7ea
Add corpus exporter (session + ES capture), record 71 spaces / 109 articles and local reichelag markdown tree with visuals. Redacted progress only in git.
334 lines
9.7 KiB
JavaScript
Executable file
334 lines
9.7 KiB
JavaScript
Executable file
#!/usr/bin/env node
|
|
/**
|
|
* Build a Bubble article/space corpus from a Playwright session and emit
|
|
* one ADR-0004 markdown tree. Output stays under XDG auth exports (not git).
|
|
*
|
|
* ./scripts/capture-auth-state.sh
|
|
* node scripts/export-bubble-corpus.mjs
|
|
*/
|
|
import { chromium } from "playwright";
|
|
import {
|
|
mkdirSync,
|
|
writeFileSync,
|
|
chmodSync,
|
|
existsSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
createWriteStream,
|
|
} from "node:fs";
|
|
import { homedir } from "node:os";
|
|
import { join } from "node:path";
|
|
import https from "node:https";
|
|
import http from "node:http";
|
|
|
|
const STATE =
|
|
process.env.COULOMB_SOCIAL_STORAGE_STATE ||
|
|
join(homedir(), ".config", "coulomb-social", "auth", "storage-state.json");
|
|
const BASE = process.env.COULOMB_SOCIAL_URL || "https://coulomb.social";
|
|
const SEED_NET =
|
|
process.env.COULOMB_SOCIAL_SEED_NETWORK ||
|
|
join(
|
|
homedir(),
|
|
".config",
|
|
"coulomb-social",
|
|
"auth",
|
|
"exports",
|
|
"20260813-083747",
|
|
"network",
|
|
);
|
|
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 15);
|
|
const OUT =
|
|
process.env.COULOMB_SOCIAL_CORPUS_DIR ||
|
|
join(homedir(), ".config", "coulomb-social", "auth", "exports", `corpus-${stamp}`);
|
|
|
|
const SPACE_CLICKS = (
|
|
process.env.COULOMB_SOCIAL_SPACE_NAMES ||
|
|
"DeepNews,RepositoryScoping,Sandbox,CoulombConfig,IntentRoyal,DecisionGuidesLibrary,NetKingdom,TinkerBase,OpsBridge,ReichelAG,SpaceCampaign,AiFeedEditor"
|
|
).split(",").map((s) => s.trim()).filter(Boolean);
|
|
|
|
function ensure(p, mode = 0o700) {
|
|
mkdirSync(p, { recursive: true, mode });
|
|
try {
|
|
chmodSync(p, mode);
|
|
} catch {
|
|
/* best effort */
|
|
}
|
|
}
|
|
function write(p, data) {
|
|
writeFileSync(p, data, { mode: 0o600 });
|
|
try {
|
|
chmodSync(p, 0o600);
|
|
} catch {
|
|
/* best effort */
|
|
}
|
|
}
|
|
function slugify(s) {
|
|
return (
|
|
String(s || "untitled")
|
|
.toLowerCase()
|
|
.normalize("NFKD")
|
|
.replace(/[\u0300-\u036f]/g, "")
|
|
.replace(/[^a-z0-9]+/g, "-")
|
|
.replace(/^-+|-+$/g, "")
|
|
.slice(0, 80) || "untitled"
|
|
);
|
|
}
|
|
function normId(id) {
|
|
if (!id || typeof id !== "string") return id;
|
|
const m = id.match(/LOOKUP__([0-9]+x[0-9]+)/);
|
|
return m ? m[1] : id;
|
|
}
|
|
function ingest(obj, spaces, articles) {
|
|
if (Array.isArray(obj)) {
|
|
for (const x of obj) ingest(x, spaces, articles);
|
|
return;
|
|
}
|
|
if (!obj || typeof obj !== "object") return;
|
|
const t = obj._type;
|
|
const src = obj._source && typeof obj._source === "object" ? obj._source : null;
|
|
const id = obj._id || (src && src._id);
|
|
if (t === "custom.space" && src && id) {
|
|
spaces[normId(id)] = { ...src, _id: normId(id) };
|
|
}
|
|
if (t === "custom.article" && src && id) {
|
|
const a = { ...src, _id: normId(id) };
|
|
if (a.space_custom_space) a.space_custom_space = normId(a.space_custom_space);
|
|
articles[normId(id)] = a;
|
|
}
|
|
if (Array.isArray(obj.responses)) {
|
|
for (const r of obj.responses) ingest(r, spaces, articles);
|
|
}
|
|
if (obj.hits && typeof obj.hits === "object") {
|
|
for (const h of obj.hits.hits || []) ingest(h, spaces, articles);
|
|
}
|
|
for (const v of Object.values(obj)) {
|
|
if (v && typeof v === "object") ingest(v, spaces, articles);
|
|
}
|
|
}
|
|
function parseBody(text, spaces, articles) {
|
|
try {
|
|
ingest(JSON.parse(text), spaces, articles);
|
|
return;
|
|
} catch {
|
|
/* try NDJSON */
|
|
}
|
|
for (const line of text.split("\n")) {
|
|
try {
|
|
ingest(JSON.parse(line), spaces, articles);
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
}
|
|
}
|
|
function download(url, dest) {
|
|
if (!url || typeof url !== "string") return Promise.resolve(null);
|
|
let u = url;
|
|
if (u.startsWith("//")) u = "https:" + u;
|
|
if (u.startsWith("/")) u = BASE.replace(/\/$/, "") + u;
|
|
if (!u.startsWith("http")) return Promise.resolve(null);
|
|
return new Promise((resolve) => {
|
|
const lib = u.startsWith("https") ? https : http;
|
|
const file = createWriteStream(dest, { mode: 0o600 });
|
|
const req = lib.get(u, { headers: { "User-Agent": "coulomb-export/1.0" } }, (res) => {
|
|
if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
|
|
file.close();
|
|
download(res.headers.location, dest).then(resolve);
|
|
return;
|
|
}
|
|
if (res.statusCode && res.statusCode >= 400) {
|
|
file.close();
|
|
resolve(null);
|
|
return;
|
|
}
|
|
res.pipe(file);
|
|
file.on("finish", () => {
|
|
file.close();
|
|
resolve(dest);
|
|
});
|
|
});
|
|
req.on("error", () => {
|
|
try {
|
|
file.close();
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
resolve(null);
|
|
});
|
|
});
|
|
}
|
|
|
|
if (!existsSync(STATE)) {
|
|
console.error("No session. Run ./scripts/capture-auth-state.sh first.");
|
|
process.exit(1);
|
|
}
|
|
|
|
const spaces = {};
|
|
const articles = {};
|
|
if (existsSync(SEED_NET)) {
|
|
for (const name of readdirSync(SEED_NET)) {
|
|
if (!name.endsWith(".body.txt")) continue;
|
|
parseBody(readFileSync(join(SEED_NET, name), "utf8"), spaces, articles);
|
|
}
|
|
console.log("seeded", Object.keys(spaces).length, "spaces", Object.keys(articles).length, "articles");
|
|
}
|
|
|
|
ensure(OUT);
|
|
const browser = await chromium.launch({ headless: true });
|
|
const context = await browser.newContext({ storageState: STATE });
|
|
const page = await context.newPage();
|
|
page.on("response", async (response) => {
|
|
try {
|
|
if (!response.url().includes("elasticsearch")) return;
|
|
const text = await response.text();
|
|
if (text && text.length < 5_000_000) parseBody(text, spaces, articles);
|
|
} catch {
|
|
/* ignore */
|
|
}
|
|
});
|
|
|
|
async function loadSpacesList() {
|
|
await page.goto(`${BASE}/vw_spaces`, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
await page.waitForTimeout(3500);
|
|
for (let i = 0; i < 6; i++) {
|
|
await page.mouse.wheel(0, 2000);
|
|
await page.waitForTimeout(600);
|
|
}
|
|
}
|
|
|
|
for (const name of SPACE_CLICKS) {
|
|
try {
|
|
await loadSpacesList();
|
|
const loc = page.getByText(name, { exact: true }).first();
|
|
if ((await loc.count()) === 0) {
|
|
console.log("skip", name);
|
|
continue;
|
|
}
|
|
await loc.click({ timeout: 5000 });
|
|
await page.waitForTimeout(2500);
|
|
await page.goto(`${BASE}/vw_pages`, { waitUntil: "domcontentloaded", timeout: 60000 });
|
|
await page.waitForTimeout(3000);
|
|
for (let i = 0; i < 4; i++) {
|
|
await page.mouse.wheel(0, 1800);
|
|
await page.waitForTimeout(400);
|
|
}
|
|
console.log("ok", name, "articles", Object.keys(articles).length);
|
|
} catch (e) {
|
|
console.log("fail", name, String(e.message || e).slice(0, 60));
|
|
}
|
|
}
|
|
await browser.close();
|
|
|
|
const bySpace = {};
|
|
for (const a of Object.values(articles)) {
|
|
const sid = a.space_custom_space;
|
|
if (!sid) continue;
|
|
(bySpace[sid] ||= []).push(a);
|
|
}
|
|
|
|
function spaceLabel(sid) {
|
|
const sp = spaces[sid] || {};
|
|
const rawSlug = sp.Slug || "";
|
|
const arts = bySpace[sid] || [];
|
|
const nomers = arts.map((a) => a.nomer_text).filter(Boolean);
|
|
let title =
|
|
nomers.find((n) => n === "ReichelAG") ||
|
|
nomers.find((n) => !/^\d{6}-/.test(n) && !/^new /i.test(n) && n.length < 48) ||
|
|
rawSlug ||
|
|
sid.slice(-12);
|
|
return { slug: slugify(title), title, rawSlug };
|
|
}
|
|
|
|
const ranked = Object.entries(bySpace).sort((a, b) => b[1].length - a[1].length);
|
|
const [sampleId, sampleArts] = ranked[0] || [null, []];
|
|
const lab = spaceLabel(sampleId);
|
|
|
|
const index = {
|
|
exported_at: new Date().toISOString(),
|
|
space_count: Object.keys(spaces).length,
|
|
article_count: Object.keys(articles).length,
|
|
sample_tree: sampleId ? `trees/${lab.slug}` : null,
|
|
spaces: Object.entries(spaces).map(([id]) => {
|
|
const L = spaceLabel(id);
|
|
return {
|
|
id,
|
|
slug: L.slug,
|
|
bubble_slug: L.rawSlug,
|
|
title_guess: L.title,
|
|
article_count: (bySpace[id] || []).length,
|
|
};
|
|
}),
|
|
};
|
|
write(join(OUT, "index.json"), JSON.stringify(index, null, 2));
|
|
write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
|
|
write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));
|
|
|
|
const treeRoot = join(OUT, "trees", lab.slug);
|
|
ensure(join(treeRoot, "pages"));
|
|
ensure(join(treeRoot, "assets"));
|
|
const used = new Set();
|
|
let visuals = 0;
|
|
for (const a of sampleArts) {
|
|
let pageSlug = slugify(a.nomer_text || a._id);
|
|
let n = 2;
|
|
while (used.has(pageSlug)) pageSlug = `${slugify(a.nomer_text || a._id)}-${n++}`;
|
|
used.add(pageSlug);
|
|
let visualRel = "";
|
|
if (a.logo_image) {
|
|
const lower = String(a.logo_image).toLowerCase();
|
|
const ext = lower.includes(".png")
|
|
? "png"
|
|
: lower.includes(".jpg") || lower.includes(".jpeg")
|
|
? "jpg"
|
|
: lower.includes(".webp")
|
|
? "webp"
|
|
: "bin";
|
|
const assetName = `${pageSlug}-visual.${ext}`;
|
|
const dest = join(treeRoot, "assets", assetName);
|
|
const ok = await download(a.logo_image, dest);
|
|
if (ok && existsSync(dest)) {
|
|
visualRel = `assets/${assetName}`;
|
|
visuals++;
|
|
}
|
|
}
|
|
const md = `---
|
|
id: bubble:${a._id}
|
|
title: ${JSON.stringify(a.nomer_text || "")}
|
|
abstractor: ${JSON.stringify(a.abstractor_text || "")}
|
|
visual: ${JSON.stringify(visualRel)}
|
|
space: ${JSON.stringify(lab.slug)}
|
|
bubble_type: article
|
|
archived: ${Boolean(a.archived_boolean)}
|
|
parent: ${JSON.stringify(a.context_custom_article ? `bubble:${normId(String(a.context_custom_article))}` : "")}
|
|
---
|
|
|
|
${a.content_text || ""}
|
|
`;
|
|
write(join(treeRoot, "pages", `${pageSlug}.md`), md);
|
|
}
|
|
write(
|
|
join(treeRoot, "pages", "index.md"),
|
|
`---
|
|
id: bubble-space:${sampleId}
|
|
title: ${JSON.stringify(lab.title)}
|
|
abstractor: ${JSON.stringify(`Imported sample (${sampleArts.length} articles)`)}
|
|
visual: ""
|
|
space: ${JSON.stringify(lab.slug)}
|
|
bubble_type: space
|
|
---
|
|
|
|
# ${lab.title}
|
|
`,
|
|
);
|
|
write(
|
|
join(OUT, "SUMMARY.md"),
|
|
`# Corpus ${stamp}
|
|
|
|
- spaces: ${Object.keys(spaces).length}
|
|
- articles: ${Object.keys(articles).length}
|
|
- sample: trees/${lab.slug}/ (${sampleArts.length} pages, ${visuals} visuals)
|
|
|
|
Do not commit — may contain private content.
|
|
`,
|
|
);
|
|
console.log("OUT", OUT);
|
|
console.log("tree", treeRoot, "pages", sampleArts.length, "visuals", visuals);
|