#!/usr/bin/env node
/**
 * Build a Bubble article/space corpus from a Playwright session and emit
 * one ADR-0004 markdown tree. Output stays under XDG auth exports (not git).
 *
 *   ./scripts/capture-auth-state.sh
 *   node scripts/export-bubble-corpus.mjs
 */
import { chromium } from "playwright";
import { assertSessionFresh, normalizeBubbleRef, mergeCapturedRecord,
  selectTargetArticles, rankSpaceLinks, ingestCapturedBody as parseBody,
  boundedCaptureBody } from "./bubble-corpus-utils.mjs";
import {
  mkdirSync,
  writeFileSync,
  chmodSync,
  existsSync,
  readFileSync,
  readdirSync,
  createWriteStream,
} from "node:fs";
import { homedir } from "node:os";
import { createHash } from "node:crypto";
import { join } from "node:path";
import https from "node:https";
import http from "node:http";

const STATE =
  process.env.COULOMB_SOCIAL_STORAGE_STATE ||
  join(homedir(), ".config", "coulomb-social", "auth", "storage-state.json");
const BASE = process.env.COULOMB_SOCIAL_URL || "https://coulomb.social";
const TARGET_LABEL = process.env.COULOMB_SOCIAL_TARGET_SPACE_LABEL || "";
const ALL_SPACES = process.env.COULOMB_SOCIAL_ALL_SPACES === "1";
const ORDER_SPACES_FILE = process.env.COULOMB_SOCIAL_ORDER_SPACES_FILE || "";
const orderSource = ORDER_SPACES_FILE ? readFileSync(ORDER_SPACES_FILE) : null;
const orderSpaces = orderSource ? JSON.parse(orderSource.toString("utf8")) : {};
const TARGET_ROUTE = process.env.COULOMB_SOCIAL_CAPTURE_ROUTE || "/vw_pages";
const SEED_NET =
  process.env.COULOMB_SOCIAL_SEED_NETWORK ||
  join(
    homedir(),
    ".config",
    "coulomb-social",
    "auth",
    "exports",
    "20260813-083747",
    "network",
  );
const stamp = new Date().toISOString().replace(/[-:TZ.]/g, "").slice(0, 15);
const OUT =
  process.env.COULOMB_SOCIAL_CORPUS_DIR ||
  join(homedir(), ".config", "coulomb-social", "auth", "exports", `corpus-${stamp}`);

const SPACE_CLICKS = (
  process.env.COULOMB_SOCIAL_SPACE_NAMES ||
  "DeepNews,RepositoryScoping,Sandbox,CoulombConfig,IntentRoyal,DecisionGuidesLibrary,NetKingdom,TinkerBase,OpsBridge,ReichelAG,SpaceCampaign,AiFeedEditor"
).split(",").map((s) => s.trim()).filter(Boolean);

function ensure(p, mode = 0o700) {
  mkdirSync(p, { recursive: true, mode });
  try {
    chmodSync(p, mode);
  } catch {
    /* best effort */
  }
}
function write(p, data) {
  writeFileSync(p, data, { mode: 0o600 });
  try {
    chmodSync(p, 0o600);
  } catch {
    /* best effort */
  }
}
function slugify(s) {
  return (
    String(s || "untitled")
      .toLowerCase()
      .normalize("NFKD")
      .replace(/[\u0300-\u036f]/g, "")
      .replace(/[^a-z0-9]+/g, "-")
      .replace(/^-+|-+$/g, "")
      .slice(0, 80) || "untitled"
  );
}
function normId(id) { return normalizeBubbleRef(id); }

function download(url, dest) {
  if (!url || typeof url !== "string") return Promise.resolve(null);
  let u = url;
  if (u.startsWith("//")) u = "https:" + u;
  if (u.startsWith("/")) u = BASE.replace(/\/$/, "") + u;
  if (!u.startsWith("http")) return Promise.resolve(null);
  return new Promise((resolve) => {
    const lib = u.startsWith("https") ? https : http;
    const file = createWriteStream(dest, { mode: 0o600 });
    const req = lib.get(u, { headers: { "User-Agent": "coulomb-export/1.0" } }, (res) => {
      if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
        file.close();
        download(res.headers.location, dest).then(resolve);
        return;
      }
      if (res.statusCode && res.statusCode >= 400) {
        file.close();
        resolve(null);
        return;
      }
      res.pipe(file);
      file.on("finish", () => {
        file.close();
        resolve(dest);
      });
    });
    req.on("error", () => {
      try {
        file.close();
      } catch {
        /* ignore */
      }
      resolve(null);
    });
    req.setTimeout(20000, () => req.destroy());
  });
}

if (!existsSync(STATE)) {
  console.error("No session. Run ./scripts/capture-auth-state.sh first.");
  process.exit(1);
}

try {
  assertSessionFresh(JSON.parse(readFileSync(STATE, "utf8")));
} catch {
  console.error("Bubble session unavailable or expired. Run ./scripts/capture-auth-state.sh.");
  process.exit(2);
}

const spaces = {};
const articles = {};
if (!ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET)) {
  for (const name of readdirSync(SEED_NET)) {
    if (!name.endsWith(".body.txt")) continue;
    parseBody(readFileSync(join(SEED_NET, name), "utf8"), spaces, articles);
  }
  console.log("seeded", Object.keys(spaces).length, "spaces", Object.keys(articles).length, "articles");
}

ensure(OUT);
const captureAttempts = [];
let responseTimeouts = 0;
let responseFailures = 0;
function checkpoint(status = "capturing") {
  write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
  write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));
  write(join(OUT, "capture-state.json"), JSON.stringify({
    status, all_visible_spaces_requested: ALL_SPACES,
    response_timeouts: responseTimeouts, response_failures: responseFailures,
    capture_attempts: captureAttempts, space_count: Object.keys(spaces).length,
    article_count: Object.keys(articles).length, live_source_completeness_verified: false,
  }, null, 2));
}
const browser = await chromium.launch({ headless: true });
const context = await browser.newContext({ storageState: STATE });
const page = await context.newPage();
const pendingResponses = [];
page.on("response", (response) => {
  pendingResponses.push((async () => {
  try {
    if (!response.url().includes("elasticsearch")) return;
    const result = await boundedCaptureBody(response);
    if (result.status === "timeout") responseTimeouts++;
    if (result.status === "failed") responseFailures++;
    const text = result.text;
    if (text && text.length < 5_000_000) {
      parseBody(text, spaces, articles);
      checkpoint();
    }
  } catch {
    /* ignore */
  }
  })());
});

async function loadSpacesList() {
  await page.goto(`${BASE}/vw_spaces`, { waitUntil: "domcontentloaded", timeout: 60000 });
  await page.waitForTimeout(3500);
  for (let i = 0; i < 6; i++) {
    await page.mouse.wheel(0, 2000);
    await page.waitForTimeout(600);
  }
}

let targetRootId = "";
let targetCaptureStep = "space_list";
if (TARGET_LABEL) {
  try {
    await loadSpacesList();
    targetCaptureStep = "space_link";
    const link = page.getByRole("link", { name: TARGET_LABEL, exact: true })
      .and(page.locator('a[href*="/pg_chunk/"]'));
    if (await link.count() !== 1) throw new Error("target_space_link_not_unique");
    const href = await link.getAttribute("href");
    if (!href || !new URL(href, BASE).pathname.startsWith("/pg_chunk/")) {
      throw new Error("target_space_root_not_found");
    }
    targetRootId = new URL(href, BASE).pathname.split("/").pop();
    targetCaptureStep = "protocol_route";
    await page.goto(`${BASE}${TARGET_ROUTE}`, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForTimeout(4000);
    targetCaptureStep = "filter_control";
    const filter = page.locator("button").filter({
      has: page.locator("svg text").filter({ hasText: /^\s*filter_list\s*$/ }),
    });
    await filter.first().click();
    targetCaptureStep = "space_filter";
    await page.waitForFunction(label =>
      [...document.querySelectorAll("select option")]
        .some(o => o.textContent.trim() === label), TARGET_LABEL, { timeout: 10000 });
    const selects = page.locator("select");
    let selected = false;
    for (let i = 0; i < await selects.count(); i++) {
      const matches = await selects.nth(i).evaluate((el, label) =>
        [...el.options].some(o => o.textContent.trim() === label), TARGET_LABEL);
      if (!matches) continue;
      await selects.nth(i).selectOption({ label: TARGET_LABEL });
      selected = true;
      break;
    }
    if (!selected) throw new Error("target_space_filter_not_found");
    targetCaptureStep = "search_control";
    await page.getByRole("button", { name: "Search", exact: true }).click();
    await page.waitForTimeout(5000);
    for (let i = 0; i < 24; i++) {
      await page.mouse.wheel(0, 1800);
      await page.waitForTimeout(500);
    }
  } catch {
    await browser.close();
    console.error(`Target capture failed at ${targetCaptureStep}; no import tree published.`);
    process.exit(3);
  }
} else {
let targets = SPACE_CLICKS.map(label => ({ label }));
if (ALL_SPACES) {
  await loadSpacesList();
  await Promise.allSettled(pendingResponses);
  const links = await page.locator('a[href*="/pg_chunk/"]').evaluateAll(elements =>
    elements.map(el => ({ href: el.href, label: el.textContent })));
  targets = rankSpaceLinks(links, { ...orderSpaces, ...spaces }, articles);
  checkpoint();
  if (!targets.length) {
    await browser.close();
    console.error("No space roots found; no capture published.");
    process.exit(5);
  }
}
for (const target of targets) {
  const name = target.label;
  let capturePhase = "space_list";
  try {
    await loadSpacesList();
    // Bubble's open-in-new-tab anchor is hidden until hover. Select the visible
    // space title within the card containing that exact source-root address.
    const loc = target.rootId
      ? page.locator('.group-item:visible').filter({
        has: page.locator(`a[href$="/pg_chunk/${target.rootId}"]`),
      }).first().getByText(name, { exact: true }).filter({ visible: true }).first()
      : page.getByText(name, { exact: true }).first();
    if ((await loc.count()) !== 1) {
      captureAttempts.push({ ...target, status: "root_not_visible" });
      checkpoint();
      continue;
    }
    capturePhase = "root_click";
    await loc.click({ timeout: 5000 });
    await page.waitForTimeout(2500);
    capturePhase = "pages_navigation";
    await page.goto(`${BASE}/vw_pages`, { waitUntil: "domcontentloaded", timeout: 60000 });
    await page.waitForTimeout(3000);
    capturePhase = "pages_scroll";
    for (let i = 0; i < 4; i++) {
      await page.mouse.wheel(0, 1800);
      await page.waitForTimeout(400);
    }
    captureAttempts.push({ ...target, status: "visited" });
    checkpoint();
    console.log("visited", captureAttempts.length, "articles", Object.keys(articles).length);
  } catch (e) {
    captureAttempts.push({ ...target, status: "capture_step_failed", phase: capturePhase,
      error_type: e.name });
    checkpoint();
    console.log("capture_step_failed", captureAttempts.length);
    if (ALL_SPACES && captureAttempts.slice(-3).every(attempt =>
      attempt.status === "capture_step_failed") && captureAttempts.length >= 3) {
      console.log("stopped_after_repeated_failures");
      break;
    }
  }
}
}
await Promise.allSettled(pendingResponses);
await browser.close();

const bySpace = {};
for (const a of Object.values(articles)) {
  const sid = a.space_custom_space;
  if (!sid) continue;
  (bySpace[sid] ||= []).push(a);
}

function spaceLabel(sid) {
  const sp = spaces[sid] || {};
  const rawSlug = sp.Slug || "";
  const arts = bySpace[sid] || [];
  const nomers = arts.map((a) => a.nomer_text).filter(Boolean);
  let title =
    nomers.find((n) => n === "ReichelAG") ||
    nomers.find((n) => !/^\d{6}-/.test(n) && !/^new /i.test(n) && n.length < 48) ||
    rawSlug ||
    sid.slice(-12);
  return { slug: slugify(title), title, rawSlug };
}

const ranked = Object.entries(bySpace).sort((a, b) => b[1].length - a[1].length);
let sampleId, sampleArts;
if (TARGET_LABEL) {
  try {
    [sampleId, sampleArts] = selectTargetArticles(articles, targetRootId);
  } catch {
    console.error("Target protocols unavailable; no import tree published. Renew Bubble login.");
    process.exit(4);
  }
} else {
  [sampleId, sampleArts] = ranked[0] || [null, []];
}
const lab = TARGET_LABEL ? { ...spaceLabel(sampleId), slug: slugify(TARGET_LABEL),
  title: TARGET_LABEL } : spaceLabel(sampleId);

const index = {
  exported_at: new Date().toISOString(),
  all_visible_spaces_requested: ALL_SPACES,
  ordering_inventory_sha256: orderSource ? createHash("sha256").update(orderSource).digest("hex") : null,
  ordering_inventory_used_for_dates_only: Boolean(orderSource),
  capture_attempts: captureAttempts,
  live_source_completeness_verified: false,
  target_requested: TARGET_LABEL || null,
  source_route: TARGET_LABEL ? TARGET_ROUTE : null,
  old_seed_used: !ALL_SPACES && !TARGET_LABEL && process.env.COULOMB_SOCIAL_FRESH_ONLY !== "1" && existsSync(SEED_NET),
  space_count: Object.keys(spaces).length,
  article_count: Object.keys(articles).length,
  sample_tree: sampleId ? `trees/${lab.slug}` : null,
  spaces: Object.entries(spaces).map(([id]) => {
    const L = spaceLabel(id);
    return {
      id,
      slug: L.slug,
      bubble_slug: L.rawSlug,
      title_guess: L.title,
      source_created_at: spaces[id]["Created Date"] ?? null,
      source_modified_at: spaces[id]["Modified Date"] ?? null,
      article_count: (bySpace[id] || []).length,
    };
  }),
};
write(join(OUT, "index.json"), JSON.stringify(index, null, 2));
write(join(OUT, "spaces.json"), JSON.stringify(spaces, null, 2));
write(join(OUT, "articles.json"), JSON.stringify(articles, null, 2));

const treeRoot = join(OUT, "trees", lab.slug);
ensure(join(treeRoot, "pages"));
ensure(join(treeRoot, "assets"));
const used = new Set();
let visuals = 0;
let targetVisual = "";
for (const a of sampleArts) {
  let pageSlug = slugify(a.nomer_text || a._id);
  let n = 2;
  while (used.has(pageSlug)) pageSlug = `${slugify(a.nomer_text || a._id)}-${n++}`;
  used.add(pageSlug);
  let visualRel = "";
  if (a.logo_image) {
    const lower = String(a.logo_image).toLowerCase();
    const ext = lower.includes(".png")
      ? "png"
      : lower.includes(".jpg") || lower.includes(".jpeg")
        ? "jpg"
        : lower.includes(".webp")
          ? "webp"
          : "bin";
    const assetName = `${pageSlug}-visual.${ext}`;
    const dest = join(treeRoot, "assets", assetName);
    const ok = await download(a.logo_image, dest);
    if (ok && existsSync(dest)) {
      visualRel = `assets/${assetName}`;
      visuals++;
    }
  }
  if (normId(a._id) === targetRootId) targetVisual = visualRel;
  const md = `---
id: bubble:${a._id}
title: ${JSON.stringify(a.nomer_text || "")}
abstractor: ${JSON.stringify(a.abstractor_text || "")}
visual: ${JSON.stringify(visualRel)}
space: ${JSON.stringify(lab.slug)}
bubble_type: article
source_created_at: ${JSON.stringify(a["Created Date"] ?? null)}
source_modified_at: ${JSON.stringify(a["Modified Date"] ?? null)}
archived: ${Boolean(a.archived_boolean)}
parent: ${JSON.stringify(a.context_custom_article ? `bubble:${normId(String(a.context_custom_article))}` : "")}
---

${a.content_text || ""}
`;
  write(join(treeRoot, "pages", `${pageSlug}.md`), md);
}
write(
  join(treeRoot, "pages", "index.md"),
  `---
id: bubble-space:${sampleId}
title: ${JSON.stringify(lab.title)}
abstractor: ${JSON.stringify(TARGET_LABEL
  ? (articles[targetRootId]?.abstractor_text || "")
  : `Imported sample (${sampleArts.length} articles)`)}
visual: ${JSON.stringify(targetVisual)}
space: ${JSON.stringify(lab.slug)}
bubble_type: space
source_created_at: ${JSON.stringify(spaces[sampleId]?.["Created Date"] ?? null)}
source_modified_at: ${JSON.stringify(spaces[sampleId]?.["Modified Date"] ?? null)}
---

${TARGET_LABEL ? (articles[targetRootId]?.content_text || `# ${lab.title}`) : `# ${lab.title}`}
`,
);
write(
  join(OUT, "SUMMARY.md"),
  `# Corpus ${stamp}

- spaces: ${Object.keys(spaces).length}
- articles: ${Object.keys(articles).length}
- sample: trees/${lab.slug}/ (${sampleArts.length} pages, ${visuals} visuals)

Do not commit — may contain private content.
`,
);
console.log("OUT", OUT);
console.log("tree", treeRoot, "pages", sampleArts.length, "visuals", visuals);
checkpoint("completed_capture_pass");