coulomb-social/scripts/bubble-corpus-utils.mjs

/** Pure capture guards; fixtures must never contain real cookies or notes. */
export function normalizeBubbleRef(value) {
  if (typeof value !== 'string') return value;
  if (!value.includes('LOOKUP__')) return value;
  return value.split('LOOKUP__').at(-1).match(/[0-9]+x[0-9]+/)?.[0] || value;
}

export async function boundedCaptureBody(response, milliseconds = 8000) {
  let timer;
  try {
    return await Promise.race([
      response.text().then(text => ({status:'read', text})),
      new Promise(resolve => { timer = setTimeout(() => resolve({status:'timeout'}), milliseconds); }),
    ]);
  } catch { return {status:'failed'}; }
  finally { clearTimeout(timer); }
}

/** Include direct-page initialization, restricted to the requested source origin. */
export function isCorpusResponse(url, base = 'https://coulomb.social') {
  try {
    const target = new URL(url);
    return target.origin === new URL(base).origin && (
      target.pathname === '/api/1.1/init/data' ||
      target.pathname.startsWith('/elasticsearch/'));
  } catch { return false; }
}

export function assertSessionFresh(state, now = Date.now() / 1000) {
  if ((state.cookies || []).some(c => c.name.endsWith('.sig') &&
      c.expires > 0 && c.expires <= now)) {
    throw new Error('bubble_session_expired');
  }
}

export function mergeCapturedRecord(existing, source, id) {
  return { ...(existing || {}), ...source, _id: normalizeBubbleRef(id) };
}

export function selectTargetArticles(articles, rootId) {
  const root = articles[normalizeBubbleRef(rootId)];
  const spaceId = normalizeBubbleRef(root?.space_custom_space);
  if (!root || !spaceId) throw new Error('target_space_identity_missing');
  const selected = Object.values(articles).filter(a =>
    normalizeBubbleRef(a.space_custom_space) === spaceId);
  if (selected.length <= 1) throw new Error('target_protocols_not_captured');
  if (selected.some(a => !Object.hasOwn(a, 'content_text'))) {
    throw new Error('target_protocol_bodies_incomplete');
  }
  return [spaceId, selected];
}

/** UI root links map through captured root articles to *space* creation dates. */
export function rankSpaceLinks(links, spaces, articles) {
  const unique = new Map();
  for (const link of links) {
    const path = new URL(link.href, 'https://source.invalid').pathname;
    const match = path.match(/^\/pg_chunk\/([A-Za-z0-9_-]+)$/);
    if (!match) continue;
    const rootId = normalizeBubbleRef(match[1]);
    const spaceId = normalizeBubbleRef(articles[rootId]?.space_custom_space);
    const created = spaces[spaceId]?.['Created Date'];
    const createdMs = Number.isSafeInteger(created) ? created : null;
    // One deterministic entry per source root, even when the UI repeats cards.
    const entry = { href: path, rootId, spaceId: spaceId || null, createdMs,
      label: String(articles[rootId]?.nomer_text || link.label || '').trim() };
    if (!unique.has(rootId) || entry.label < unique.get(rootId).label) unique.set(rootId, entry);
  }
  return [...unique.values()].sort((a, b) =>
    (b.createdMs ?? -Infinity) - (a.createdMs ?? -Infinity) ||
    (a.rootId < b.rootId ? -1 : a.rootId > b.rootId ? 1 : 0));
}

/** Follow only explicit source references or records missing their body/date. */
export function pendingCaptureReferences(spaces, articles, inventory = {}) {
  const needed = new Set();
  const add = value => {
    if (typeof value !== 'string') return;
    const id = normalizeBubbleRef(value);
    if (/^[0-9]+x[0-9]+$/.test(id)) needed.add(id);
  };
  for (const [id, space] of Object.entries({...inventory, ...spaces})) {
    const root = normalizeBubbleRef(space._chunk_custom_article || inventory[id]?._chunk_custom_article);
    if (root && (!articles[root] || !spaces[id] || spaces[id]['Created Date'] == null)) add(root);
  }
  for (const [id, article] of Object.entries(articles)) {
    if (typeof article.content_text !== 'string' || article['Created Date'] == null) add(id);
    for (const field of ['chunks_list_custom_article', 'contents_list_custom_article']) {
      for (const ref of Array.isArray(article[field]) ? article[field] : []) {
        if (!articles[normalizeBubbleRef(ref)]) add(ref);
      }
    }
    const parent = normalizeBubbleRef(article.context_custom_article);
    if (parent && !articles[parent]) add(parent);
  }
  return [...needed].sort();
}

function ingest(obj, spaces, articles) {
  if (Array.isArray(obj)) {
    for (const x of obj) ingest(x, spaces, articles);
    return;
  }
  if (!obj || typeof obj !== "object") return;
  // Direct page loads deliver typed initialization envelopes rather than search
  // hits. An empty data object is unresolved, never an invented empty article.
  if (typeof obj.type === 'string' && Object.hasOwn(obj, 'data')) {
    const data = obj.data;
    if (['custom.space', 'custom.article'].includes(obj.type) &&
        data && typeof data === 'object' && !Array.isArray(data) &&
        typeof obj.id === 'string' && data._type === obj.type &&
        normalizeBubbleRef(data._id) === normalizeBubbleRef(obj.id)) {
      ingest({_type:obj.type, _id:obj.id, _source:data}, spaces, articles);
    }
    return; // User/session envelopes do not belong in the content corpus.
  }
  const t = obj._type;
  const src = obj._source && typeof obj._source === "object" ? obj._source : null;
  const id = obj._id || (src && src._id);
  if (t === "custom.space" && src && id) {
    spaces[normalizeBubbleRef(id)] = mergeCapturedRecord(spaces[normalizeBubbleRef(id)], src, id);
  }
  if (t === "custom.article" && src && id) {
    const a = mergeCapturedRecord(articles[normalizeBubbleRef(id)], src, id);
    if (a.space_custom_space) a.space_custom_space = normalizeBubbleRef(a.space_custom_space);
    articles[normalizeBubbleRef(id)] = a;
  }
  if (Array.isArray(obj.responses)) {
    for (const r of obj.responses) ingest(r, spaces, articles);
  }
  if (obj.hits && typeof obj.hits === "object") {
    for (const h of obj.hits.hits || []) ingest(h, spaces, articles);
  }
  for (const v of Object.values(obj)) {
    if (v && typeof v === "object") ingest(v, spaces, articles);
  }
}
export function ingestCapturedBody(text, spaces, articles) {
  try {
    ingest(JSON.parse(text), spaces, articles);
    return;
  } catch {
    /* try NDJSON */
  }
  for (const line of text.split("\n")) {
    try {
      ingest(JSON.parse(line), spaces, articles);
    } catch {
      /* ignore */
    }
  }
}