coulomb-social/scripts/capture-bubble-references.mjs

#!/usr/bin/env node
/** Continue a private checkpoint through explicit UI references, without a full recrawl. */
import { chromium } from 'playwright';
import { readFileSync, writeFileSync, mkdirSync, renameSync } from 'node:fs';
import { homedir } from 'node:os';
import { join, resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createHash } from 'node:crypto';
import { parseArgs } from 'node:util';
import { assertSessionFresh, ingestCapturedBody, boundedCaptureBody,
  pendingCaptureReferences, isCorpusResponse } from './bubble-corpus-utils.mjs';

const {values} = parseArgs({options: {
  capture: {type:'string'}, output: {type:'string'},
  'inventory-spaces': {type:'string'}, 'max-visits': {type:'string', default:'200'},
}});
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const BASE = 'https://coulomb.social';
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
let browser;
try {
  if (!values.capture || !values.output) throw new Error('private_input_output_required');
  const out = resolve(values.output);
  if (out === ROOT || out.startsWith(ROOT + '/')) throw new Error('output_must_be_outside_repo');
  const limit = Number(values['max-visits']);
  if (!Number.isInteger(limit) || limit < 1 || limit > 2000) throw new Error('invalid_limit');
  const spacesBytes = readFileSync(join(values.capture, 'spaces.json'));
  const articlesBytes = readFileSync(join(values.capture, 'articles.json'));
  const spaces = JSON.parse(spacesBytes), articles = JSON.parse(articlesBytes);
  const inventoryBytes = values['inventory-spaces'] ? readFileSync(values['inventory-spaces']) : null;
  const inventory = inventoryBytes ? JSON.parse(inventoryBytes) : {};
  const state = process.env.COULOMB_SOCIAL_STORAGE_STATE ||
    join(homedir(), '.config/coulomb-social/auth/storage-state.json');
  assertSessionFresh(JSON.parse(readFileSync(state, 'utf8')));
  mkdirSync(out, {mode:0o700}); // Refuse overwrite of any earlier checkpoint.
  const attempts = [];
  let responseTimeouts = 0;
  function write(name, value) {
    const temporary = join(out, name + '.partial');
    writeFileSync(temporary, JSON.stringify(value, null, 2) + '\n', {mode:0o600});
    renameSync(temporary, join(out, name));
  }
  function checkpoint(status = 'capturing_references') {
    write('spaces.json', spaces); write('articles.json', articles);
    write('capture-state.json', {
      status, space_count:Object.keys(spaces).length, article_count:Object.keys(articles).length,
      attempts, response_timeouts:responseTimeouts,
      remaining_references:pendingCaptureReferences(spaces, articles, inventory),
      base_spaces_sha256:createHash('sha256').update(spacesBytes).digest('hex'),
      base_articles_sha256:createHash('sha256').update(articlesBytes).digest('hex'),
      inventory_sha256:inventoryBytes ? createHash('sha256').update(inventoryBytes).digest('hex') : null,
      live_source_completeness_verified:false, production_imported:false,
    });
  }
  checkpoint();
  browser = await chromium.launch({headless:true});
  const context = await browser.newContext({storageState:state});
  const page = await context.newPage();
  const pending = new Set();
  const observer = response => {
    if (!isCorpusResponse(response.url(), BASE)) return;
    const operation = (async () => {
      try {
        const result = await boundedCaptureBody(response);
        if (result.status === 'timeout') responseTimeouts++;
        const text = result.text;
        if (typeof text === 'string' && text.length < 5_000_000) {
          ingestCapturedBody(text, spaces, articles); checkpoint();
        }
      } catch { /* A cancelled/denied body leaves its reference unresolved. */ }
    })();
    pending.add(operation); operation.finally(() => pending.delete(operation));
  };
  page.on('response', observer);
  const attempted = new Set();
  while (attempted.size < limit) {
    const next = pendingCaptureReferences(spaces, articles, inventory)
      .find(id => !attempted.has(id));
    if (!next) break;
    attempted.add(next);
    let status = 'visited';
    try {
      await page.goto(BASE + '/pg_chunk/' + next,
        {waitUntil:'domcontentloaded', timeout:20000});
      await delay(2000);
    } catch { status = 'navigation_failed'; }
    await Promise.allSettled([...pending]); // Each response read has its own bound.
    attempts.push({id:next,status}); checkpoint();
    console.log(JSON.stringify({visits:attempted.size, spaces:Object.keys(spaces).length,
      articles:Object.keys(articles).length}));
  }
  page.off('response', observer);
  await Promise.allSettled([...pending]);
  await browser.close(); browser = null;
  checkpoint('completed_reference_pass');
  console.log(JSON.stringify({status:'completed_reference_pass', visits:attempts.length,
    spaces:Object.keys(spaces).length, articles:Object.keys(articles).length,
    remaining_references:pendingCaptureReferences(spaces, articles, inventory).length,
    live_source_completeness_verified:false, production_imported:false}));
} catch {
  if (browser) await browser.close();
  console.error('Reference capture refused or stopped; private checkpoints retained');
  process.exitCode = 1;
}