coulomb-social/scripts/capture-bubble-resources.mjs

#!/usr/bin/env node
/** Bounded read-only page visits for retained scrap/attachment source gaps. */
import {chromium} from 'playwright';
import {readFileSync, mkdirSync, openSync, writeFileSync, fsyncSync, closeSync,
  renameSync, lstatSync} from 'node:fs';
import {resolve, dirname, join} from 'node:path';
import {homedir} from 'node:os';
import {fileURLToPath} from 'node:url';
import {createHash} from 'node:crypto';
import {parseArgs} from 'node:util';
import {assertSessionFresh, boundedCaptureBody, isCorpusResponse,
  ingestCapturedBody} from './bubble-corpus-utils.mjs';
import {resourceParents, ingestResourceBody} from './bubble-resource-utils.mjs';

const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..');
const BASE='https://coulomb.social';
const sha=raw=>createHash('sha256').update(raw).digest('hex');
const {values}=parseArgs({options:{articles:{type:'string'},output:{type:'string'},
  'expect-articles-sha256':{type:'string'}}});
let browser;
function noSymlinks(path) {
  for(let p=path;;p=dirname(p)) {
    if(lstatSync(p).isSymbolicLink()) throw new Error('symlink_refused');
    if(dirname(p)===p) break;
  }
}
function privateRead(path) {
  noSymlinks(path);
  const info=lstatSync(path);
  if(!info.isFile() || info.uid!==process.getuid() || (info.mode&0o077) ||
      info.nlink!==1 || info.size>10_000_000) throw new Error('private_input_required');
  return readFileSync(path);
}
try {
  if(!values.articles || !values.output || !/^[a-f0-9]{64}$/.test(
      values['expect-articles-sha256']||'')) throw new Error('exact_source_required');
  const input=resolve(values.articles),out=resolve(values.output);
  const raw=privateRead(input);
  if(sha(raw)!==values['expect-articles-sha256']) throw new Error('source_drift');
  const targets=resourceParents(JSON.parse(raw));
  if(!targets.length || targets.length>10) throw new Error('bounded_targets_required');
  if(out===ROOT || out.startsWith(ROOT+'/') || out.startsWith(dirname(input)+'/'))
    throw new Error('output_overlap');
  noSymlinks(dirname(out));
  const parent=lstatSync(dirname(out));
  if(parent.uid!==process.getuid() || (parent.mode&0o077)) throw new Error('private_parent_required');
  const state=JSON.parse(privateRead(join(homedir(),'.config/coulomb-social/auth/storage-state.json')));
  assertSessionFresh(state);
  const result={schema:'coulomb.bubble-resource-capture.v1',status:'capturing',
    started_at:new Date().toISOString(),source_articles_sha256:sha(raw),target_article_ids:targets,
    attempts:[],resources:{records:{},empty:{}},parent_articles:{},
    response_status_counts:{},response_timeouts:0,response_failures:0,
    production_imported:false,resource_completeness_verified:false,ui_controls_activated:false};
  mkdirSync(out,{mode:0o700});
  function checkpoint() {
    const file=join(out,'capture.json.partial');
    const fd=openSync(file,'w',0o600);
    try {writeFileSync(fd,JSON.stringify(result,null,2)+'\n');fsyncSync(fd);} finally {closeSync(fd);}
    renameSync(file,join(out,'capture.json'));
    const directory=openSync(out,'r');try {fsyncSync(directory);} finally {closeSync(directory);}
  }
  checkpoint();
  browser=await chromium.launch({headless:true});
  const context=await browser.newContext({storageState:state});
  const page=await context.newPage();
  const pending=new Set();
  const observer=response=>{
    if(!isCorpusResponse(response.url(),BASE)) return;
    const operation=(async()=>{
      const status=String(response.status());
      result.response_status_counts[status]=(result.response_status_counts[status]||0)+1;
      const body=await boundedCaptureBody(response);
      if(body.status==='timeout') result.response_timeouts++;
      else if(body.status==='failed') result.response_failures++;
      else if(body.text.length<5_000_000 && response.ok()) {
        ingestResourceBody(body.text,result.resources);
        const spaces={},articles={};ingestCapturedBody(body.text,spaces,articles);
        for(const id of targets) if(articles[id]) result.parent_articles[id]=articles[id];
      }
      checkpoint();
    })();
    pending.add(operation);operation.finally(()=>pending.delete(operation));
  };
  page.on('response',observer);
  for(const id of targets) {
    let status='visited',routeMatched=false,resourceControlCandidates=0;
    try {
      await page.goto(BASE+'/pg_chunk/'+id,{waitUntil:'domcontentloaded',timeout:20000});
      await page.waitForTimeout(2000);
      routeMatched=page.url().split(/[?#]/)[0]===BASE+'/pg_chunk/'+id;
      // Count only exact resource labels; never print titles, bodies or account names.
      resourceControlCandidates=await page.getByRole('button',{
        name:/^(attachments?|scraps?|anhänge|anhang)$/i}).count();
    } catch {status='navigation_failed';}
    await Promise.allSettled([...pending]);
    result.attempts.push({id,status,route_matched:routeMatched,
      resource_control_candidates:resourceControlCandidates});checkpoint();
    console.log(JSON.stringify({visits:result.attempts.length,
      typed_resources:Object.keys(result.resources.records).length}));
  }
  page.off('response',observer);await Promise.allSettled([...pending]);
  await browser.close();browser=null;
  if(sha(privateRead(input))!==result.source_articles_sha256) throw new Error('source_changed');
  result.status='completed_bounded_observation';result.completed_at=new Date().toISOString();
  checkpoint();
  console.log(JSON.stringify({status:result.status,visits:result.attempts.length,
    typed_resources:Object.keys(result.resources.records).length,
    empty_resource_envelopes:Object.keys(result.resources.empty).length,
    observed_parent_articles:Object.keys(result.parent_articles).length,
    response_timeouts:result.response_timeouts,response_failures:result.response_failures,
    resource_control_candidates:result.attempts.reduce((n,a)=>n+a.resource_control_candidates,0),
    source_articles_sha256:result.source_articles_sha256,
    private_capture_sha256:sha(readFileSync(join(out,'capture.json'))),
    resource_completeness_verified:false,production_imported:false}));
} catch {
  if(browser) await browser.close();
  console.error('Resource capture refused or stopped; private checkpoint retained, no completeness inferred.');
  process.exitCode=1;
}