#!/usr/bin/env node /** Bounded read-only page visits for retained scrap/attachment source gaps. */ import {chromium} from 'playwright'; import {readFileSync, mkdirSync, openSync, writeFileSync, fsyncSync, closeSync, renameSync, lstatSync} from 'node:fs'; import {resolve, dirname, join} from 'node:path'; import {homedir} from 'node:os'; import {fileURLToPath} from 'node:url'; import {createHash} from 'node:crypto'; import {parseArgs} from 'node:util'; import {assertSessionFresh, boundedCaptureBody, isCorpusResponse, ingestCapturedBody} from './bubble-corpus-utils.mjs'; import {resourceParents, ingestResourceBody} from './bubble-resource-utils.mjs'; const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..'); const BASE='https://coulomb.social'; const sha=raw=>createHash('sha256').update(raw).digest('hex'); const {values}=parseArgs({options:{articles:{type:'string'},output:{type:'string'}, 'expect-articles-sha256':{type:'string'}}}); let browser; function noSymlinks(path) { for(let p=path;;p=dirname(p)) { if(lstatSync(p).isSymbolicLink()) throw new Error('symlink_refused'); if(dirname(p)===p) break; } } function privateRead(path) { noSymlinks(path); const info=lstatSync(path); if(!info.isFile() || info.uid!==process.getuid() || (info.mode&0o077) || info.nlink!==1 || info.size>10_000_000) throw new Error('private_input_required'); return readFileSync(path); } try { if(!values.articles || !values.output || !/^[a-f0-9]{64}$/.test( values['expect-articles-sha256']||'')) throw new Error('exact_source_required'); const input=resolve(values.articles),out=resolve(values.output); const raw=privateRead(input); if(sha(raw)!==values['expect-articles-sha256']) throw new Error('source_drift'); const targets=resourceParents(JSON.parse(raw)); if(!targets.length || targets.length>10) throw new Error('bounded_targets_required'); if(out===ROOT || out.startsWith(ROOT+'/') || out.startsWith(dirname(input)+'/')) throw new Error('output_overlap'); noSymlinks(dirname(out)); const parent=lstatSync(dirname(out)); if(parent.uid!==process.getuid() || (parent.mode&0o077)) throw new Error('private_parent_required'); const state=JSON.parse(privateRead(join(homedir(),'.config/coulomb-social/auth/storage-state.json'))); assertSessionFresh(state); const result={schema:'coulomb.bubble-resource-capture.v1',status:'capturing', started_at:new Date().toISOString(),source_articles_sha256:sha(raw),target_article_ids:targets, attempts:[],resources:{records:{},empty:{}},parent_articles:{}, response_status_counts:{},response_timeouts:0,response_failures:0, production_imported:false,resource_completeness_verified:false,ui_controls_activated:false}; mkdirSync(out,{mode:0o700}); function checkpoint() { const file=join(out,'capture.json.partial'); const fd=openSync(file,'w',0o600); try {writeFileSync(fd,JSON.stringify(result,null,2)+'\n');fsyncSync(fd);} finally {closeSync(fd);} renameSync(file,join(out,'capture.json')); const directory=openSync(out,'r');try {fsyncSync(directory);} finally {closeSync(directory);} } checkpoint(); browser=await chromium.launch({headless:true}); const context=await browser.newContext({storageState:state}); const page=await context.newPage(); const pending=new Set(); const observer=response=>{ if(!isCorpusResponse(response.url(),BASE)) return; const operation=(async()=>{ const status=String(response.status()); result.response_status_counts[status]=(result.response_status_counts[status]||0)+1; const body=await boundedCaptureBody(response); if(body.status==='timeout') result.response_timeouts++; else if(body.status==='failed') result.response_failures++; else if(body.text.length<5_000_000 && response.ok()) { ingestResourceBody(body.text,result.resources); const spaces={},articles={};ingestCapturedBody(body.text,spaces,articles); for(const id of targets) if(articles[id]) result.parent_articles[id]=articles[id]; } checkpoint(); })(); pending.add(operation);operation.finally(()=>pending.delete(operation)); }; page.on('response',observer); for(const id of targets) { let status='visited',routeMatched=false,resourceControlCandidates=0; try { await page.goto(BASE+'/pg_chunk/'+id,{waitUntil:'domcontentloaded',timeout:20000}); await page.waitForTimeout(2000); routeMatched=page.url().split(/[?#]/)[0]===BASE+'/pg_chunk/'+id; // Count only exact resource labels; never print titles, bodies or account names. resourceControlCandidates=await page.getByRole('button',{ name:/^(attachments?|scraps?|anhänge|anhang)$/i}).count(); } catch {status='navigation_failed';} await Promise.allSettled([...pending]); result.attempts.push({id,status,route_matched:routeMatched, resource_control_candidates:resourceControlCandidates});checkpoint(); console.log(JSON.stringify({visits:result.attempts.length, typed_resources:Object.keys(result.resources.records).length})); } page.off('response',observer);await Promise.allSettled([...pending]); await browser.close();browser=null; if(sha(privateRead(input))!==result.source_articles_sha256) throw new Error('source_changed'); result.status='completed_bounded_observation';result.completed_at=new Date().toISOString(); checkpoint(); console.log(JSON.stringify({status:result.status,visits:result.attempts.length, typed_resources:Object.keys(result.resources.records).length, empty_resource_envelopes:Object.keys(result.resources.empty).length, observed_parent_articles:Object.keys(result.parent_articles).length, response_timeouts:result.response_timeouts,response_failures:result.response_failures, resource_control_candidates:result.attempts.reduce((n,a)=>n+a.resource_control_candidates,0), source_articles_sha256:result.source_articles_sha256, private_capture_sha256:sha(readFileSync(join(out,'capture.json'))), resource_completeness_verified:false,production_imported:false})); } catch { if(browser) await browser.close(); console.error('Resource capture refused or stopped; private checkpoint retained, no completeness inferred.'); process.exitCode=1; }