tegwick
37e3bfc2c7
Some checks failed
Use-case regression gate / usecase-regressions (push) Failing after 7m29s
Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a192e9-c28b-7533-bf25-e2341364c22a
91 lines
8.1 KiB
JavaScript
91 lines
8.1 KiB
JavaScript
#!/usr/bin/env node
|
|
/** Recover existing files through the user-confirmed Bubble attachment modal. */
|
|
import {chromium} from 'playwright';
|
|
import {readFileSync,mkdirSync,writeFileSync,lstatSync,openSync,writeSync,fsyncSync,
|
|
closeSync,renameSync,unlinkSync} from 'node:fs';
|
|
import {resolve,dirname,join} from 'node:path';
|
|
import {homedir} from 'node:os';
|
|
import {fileURLToPath} from 'node:url';
|
|
import {createHash} from 'node:crypto';
|
|
import {parseArgs} from 'node:util';
|
|
import {assertSessionFresh,isCorpusResponse,boundedCaptureBody,ingestCapturedBody,
|
|
normalizeBubbleRef} from './bubble-corpus-utils.mjs';
|
|
import {resourceParents,ingestResourceBody,attachmentRows} from './bubble-resource-utils.mjs';
|
|
const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..'),BASE='https://coulomb.social';
|
|
const MAX_FILE=64*1024*1024,MAX_TOTAL=512*1024*1024;
|
|
const sha=raw=>createHash('sha256').update(raw).digest('hex');
|
|
const {values}=parseArgs({options:{articles:{type:'string'},output:{type:'string'},
|
|
'expect-articles-sha256':{type:'string'},'example-article':{type:'string'}}});
|
|
function noLinks(path){for(let p=path;;p=dirname(p)){if(lstatSync(p).isSymbolicLink())throw Error('symlink');if(dirname(p)===p)break;}}
|
|
function privateRead(path){noLinks(path);const s=lstatSync(path);if(!s.isFile()||s.uid!==process.getuid()||(s.mode&0o077)||s.nlink!==1||s.size>10000000)throw Error('private_input');return readFileSync(path);}
|
|
let browser,total=0;
|
|
try{
|
|
if(!values.articles||!values.output||!/^[a-f0-9]{64}$/.test(values['expect-articles-sha256']||''))throw Error('source_binding');
|
|
const source=resolve(values.articles),out=resolve(values.output),raw=privateRead(source);
|
|
if(sha(raw)!==values['expect-articles-sha256'])throw Error('source_drift');
|
|
const example=values['example-article'];if(example&&!/^\d+x\d+$/.test(example))throw Error('example_id');
|
|
const parents=[...new Set([...(example?[example]:[]),...resourceParents(JSON.parse(raw))])];
|
|
if(!parents.length||parents.length>10)throw Error('target_bound');
|
|
if(out===ROOT||out.startsWith(ROOT+'/')||out.startsWith(dirname(source)+'/'))throw Error('output_overlap');
|
|
noLinks(dirname(out));const parent=lstatSync(dirname(out));if(parent.uid!==process.getuid()||(parent.mode&0o077))throw Error('private_parent');
|
|
const state=JSON.parse(privateRead(join(homedir(),'.config/coulomb-social/auth/storage-state.json')));assertSessionFresh(state);
|
|
mkdirSync(out,{mode:0o700});mkdirSync(join(out,'files'),{mode:0o700});
|
|
const result={schema:'coulomb.bubble-attachment-modal-capture.v1',status:'capturing',
|
|
started_at:new Date().toISOString(),source_articles_sha256:sha(raw),target_article_ids:parents,
|
|
resources:{records:{},empty:{}},articles:{},attempts:[],files:{},
|
|
resource_completeness_verified:false,source_mutations:false,production_imported:false};
|
|
function checkpoint(){const f=join(out,'capture.json.partial'),fd=openSync(f,'w',0o600);try{writeFileSync(fd,JSON.stringify(result,null,2)+'\n');fsyncSync(fd);}finally{closeSync(fd);}renameSync(f,join(out,'capture.json'));const d=openSync(out,'r');try{fsyncSync(d);}finally{closeSync(d);}}
|
|
checkpoint();
|
|
browser=await chromium.launch({headless:true});const context=await browser.newContext({storageState:state,acceptDownloads:true});
|
|
const page=await context.newPage(),pending=new Set(),observedArticles={},observedResources={records:{},empty:{}};
|
|
let responseFailures=0;
|
|
const observer=r=>{if(!isCorpusResponse(r.url(),BASE))return;const p=(async()=>{const b=await boundedCaptureBody(r);if(b.text&&r.ok()){ingestResourceBody(b.text,observedResources);ingestCapturedBody(b.text,{},observedArticles);}else responseFailures++;})();pending.add(p);p.finally(()=>pending.delete(p));};
|
|
page.on('response',observer);
|
|
async function retain(url){
|
|
if(result.files[url])return result.files[url];
|
|
const response=await fetch(url,{redirect:'error',signal:AbortSignal.timeout(90000)});
|
|
if(!response.ok||!response.body||Number(response.headers.get('content-length'))>MAX_FILE)throw Error('download_refused');
|
|
const path='files/'+sha(Buffer.from(url)),fd=openSync(join(out,path),'wx',0o600),hash=createHash('sha256');let size=0;
|
|
try{for await(const chunk of response.body){size+=chunk.length;total+=chunk.length;if(size>MAX_FILE||total>MAX_TOTAL)throw Error('download_bound');hash.update(chunk);let offset=0;while(offset<chunk.length)offset+=writeSync(fd,chunk,offset,chunk.length-offset);}if(!size)throw Error('empty_download');fsyncSync(fd);}
|
|
catch(error){closeSync(fd);unlinkSync(join(out,path));throw error;}
|
|
closeSync(fd);const d=openSync(join(out,'files'),'r');try{fsyncSync(d);}finally{closeSync(d);}
|
|
const file={path,bytes:size,sha256:hash.digest('hex'),content_type:response.headers.get('content-type')||''};result.files[url]=file;return file;
|
|
}
|
|
for(const id of parents){
|
|
await page.goto(BASE+'/pg_chunk/'+id,{waitUntil:'domcontentloaded',timeout:30000});await page.waitForTimeout(2500);
|
|
if(page.url().split(/[?#]/)[0]!==BASE+'/pg_chunk/'+id)throw Error('source_route');
|
|
const opener=page.locator('[title="Add attachment"]:visible');if(await opener.count()!==1)throw Error('unique_modal_opener');
|
|
await opener.click();await page.waitForTimeout(2500);await Promise.allSettled([...pending]);
|
|
const modal=page.locator('.bubble-element.Popup:visible');if(await modal.count()!==1)throw Error('unique_modal');
|
|
const buttons=modal.locator('[title="Open file"]:visible'),count=await buttons.count();
|
|
const rows=attachmentRows(id,observedResources,observedArticles);if(count!==rows.length)throw Error('visible_row_binding');
|
|
const attempt={article_id:id,modal_opened:true,visible_file_actions:count,rows:[],response_failures:responseFailures};
|
|
for(let i=0;i<count;i++){
|
|
const downloads=[];const onDownload=d=>downloads.push(d);
|
|
const onPage=p=>p.on('download',onDownload);page.on('download',onDownload);context.on('page',onPage);
|
|
try{
|
|
await buttons.nth(i).click();
|
|
for(let n=0;n<60&&!downloads.length;n++)await page.waitForTimeout(100);
|
|
if(downloads.length!==1)throw Error('single_ui_download_required');
|
|
const download=downloads[0],url=download.url();const matches=rows.filter(r=>r.url===url);
|
|
if(matches.length!==1)throw Error('ui_source_binding');
|
|
const row=matches[0],file=await retain(url);
|
|
const received=await download.path();
|
|
if(!received||lstatSync(received).size!==file.bytes||sha(readFileSync(received))!==file.sha256)
|
|
throw Error('independent_download_byte_mismatch');
|
|
attempt.rows.push({...row,...file,suggested_filename:download.suggestedFilename(),
|
|
ui_download_url_verified:true,ui_and_independent_download_bytes_match:true});
|
|
}finally{page.off('download',onDownload);context.off('page',onPage);for(const p of context.pages())if(p!==page)await p.close();}
|
|
}
|
|
if(new Set(attempt.rows.map(r=>r.relation_id)).size!==rows.length)throw Error('duplicate_or_missing_relation');
|
|
for(const row of rows){const key='custom.relattachment:'+row.relation_id;result.resources.records[key]=observedResources.records[key];result.articles[row.article_id]=observedArticles[row.article_id];}
|
|
if(!observedArticles[id])throw Error('parent_source_missing');result.articles[id]=observedArticles[id];
|
|
result.attempts.push(attempt);checkpoint();
|
|
console.log(JSON.stringify({visited:result.attempts.length,modal_files:count,retained_files:Object.keys(result.files).length,retained_bytes:total}));
|
|
}
|
|
page.off('response',observer);await Promise.allSettled([...pending]);
|
|
await browser.close();browser=null;
|
|
if(sha(privateRead(source))!==result.source_articles_sha256)throw Error('source_changed');
|
|
result.status='completed_bounded_attachment_modal_capture';result.completed_at=new Date().toISOString();checkpoint();
|
|
console.log(JSON.stringify({status:result.status,visited:result.attempts.length,relations:Object.keys(result.resources.records).length,files:Object.keys(result.files).length,bytes:total,private_capture_sha256:sha(readFileSync(join(out,'capture.json'))),source_mutations:false,production_imported:false,resource_completeness_verified:false}));
|
|
}catch(error){if(browser)await browser.close();console.error('Attachment modal capture stopped: '+(error.message||'unknown').replace(/[^a-zA-Z0-9_]/g,'_').slice(0,80));process.exitCode=1;}
|