From 0ee31b596e5f5085bf64c4150dd076c3c78faf49 Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 31 Oct 2026 15:37:08 +0100 Subject: [PATCH] Capture bounded source resource evidence and expose navigation gap Assistant: codex Assistant-Model: gpt-6-astra Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3 --- docs/migration/bubble-auth-and-export.md | 22 ++++ .../bubble-resource-capture-2026-10-31.json | 102 +++++++++++++++ scripts/bubble-resource-utils.mjs | 53 ++++++++ scripts/capture-bubble-resources.mjs | 119 ++++++++++++++++++ scripts/tests/bubble-resource-utils.test.mjs | 47 +++++++ tests/test_bubble_capture_guards.py | 7 +- .../CSOC-WP-0013-sharing-content-hardening.md | 8 ++ ...-WP-0014-cohort-transfer-bubble-offramp.md | 18 ++- 8 files changed, 373 insertions(+), 3 deletions(-) create mode 100644 docs/release/bubble-resource-capture-2026-10-31.json create mode 100644 scripts/bubble-resource-utils.mjs create mode 100644 scripts/capture-bubble-resources.mjs create mode 100644 scripts/tests/bubble-resource-utils.test.mjs diff --git a/docs/migration/bubble-auth-and-export.md b/docs/migration/bubble-auth-and-export.md index 619d46c..eb47e7a 100644 --- a/docs/migration/bubble-auth-and-export.md +++ b/docs/migration/bubble-auth-and-export.md @@ -218,3 +218,25 @@ that the attachment flag enumerates all resources. The supplemental report is `~/.config/coulomb-social/auth/exports/body-resources-20261031.json`; the existing original and Visual bundle receipts remain unchanged. Public proof: `docs/release/bubble-body-resources-2026-10-31.json`. + +### Native resource follow-up + +The separate `resources-20261031` private directory records a bounded source +observation of the five affected articles. Their fresh parent records match the +retained capture, but the page visits returned no typed scrap/attachment records. +Two response reads failed, so zero captured records must not be read as zero +existing resources. No prior capture was overwritten. The private `index.html` +links all five original pages for the operator to identify the existing-resource +view/download path. Add attachment controls were observed but not activated. + +The collector is `scripts/capture-bubble-resources.mjs`, taking `--articles`, +`--expect-articles-sha256` and a new private `--output` directory. It visits at +most ten explicit affected article IDs, selects only typed scrap/attachment +content envelopes from the existing same-origin read responses, and excludes +user/session envelopes. Multiple observed resource versions remain separate; +empty initialization envelopes remain unresolved. It does not decode or invent +provider query payloads, click controls, submit forms, fetch arbitrary attachment +URLs or create an import manifest. `completed_bounded_observation` describes only +the terminal visit pass; completeness remains false. Source hashes, response +failures and private checkpoint hashes are in +`docs/release/bubble-resource-capture-2026-10-31.json`. diff --git a/docs/release/bubble-resource-capture-2026-10-31.json b/docs/release/bubble-resource-capture-2026-10-31.json new file mode 100644 index 0000000..e63640c --- /dev/null +++ b/docs/release/bubble-resource-capture-2026-10-31.json @@ -0,0 +1,102 @@ +{ + "schema": "coulomb.bubble-resource-observation-result.v1", + "recorded_at": "2026-10-31T14:35:31.435479+00:00", + "task": "CSOC-WP-0014-T01", + "status": "completed_bounded_observation", + "source_articles_sha256": "f33967c79c0f8a60a7d504f5d5e4082f8bc9e3c34f6a3160cb200fd380831d62", + "visited_articles": 5, + "route_matched_visits": 5, + "fresh_unchanged_parent_records": 5, + "typed_resources_observed": 0, + "empty_resource_envelopes": 0, + "response_status_counts": { + "200": 41 + }, + "response_timeouts": 0, + "response_read_failures": 2, + "retained_network_documents_checked": 294, + "retained_typed_resources_found": 0, + "linked_descriptor_targets_found": 0, + "settled_resource_controls_checked": 2, + "settled_control_label": "Add attachment", + "controls_activated": false, + "provider_queries_fabricated": false, + "resource_completeness_verified": false, + "resource_absence_inferred": false, + "operator_navigation_question_pending": true, + "original_capture_changed": false, + "production_imported": false, + "validation": { + "full_gate": "make check passed", + "junit": { + "tests": 596, + "errors": 0, + "failures": 0, + "skipped": 0 + }, + "synthetic_resource_node_tests": 4 + }, + "private_files": [ + { + "path": "capture.json", + "sha256": "81a8ee56ee6885e5d54ebd184bf309effb01fade3ff5d7ff6229e9d96f973751" + }, + { + "path": "index.html", + "sha256": "9218781bcf9273e2095494296d1d3f23d324e671c6d21920d69207b36181122f" + }, + { + "path": "ui-resource-controls-settled.json", + "sha256": "ad0b3823db1d241ce1473df83bd2a7bba32da49e44dc48e0e1d5954bd8a6faa9" + }, + { + "path": "ui-resource-controls.json", + "sha256": "4050f519bca5e4791ac0f0680a63eee2167c2b0e89c530ad045514bcb26a5da3" + }, + { + "path": "ui-resource-observation.json", + "sha256": "704363f4bb6ea5dcc606da13b667b979dd419d631b14b2c8517deb90723ccd1e" + } + ], + "implementation_files": [ + { + "path": "scripts/capture-bubble-resources.mjs", + "sha256": "647691c4283ee09eb8b320bbf3ef943eb8ad6be8d365e98a6b28dfd2a43bc92a" + }, + { + "path": "scripts/bubble-resource-utils.mjs", + "sha256": "e0cbd131bc28f67b3a88c45fb1e07cf12b33a71db051b5507e2e7ac14bf30170" + }, + { + "path": "scripts/bubble-corpus-utils.mjs", + "sha256": "aeb867c5a4fc332e3c994a24e47c7f93cf9d1b8f95738871ef9971f81571f502" + }, + { + "path": "scripts/tests/bubble-resource-utils.test.mjs", + "sha256": "d52dfc4fe8c3630ed020e9d6456e6836302c8f3e378bda424a1e3007873dee63" + }, + { + "path": "tests/test_bubble_capture_guards.py", + "sha256": "384e7da7eee76ca7f55cfcb45f9e2d8722509c9d0f3668288053b56f410047e1" + } + ], + "statehub_readback": { + "tasks": [ + { + "task_id": "0002882d-f156-5a5d-b524-d04793f4684d", + "status": "progress", + "resource_evidence_synced": true + }, + { + "task_id": "8d06cdee-e0ce-5c3e-bf98-705deed0ee62", + "status": "progress", + "resource_evidence_synced": true + } + ], + "t01_intervention_note_matches_source": true, + "fix_consistency_exit_code": 0, + "automation_errors": 0, + "assessment_failures": 0, + "warnings": 7 + } +} diff --git a/scripts/bubble-resource-utils.mjs b/scripts/bubble-resource-utils.mjs new file mode 100644 index 0000000..0ab390b --- /dev/null +++ b/scripts/bubble-resource-utils.mjs @@ -0,0 +1,53 @@ +/** Select typed content resources; never retain user/session envelopes or raw responses. */ +import {normalizeBubbleRef} from './bubble-corpus-utils.mjs'; + +export const RESOURCE_TYPES = new Set(['custom.scrap', 'custom.relattachment', 'custom.attachment']); +const validId = value => typeof value === 'string' && /^[0-9]+x[0-9]+$/.test(value); + +export function resourceParents(articles) { + return Object.entries(articles).filter(([,a]) => a.hasattachments_boolean === true || + (Object.hasOwn(a, '_list_custom_scrap') && + !(Array.isArray(a._list_custom_scrap) && a._list_custom_scrap.length === 0))) + .map(([id]) => normalizeBubbleRef(id)).filter(validId).sort(); +} + +export function ingestResourceBody(text, output) { + const visit = obj => { + if (Array.isArray(obj)) { obj.forEach(visit); return; } + if (!obj || typeof obj !== 'object') return; + // Typed initialization is a terminal envelope. Never walk into unrelated + // user/session data, even if it contains something resembling a resource. + if (typeof obj.type === 'string' && Object.hasOwn(obj, 'data')) { + const id = normalizeBubbleRef(obj.id), data = obj.data; + if (!RESOURCE_TYPES.has(obj.type) || !validId(id)) return; + if (data && typeof data === 'object' && !Array.isArray(data) && + Object.keys(data).length === 0) { + output.empty[`${obj.type}:${id}`] = {type:obj.type,id}; + } else if (data && data._type === obj.type && normalizeBubbleRef(data._id) === id) { + visit({_type:obj.type,_id:id,_source:data}); + } + return; + } + if (RESOURCE_TYPES.has(obj._type)) { + const id = normalizeBubbleRef(obj._id), data = obj._source; + if (validId(id) && data && typeof data === 'object' && !Array.isArray(data) && + (!data._id || normalizeBubbleRef(data._id) === id) && + (!data._type || data._type === obj._type)) { + const key = `${obj._type}:${id}`; + // Keep source observations, not a guessed union of different versions. + const observed = {_type:obj._type,_id:id,...data}; + const versions = output.records[key] ||= []; + if (!versions.some(v => JSON.stringify(v) === JSON.stringify(observed))) versions.push(observed); + } + return; + } + if (Array.isArray(obj.responses)) obj.responses.forEach(visit); + if (obj.hits && Array.isArray(obj.hits.hits)) obj.hits.hits.forEach(visit); + }; + try { visit(JSON.parse(text)); } + catch { + for (const line of text.split('\n')) { + try { visit(JSON.parse(line)); } catch { /* Not a JSON record. */ } + } + } +} diff --git a/scripts/capture-bubble-resources.mjs b/scripts/capture-bubble-resources.mjs new file mode 100644 index 0000000..152acef --- /dev/null +++ b/scripts/capture-bubble-resources.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node +/** Bounded read-only page visits for retained scrap/attachment source gaps. */ +import {chromium} from 'playwright'; +import {readFileSync, mkdirSync, openSync, writeFileSync, fsyncSync, closeSync, + renameSync, lstatSync} from 'node:fs'; +import {resolve, dirname, join} from 'node:path'; +import {homedir} from 'node:os'; +import {fileURLToPath} from 'node:url'; +import {createHash} from 'node:crypto'; +import {parseArgs} from 'node:util'; +import {assertSessionFresh, boundedCaptureBody, isCorpusResponse, + ingestCapturedBody} from './bubble-corpus-utils.mjs'; +import {resourceParents, ingestResourceBody} from './bubble-resource-utils.mjs'; + +const ROOT=resolve(dirname(fileURLToPath(import.meta.url)),'..'); +const BASE='https://coulomb.social'; +const sha=raw=>createHash('sha256').update(raw).digest('hex'); +const {values}=parseArgs({options:{articles:{type:'string'},output:{type:'string'}, + 'expect-articles-sha256':{type:'string'}}}); +let browser; +function noSymlinks(path) { + for(let p=path;;p=dirname(p)) { + if(lstatSync(p).isSymbolicLink()) throw new Error('symlink_refused'); + if(dirname(p)===p) break; + } +} +function privateRead(path) { + noSymlinks(path); + const info=lstatSync(path); + if(!info.isFile() || info.uid!==process.getuid() || (info.mode&0o077) || + info.nlink!==1 || info.size>10_000_000) throw new Error('private_input_required'); + return readFileSync(path); +} +try { + if(!values.articles || !values.output || !/^[a-f0-9]{64}$/.test( + values['expect-articles-sha256']||'')) throw new Error('exact_source_required'); + const input=resolve(values.articles),out=resolve(values.output); + const raw=privateRead(input); + if(sha(raw)!==values['expect-articles-sha256']) throw new Error('source_drift'); + const targets=resourceParents(JSON.parse(raw)); + if(!targets.length || targets.length>10) throw new Error('bounded_targets_required'); + if(out===ROOT || out.startsWith(ROOT+'/') || out.startsWith(dirname(input)+'/')) + throw new Error('output_overlap'); + noSymlinks(dirname(out)); + const parent=lstatSync(dirname(out)); + if(parent.uid!==process.getuid() || (parent.mode&0o077)) throw new Error('private_parent_required'); + const state=JSON.parse(privateRead(join(homedir(),'.config/coulomb-social/auth/storage-state.json'))); + assertSessionFresh(state); + const result={schema:'coulomb.bubble-resource-capture.v1',status:'capturing', + started_at:new Date().toISOString(),source_articles_sha256:sha(raw),target_article_ids:targets, + attempts:[],resources:{records:{},empty:{}},parent_articles:{}, + response_status_counts:{},response_timeouts:0,response_failures:0, + production_imported:false,resource_completeness_verified:false,ui_controls_activated:false}; + mkdirSync(out,{mode:0o700}); + function checkpoint() { + const file=join(out,'capture.json.partial'); + const fd=openSync(file,'w',0o600); + try {writeFileSync(fd,JSON.stringify(result,null,2)+'\n');fsyncSync(fd);} finally {closeSync(fd);} + renameSync(file,join(out,'capture.json')); + const directory=openSync(out,'r');try {fsyncSync(directory);} finally {closeSync(directory);} + } + checkpoint(); + browser=await chromium.launch({headless:true}); + const context=await browser.newContext({storageState:state}); + const page=await context.newPage(); + const pending=new Set(); + const observer=response=>{ + if(!isCorpusResponse(response.url(),BASE)) return; + const operation=(async()=>{ + const status=String(response.status()); + result.response_status_counts[status]=(result.response_status_counts[status]||0)+1; + const body=await boundedCaptureBody(response); + if(body.status==='timeout') result.response_timeouts++; + else if(body.status==='failed') result.response_failures++; + else if(body.text.length<5_000_000 && response.ok()) { + ingestResourceBody(body.text,result.resources); + const spaces={},articles={};ingestCapturedBody(body.text,spaces,articles); + for(const id of targets) if(articles[id]) result.parent_articles[id]=articles[id]; + } + checkpoint(); + })(); + pending.add(operation);operation.finally(()=>pending.delete(operation)); + }; + page.on('response',observer); + for(const id of targets) { + let status='visited',routeMatched=false,resourceControlCandidates=0; + try { + await page.goto(BASE+'/pg_chunk/'+id,{waitUntil:'domcontentloaded',timeout:20000}); + await page.waitForTimeout(2000); + routeMatched=page.url().split(/[?#]/)[0]===BASE+'/pg_chunk/'+id; + // Count only exact resource labels; never print titles, bodies or account names. + resourceControlCandidates=await page.getByRole('button',{ + name:/^(attachments?|scraps?|anhänge|anhang)$/i}).count(); + } catch {status='navigation_failed';} + await Promise.allSettled([...pending]); + result.attempts.push({id,status,route_matched:routeMatched, + resource_control_candidates:resourceControlCandidates});checkpoint(); + console.log(JSON.stringify({visits:result.attempts.length, + typed_resources:Object.keys(result.resources.records).length})); + } + page.off('response',observer);await Promise.allSettled([...pending]); + await browser.close();browser=null; + if(sha(privateRead(input))!==result.source_articles_sha256) throw new Error('source_changed'); + result.status='completed_bounded_observation';result.completed_at=new Date().toISOString(); + checkpoint(); + console.log(JSON.stringify({status:result.status,visits:result.attempts.length, + typed_resources:Object.keys(result.resources.records).length, + empty_resource_envelopes:Object.keys(result.resources.empty).length, + observed_parent_articles:Object.keys(result.parent_articles).length, + response_timeouts:result.response_timeouts,response_failures:result.response_failures, + resource_control_candidates:result.attempts.reduce((n,a)=>n+a.resource_control_candidates,0), + source_articles_sha256:result.source_articles_sha256, + private_capture_sha256:sha(readFileSync(join(out,'capture.json'))), + resource_completeness_verified:false,production_imported:false})); +} catch { + if(browser) await browser.close(); + console.error('Resource capture refused or stopped; private checkpoint retained, no completeness inferred.'); + process.exitCode=1; +} diff --git a/scripts/tests/bubble-resource-utils.test.mjs b/scripts/tests/bubble-resource-utils.test.mjs new file mode 100644 index 0000000..5270e86 --- /dev/null +++ b/scripts/tests/bubble-resource-utils.test.mjs @@ -0,0 +1,47 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import {ingestResourceBody, resourceParents} from '../bubble-resource-utils.mjs'; +const id='1700000000000x1', other='1700000000000x2'; +const output=()=>({records:{},empty:{}}); + +test('only explicit affected parents are visited; descriptors are not article targets',()=>{ + assert.deepEqual(resourceParents({[id]:{_list_custom_scrap:'LOOKUP__'+other}, + [other]:{hasattachments_boolean:true}, '1700000000000x3':{_list_custom_scrap:[]}, + invalid:{hasattachments_boolean:true}}),[id,other]); +}); + +test('typed search and initialization resource observations preserve versions',()=>{ + const result=output(); + ingestResourceBody(JSON.stringify({responses:[{hits:{hits:[{_type:'custom.scrap',_id:id, + _source:{content_text:'Synthetic first'}}]}}]}),result); + const envelope={type:'custom.scrap',id,data:{_type:'custom.scrap',_id:id, + content_text:'Synthetic second'}}; + ingestResourceBody(JSON.stringify(envelope),result); + ingestResourceBody(JSON.stringify(envelope),result); + assert.equal(result.records['custom.scrap:'+id].length,2); + assert.equal(result.records['custom.scrap:'+id][0].content_text,'Synthetic first'); +}); + +test('empty envelopes remain unresolved; mixed ids/types and nested users are excluded',()=>{ + const result=output(); + const resource={_type:'custom.scrap',_id:id,_source:{content_text:'Synthetic'}}; + ingestResourceBody(JSON.stringify([ + {type:'custom.scrap',id,data:{}}, + {type:'custom.scrap',id,data:{_type:'custom.scrap',_id:other}}, + {type:'user',id,data:resource}, + {_type:'user',_id:id,_source:resource}, + {...resource,_source:{_type:'custom.article'}}, + {...resource,_source:{_id:other}}, + {_type:'custom.article',_id:id,_source:{content_text:'Synthetic article'}}, + ]),result); + assert.deepEqual(result.records,{}); + assert.deepEqual(result.empty,{'custom.scrap:1700000000000x1':{type:'custom.scrap',id}}); +}); + +test('NDJSON selects only content-resource types',()=>{ + const result=output(); + ingestResourceBody('unrelated\n'+JSON.stringify({_type:'custom.relattachment',_id:id, + _source:{url_text:'https://example.invalid/synthetic'}})+'\n'+ + JSON.stringify({_type:'session',_id:other,_source:{cookie:'synthetic'}}),result); + assert.deepEqual(Object.keys(result.records),['custom.relattachment:'+id]); +}); diff --git a/tests/test_bubble_capture_guards.py b/tests/test_bubble_capture_guards.py index 8f7a3d8..a7e916c 100644 --- a/tests/test_bubble_capture_guards.py +++ b/tests/test_bubble_capture_guards.py @@ -2,11 +2,14 @@ import subprocess from pathlib import Path +import pytest -def test_bubble_capture_guards(): + +@pytest.mark.parametrize('suite', ['bubble-corpus-utils', 'bubble-resource-utils']) +def test_bubble_capture_guards(suite): root = Path(__file__).resolve().parents[1] result = subprocess.run( - ['node', '--test', 'scripts/tests/bubble-corpus-utils.test.mjs'], + ['node', '--test', f'scripts/tests/{suite}.test.mjs'], cwd=root, capture_output=True, text=True, check=False, ) assert result.returncode == 0, result.stdout + result.stderr diff --git a/workplans/CSOC-WP-0013-sharing-content-hardening.md b/workplans/CSOC-WP-0013-sharing-content-hardening.md index a70ed59..fbfc3bf 100644 --- a/workplans/CSOC-WP-0013-sharing-content-hardening.md +++ b/workplans/CSOC-WP-0013-sharing-content-hardening.md @@ -175,6 +175,14 @@ narrows lossiness uncertainty but does not resolve those records or accept the s markup-change cases, original Bubble appearance or productive content fidelity. Evidence: `docs/release/bubble-body-resources-2026-10-31.json`. T02 remains progress. +The subsequent bounded native read recovered fresh unchanged metadata for all +five resource-affected articles, but no resource records. The visible resource +control is Add attachment, not a demonstrated path to existing content, and was +left untouched. The operator viewing/download-path question is pending under +WP-0014-T01. `docs/release/bubble-resource-capture-2026-10-31.json` records the +incomplete return and private review; neither resource absence nor fidelity is +inferred. T02 remains progress. + ## T03 — Review productive sharing and personal portability ```task diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 2f8864d..c6ac172 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -32,7 +32,7 @@ id: CSOC-WP-0014-T01 status: progress priority: high needs_human: true -intervention_note: "Renewed Bubble session verified and targeted capture completed on 2026-10-31. Four source graph exception groups still need original content/date evidence or explicit disposition; private review: ~/.config/coulomb-social/auth/exports/source-exception-review-20261031/index.html. No further login renewal is currently required; no capture job is running." +intervention_note: "Fresh source session verified on 2026-10-31. Four source graph exception groups still need original content/date evidence or explicit disposition: ~/.config/coulomb-social/auth/exports/source-exception-review-20261031/index.html. Also awaiting the existing-resource viewing/download path for four scrap relations and one attachment flag: ~/.config/coulomb-social/auth/exports/resources-20261031/index.html. Bounded capture is terminal; no further login renewal is currently required." state_hub_task_id: "0002882d-f156-5a5d-b524-d04793f4684d" ``` @@ -145,6 +145,22 @@ Proof: `docs/release/bubble-body-resources-2026-10-31.json`. Actual scrap/attach capture or disposition, identity/grants, source exceptions and productive transfer remain open. T01 stays progress; link serialization proof is not a transfer verdict. +2026-10-31 resource recovery attempt: 294 retained network documents contain no +typed scrap/attachment records. A fresh, bounded read-only visit of the five +affected articles returned all five parent records unchanged, but no typed +resources or empty-resource envelopes. There were 41 HTTP 200 source responses, +zero read timeouts and two response-read failures; this is not evidence that +resources do not exist. DOM inspection found no links to the four descriptor IDs. +After allowing the UI to settle on one scrap parent and the attachment parent, +the resource-related controls are Add attachment, which were not activated. +The observed read endpoints use encoded request bodies; no guessed lookup or +broader query was submitted. The operator has been asked where existing resources +can be viewed/downloaded, with all five original pages linked in the private +review above. No deletion, empty list or discontinuation inferred. Original +capture and Visual preparation are unchanged. Evidence: +`docs/release/bubble-resource-capture-2026-10-31.json`. T01 stays progress and the +existing source-resource gaps remain explicit until recovery or disposition. + ## T02 — Review remaining journeys and record transfer decisions ```task