diff --git a/docs/migration/bubble-auth-and-export.md b/docs/migration/bubble-auth-and-export.md index b40deae..01932ab 100644 --- a/docs/migration/bubble-auth-and-export.md +++ b/docs/migration/bubble-auth-and-export.md @@ -142,3 +142,38 @@ Set COULOMB_SOCIAL_CORPUS_DIR to a private directory if needed. Do not put raw exports in the checkout. Default target route is /vw_pages; override via COULOMB_SOCIAL_CAPTURE_ROUTE only after confirming the actual authenticated protocol surface. A successful public-route load is not authentication proof. + + +## Retained Visual capture and review bundle + +The October31 all-space bundle now has a separate private Visual preparation at +`~/.config/coulomb-social/auth/exports/visual-bundle-20261031`. The original +`review-bundle-20261031` remains unchanged.242Visual references were captured from +108URLs on the one recorded Bubble CDN, producing90distinct images.236draft +article pages and67space indexes link local hashed assets. All90images decode in +an offline browser; this is not a source/product appearance verdict or an import. + +The reusable collector is `scripts/capture-bubble-visuals.py ORIGINAL --output NEW`. +It requires a complete hash-bound original bundle, a new private directory outside +repositories, and the exact known CDN over HTTPS. It refuses redirects, alternate +origins, credentials in URLs, active/unknown image formats, source drift, oversized +responses and existing output. No stored browser credentials are read or sent. +A missing final reconciliation.json means partial output, which must be retained +for review rather than used as an import bundle. Original body/date bytes are not +rewritten; only the compiler's empty Visual metadata is replaced. + +Verification for the completed bundle (read-only): + +```bash +python3 scripts/verify-bubble-visual-preservation.py \ + ~/.config/coulomb-social/auth/exports/review-bundle-20261031 \ + ~/.config/coulomb-social/auth/exports/visual-bundle-20261031 +node scripts/verify-bubble-visuals.mjs \ + ~/.config/coulomb-social/auth/exports/visual-bundle-20261031 +``` + +Only aggregate results belong in Git/chat. Images, titles, source URLs and original +records remain private. The new bundle retains all71spaces and the existing +graph/grant/fidelity gates.18body links and one attachment flag are still +unreconciled. Captured Visual bytes do not authorize copying memberships or +productive import. Public proof: docs/release/bubble-visual-preparation-2026-10-31.json. diff --git a/docs/release/bubble-visual-preparation-2026-10-31.json b/docs/release/bubble-visual-preparation-2026-10-31.json new file mode 100644 index 0000000..fee8d0c --- /dev/null +++ b/docs/release/bubble-visual-preparation-2026-10-31.json @@ -0,0 +1,83 @@ +{ + "schema": "coulomb.bubble-visual-preparation.v1", + "status": "captured_and_privately_prepared_not_imported", + "owner_task": "CSOC-WP-0014-T08", + "private_bundle": "~/.config/coulomb-social/auth/exports/visual-bundle-20261031", + "source_report_sha256": "75fcd87d17ea0c5042e903efd60e65839da7716e5399ff431a345486ddd10f01", + "capture": { + "references": 242, + "unique_urls": 108, + "downloaded_bytes": 168139156, + "materialized_pages": 236, + "body_rewriting_performed": false, + "grants_changed": false, + "browser_render_verified": false, + "production_imported": false + }, + "browser": { + "schema": "coulomb.private-visual-decode.v1", + "status": "all_captured_visuals_decoded", + "decoded_images": 90, + "bundle_sha256": "a255df2237d95588b21a5db059487ca0178d1df28c810ebaaf744c5e6491267a", + "browser_network_disabled": true, + "productive_fidelity_accepted": false + }, + "preservation": { + "schema": "coulomb.visual-preservation-proof.v1", + "verified_output_files": 738, + "verified_original_files": 545, + "frontmatter_only_changes": 303, + "all_source_bodies_and_dates_preserved": true, + "spaces_preserved": 71, + "original_blockers_preserved": true, + "unassigned_records_preserved": true, + "private_modes_verified": true, + "private_bundle_sha256": "a255df2237d95588b21a5db059487ca0178d1df28c810ebaaf744c5e6491267a", + "production_imported": false + }, + "boundaries": { + "cdn_origins": 1, + "https_only": true, + "redirects_allowed": false, + "cookies_or_authorization_sent": false, + "live_application_or_grants_changed": false, + "source_completeness_or_fidelity_accepted": false, + "all_other_links_attachments_reconciled": false + }, + "tests": { + "focused_passed": 24, + "full_gate_passed": 570, + "failures": 0, + "errors": 0, + "skipped": 0, + "warnings": 69, + "full_junit_sha256": "d11afa225571077d1d59c68711a5f0eb402de8ceff7f2f50c279fbba7f923349", + "final_verification_script": "native private bundle pass; final all-repo Ruff pass" + }, + "remaining": { + "source_graph_exception_groups": 4, + "unassigned_article_records": 1, + "body_link_references": 18, + "attachment_review_flags": 1, + "identity_and_explicit_grants": "unresolved", + "productive_fidelity_verdict": "pending" + }, + "sources": [ + { + "path": "scripts/capture-bubble-visuals.py", + "sha256": "c16c8cf5013a6f02c9a9f79f444aea82ced7d8642b0fb2d2dc5315fabe92be7c" + }, + { + "path": "scripts/verify-bubble-visuals.mjs", + "sha256": "2cb9625b0738028508957b8bc40a69b7ab2719046d20e0a69a0ac1a9394f56c5" + }, + { + "path": "scripts/verify-bubble-visual-preservation.py", + "sha256": "9fbc212bd668126a0f6ce6a458b1ca3fba1e14d4b44438c0697f63dc34b5bc3d" + }, + { + "path": "tests/test_bubble_visuals.py", + "sha256": "707fc9aa4ac0de5e40d33ee76571d74c141b361154a659f10c723a6597262006" + } + ] +} diff --git a/docs/release/goal-progress-audit-2026-10-31.md b/docs/release/goal-progress-audit-2026-10-31.md index 684a31f..fc34226 100644 --- a/docs/release/goal-progress-audit-2026-10-31.md +++ b/docs/release/goal-progress-audit-2026-10-31.md @@ -150,3 +150,14 @@ retains actual ops registration, client/app/instance finance attribution/report, current SLO and stage-1 smoke. This makes the source checklist13of32done; it is an additional independently verified requirement, not a new product scope or an updated effort estimate. The rough remaining-work estimate is unchanged. + + +## Visual preservation checkpoint + +WP-0014-T08 completes the previously missing private Visual capture and draft-tree +materialization:242references,108URLs,90distinct images,236article pages and67 +space indexes. Offline decoding and independent body/date/source preservation +checks pass. This advances the source checklist to14of33done, not an effort +percentage. Source exceptions, identity/grant mapping, links/attachments and +productive transfer/verdicts remain open; the rough remaining-effort estimate +has not been recalculated from these counts. diff --git a/docs/usecases/sharing-portability-review.md b/docs/usecases/sharing-portability-review.md index 3abaf51..fe86fa8 100644 --- a/docs/usecases/sharing-portability-review.md +++ b/docs/usecases/sharing-portability-review.md @@ -136,3 +136,15 @@ bounds. Incremental provider cost is unpriced; no EUR autopilot budget is claime Application writes pause during paired capture and the Recreate rollout. Evidence: `../release/content-lifecycle-release-preparation-2026-10-31.json`, its proposal, native target/preflight/row-drift receipts, and the candidate recovery receipt. + + +### October31 Visual preservation follow-through + +A new private `~/.config/coulomb-social/auth/exports/visual-bundle-20261031` retains +242Visual references as90distinct local images and populates236draft article +Visuals plus67space indexes. The original rendering cards and source capture are +unchanged. Offline image decoding and exact body/date preservation pass; neither +proves appearance fidelity to Bubble. Use these retained local assets for the +source/product comparison after each mapping is ready. Graph exceptions, grants, +18body links and one attachment flag remain explicit. No productive import or +user verdict is inferred. Evidence: ../release/bubble-visual-preparation-2026-10-31.json. diff --git a/scripts/capture-bubble-visuals.py b/scripts/capture-bubble-visuals.py new file mode 100644 index 0000000..170ef44 --- /dev/null +++ b/scripts/capture-bubble-visuals.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Capture retained Bubble Visual URLs and prepare a private, hash-bound review bundle.""" + +import argparse +import copy +import hashlib +import json +import os +import re +import stat +import sys +import urllib.request +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlsplit + +CDN = '13a55be71f3a6a4c9ea356833df823c4.cdn.bubble.io' +MAX_ASSET = 16 * 1024 * 1024 +MAX_TOTAL = 256 * 1024 * 1024 +ROOT = Path(__file__).resolve().parents[1] + + +class Refused(Exception): + pass + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def canonical_url(value): + if not isinstance(value, str) or any(ord(c) <= 32 for c in value): + raise Refused('invalid_visual_url') + value = 'https:' + value if value.startswith('//') else value + try: + url = urlsplit(value) + valid = (url.scheme == 'https' and url.hostname == CDN and url.port in (None, 443) + and url.username is None and url.password is None and not url.query + and not url.fragment and url.path.startswith('/') and '\\' not in value) + except ValueError: + valid = False + if not valid: + raise Refused('outside_reviewed_visual_origin') + return value + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise Refused('visual_redirect_refused') + + +def image_extension(raw, media_type): + if raw.startswith(b'\x89PNG\r\n\x1a\n') and media_type == 'image/png': + return 'png' + if raw.startswith(b'\xff\xd8\xff') and media_type == 'image/jpeg': + return 'jpg' + if (len(raw) >= 12 and raw[:4] == b'RIFF' and raw[8:12] == b'WEBP' + and media_type == 'image/webp'): + return 'webp' + raise Refused('unsupported_visual_bytes_or_type') + + +def fetch(url): + url = canonical_url(url) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + request = urllib.request.Request(url, headers={'User-Agent': 'coulomb-visual-capture/1.0'}) + with opener.open(request, timeout=20) as response: + if response.status != 200 or response.geturl() != url: + raise Refused('unexpected_visual_response') + length = response.headers.get('Content-Length') + if length and (not length.isdigit() or int(length) > MAX_ASSET): + raise Refused('visual_size_limit') + raw = response.read(MAX_ASSET + 1) + if len(raw) > MAX_ASSET: + raise Refused('visual_size_limit') + if length and int(length) != len(raw): + raise Refused('incomplete_visual_response') + extension = image_extension(raw, response.headers.get_content_type()) + return raw, extension + + +def read_private(root, name, expected=None): + relative = Path(name) + if relative.is_absolute() or '..' in relative.parts or not relative.parts: + raise Refused('unsafe_bundle_path') + path = root / relative + if any(p.is_symlink() for p in (path, *path.parents)): + raise Refused('bundle_symlink_refused') + info = path.stat() + if (not stat.S_ISREG(info.st_mode) or info.st_uid != os.getuid() + or info.st_mode & 0o077 or info.st_nlink != 1 or info.st_size > MAX_ASSET): + raise Refused('private_bounded_source_required') + raw = path.read_bytes() + if expected is not None and sha(raw) != expected: + raise Refused('source_file_changed') + return raw + + +def publish(root, name, raw): + relative = Path(name) + if relative.is_absolute() or '..' in relative.parts or not relative.parts: + raise Refused('unsafe_output_path') + path = root / relative + if any(p.is_symlink() for p in (path, *path.parents)): + raise Refused('output_symlink_refused') + parent = root + for part in relative.parts[:-1]: + parent = parent / part + parent.mkdir(mode=0o700, exist_ok=True) + info = parent.stat() + if info.st_uid != os.getuid() or info.st_mode & 0o077: + raise Refused('private_output_directory_required') + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, 'wb') as stream: + stream.write(raw) + stream.flush() + os.fsync(stream.fileno()) + if read_private(root, name) != raw: + raise Refused('capture_readback_failed') + + +def with_visual(raw, relative, *, index=False): + # Replace only the compiler's known empty field; never parse/reformat the body. + head, body = raw.split(b'\n---\n', 1) + if not head.startswith(b'---\n'): + raise Refused('draft_frontmatter_required') + new = b'visual: ' + json.dumps(relative).encode() + if index: + if re.search(rb'^visual:', head, re.M): + raise Refused('unexpected_index_visual') + head += b'\n' + new + else: + if len(re.findall(rb'^visual: ""$', head, re.M)) != 1: + raise Refused('empty_draft_visual_required') + head = re.sub(rb'^visual: ""$', lambda _: new, head, flags=re.M) + return head + b'\n---\n' + body + + +def capture(bundle, output, fetcher=fetch): + bundle = Path(os.path.abspath(bundle.expanduser())) + output = Path(os.path.abspath(output.expanduser())) + if (output.is_relative_to(ROOT) or output.is_relative_to(bundle) + or bundle.is_relative_to(output) + or any(p.is_symlink() for p in (output, *output.parents)) + or any((p / '.git').exists() for p in (output, *output.parents))): + raise Refused('private_separate_output_required') + parent = output.parent.stat() + if parent.st_uid != os.getuid() or parent.st_mode & 0o077: + raise Refused('private_owned_output_parent_required') + original = read_private(bundle, 'reconciliation.json') + report = json.loads(original) + if (report.get('schema') != 'coulomb.bubble-review-bundle.v1' + or report.get('status') != 'completed_offline_preparation'): + raise Refused('completed_original_bundle_required') + # Verify every declared file and the captured source bindings before any network read. + files = {name: read_private(bundle, name, digest) + for name, digest in report['file_sha256'].items()} + if any(not name.startswith(('source-bodies/', 'review-trees/')) for name in files): + raise Refused('unexpected_original_bundle_namespace') + for space in report['spaces']: + tree = space['tree'] + if space.get('import_ready') is not False: + raise Refused('unapproved_original_readiness') + if tree is None: + continue + if not re.fullmatch(r'review-trees/bubble-[0-9a-f]{24}', tree): + raise Refused('unexpected_review_tree_path') + for page in space['articles']: + if not re.fullmatch(re.escape(tree) + r'/pages/article-[0-9a-f]{24}\.md', + page['file']): + raise Refused('unexpected_review_page_path') + for name, digest in report['source_sha256'].items(): + files['source-capture/' + name + '.json'] = read_private( + bundle, 'source-capture/' + name + '.json', digest) + visuals = {} + for article in report['articles']: + refs = [ref['source'] for ref in article['references'] if ref['kind'] == 'visual'] + if len(refs) > 1: + raise Refused('ambiguous_visual_reference') + if refs: + visuals[article['id']] = canonical_url(refs[0]) + output.mkdir(mode=0o700) # Existing output always refuses; never replace a capture. + captured, total = {}, 0 + for url in sorted(set(visuals.values())): + raw, extension = fetcher(url) + if len(raw) > MAX_ASSET or total + len(raw) > MAX_TOTAL: + raise Refused('visual_capture_budget_exceeded') + total += len(raw) + name = 'captured-visuals/' + sha(raw) + '.' + extension + if name not in files: + files[name] = raw + captured[url] = {'file': name, 'sha256': sha(raw), 'bytes': len(raw)} + # Checkpoint captured bytes immediately; no final report means incomplete output. + if not (output / name).exists(): + publish(output, name, raw) + updated = copy.deepcopy(report) + by_id = {a['id']: a for a in updated['articles']} + materialized = 0 + for space in updated['spaces']: + tree = space['tree'] + if tree is None: + continue + for page in space['articles']: + url = visuals.get(page['id']) + if not url: + continue + visual = captured[url] + relative = 'assets/' + Path(visual['file']).name + files[tree + '/' + relative] = files[visual['file']] + files[page['file']] = with_visual(files[page['file']], relative) + page['file_sha256'] = sha(files[page['file']]) + materialized += 1 + if page['id'] == space['root_article_id']: + index = tree + '/pages/index.md' + files[index] = with_visual(files[index], relative, index=True) + # Keep all unresolved gates: retained bytes do not certify source/UI fidelity or grants. + space['import_ready'] = False + for identity, url in visuals.items(): + article = by_id[identity] + article['captured_visual'] = captured[url] + for ref in article['references']: + if ref['kind'] == 'visual': + ref['capture_status'] = 'bytes_retained_and_hashed' + ref['resolved'] = False + # Recheck the immutable inputs after fetching; source drift cannot publish completion. + if read_private(bundle, 'reconciliation.json') != original: + raise Refused('source_report_changed_during_capture') + for name, digest in report['file_sha256'].items(): + read_private(bundle, name, digest) + for name, digest in report['source_sha256'].items(): + read_private(bundle, 'source-capture/' + name + '.json', digest) + files['source-capture/reconciliation-before-visuals.json'] = original + for name, raw in sorted(files.items()): + if not name.startswith('captured-visuals/'): + publish(output, name, raw) + updated.update(status='completed_private_visual_preparation', + original_report_sha256=sha(original), + captured_at=datetime.now(UTC).isoformat(), + visual_capture={'references': len(visuals), 'unique_urls': len(captured), + 'downloaded_bytes': total, 'materialized_pages': materialized, + 'body_rewriting_performed': False, 'grants_changed': False, + 'browser_render_verified': False, 'production_imported': False}, + network_requests=len(captured), + file_sha256={name: sha(raw) for name, raw in sorted(files.items())}) + for directory in sorted([output, *(p for p in output.rglob('*') if p.is_dir())], + key=lambda p: len(p.parts), reverse=True): + fd = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + raw_report = (json.dumps(updated, indent=2, sort_keys=True) + '\n').encode() + publish(output, 'reconciliation.json', raw_report) + fd = os.open(output, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + return updated['visual_capture'] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('bundle', type=Path) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + try: + print(json.dumps(capture(args.bundle, args.output))) + except Exception: # Private URLs, titles and provider errors never enter stdout/stderr. + print('Visual capture refused or incomplete; retain any partial output for review. ' + 'No final reconciliation.json means it is not a completed bundle.', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/verify-bubble-visual-preservation.py b/scripts/verify-bubble-visual-preservation.py new file mode 100644 index 0000000..ca4587c --- /dev/null +++ b/scripts/verify-bubble-visual-preservation.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Independently verify Visual-only frontmatter changes against the original private bundle.""" + +import argparse +import hashlib +import json +import os +import re +from pathlib import Path + + +def read(root, name): + path = root / name + if ( + Path(name).is_absolute() + or ".." in Path(name).parts + or any(p.is_symlink() for p in (path, *path.parents)) + ): + raise ValueError("unsafe source path") + info = path.stat() + if ( + not path.is_file() + or info.st_uid != os.getuid() + or info.st_mode & 0o077 + or info.st_nlink != 1 + or info.st_size > 16 * 1024 * 1024 + ): + raise ValueError("private bounded source required") + return path.read_bytes() + + +def verify(source, out): + if not __debug__: + raise ValueError("verification assertions required") + old = json.loads(read(source, "reconciliation.json")) + new = json.loads(read(out, "reconciliation.json")) + def sha(raw): + return hashlib.sha256(raw).hexdigest() + assert new["original_report_sha256"] == sha(read(source, "reconciliation.json")) + assert read(out, "source-capture/reconciliation-before-visuals.json") == read( + source, "reconciliation.json" + ) + for name, digest in new["file_sha256"].items(): + assert sha(read(out, name)) == digest + changed = 0 + for name, digest in old["file_sha256"].items(): + original = read(source, name) + current = read(out, name) + assert sha(original) == digest + if current != original: + assert "/pages/" in name and name.endswith(".md") + before, body = original.split(b"\n---\n", 1) + after, after_body = current.split(b"\n---\n", 1) + assert body == after_body + assert ( + len(re.findall(rb'^visual: "assets/[a-f0-9]{64}\.(?:png|jpg|webp)"$', after, re.M)) + == 1 + ) + if name.endswith("/index.md"): + restored = re.sub( + rb'\nvisual: "assets/[a-f0-9]{64}\.(?:png|jpg|webp)"$', b"", after + ) + else: + restored = re.sub( + rb'^visual: "assets/[a-f0-9]{64}\.(?:png|jpg|webp)"$', + b'visual: ""', + after, + flags=re.M, + ) + assert restored == before + changed += 1 + for axis in ["unassigned_article_ids", "source_sha256"]: + assert new[axis] == old[axis] + assert len(old["spaces"]) == len(new["spaces"]) + for a, b in zip(old["spaces"], new["spaces"], strict=True): + assert a["blockers"] == b["blockers"] and not b["import_ready"] + for key in a: + if key != "articles": + assert a[key] == b[key], key + for x, y in zip(a["articles"], b["articles"], strict=True): + assert {k: v for k, v in x.items() if k != "file_sha256"} == { + k: v for k, v in y.items() if k != "file_sha256" + } + for p in out.rglob("*"): + assert p.stat().st_mode & 0o077 == 0 + return { + "schema": "coulomb.visual-preservation-proof.v1", + "verified_output_files": len(new["file_sha256"]), + "verified_original_files": len(old["file_sha256"]), + "frontmatter_only_changes": changed, + "all_source_bodies_and_dates_preserved": True, + "spaces_preserved": len(new["spaces"]), + "original_blockers_preserved": True, + "unassigned_records_preserved": True, + "private_modes_verified": True, + "private_bundle_sha256": sha(read(out, "reconciliation.json")), + "production_imported": False, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("original", type=Path) + parser.add_argument("prepared", type=Path) + args = parser.parse_args() + try: + print(json.dumps(verify(args.original, args.prepared))) + except Exception: + print("Private visual preservation verification failed; no acceptance inferred.") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-bubble-visuals.mjs b/scripts/verify-bubble-visuals.mjs new file mode 100644 index 0000000..9ea4d64 --- /dev/null +++ b/scripts/verify-bubble-visuals.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +/** Decode only locally retained Visual files in an offline disposable browser; emit counts only. */ +import { readFileSync, lstatSync, realpathSync } from 'node:fs'; +import { resolve, join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { chromium } from 'playwright'; + +let browser; +try { + const root = resolve(process.argv[2]); + const reportBytes = readFileSync(join(root, 'reconciliation.json')); + const report = JSON.parse(reportBytes); + if (report.status !== 'completed_private_visual_preparation') throw new Error('unfinished_bundle'); + const entries = Object.entries(report.file_sha256).filter(([name]) => name.startsWith('captured-visuals/')); + browser = await chromium.launch({ headless: true }); + const context = await browser.newContext({ serviceWorkers: 'block', offline: true }); + await context.route('**/*', route => route.abort()); + const page = await context.newPage(); + let decoded = 0; + for (const [name, hash] of entries) { + if (!/^captured-visuals\/[a-f0-9]{64}\.(png|jpg|webp)$/.test(name)) throw new Error('invalid_asset_path'); + const path = join(root, name), info = lstatSync(path); + if (!info.isFile() || info.isSymbolicLink() || info.size > 16 * 1024 * 1024 || + (info.mode & 0o077) || info.uid !== process.getuid() || + realpathSync(path) !== path) throw new Error('unsafe_asset'); + const data = readFileSync(path); + if (createHash('sha256').update(data).digest('hex') !== hash) throw new Error('changed_asset'); + const mime = name.endsWith('.jpg') ? 'image/jpeg' : name.endsWith('.webp') ? 'image/webp' : 'image/png'; + const ok = await page.evaluate(async ({data, mime}) => { + const bytes = Uint8Array.from(atob(data), x => x.charCodeAt(0)); + const url = URL.createObjectURL(new Blob([bytes], {type: mime})); + try { + const img = new Image(); img.src = url; + await Promise.race([img.decode(), new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), 10000))]); + return img.naturalWidth > 0 && img.naturalHeight > 0; + } finally { URL.revokeObjectURL(url); } + }, {data: data.toString('base64'), mime}); + if (!ok) throw new Error('image_decode_failed'); + decoded++; + } + console.log(JSON.stringify({schema: 'coulomb.private-visual-decode.v1', status: 'all_captured_visuals_decoded', + decoded_images: decoded, bundle_sha256: createHash('sha256').update(reportBytes).digest('hex'), + browser_network_disabled: true, productive_fidelity_accepted: false})); +} catch { + console.error('Private visual decode verification failed; no acceptance inferred.'); + process.exitCode = 1; +} finally { + if (browser) await browser.close(); +} diff --git a/tests/test_bubble_visuals.py b/tests/test_bubble_visuals.py new file mode 100644 index 0000000..aadec2b --- /dev/null +++ b/tests/test_bubble_visuals.py @@ -0,0 +1,157 @@ +"""Synthetic visual capture: origin limits, immutable source, complete private output.""" + +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + +from coulomb_social.apps.spaces.bubble_bundle import prepare_bundle +from tests.test_bubble_bundle import capture as source_capture + +SPEC = importlib.util.spec_from_file_location( + 'bubble_visuals', Path(__file__).resolve().parents[1] / 'scripts/capture-bubble-visuals.py') +visuals = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(visuals) +URL = 'https://' + visuals.CDN + '/synthetic.png' +PNG = b'\x89PNG\r\n\x1a\nsynthetic-no-decode-claim' + + +def bundle(tmp_path): + tmp_path.chmod(0o700) + source = tmp_path / 'source' + source.mkdir(mode=0o700) + spaces, articles = source_capture() + articles['r']['logo_image'] = '//' + visuals.CDN + '/synthetic.png' + articles['c']['logo_image'] = URL + # Preserve the blocked and unassigned records as well as all eligible draft pages. + articles['orphan'] = {'logo_image': URL, 'content_text': 'Unassigned synthetic body'} + report, files = prepare_bundle(spaces, articles, inventory_spaces={'missing': {}}) + inputs = {k: json.dumps(v).encode() for k, v in [('spaces', spaces), ('articles', articles)]} + report['source_sha256'] = {k: hashlib.sha256(v).hexdigest() for k, v in inputs.items()} + report['status'] = 'completed_offline_preparation' + for name, raw in {**files, **{'source-capture/' + k + '.json': v + for k, v in inputs.items()}}.items(): + visuals.publish(source, name, raw) + visuals.publish(source, 'reconciliation.json', json.dumps(report).encode()) + return source, report, files + + +def test_shared_visuals_private_bundle_preserves_bodies_dates_gates_and_source(tmp_path): + source, report, files = bundle(tmp_path) + before = {str(p.relative_to(source)): p.read_bytes() for p in source.rglob('*') if p.is_file()} + calls = [] + def fetch(url): + calls.append(url) + return PNG, 'png' + out = tmp_path / 'output' + result = visuals.capture(source, out, fetch) + assert calls == [URL] + assert result['references'] == 3 and result['unique_urls'] == 1 + assert result['materialized_pages'] == 2 + after = json.loads((out / 'reconciliation.json').read_bytes()) + assert after['unassigned_article_ids'] == report['unassigned_article_ids'] + assert len(after['spaces']) == len(report['spaces']) + for original, updated in zip(report['spaces'], after['spaces'], strict=True): + assert updated['blockers'] == original['blockers'] and not updated['import_ready'] + for page in updated['articles']: + if page['file']: + raw = (out / page['file']).read_bytes() + assert raw.split(b'\n---\n', 1)[1] == files[page['file']].split(b'\n---\n', 1)[1] + assert b'source_created_at:' in raw and b'visual: "assets/' in raw + for name, digest in after['file_sha256'].items(): + assert visuals.sha((out / name).read_bytes()) == digest + assert before == {str(p.relative_to(source)): p.read_bytes() + for p in source.rglob('*') if p.is_file()} + for p in out.rglob('*'): + assert p.stat().st_mode & 0o077 == 0 + with pytest.raises(FileExistsError): + visuals.capture(source, out, fetch) + assert calls == [URL] + + +@pytest.mark.parametrize('value', ['http://' + visuals.CDN + '/x', + 'https://example.invalid/x', 'https://user@' + visuals.CDN + '/x', + 'https://' + visuals.CDN + ':8443/x', 'https://' + visuals.CDN + '/x?q=secret', + 'https://' + visuals.CDN + '/x#fragment', 'https://' + visuals.CDN + '/x\n', + 'https://' + visuals.CDN + '.example.invalid/x', '//127.0.0.1/x']) +def test_unreviewed_urls_refused(value): + with pytest.raises(visuals.Refused): + visuals.canonical_url(value) + + +def test_redirects_and_active_content_refused(): + with pytest.raises(visuals.Refused): + visuals.NoRedirect().redirect_request(None, None, 302, '', {}, URL) + for raw, kind in [(b'', 'image/svg+xml'), (PNG, 'text/html'), (b'html', 'image/png')]: + with pytest.raises(visuals.Refused): + visuals.image_extension(raw, kind) + + +@pytest.mark.parametrize('defect', ['file-drift', 'outside-tree', 'symlink', 'public-source']) +def test_bad_source_refuses_before_fetch_or_output(tmp_path, defect): + source, report, _ = bundle(tmp_path) + name = next(iter(report['file_sha256'])) + if defect == 'file-drift': + (source / name).write_bytes(b'changed') + elif defect == 'outside-tree': + report['spaces'][0]['tree'] = '/outside' + (source / 'reconciliation.json').write_text(json.dumps(report)) + elif defect == 'symlink': + (source / name).unlink() + (source / name).symlink_to(tmp_path / 'missing') + else: + (source / name).chmod(0o644) + out = tmp_path / 'out' + with pytest.raises((visuals.Refused, KeyError)): + visuals.capture(source, out, lambda u: pytest.fail('network called')) + assert not out.exists() + + +@pytest.mark.parametrize('defect', ['fetch-failure', 'during-fetch-drift', 'budget']) +def test_partial_capture_never_publishes_completion(tmp_path, monkeypatch, defect): + source, report, _ = bundle(tmp_path) + if defect == 'budget': + monkeypatch.setattr(visuals, 'MAX_TOTAL', 1) + def fetch(url): + if defect == 'fetch-failure': + raise RuntimeError('private-url-do-not-print') + if defect == 'during-fetch-drift': + (source / next(iter(report['file_sha256']))).write_bytes(b'changed') + return PNG, 'png' + out = tmp_path / 'out' + with pytest.raises((visuals.Refused, RuntimeError)): + visuals.capture(source, out, fetch) + assert not (out / 'reconciliation.json').exists() + + +@pytest.mark.parametrize('defect', ['status', 'redirected-url', 'oversize-header', + 'oversize-body', 'short-body', 'active-type']) +def test_fetch_rejects_invalid_transport_results_without_followup(monkeypatch, defect): + from email.message import Message + class Response: + status = 404 if defect == 'status' else 200 + headers = Message() + headers['Content-Type'] = 'text/html' if defect == 'active-type' else 'image/png' + headers['Content-Length'] = str( + visuals.MAX_ASSET + 1 if defect == 'oversize-header' else len(PNG) + 1 + if defect == 'short-body' else len(PNG)) + def __enter__(self): + return self + def __exit__(self, *args): + pass + def geturl(self): + return 'https://example.invalid/x' if defect == 'redirected-url' else URL + def read(self, maximum): + assert maximum == visuals.MAX_ASSET + 1 + return b'x' * maximum if defect == 'oversize-body' else PNG + class Opener: + def open(self, request, timeout): + assert request.full_url == URL and timeout == 20 + assert not any(k.lower() in {'authorization', 'cookie', 'referer'} + for k in request.headers) + return Response() + monkeypatch.setattr(visuals.urllib.request, 'build_opener', lambda *args: Opener()) + with pytest.raises(visuals.Refused): + visuals.fetch(URL) diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 87baca8..f36d428 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -18,7 +18,7 @@ Residual acceptance from CSOC-WP-0008, split at the user's request. Existing imp ## Current execution map — 2026-10-31 -T05 records completed capture/tooling preparation; T04 now closes the independently verified accepted-cohort date correction. T01 still owns source exceptions, currentness and identity/grant/assets reconciliation. Remaining date verification is T06 and actual further-cohort migration is T07. T02 productive journey/transfer verdicts and T03 the separate cutover decision remain open. Bubble login renewal is currently cleared; four source graph exceptions and one unassigned record remain. +T05 records completed capture/tooling preparation; T08 adds verified private Visual capture and draft-tree materialization. T04 now closes the independently verified accepted-cohort date correction. T01 still owns source exceptions, currentness and identity/grant/assets reconciliation. Remaining date verification is T06 and actual further-cohort migration is T07. T02 productive journey/transfer verdicts and T03 the separate cutover decision remain open. Bubble login renewal is currently cleared; four source graph exceptions and one unassigned record remain. The [four-workplan assessment](../docs/release/goal-progress-audit-2026-10-31.md) records the scope split and effort estimate. Task counts are not effort percentages. @@ -334,3 +334,40 @@ After each source/identity/grant/assets mapping is ready under T01 and content r ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records. + + +## T08 — Capture and prepare the retained Visual images + +```task +id: CSOC-WP-0014-T08 +status: done +priority: high +needs_human: false +state_hub_task_id: "b7464ad5-9087-54a9-b3da-991b87bcefc8" +``` + +Completed2026-10-31. The previous67-tree review bundle retained Visual URLs but +left its Visual fields empty. Captured242 references through108 distinct HTTPS +URLs on the single recorded Bubble CDN;90 distinct binary images are retained +privately (168,139,156 downloaded bytes, including repeated content). No cookies, +authorization headers, arbitrary origins or redirects are used. Private capture +has per-image/aggregate limits and a final completion receipt; partial captures +cannot be mistaken for completed output. No browser login renewal was needed. + +A separate new bundle links local images on236 eligible draft pages and their67 +space indexes. All71 spaces, unassigned records and original gates remain. An +independent verifier checks738 output files against manifests, preserves all545 +original files modulo303 Visual-only frontmatter changes, and proves unchanged +bodies, original dates and all other metadata. All90 images decode in an offline +browser. This does not establish original Bubble rendering fidelity, current source +completeness, grants, or productive acceptance. Blocked/unassigned Visuals remain +retained even when no draft destination exists. Original capture is unchanged. + +Evidence: docs/release/bubble-visual-preparation-2026-10-31.json; private bundle +~/.config/coulomb-social/auth/exports/visual-bundle-20261031.24focused checks and +full make check570pass, no failures/errors/skips; final verifier and Ruff pass. +This extracts completed Visual preservation from T01/T05 without closing the +broader mapping/transfer requirement. T01 retains18body links, one attachment +flag, four graph exception groups, one unassigned record, source currentness, +identity/explicit grants and destination reconciliation. T07 remains unstarted +for productive transfer; T02/WP-0013 retain productive fidelity/verdicts.