diff --git a/coulomb_social/apps/spaces/bubble_bundle.py b/coulomb_social/apps/spaces/bubble_bundle.py index 590cc00..66c1767 100644 --- a/coulomb_social/apps/spaces/bubble_bundle.py +++ b/coulomb_social/apps/spaces/bubble_bundle.py @@ -143,6 +143,18 @@ def prepare_bundle(spaces, articles, *, inventory_spaces=None): detail["relations"]["context_custom_article"] = normalize_ref(parent) if parent else None if article.get("hasattachments_boolean"): detail["review_flags"].append("attachment_records_require_capture") + if "_list_custom_scrap" in article: + # Bubble may return a deferred LOOKUP string instead of the actual + # list. Preserve it privately; it is not a captured scrap record or + # an article ID that may be traversed as a chunk. + raw_scraps = article["_list_custom_scrap"] + detail["scrap_relation"] = { + "raw": raw_scraps, + "state": "observed_empty_list" if raw_scraps == [] else "unresolved", + "records_captured": False, + } + if raw_scraps != []: + detail["review_flags"].append("scrap_relation_records_require_capture") article_details[identity] = detail for entry in plan["spaces"]: diff --git a/coulomb_social/apps/spaces/rendering_audit.py b/coulomb_social/apps/spaces/rendering_audit.py index 9886745..e664e92 100644 --- a/coulomb_social/apps/spaces/rendering_audit.py +++ b/coulomb_social/apps/spaces/rendering_audit.py @@ -34,6 +34,11 @@ def audit_body(body: str): rendered = render_markdown(body) before, after = _Structure(raw), _Structure(rendered) removed_attrs = before.attributes - after.attributes + reference_attributes = {"href", "src", "poster", "data", "action"} + before_refs = Counter({key: count for key, count in before.attributes.items() + if key[1] in reference_attributes}) + after_refs = Counter({key: count for key, count in after.attributes.items() + if key[1] in reference_attributes}) text_changed = " ".join("".join(before.text).split()) != ( " ".join("".join(after.text).split())) markup_changed = before.tags != after.tags or before.attributes != after.attributes @@ -43,6 +48,11 @@ def audit_body(body: str): "sanitization_changed_html": raw != rendered, "markup_changed_by_sanitization": markup_changed, "serialization_only_change": raw != rendered and not markup_changed and not text_changed, + "reference_occurrences_before": sum(before_refs.values()), + "reference_occurrences_after": sum(after_refs.values()), + "reference_attributes_preserved": before_refs == after_refs, + "removed_reference_occurrences": sum((before_refs - after_refs).values()), + "added_reference_occurrences": sum((after_refs - before_refs).values()), "removed_tag_counts": dict(sorted((before.tags - after.tags).items())), "added_tag_counts": dict(sorted((after.tags - before.tags).items())), "removed_attributes": [ diff --git a/docs/migration/bubble-auth-and-export.md b/docs/migration/bubble-auth-and-export.md index 51f0f21..619d46c 100644 --- a/docs/migration/bubble-auth-and-export.md +++ b/docs/migration/bubble-auth-and-export.md @@ -196,3 +196,25 @@ membership field is unknown, not an empty list. Source role names and visibility values are preserved for review; no equivalence with target owner/member/viewer permissions is assumed. The accepted ReichelAG owner grant remains evidence for that cohort alone. See `docs/release/bubble-access-audit-2026-10-31.json`. + +## Body links and deferred scrap relations + +`scripts/verify-bubble-body-resources.py ORIGINAL VISUAL --output PRIVATE_JSON` +checks source hashes, recompiles original draft bytes, checks prepared bodies and +compares reference attributes before and after the product sanitizer. It neither +fetches destinations nor changes source files. Its stdout receipt contains only +aggregate counts and hashes. Raw references and deferred scrap descriptors belong +in the separate private report, outside both bundles and the checkout. + +The October 31 proof verifies 18 distinct body-link entries with 28 occurrences, +all preserved by the product renderer. It also verifies 242 source bodies and 236 +prepared pages. This does not prove remote availability or Bubble appearance. +Four `_list_custom_scrap` values are deferred strings, not captured record lists; +the compiler now preserves them as unresolved review flags instead of silently +omitting them. Explicit empty lists remain distinct from missing or deferred data. +These four records and the separate attachment-flagged article still need source +recovery or disposition. Do not interpret the strings as article IDs or infer +that the attachment flag enumerates all resources. The supplemental report is +`~/.config/coulomb-social/auth/exports/body-resources-20261031.json`; the existing +original and Visual bundle receipts remain unchanged. Public proof: +`docs/release/bubble-body-resources-2026-10-31.json`. diff --git a/docs/release/bubble-body-resources-2026-10-31.json b/docs/release/bubble-body-resources-2026-10-31.json new file mode 100644 index 0000000..273978d --- /dev/null +++ b/docs/release/bubble-body-resources-2026-10-31.json @@ -0,0 +1,98 @@ +{ + "schema": "coulomb.bubble-body-resource-proof.v1", + "source_report_sha256": "75fcd87d17ea0c5042e903efd60e65839da7716e5399ff431a345486ddd10f01", + "visual_report_sha256": "a255df2237d95588b21a5db059487ca0178d1df28c810ebaaf744c5e6491267a", + "source_sha256": { + "articles": "f33967c79c0f8a60a7d504f5d5e4082f8bc9e3c34f6a3160cb200fd380831d62", + "retained_spaces": "92038249ec2e4eed4c8d61ffcdbe030232cebc52e59ba005baee506a16930550", + "spaces": "ae20b0cfd1705f486c0b67ef48ac84b396f953cbcef09f9a8f07d07a66f1bf71" + }, + "captured_articles": 245, + "source_bodies_verified": 242, + "missing_bodies": 3, + "prepared_pages_verified": 236, + "original_draft_files_recompiled_unchanged": 545, + "body_reference_inventory_entries": 18, + "body_reference_records": 1, + "body_reference_classifications": { + "external": 18 + }, + "reference_occurrences_before": 28, + "reference_occurrences_after": 28, + "records_with_changed_reference_attributes": 0, + "unresolved_scrap_relation_records": 4, + "attachment_flag_records": 1, + "live_destinations_checked": false, + "original_bubble_render_compared": false, + "productive_fidelity_accepted": false, + "production_imported": false, + "network_requests": 0, + "private_report_sha256": "4caec573982dafc82e7228d9221c941f585522e235d64d1f6edac3b43913cf26", + "recorded_at": "2026-10-31T14:11:42.653868+00:00", + "tasks": [ + "CSOC-WP-0014-T01", + "CSOC-WP-0013-T02" + ], + "task_statuses_changed": false, + "implementation_files": [ + { + "path": "scripts/verify-bubble-body-resources.py", + "sha256": "8c9a56964ea59422830481d5aacb668e8589f6096ca5130fb6d541edc0ab4d8d" + }, + { + "path": "scripts/verify-bubble-visual-preservation.py", + "sha256": "9fbc212bd668126a0f6ce6a458b1ca3fba1e14d4b44438c0697f63dc34b5bc3d" + }, + { + "path": "coulomb_social/apps/spaces/rendering_audit.py", + "sha256": "093f293f6651a759d77ce7d444ba83fdfaead016b353a167693b2e36c5807894" + }, + { + "path": "coulomb_social/apps/spaces/bubble_bundle.py", + "sha256": "fa928e049ba1052b736b13edf78883b249735312c6b98486e5dd16d4f2c96858" + }, + { + "path": "tests/test_rendering_audit.py", + "sha256": "5e1aa72a322409a9576962c817d8714ee5eed042bbc38f22cb656a8319f52f67" + }, + { + "path": "tests/test_bubble_bundle.py", + "sha256": "2ad0453be2c1c1643d4ef6af2e40dc270f70f863522aa221d1d4bb018a4f1f59" + }, + { + "path": "tests/test_bubble_body_resources.py", + "sha256": "b3c420b62bceb253cd7228f3005b235c2d3bfd170136b960968311eb8fe48077" + } + ], + "validation": { + "focused_tests_passed": 24, + "full_gate": "make check passed", + "current_verifier_reproduces_private_report": true, + "all_original_draft_bytes_verified": 545, + "junit": { + "tests": 595, + "errors": 0, + "failures": 0, + "skipped": 0 + }, + "focused_result_source": "full acceptance gate JUnit" + }, + "statehub_readback": { + "tasks": [ + { + "task_id": "0002882d-f156-5a5d-b524-d04793f4684d", + "status": "progress", + "body_resource_evidence_synced": true + }, + { + "task_id": "8d06cdee-e0ce-5c3e-bf98-705deed0ee62", + "status": "progress", + "body_resource_evidence_synced": true + } + ], + "fix_consistency_exit_code": 0, + "automation_errors": 0, + "assessment_failures": 0, + "warnings": 7 + } +} diff --git a/docs/usecases/sharing-portability-review.md b/docs/usecases/sharing-portability-review.md index fe86fa8..a7c7f68 100644 --- a/docs/usecases/sharing-portability-review.md +++ b/docs/usecases/sharing-portability-review.md @@ -148,3 +148,14 @@ proves appearance fidelity to Bubble. Use these retained local assets for the source/product comparison after each mapping is ready. Graph exceptions, grants, 18body links and one attachment flag remain explicit. No productive import or user verdict is inferred. Evidence: ../release/bubble-visual-preparation-2026-10-31.json. + +### October31 body-resource follow-through + +The later offline check verifies exact destination preservation for all 18 body-link +entries and 28 occurrences through the product renderer. It does not contact those +destinations or accept original appearance. Four deferred scrap relation fields +are now explicitly unresolved alongside the separate attachment flag; they were +present in raw source but omitted from the prior review flags. They must be +captured or given an explicit disposition before migration can be accepted. +Private supplemental report: `~/.config/coulomb-social/auth/exports/body-resources-20261031.json`. +Public proof: `../release/bubble-body-resources-2026-10-31.json`. diff --git a/scripts/verify-bubble-body-resources.py b/scripts/verify-bubble-body-resources.py new file mode 100644 index 0000000..07bc0f4 --- /dev/null +++ b/scripts/verify-bubble-body-resources.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Verify retained body references through the product renderer, without network or import.""" + +import argparse +import hashlib +import importlib.util +import json +import os +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +os.environ['DJANGO_SETTINGS_MODULE'] = 'coulomb_social.settings.test' +import django # noqa: E402 + +django.setup() +from coulomb_social.apps.spaces.bubble_bundle import prepare_bundle # noqa: E402 +from coulomb_social.apps.spaces.rendering_audit import audit_body # noqa: E402 + +# Same bounded, owned-file and no-symlink reader used for the independent +# Visual-only preservation proof. It performs no initialization or network I/O. +SPEC = importlib.util.spec_from_file_location( + 'visual_preservation', ROOT / 'scripts/verify-bubble-visual-preservation.py') +reader = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(reader) + + +def sha(raw): + return hashlib.sha256(raw).hexdigest() + + +def verify(original, prepared): + raw_report = reader.read(original, 'reconciliation.json') + before = json.loads(raw_report) + visual_raw = reader.read(prepared, 'reconciliation.json') + visual = json.loads(visual_raw) + if (before['schema'] != 'coulomb.bubble-review-bundle.v1' + or before['status'] != 'completed_offline_preparation' + or visual['original_report_sha256'] != sha(raw_report)): + raise ValueError('source report mismatch') + inputs = {} + for name in ['spaces', 'articles', 'retained_spaces']: + raw = reader.read(original, 'source-capture/' + name + '.json') + if sha(raw) != before['source_sha256'][name]: + raise ValueError('source digest mismatch') + inputs[name] = json.loads(raw) + fresh, files = prepare_bundle(inputs['spaces'], inputs['articles'], + inventory_spaces=inputs['retained_spaces']) + # New report guards must not alter the original draft content or sidecars. + if {name: sha(raw) for name, raw in files.items()} != before['file_sha256']: + raise ValueError('draft file transformation changed') + for name, raw in files.items(): + if reader.read(original, name) != raw: + raise ValueError('original draft file drift') + bodies = {} + for article in fresh['articles']: + if article['body_file']: + raw = reader.read(original, article['body_file']) + if sha(raw) != article['body_sha256']: + raise ValueError('body sidecar drift') + bodies[article['id']] = raw + prepared_pages = 0 + for space in visual['spaces']: + for page in space['articles']: + if page['file']: + raw = reader.read(prepared, page['file']) + if sha(raw) != visual['file_sha256'][page['file']]: + raise ValueError('prepared page drift') + if raw.split(b'\n---\n', 1)[1] != bodies[page['id']]: + raise ValueError('prepared source body changed') + prepared_pages += 1 + details = [] + for article in fresh['articles']: + detail = {'id': article['id'], 'body_present': article['id'] in bodies, + 'references': [r for r in article['references'] if r['kind'] != 'visual'], + 'review_flags': article['review_flags']} + if 'scrap_relation' in article: + detail['scrap_relation'] = article['scrap_relation'] + if detail['body_present']: + detail['rendering'], _ = audit_body(bodies[article['id']].decode()) + details.append(detail) + rendered = [a['rendering'] for a in details if a['body_present']] + refs = [r for a in details for r in a['references']] + summary = { + 'schema': 'coulomb.bubble-body-resource-proof.v1', + 'source_report_sha256': sha(raw_report), 'visual_report_sha256': sha(visual_raw), + 'source_sha256': before['source_sha256'], + 'captured_articles': len(details), 'source_bodies_verified': len(bodies), + 'missing_bodies': len(details) - len(bodies), 'prepared_pages_verified': prepared_pages, + 'original_draft_files_recompiled_unchanged': len(files), + 'body_reference_inventory_entries': len(refs), + 'body_reference_records': sum(bool(a['references']) for a in details), + 'body_reference_classifications': dict(Counter(r['classification'] for r in refs)), + 'reference_occurrences_before': sum(r['reference_occurrences_before'] for r in rendered), + 'reference_occurrences_after': sum(r['reference_occurrences_after'] for r in rendered), + 'records_with_changed_reference_attributes': sum( + not r['reference_attributes_preserved'] for r in rendered), + 'unresolved_scrap_relation_records': sum( + a.get('scrap_relation', {}).get('state') == 'unresolved' for a in details), + 'attachment_flag_records': sum('attachment_records_require_capture' in a['review_flags'] + for a in details), + 'live_destinations_checked': False, 'original_bubble_render_compared': False, + 'productive_fidelity_accepted': False, 'production_imported': False, + 'network_requests': 0, + } + return summary, {'summary': summary, 'records': details, + 'fresh_space_blockers': [{'id': s['id'], 'blockers': s['blockers']} + for s in fresh['spaces']]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('original', type=Path) + parser.add_argument('prepared', type=Path) + parser.add_argument('--output', type=Path, required=True, help='New private JSON outside repo') + args = parser.parse_args() + try: + out = args.output + if (out.exists() or any(p.is_symlink() for p in (out, *out.parents)) + or any(out.resolve().is_relative_to(p.resolve()) + for p in [ROOT, args.original, args.prepared])): + raise ValueError('separate new output required') + info = out.parent.stat() + if info.st_uid != os.getuid() or info.st_mode & 0o077: + raise ValueError('private owned output parent required') + summary, private = verify(args.original, args.prepared) + raw = json.dumps(private, indent=2).encode() + with os.fdopen(os.open(out, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb') as f: + f.write(raw) + f.flush() + os.fsync(f.fileno()) + summary['private_report_sha256'] = sha(raw) + print(json.dumps(summary)) + except Exception: + print('Body resource proof refused; no completeness or acceptance inferred.', + file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_bubble_body_resources.py b/tests/test_bubble_body_resources.py new file mode 100644 index 0000000..de7fb88 --- /dev/null +++ b/tests/test_bubble_body_resources.py @@ -0,0 +1,65 @@ +"""Body-resource evidence must bind both source bytes and the prepared pages.""" + +import hashlib +import importlib.util +import json +from pathlib import Path + +import pytest + +from tests.test_bubble_visuals import PNG, bundle, visuals + +SPEC = importlib.util.spec_from_file_location( + 'body_resources', + Path(__file__).resolve().parents[1] / 'scripts/verify-bubble-body-resources.py') +resources = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resources) + + +def prepared(tmp_path): + source, report, _ = bundle(tmp_path) + raw = json.dumps({'missing': {}}).encode() + visuals.publish(source, 'source-capture/retained_spaces.json', raw) + report['source_sha256']['retained_spaces'] = hashlib.sha256(raw).hexdigest() + (source / 'reconciliation.json').write_text(json.dumps(report)) + out = tmp_path / 'visual' + visuals.capture(source, out, lambda _: (PNG, 'png')) + return source, out + + +def test_resource_proof_preserves_scope_and_never_accepts_productive_fidelity(tmp_path): + source, out = prepared(tmp_path) + summary, private = resources.verify(source, out) + assert summary['captured_articles'] == 3 + assert summary['source_bodies_verified'] == 3 + assert summary['prepared_pages_verified'] == 2 + assert summary['network_requests'] == 0 + assert not summary['productive_fidelity_accepted'] + assert not summary['live_destinations_checked'] + assert not summary['production_imported'] + assert len(private['fresh_space_blockers']) == 2 + + +@pytest.mark.parametrize('defect', ['source-drift', 'sidecar-drift', 'prepared-body-drift', + 'original-report-drift', 'original-draft-drift']) +def test_resource_proof_refuses_source_or_prepared_drift(tmp_path, defect): + source, out = prepared(tmp_path) + if defect == 'source-drift': + (source / 'source-capture/articles.json').write_text('{}') + elif defect == 'sidecar-drift': + next((source / 'source-bodies').glob('*.txt')).write_text('Changed') + elif defect == 'original-draft-drift': + next((source / 'review-trees').glob('*/pages/*.md')).write_text('Changed') + elif defect == 'original-report-drift': + p = source / 'reconciliation.json' + p.write_text(p.read_text() + '\n') + else: + report = json.loads((out / 'reconciliation.json').read_text()) + name = next(p['file'] for s in report['spaces'] for p in s['articles'] if p['file']) + changed = (out / name).read_bytes() + b'Changed' + (out / name).write_bytes(changed) + # Even a matching prepared-file hash cannot authorize a changed source body. + report['file_sha256'][name] = hashlib.sha256(changed).hexdigest() + (out / 'reconciliation.json').write_text(json.dumps(report)) + with pytest.raises(ValueError): + resources.verify(source, out) diff --git a/tests/test_bubble_bundle.py b/tests/test_bubble_bundle.py index a66fae0..4179444 100644 --- a/tests/test_bubble_bundle.py +++ b/tests/test_bubble_bundle.py @@ -101,6 +101,33 @@ def test_destination_names_survive_label_changes_and_additional_records(): assert all(".." not in Path(path).parts for path in files) +@pytest.mark.parametrize('scraps', [None, 'LOOKUP__custom.scrap__1x2', ['1x2'], {}]) +def test_scrap_descriptors_never_become_captured_articles_or_disappear(scraps): + spaces, articles = capture() + articles['r']['_list_custom_scrap'] = scraps + report, files = prepare_bundle(spaces, articles) + detail = next(a for a in report['articles'] if a['id'] == 'r') + assert detail['scrap_relation'] == { + 'raw': scraps, 'state': 'unresolved', 'records_captured': False} + assert 'scrap_relation_records_require_capture' in detail['review_flags'] + assert 'unsupported_content_or_attachments' in report['spaces'][0]['blockers'] + assert len(report['articles']) == 2 + assert all(not entry['import_ready'] for entry in report['spaces']) + assert files[detail['body_file']] == articles['r']['content_text'].encode() + + +def test_explicit_empty_scrap_list_is_distinct_from_missing_or_deferred(): + spaces, articles = capture() + articles['r']['_list_custom_scrap'] = [] + report, _ = prepare_bundle(spaces, articles) + root = next(a for a in report['articles'] if a['id'] == 'r') + child = next(a for a in report['articles'] if a['id'] == 'c') + assert root['scrap_relation']['state'] == 'observed_empty_list' + assert not root['scrap_relation']['records_captured'] + assert 'scrap_relation_records_require_capture' not in root['review_flags'] + assert 'scrap_relation' not in child + + def test_shared_source_reference_never_selects_an_arbitrary_destination(): spaces, articles = capture() spaces["another"] = source(1700000000000, _chunk_custom_article="c") diff --git a/tests/test_rendering_audit.py b/tests/test_rendering_audit.py index 562ae8d..ec7e902 100644 --- a/tests/test_rendering_audit.py +++ b/tests/test_rendering_audit.py @@ -26,6 +26,23 @@ def test_audit_uses_product_renderer_and_reports_custom_tags_attributes_and_sche assert audit["body_sha256"] == hashlib.sha256(body.encode()).hexdigest() assert not audit["original_bubble_render_compared"] assert not audit["productive_fidelity_accepted"] + assert audit["reference_occurrences_before"] == 2 + assert audit["reference_occurrences_after"] == 1 + assert not audit["reference_attributes_preserved"] + assert audit["removed_reference_occurrences"] == 1 + + +def test_link_entities_duplicates_and_relative_assets_preserve_exact_reference_attributes(): + body = ('[first](https://example.invalid/?a=1&b=2#part)\n\n' + '[second](https://example.invalid/?a=1&b=2#part)\n\n' + 'third' + '') + audit, _ = audit_body(body) + assert audit['reference_occurrences_before'] == 4 + assert audit['reference_occurrences_after'] == 4 + assert audit['reference_attributes_preserved'] + assert audit['removed_reference_occurrences'] == 0 + assert audit['added_reference_occurrences'] == 0 def test_serialized_line_break_difference_is_not_reported_as_lost_markup(): diff --git a/workplans/CSOC-WP-0013-sharing-content-hardening.md b/workplans/CSOC-WP-0013-sharing-content-hardening.md index 0f5ad3f..a70ed59 100644 --- a/workplans/CSOC-WP-0013-sharing-content-hardening.md +++ b/workplans/CSOC-WP-0013-sharing-content-hardening.md @@ -166,6 +166,15 @@ of incorrectly describing storage hardening as unreleased. Source/capture proof: docs/release/bubble-source-continuation-2026-10-31.json. Productive UC-003–006 verdicts and remaining content/revision obligations are still open. +2026-10-31 body-resource check: 18 distinct captured link entries and all 28 +occurrences preserve exact reference attributes through the current product +renderer. Verification binds 242 source bodies and 236 prepared article pages; +no remote link was fetched. The compiler now explicitly retains and blocks four +deferred scrap relation fields in addition to the one attachment flag. This +narrows lossiness uncertainty but does not resolve those records or accept the six +markup-change cases, original Bubble appearance or productive content fidelity. +Evidence: `docs/release/bubble-body-resources-2026-10-31.json`. T02 remains progress. + ## T03 — Review productive sharing and personal portability ```task diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 8799485..2f8864d 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -131,6 +131,20 @@ identity mapping, grant proposal or import manifest was inferred. T01 remains progress until source exceptions and the actual identity/grant/assets/link reconciliation are resolved; no extra completion task was added for this audit. +2026-10-31 body-resource follow-through: all 18 distinct captured body-link entries +(28 occurrences on one article) retain exact reference attributes through the +current product Markdown/sanitizer path. All 242 captured bodies and 236 prepared +article pages match source bytes; the compiler reproduces all 545 original draft +files unchanged. Destination availability and original Bubble appearance were not +tested. Four `_list_custom_scrap` fields contain deferred LOOKUP strings rather +than captured record lists. Preparation now retains these values privately and +marks them as unresolved, including an explicit affected-space blocker. They are +separate from the one attachment-flagged article. The immutable original and Visual +bundles remain untouched; the supplemental private report records the new blockers. +Proof: `docs/release/bubble-body-resources-2026-10-31.json`. Actual scrap/attachment +capture or disposition, identity/grants, source exceptions and productive transfer +remain open. T01 stays progress; link serialization proof is not a transfer verdict. + ## T02 — Review remaining journeys and record transfer decisions ```task