diff --git a/coulomb_social/apps/spaces/content.py b/coulomb_social/apps/spaces/content.py index b463f4f..dbebc5b 100644 --- a/coulomb_social/apps/spaces/content.py +++ b/coulomb_social/apps/spaces/content.py @@ -87,6 +87,15 @@ def page_path_for(space: Space, page_slug: str) -> str: return f"{root}/{filename}" if root else filename +def render_markdown_unfiltered(text: str) -> str: + """Intermediate HTML for sanitization/auditing; never serve this directly.""" + return md_lib.markdown( + text, + extensions=["fenced_code", "tables", "nl2br", "sane_lists"], + output_format="html", + ) + + def render_markdown(text: str) -> str: """Render the supported Markdown subset. @@ -95,13 +104,8 @@ def render_markdown(text: str) -> str: Relative asset URLs are kept so copied page images still resolve. This is not acceptance of a private corpus or of arbitrary Bubble markup. """ - raw_html = md_lib.markdown( - text, - extensions=["fenced_code", "tables", "nl2br", "sane_lists"], - output_format="html", - ) return bleach.clean( - raw_html, + render_markdown_unfiltered(text), tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRS, protocols=_ALLOWED_PROTOCOLS, diff --git a/coulomb_social/apps/spaces/rendering_audit.py b/coulomb_social/apps/spaces/rendering_audit.py new file mode 100644 index 0000000..9886745 --- /dev/null +++ b/coulomb_social/apps/spaces/rendering_audit.py @@ -0,0 +1,103 @@ +"""Private corpus rendering evidence using the actual product renderer.""" + +from __future__ import annotations + +import hashlib +import html +from collections import Counter +from html.parser import HTMLParser +from urllib.parse import quote + +from .content import render_markdown, render_markdown_unfiltered + + +class _Structure(HTMLParser): + def __init__(self, value): + super().__init__(convert_charrefs=True) + self.tags = Counter() + self.attributes = Counter() + self.text = [] + self.feed(value) + + def handle_starttag(self, tag, attrs): + self.tags[tag] += 1 + self.attributes.update((tag, key, value) for key, value in attrs) + + handle_startendtag = handle_starttag + + def handle_data(self, value): + self.text.append(value) + + +def audit_body(body: str): + raw = render_markdown_unfiltered(body) + rendered = render_markdown(body) + before, after = _Structure(raw), _Structure(rendered) + removed_attrs = before.attributes - after.attributes + text_changed = " ".join("".join(before.text).split()) != ( + " ".join("".join(after.text).split())) + markup_changed = before.tags != after.tags or before.attributes != after.attributes + return { + "body_sha256": hashlib.sha256(body.encode()).hexdigest(), + "rendered_sha256": hashlib.sha256(rendered.encode()).hexdigest(), + "sanitization_changed_html": raw != rendered, + "markup_changed_by_sanitization": markup_changed, + "serialization_only_change": raw != rendered and not markup_changed and not text_changed, + "removed_tag_counts": dict(sorted((before.tags - after.tags).items())), + "added_tag_counts": dict(sorted((after.tags - before.tags).items())), + "removed_attributes": [ + {"tag": tag, "attribute": key, "value": value, "count": count} + for (tag, key, value), count in sorted( + removed_attrs.items(), key=lambda item: repr(item[0])) + ], + "visible_text_changed_by_sanitization": text_changed, + "original_bubble_render_compared": False, + "productive_fidelity_accepted": False, + }, rendered + + +def review_card(title: str, body: str, rendered: str, audit: dict, *, source_id="") -> bytes: + # Private review files must not load source images, frames, styles or scripts. + # Inert also prevents links/forms in the rendered sample from being activated. + original = ('

Open original Bubble view (opens only when clicked)

' + ) if source_id else "" + return ("" + '' + "Private rendering review

" + html.escape(title) + "

" + "Product rendering only; compare with the original Bubble view before accepting. " + "Resources and interactive links within the sample are disabled.

" + original + + "

Sanitization changed HTML: " + + str(audit["sanitization_changed_html"]).lower() + + "

Exact captured source

" + html.escape(body)
+            + "

Current product rendering

" + + rendered + "
").encode() + + +def review_index(records: list[dict]) -> bytes: + rows = [] + for record in sorted(records, key=lambda r: ( + not (r.get("markup_changed_by_sanitization") or r["status"] == "body_missing"), r["id"] + )): + label = html.escape(record["title"]) + if record.get("card"): + label = '' + label + "" + status = ("body missing" if record["status"] == "body_missing" else ( + "markup change — review" if record["markup_changed_by_sanitization"] else ( + "text change — review" if record["visible_text_changed_by_sanitization"] + else "no structural sanitizer change; original comparison still required"))) + rows.append("
  • " + label + " — " + status + "
    • ") + return ('' + '' + 'Private corpus rendering review' + '

    Private corpus rendering review

    Compare each card with the original ' + 'Bubble view. Sanitizer differences are observations, not acceptance verdicts. ' + 'Body-missing and markup-change records come first.

      ' + + "".join(rows) + '
    ').encode() diff --git a/docs/migration/corpus-import.md b/docs/migration/corpus-import.md index eb312cc..6ad79d4 100644 --- a/docs/migration/corpus-import.md +++ b/docs/migration/corpus-import.md @@ -136,6 +136,22 @@ existing destinations and explicit tenant/identity/grants are reconciled. Keep all drafts private. Do not feed the whole bundle to `import_content_corpus` or replace a live tree to backfill dates. +Prepare private source/product rendering cards without contacting Bubble: + +```bash +uv run python scripts/audit-bubble-rendering.py /private/checkpoint/articles.json \ + --output /private/new-rendering-review +``` + +The tool invokes the actual product renderer under integration-disabled test +settings. It records body/render/card hashes and structural sanitizer changes, +retains missing-body records, and writes private `index.html`/cards plus a final +`audit.json` completion receipt. Output is new, outside the checkout, mode 0700 +with files mode 0600. Samples have a restrictive resource policy and inert body +links/forms. The original Bubble view opens only through an explicit review link. +An unchanged sanitizer structure is not an original-rendering comparison or +productive fidelity acceptance. Use the UC-003–006 review walkthrough for verdicts. + Packaged SQL migration rehearsal (internal disposable PostgreSQL, no production): ```bash diff --git a/docs/release/corpus-rendering-audit-2026-10-28.json b/docs/release/corpus-rendering-audit-2026-10-28.json new file mode 100644 index 0000000..f5a0d4b --- /dev/null +++ b/docs/release/corpus-rendering-audit-2026-10-28.json @@ -0,0 +1,27 @@ +{ + "captured_records": 244, + "date": "2026-10-28", + "markup_changed_by_sanitization": 6, + "missing_bodies": 3, + "original_bubble_render_compared": false, + "previous_renderer_comparison_bodies": 241, + "previous_renderer_output_mismatches": 0, + "previous_renderer_source_commit": "ed2c8e2f4d3882b2933aa8b57f82af5654286de0", + "private_audit_sha256": "8363c619bfaf663f502a80f94108d3a4d5e26c997c51579c090bc8fea145a744", + "production_content_modified": false, + "productive_fidelity_accepted": false, + "removed_attributes_total": 17, + "removed_tags_total": 3, + "rendered_bodies": 241, + "renderer_policy_unchanged": true, + "sanitization_changed_html": 6, + "schema": "coulomb.corpus-rendering-summary.v1", + "serialization_only_changes": 0, + "source_sha256": "b00f84adb5de60c724b004e3f05d64f44c8f4ca4bd0f5fef826c4e323841e87a", + "visible_text_changed_by_sanitization": 0, + "browser_card_check": { + "review_cards_opened": 6, + "external_request_attempts": 0, + "card_samples_inert": true + } +} diff --git a/docs/release/recovery-custody-review-2026-10-28.md b/docs/release/recovery-custody-review-2026-10-28.md new file mode 100644 index 0000000..7f0ef07 --- /dev/null +++ b/docs/release/recovery-custody-review-2026-10-28.md @@ -0,0 +1,41 @@ +# Coulomb independent recovery custody — review packet + +WP-0012 cannot accept the first offsite recovery point until its key, version +receipt, image/tools and recovery read authority survive loss of both the primary +node and this workstation. The scoped publisher and its own pair delivery are +already admitted under decision 9557e2b8-2b37-5b94-a5e6-e5fe08d0351e; this packet +does not reopen that approval or the daily 03:15 UTC schedule. + +## Proposed additional admission + +| Boundary | Concrete proposed result | +| --- | --- | +| Owner | Railiance-platform, with platform-root as attended recovery custodian and failure recipient, matching the existing owner-operated recovery mechanism. | +| Application scope | Only `platform-pg/application-archives/coulomb-social/` in the existing primary bucket; no email-connect or native database archive access. | +| Recovery reader | A separately held provider caller allowed only exact-prefix/version reads and necessary listing. Prove denied creation, deletion/version deletion, sibling access and IAM/bucket administration. Do not reuse the scheduled publisher pair as the independent reader. | +| Encryption | A dedicated Coulomb age recipient/identity, generated and retained through platform-owned custody. The worker receives only the public recipient. Do not retrieve or reuse email-connect's private identity or reader. | +| Independent kit | A Coulomb-only protected location outside the primary node/workstation stores the private identity under custodian access, exact image/recovery tools and public version/hash/size receipts. The publisher cannot read escrow or alter/delete protected recovery material. | +| Retention | Rolling archives: retain the existing proposed 14 daily/4 weekly selections only after lifecycle/headroom compatibility is proved. Store the exact last successfully recovered point and its kit outside rolling expiration; protected kit/point has no automatic expiry until replaced and verified. | +| Local historical copies | Preserve existing private production-pair/drill copies. Inventory their locations/counts privately; obtain explicit owner disposition rather than deleting them as part of activation. | +| Acceptance | Native positive/negative caller checks, independently retrieved key/kit/receipt, exact-version archive retrieval, decryption and isolated app/content/membership/denied-read rehearsal, measured timing and received failure/stale evidence. Only then activate timers. | + +This adds recovery custody and read authority; the previous publisher approval +does not itself admit these boundaries. No new identity, provider grant, key, +protected location, capture/upload, timer or lifecycle has been created by this +packet. No credentials belong in a reply. + +## Execution after admission + +Reuse RPF-WP-0053-T02/T03/T04 and its governed platform custody front door. +Record exact owner approval and source references before provisioning/delivery. +Choose and record actual provider/project/protected-prefix coordinates through +that owner, verify their boundaries, then bind the public recipient and recovery +kit to the consumer. Do not infer a current grant from the older generic Nextcloud +route or from the separately accepted email-ledger recovery. + +Run the already admitted coherent first capture/publication with the application +resumed before transfer. Retain its exact object version/hash receipt independently, +retrieve through the new reader, decrypt only in the custodian environment and +restore into an isolated instance. No production restore is included. Refresh +the snapshot helper's pinned image after any later app release, and repeat the +paired recovery proof when its schema changes materially. diff --git a/docs/usecases/sharing-portability-review.md b/docs/usecases/sharing-portability-review.md index 8e97c2a..d5c8004 100644 --- a/docs/usecases/sharing-portability-review.md +++ b/docs/usecases/sharing-portability-review.md @@ -18,3 +18,25 @@ For each journey reply `accepted`, `gap` or `deferred`, with a short explanation An observed gap becomes a live task before that journey is accepted. Weekly-call acceptance remains UC-008 only. A source-date check for newly migrated pages and all their chunks is also required under the user's October 27 migration instruction. + +## Private corpus rendering cards — 2026-10-28 + +`scripts/audit-bubble-rendering.py` now prepares private source/product cards and +an index using the actual product renderer. The current capture has 244 article +records: 241 bodies render and three bodies are missing. Six records change +markup under sanitization (three removed custom tags and 17 removed attributes +in total); no extracted-text change was observed between intermediate Markdown +HTML and sanitized HTML. These counts do not compare with Bubble's own rendering +and do not supply an acceptance verdict. Source text/bytes remain preserved. + +In the private index, review the six markup-change cards first. Each shows the +exact captured source beside the current product rendering and provides an +explicit link to the original Bubble view. Compare the same record there: +meaning and placeholders, line breaks, lists/tables, Visual and any body-linked +resources. Record `accepted`, `gap` or `deferred` per record, keeping its private +source ID in the private report. Open gaps remain WP-0013-T02; resolve or explicitly +accept intended differences before marking the corresponding cohort transferred. +Review the body-missing records as source exceptions; never substitute an empty +page. Card samples disable resource loading and interaction, so image/resource +fidelity needs the separate authorized asset review. Then apply UC-003–006 above +to disposable pages on the exact intended release. diff --git a/scripts/audit-bubble-rendering.py b/scripts/audit-bubble-rendering.py new file mode 100644 index 0000000..571cd74 --- /dev/null +++ b/scripts/audit-bubble-rendering.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Offline private rendering cards and lossiness report; never grants acceptance.""" + +import argparse +import hashlib +import json +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +# The standalone tool uses an integration-disabled, in-memory configuration. +os.environ["DJANGO_SETTINGS_MODULE"] = "coulomb_social.settings.test" +import django # noqa: E402 + +django.setup() + +from coulomb_social.apps.spaces.bubble_transfer import _records # noqa: E402 +from coulomb_social.apps.spaces.rendering_audit import ( # noqa: E402 + audit_body, + review_card, + review_index, +) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("articles", type=Path, help="Private captured articles.json") + parser.add_argument("--output", type=Path, required=True, help="New private directory") + args = parser.parse_args() + try: + output = args.output.expanduser().resolve() + source = args.articles.expanduser().resolve() + repo = Path(__file__).resolve().parents[1] + if output.is_relative_to(repo) or source.is_relative_to(output): + raise ValueError("Unsafe output location") + captured = source.read_bytes() + records = _records(json.loads(captured)) + report = {"schema": "coulomb.bubble-rendering-audit.v1", + "source_sha256": hashlib.sha256(captured).hexdigest(), + "original_bubble_render_compared": False, + "productive_fidelity_accepted": False, "records": []} + cards = {} + for identity, article in sorted(records.items()): + body = article.get("content_text") + title = str(article.get("nomer_text") or identity) + if not isinstance(body, str): + report["records"].append({"id": identity, "title": title, "status": "body_missing"}) + continue + audit, rendered = audit_body(body) + card = hashlib.sha256(identity.encode()).hexdigest() + ".html" + cards[card] = review_card(title, body, rendered, audit, source_id=identity) + report["records"].append({"id": identity, "title": title, + "status": "rendered", "card": card, + "card_sha256": hashlib.sha256(cards[card]).hexdigest(), + **audit}) + output.mkdir(mode=0o700) + for name, data in sorted({**cards, "index.html": review_index(report["records"])}.items()): + fd = os.open(output / name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as stream: + stream.write(data) + report["status"] = "completed_offline_rendering_audit" + fd = os.open(output / "audit.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(report, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({ + "records": len(records), "rendered_bodies": len(cards), + "sanitization_changed_html": sum(r.get("sanitization_changed_html", False) + for r in report["records"]), + "markup_changed_by_sanitization": sum(r.get("markup_changed_by_sanitization", False) + for r in report["records"]), + "serialization_only_changes": sum(r.get("serialization_only_change", False) + for r in report["records"]), + "productive_fidelity_accepted": False, + })) + except (ValueError, OSError, TypeError): + print("Rendering audit refused; private files retained. " + "Without audit.json output is incomplete.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_rendering_audit.py b/tests/test_rendering_audit.py new file mode 100644 index 0000000..562ae8d --- /dev/null +++ b/tests/test_rendering_audit.py @@ -0,0 +1,68 @@ +"""Rendering review uses synthetic notes and never accepts private corpus fidelity.""" + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +from coulomb_social.apps.spaces.content import render_markdown +from coulomb_social.apps.spaces.rendering_audit import audit_body, review_card + + +def test_audit_uses_product_renderer_and_reports_custom_tags_attributes_and_schemes(): + body = ('# Synthetic\n\nPlaceholder
    Text\n' + '' + 'Unsafe') + audit, rendered = audit_body(body) + assert rendered == render_markdown(body) + assert audit["sanitization_changed_html"] + assert audit["markup_changed_by_sanitization"] + assert not audit["serialization_only_change"] + assert audit["removed_tag_counts"] == {"topic": 1} + assert {a["attribute"] for a in audit["removed_attributes"]} == {"onerror", "href"} + assert "assets/picture.png" in rendered and "
    " in rendered + assert audit["body_sha256"] == hashlib.sha256(body.encode()).hexdigest() + assert not audit["original_bubble_render_compared"] + assert not audit["productive_fidelity_accepted"] + + +def test_serialized_line_break_difference_is_not_reported_as_lost_markup(): + audit, _ = audit_body("First
    Second") + assert audit["serialization_only_change"] + assert not audit["markup_changed_by_sanitization"] + assert not audit["visible_text_changed_by_sanitization"] + + +def test_review_card_escapes_source_and_blocks_resources_and_interaction(): + body = '' + audit, rendered = audit_body(body) + card = review_card("", body, rendered, audit, + source_id='a/"?b').decode() + assert "