diff --git a/coulomb_social/apps/spaces/date_backfill.py b/coulomb_social/apps/spaces/date_backfill.py new file mode 100644 index 0000000..d3e960d --- /dev/null +++ b/coulomb_social/apps/spaces/date_backfill.py @@ -0,0 +1,114 @@ +"""Offline date reconciliation for an explicitly selected existing destination.""" + +from __future__ import annotations + +import re +from collections import Counter + +from .bubble_transfer import _records, normalize_ref, source_timestamp + + +def _source_id(meta, prefix): + identity = meta.get("id", "") + if not isinstance(identity, str) or not identity.startswith(prefix): + return None + return normalize_ref(identity[len(prefix):]) + + +def plan_date_backfill(observation, spaces, articles, selection): + """Propose only SQL/index birthdate correction; never rewrite/import content. + + Provenance on copied pages is not destination authority. Selection must pin + the existing SQL identity, tenant, slug and original source space explicitly. + The observed file hash and SQL date are preconditions for a future executor, + not a claim that a read-only observation is a coherent backup/recovery point. + """ + if not isinstance(observation, dict) or observation.get("schema") != ( + "coulomb.source-date-observation.v1" + ): + raise ValueError("Unknown observation schema") + spaces, articles = _records(spaces), _records(articles) + required = {"space_id", "space_slug", "tenant_id", "source_space_id"} + if not isinstance(selection, dict) or set(selection) != required: + raise ValueError("Explicit destination/source selection required") + if not all(isinstance(value, str) and value for value in selection.values()): + raise ValueError("Nonempty selection required") + if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", selection["space_slug"]): + raise ValueError("Invalid destination slug") + selected = [s for s in observation["spaces"] if s["id"] == selection["space_id"]] + if len(selected) != 1: + raise ValueError("Unambiguous observed SQL identity required") + target = selected[0] + if target["slug"] != selection["space_slug"] or target["tenant_id"] != selection["tenant_id"]: + raise ValueError("Destination identity/tenant/slug mismatch") + indexes = [page for page in target["pages"] if page["slug"] == "index"] + if len(indexes) != 1: + raise ValueError("Existing source-marked index required") + index = indexes[0] + source_id = normalize_ref(selection["source_space_id"]) + if _source_id(index["meta"], "bubble-space:") != source_id or source_id not in spaces: + raise ValueError("Index/source identity mismatch or source record absent") + if not re.fullmatch(r"[a-f0-9]{64}", index["sha256"]): + raise ValueError("Observed index hash required") + raw_created = spaces[source_id].get("Created Date") + created = source_timestamp(raw_created) + before_sql = source_timestamp(target["created_at"]) + changes = [] + blockers = [] + if before_sql != created: + changes.append({ + "operation": "update_space_created_at", "space_id": target["id"], + "tenant_id": target["tenant_id"], "space_slug": target["slug"], + "expected_before": target["created_at"], "value": created.isoformat(), + }) + if "source_created_at" not in index["meta"]: + changes.append({ + "operation": "insert_frontmatter_field", "field": "source_created_at", + "value": raw_created, "file": f"spaces/{target['slug']}/pages/index.md", + "expected_sha256": index["sha256"], + "preserve_all_existing_metadata_and_body_bytes": True, + }) + else: + try: + if source_timestamp(index["meta"]["source_created_at"]) != created: + blockers.append("existing_index_date_conflicts_with_source") + except (ValueError, OverflowError, OSError): + blockers.append("existing_index_date_invalid_requires_review") + + source_ids = [_source_id(p["meta"], "bubble:") for p in target["pages"]] + counts = Counter(identity for identity in source_ids if identity) + pages = [] + for page in target["pages"]: + identity = _source_id(page["meta"], "bubble:") + if page["slug"] == "index": + continue + status = "local_page_preserved" + if identity: + article = articles.get(identity) + if article is None: + status = "source_article_missing" + elif "source_created_at" not in page["meta"]: + status = "article_date_missing" + else: + try: + status = ("article_date_matches" if source_timestamp( + page["meta"]["source_created_at"] + ) == source_timestamp(article.get("Created Date")) else "article_date_conflict") + except (ValueError, OverflowError, OSError): + status = "article_date_invalid" + if status != "article_date_matches": + blockers.append(status) + pages.append({"slug": page["slug"], "source_id": identity, "status": status, + "duplicate_provenance": bool(identity and counts[identity] > 1), + "observed_sha256": page["sha256"], "proposed_changes": []}) + return { + "schema": "coulomb.source-date-backfill-plan.v1", "selection": selection, + "source_created_at_raw": raw_created, "source_created_at": created.isoformat(), + "observed_sql_created_at": target["created_at"], + "index_sha256": index["sha256"], "proposed_changes": changes, "pages": pages, + "blockers": sorted(set(blockers)), "prepared_change_set_valid": not blockers, + "ready_to_apply": False, "production_writes": False, "content_imported": False, + "requires_fresh_paired_backup": True, "requires_writers_quiesced": True, + "requires_live_hash_and_sql_preconditions": True, + "body_bytes_rewritten": False, "memberships_changed": False, + } diff --git a/docs/migration/corpus-import.md b/docs/migration/corpus-import.md index 6ad79d4..afcbf75 100644 --- a/docs/migration/corpus-import.md +++ b/docs/migration/corpus-import.md @@ -152,6 +152,45 @@ links/forms. The original Bubble view opens only through an explicit review link An unchanged sanitizer structure is not an original-rendering comparison or productive fidelity acceptance. Use the UC-003–006 review walkthrough for verdicts. +## Existing accepted content: date reconciliation without replacement + +Use `scripts/plan-source-date-backfill.py` for an explicitly selected **existing** +destination. It plans corrections; it has no execution/import mode: + +```bash +uv run python scripts/plan-source-date-backfill.py /private/fresh-capture \ + --observation /private/current-date-observation.json \ + --selection /private/accepted-space-selection.json --output /private/new-date-plan.json +``` + +The private observation schema is `coulomb.source-date-observation.v1`: `spaces` +contains observed SQL `id`, `slug`, `tenant_id`, `created_at`, and `pages` containing +`slug`, full-file `sha256` and source frontmatter `meta`. Capture it through an +authorized read-only application context without emitting bodies/identities into +logs. This observation is not a quiesced recovery point. The selection must pin +exactly `space_id`, `space_slug`, `tenant_id`, `source_space_id`; neither a title +nor source provenance on a copied page is authority to select a destination. + +The planner normalizes old typed/prefixed Bubble IDs, binds all four inputs by +hash and refuses mismatched SQL/tenant/slug/index/source identities. It compares +article creation dates but never proposes article changes, including when a +copy retains duplicate provenance. Local pages remain excluded. Existing index +date conflicts/malformed values and source/article date gaps require review. +For a missing index field it proposes **inserting only** `source_created_at`, +preserving every existing metadata field and body byte. SQL changes target only +the selected space's `created_at`, with its observed value as a precondition. + +Output is a new mode-0600 JSON outside the checkout. It contains private IDs, +hashes and proposed values; stdout contains counts only. Every plan remains +`ready_to_apply: false`: apply requires a fresh paired rollback/recovery point, +quiesced writers, live file-hash/SQL preconditions and verified postflight. Never +use tree replacement/reimport to perform this backfill. + +The October28 accepted-space observation found 36 imported article dates already +matching fresh source, plus two preserved local pages. Its plan has only the SQL +space date correction and missing index field insertion; no productive writes. +Counts/hashes: `docs/release/accepted-source-date-reconciliation-2026-10-28.json`. + Packaged SQL migration rehearsal (internal disposable PostgreSQL, no production): ```bash diff --git a/docs/release/accepted-source-date-reconciliation-2026-10-28.json b/docs/release/accepted-source-date-reconciliation-2026-10-28.json new file mode 100644 index 0000000..82ab427 --- /dev/null +++ b/docs/release/accepted-source-date-reconciliation-2026-10-28.json @@ -0,0 +1,32 @@ +{ + "blockers": 0, + "body_bytes_rewritten": false, + "date": "2026-10-28", + "duplicate_article_provenance_pages": 0, + "memberships_changed": false, + "observation_writers_quiesced": false, + "page_status_counts": { + "article_date_matches": 36, + "local_page_preserved": 2 + }, + "prepared_change_set_valid": true, + "private_plan_sha256": "44369d5abbf92d3e6d52e909cdc1e1c3a90b8817be7027c3a43846fbfc381ef8", + "production_writes": false, + "proposed_change_counts": { + "insert_frontmatter_field": 1, + "update_space_created_at": 1 + }, + "read_observation_only": true, + "ready_to_apply": false, + "requires_fresh_paired_backup": true, + "requires_live_hash_and_sql_preconditions": true, + "requires_writers_quiesced": true, + "schema": "coulomb.accepted-source-date-reconciliation.v1", + "scope": "accepted weekly-call space", + "source_sha256": { + "articles": "b00f84adb5de60c724b004e3f05d64f44c8f4ca4bd0f5fef826c4e323841e87a", + "observation": "750038002b157db267cadadee6496d3adafd9ba7c238372cffa83327d687280e", + "selection": "21bb3446329c8ab2b93d7b3e59eb2a68b2501237b5621931123787a757681ee9", + "spaces": "ae20b0cfd1705f486c0b67ef48ac84b396f953cbcef09f9a8f07d07a66f1bf71" + } +} diff --git a/docs/release/recovery-custody-review-2026-10-28.md b/docs/release/recovery-custody-review-2026-10-28.md index ddd63f6..b2386e5 100644 --- a/docs/release/recovery-custody-review-2026-10-28.md +++ b/docs/release/recovery-custody-review-2026-10-28.md @@ -5,6 +5,17 @@ 17d0eb67f397e3d1f5b9b63880df5054aa263c5a. Approval is admission, not evidence of provisioned resources or successful recovery. No repeat approval is required. +**Access clarification:** decision 6139e02b-3fb7-4eff-be8a-efe127c45ce5 records the +user's instruction that the attended Scaleway Console login is a one-time +bootstrap. Routine access must use scoped APIs. Establish the publisher, separate +archive reader and custodian escrow API access through governed credential custody; +personal Console state is never a worker/runtime credential or the regular drill +access method. Preserve node/workstation-loss independence for those credentials. +Retire the temporary personal session after native programmatic access and custody +are verified. This does not admit a standing organization-wide operator key or IAM +administration on the publisher. The first login attempt was aborted without +saving state; a fresh attended bootstrap browser is awaiting login confirmation. + WP-0012 cannot accept the first offsite recovery point until its key, version receipt, image/tools and recovery read authority survive loss of both the primary node and this workstation. The scoped publisher and its own pair delivery are diff --git a/scripts/plan-source-date-backfill.py b/scripts/plan-source-date-backfill.py new file mode 100644 index 0000000..e60f8cf --- /dev/null +++ b/scripts/plan-source-date-backfill.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Plan an existing space's original-date correction from private read-only metadata.""" + +import argparse +import hashlib +import json +import os +import sys +from collections import Counter +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from coulomb_social.apps.spaces.date_backfill import plan_date_backfill # noqa: E402 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("capture", type=Path, help="Private spaces.json/articles.json capture") + parser.add_argument("--observation", type=Path, required=True) + parser.add_argument("--selection", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + output = args.output.expanduser().resolve() + repo = Path(__file__).resolve().parents[1] + if output.is_relative_to(repo): + raise ValueError("Output must be private outside checkout") + paths = {"spaces": args.capture / "spaces.json", "articles": args.capture / "articles.json", + "observation": args.observation, "selection": args.selection} + if any(path.resolve() == output for path in paths.values()): + raise ValueError("Output overlaps input") + raw = {name: path.read_bytes() for name, path in paths.items()} + inputs = {name: json.loads(value) for name, value in raw.items()} + plan = plan_date_backfill(inputs["observation"], inputs["spaces"], inputs["articles"], + inputs["selection"]) + plan["source_sha256"] = {name: hashlib.sha256(value).hexdigest() + for name, value in sorted(raw.items())} + fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(plan, stream, indent=2, sort_keys=True) + stream.write("\n") + print(json.dumps({ + "proposed_changes": len(plan["proposed_changes"]), + "page_status_counts": dict(sorted(Counter(p["status"] for p in plan["pages"]).items())), + "blockers": len(plan["blockers"]), "ready_to_apply": False, + "production_writes": False, + })) + except (ValueError, OSError, TypeError, KeyError, AttributeError, OverflowError): + print("Date plan refused; inspect private inputs/output locally.", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_source_date_backfill.py b/tests/test_source_date_backfill.py new file mode 100644 index 0000000..c96067f --- /dev/null +++ b/tests/test_source_date_backfill.py @@ -0,0 +1,114 @@ +"""Synthetic observations only; date plans must preserve local edits/copies.""" + +import copy +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from coulomb_social.apps.spaces.date_backfill import plan_date_backfill + + +def fixtures(): + digest = hashlib.sha256(b"Synthetic edited bytes").hexdigest() + selection = {"space_id": "destination-id", "space_slug": "accepted", + "tenant_id": "tenant:synthetic", "source_space_id": "source"} + pages = [ + {"slug": "index", "sha256": digest, + "meta": {"id": "bubble-space:source", "bubble_type": "space"}}, + {"slug": "notes", "sha256": digest, + "meta": {"id": "bubble:article", "source_created_at": 1700000001000}}, + {"slug": "copy", "sha256": digest, + "meta": {"id": "bubble:article", "source_created_at": 1700000001000}}, + {"slug": "local-notes", "sha256": digest, "meta": {}}, + ] + observation = {"schema": "coulomb.source-date-observation.v1", "spaces": [{ + "id": "destination-id", "slug": "accepted", "tenant_id": "tenant:synthetic", + "created_at": "2026-10-06T12:00:00+00:00", "pages": pages, + }]} + spaces = {"source": {"Created Date": 1700000000123}} + articles = {"article": {"Created Date": 1700000001000}} + return observation, spaces, articles, selection + + +def test_plan_only_changes_index_field_and_sql_date_keeps_copies_edits_and_local_pages(): + data = fixtures() + before = copy.deepcopy(data) + plan = plan_date_backfill(*data) + assert data == before + assert plan["prepared_change_set_valid"] + assert [c["operation"] for c in plan["proposed_changes"]] == [ + "update_space_created_at", "insert_frontmatter_field"] + assert plan["proposed_changes"][1]["preserve_all_existing_metadata_and_body_bytes"] + assert plan["proposed_changes"][0]["value"] == "2023-11-14T22:13:20.123000+00:00" + assert sum(p["duplicate_provenance"] for p in plan["pages"]) == 2 + assert all(not p["proposed_changes"] for p in plan["pages"]) + assert plan["pages"][-1]["status"] == "local_page_preserved" + assert not plan["ready_to_apply"] and not plan["production_writes"] + assert plan["requires_fresh_paired_backup"] and plan["requires_live_hash_and_sql_preconditions"] + + +@pytest.mark.parametrize("field", ["space_id", "space_slug", "tenant_id", "source_space_id"]) +def test_ambiguous_or_wrong_destination_is_not_inferred_from_provenance(field): + observation, spaces, articles, selection = fixtures() + selection[field] = "different" + with pytest.raises(ValueError): + plan_date_backfill(observation, spaces, articles, selection) + + +def test_existing_index_conflict_requires_review_and_never_proposes_overwriting_it(): + observation, spaces, articles, selection = fixtures() + observation["spaces"][0]["pages"][0]["meta"]["source_created_at"] = 1600000000000 + plan = plan_date_backfill(observation, spaces, articles, selection) + assert plan["blockers"] == ["existing_index_date_conflicts_with_source"] + assert not plan["prepared_change_set_valid"] + assert all(c["operation"] != "insert_frontmatter_field" for c in plan["proposed_changes"]) + + +def test_article_date_gap_is_reported_without_rewriting_missing_source_or_local_copies(): + observation, spaces, articles, selection = fixtures() + del observation["spaces"][0]["pages"][1]["meta"]["source_created_at"] + plan = plan_date_backfill(observation, spaces, articles, selection) + assert "article_date_missing" in plan["blockers"] + assert all(not p["proposed_changes"] for p in plan["pages"]) + + +def test_numeric_prefixed_original_index_reference_matches_normalized_source(): + observation, _, articles, selection = fixtures() + identity = "1700000000000x12345678" + selection["source_space_id"] = identity + observation["spaces"][0]["pages"][0]["meta"]["id"] = ( + "bubble-space:1234__LOOKUP__" + identity) + plan = plan_date_backfill(observation, {identity: {"Created Date": 1700000000123}}, + articles, selection) + assert plan["prepared_change_set_valid"] + + +def test_cli_private_plan_hashes_and_no_overwrite_or_checkout_output(tmp_path): + observation, spaces, articles, selection = fixtures() + capture = tmp_path / "capture" + capture.mkdir() + for name, data in [("spaces", spaces), ("articles", articles)]: + (capture / (name + ".json")).write_text(json.dumps(data)) + observed = tmp_path / "observation.json" + observed.write_text(json.dumps(observation)) + selected = tmp_path / "selection.json" + selected.write_text(json.dumps(selection)) + output = tmp_path / "plan.json" + root = Path(__file__).resolve().parents[1] + cmd = [sys.executable, str(root / "scripts/plan-source-date-backfill.py"), str(capture), + "--observation", str(observed), "--selection", str(selected), "--output", str(output)] + result = subprocess.run(cmd, capture_output=True, text=True) + assert result.returncode == 0, result.stderr + assert "destination-id" not in result.stdout + plan = json.loads(output.read_text()) + assert plan["source_sha256"]["observation"] == hashlib.sha256(observed.read_bytes()).hexdigest() + assert os.stat(output).st_mode & 0o777 == 0o600 + assert subprocess.run(cmd, capture_output=True).returncode == 1 + cmd[-1] = str(root / "var/refused-date-plan.json") + assert subprocess.run(cmd, capture_output=True).returncode == 1 + assert not Path(cmd[-1]).exists() diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index ad30a17..881d388 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -82,6 +82,16 @@ now contains exact attended Console resource/boundary steps, including isolated project selection to avoid project-wide read leakage. Native operator access and independent custody/recovery/retention proofs remain required. +User clarified that Console login may be used once more for bootstrap but must +not become later routine access. Decision 6139e02b-3fb7-4eff-be8a-efe127c45ce5: +establish scoped programmatic publisher, archive-reader and custodian escrow +access under governed custody, independent of primary node/workstation loss; +retire the temporary personal Console session after native verification. No +standing organization-wide operator key or publisher IAM administration follows. +Stopped the aborted capture (terminal exit1, no state saved), then opened a fresh +attended browser and requested login confirmation. Routine operations must not +depend on retaining that personal session. No provider resources created yet. + ## T02 — Activate and prove the first encrypted paired recovery point ```task diff --git a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md index 0eae946..9ac2d8a 100644 --- a/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md +++ b/workplans/CSOC-WP-0014-cohort-transfer-bubble-offramp.md @@ -116,12 +116,13 @@ page/chunk source timestamps, requiring them throughout Bubble trees before writes. Tests include missing index/date refusal, timezone-aware/epoch dates, edit/reimport preservation and rollback. Do not invent dates from IDs or mtime. -Still required: deterministic tree conversion/date manifest and per-source-ID -comparison on rehearsal/target; reconcile source date exceptions. Backfill the +Still required: resolve remaining tree/source-date exceptions and verify the +per-source-ID date manifest on the productive target. Backfill the accepted cohort's missing original space date without replacing its content or losing post-transfer edits, copies or aliases. Source provenance on copied local -pages is not authority to overwrite those copies during a bulk reimport. Refresh -the interim c11300a candidate for the stricter date guard before production use. +pages is not authority to overwrite those copies during a bulk reimport. Current +candidate 17d0eb6 includes the stricter date guard (WP-0013 proof); guarded rollout +still requires paired rollback/recovery evidence. Do not close on source-level tests alone. 2026-10-28: Draft indexes keep space creation dates separate from their source @@ -135,6 +136,22 @@ rehearsal embedded in bubble-offline-preparation-2026-10-28.json. This proves draft conversion/import date preservation, not rendering, source completeness, live acceptance or accepted-cohort backfill. Existing live content is untouched. +2026-10-28 accepted-cohort reconciliation: authorized read-only application +metadata observation found 39 pages in the accepted space: 36 imported articles, +its index and two local pages. All 36 article creation dates match fresh source; +no article dates need correction. The old index uses a prefixed LOOKUP source ID, +now correctly normalized to its fresh source space. Its missing source creation +field and SQL insertion-time creation date need two targeted corrections. +Implemented an offline, hash-bound planner requiring explicit SQL/tenant/slug/ +source selection. It proposes only SQL created_at and index-field insertion; +local edits/copies, every body/asset/alias and grants remain untouched. Duplicate +article provenance is audited, never used as overwrite authority. Existing date +conflicts/missing source are review blockers. Actual plan has zero date blockers, +36 matching article dates and two excluded local pages, ready_to_apply false. +Evidence: docs/release/accepted-source-date-reconciliation-2026-10-28.json. +No productive backfill performed; fresh paired rollback/recovery point, +quiescence, live hash/SQL preconditions and postflight remain required. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.