From 54eef1c4b564b14af02208261ea8a5a133172969 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 30 Oct 2026 20:39:45 +0100 Subject: [PATCH] Export tenant records and retained content with schema-derived privacy inventory Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3 --- coulomb_social/apps/members/client_export.py | 273 ++++++++++++++++++ .../apps/members/management/__init__.py | 0 .../members/management/commands/__init__.py | 0 .../commands/export_client_dataset.py | 38 +++ ...ient-data-export-rehearsal-2026-10-30.json | 41 +++ .../client-data-export-review-2026-10-30.md | 92 ++++++ ...service-contract-conformance-2026-10-27.md | 2 +- scripts/rehearse-client-export.py | 154 ++++++++++ tests/test_client_export.py | 174 +++++++++++ ...0012-durable-recovery-service-readiness.md | 16 + 10 files changed, 789 insertions(+), 1 deletion(-) create mode 100644 coulomb_social/apps/members/client_export.py create mode 100644 coulomb_social/apps/members/management/__init__.py create mode 100644 coulomb_social/apps/members/management/commands/__init__.py create mode 100644 coulomb_social/apps/members/management/commands/export_client_dataset.py create mode 100644 docs/release/client-data-export-rehearsal-2026-10-30.json create mode 100644 docs/release/client-data-export-review-2026-10-30.md create mode 100644 scripts/rehearse-client-export.py create mode 100644 tests/test_client_export.py diff --git a/coulomb_social/apps/members/client_export.py b/coulomb_social/apps/members/client_export.py new file mode 100644 index 0000000..90a8d5e --- /dev/null +++ b/coulomb_social/apps/members/client_export.py @@ -0,0 +1,273 @@ +"""Operator client-data export from an externally quiesced database/content pair. + +This is separate from personal portability. It grants no tenant access, performs +no deletion, and does not prove the operator's source snapshot/fence admission. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import tempfile +import zipfile +from pathlib import Path + +from django.apps import apps +from django.contrib.admin.models import LogEntry +from django.contrib.auth.models import Group, Permission +from django.contrib.contenttypes.models import ContentType +from django.contrib.sessions.models import Session +from django.core.serializers.json import DjangoJSONEncoder +from django.db import connection, transaction + +from coulomb_social.apps.spaces.models import Space, SpaceMembership + +from .models import Member, User + +# Explicit coverage makes a new field/model a review requirement instead of +# silently exporting a new credential or silently omitting new client data. +FIELDS = { + "members.user": { + "identity": "id username email full_name", + "activity": "last_login date_joined", + "access": "is_active is_staff is_superuser groups user_permissions", + "credential_excluded": "password", + }, + "members.member": { + "identity": "id tenant_id user user_engine_user_id issuer subject display_name email", + "activity": "created_at updated_at last_login_at", + }, + "spaces.space": { + "client_content": "id tenant_id slug title description visual content_root", + "identity": "created_by", + "activity": "created_at updated_at", + "access": "is_active", + "external_reference": "forgejo_owner forgejo_repo default_branch", + }, + "spaces.spacemembership": { + "identity": "id member", + "access": "space role", + "activity": "created_at", + }, +} + + +class ExportRefused(ValueError): + pass + + +def schema_inventory(): + models = {model._meta.label_lower: model for model in apps.get_models() + if model.__module__.startswith("coulomb_social.")} + if set(models) != set(FIELDS): + raise ExportRefused("Unclassified application model") + result = [] + for label, model in sorted(models.items()): + classified = {field: category for category, names in FIELDS[label].items() + for field in names.split()} + fields = [*model._meta.concrete_fields, *model._meta.many_to_many] + if set(classified) != {field.name for field in fields}: + raise ExportRefused("Unclassified application field") + result.append({"model": label, "table": model._meta.db_table, "fields": [ + {"name": field.name, "column": field.column, "type": field.get_internal_type(), + "category": classified[field.name], "nullable": field.null, + "related_model": field.related_model._meta.label_lower + if field.is_relation else None} for field in fields]}) + return { + "schema": "coulomb.client-data-inventory.v1", "models": result, + "unstructured_personal_data": [ + "Page metadata and bodies, assets and filenames can contain arbitrary personal data.", + "Immutable revisions retain prior and deleted page content.", + "Aliases retain prior page addresses; admin logs retain operator/object descriptions.", + ], + "operational_stores": { + "django_session": "Credential-bearing signed sessions and OIDC state; never exported.", + "django_admin_log": "Export only events attributable to selected client objects.", + "process_logs": "Operator retention applies; outside application data export.", + "backup_copies": "Independent retention/expiry requires operator disposition.", + }, + "operational_schema": [ + {"model": model._meta.label_lower, "table": model._meta.db_table, + "fields": [{"name": f.name, "column": f.column, + "type": f.get_internal_type(), "nullable": f.null} + for f in model._meta.concrete_fields]} + for model in (Session, LogEntry, Group, Permission, ContentType, + User.groups.through, User.user_permissions.through, + Group.permissions.through) + ], + } + + +def _hash_stream(stream): + digest = hashlib.sha256() + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _hash_file(path): + with path.open("rb") as stream: + return _hash_stream(stream) + + +def _rows(queryset, *, exclude=()): + fields = [f.attname for f in queryset.model._meta.concrete_fields if f.name not in exclude] + return list(queryset.order_by("pk").values(*fields)) + + +def _regular_tree(root): + if any(p.is_symlink() for p in [root, *root.parents]) or not root.is_dir(): + raise ExportRefused("Real content root required") + files = {} + for path in sorted(root.rglob("*")): + mode = path.lstat().st_mode + if stat.S_ISDIR(mode): + continue + if not stat.S_ISREG(mode) or path.stat().st_nlink != 1: + raise ExportRefused("Content contains a link or special file") + files[path.relative_to(root).as_posix()] = path + return files + + +def _content_files(root, all_spaces, selected): + files = _regular_tree(root) + known = {s.slug for s in all_spaces} + if len(known) != len(all_spaces): + raise ExportRefused("Ambiguous storage slug") + for space in all_spaces: + if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", space.slug): + raise ExportRefused("Invalid storage slug") + own = {s.slug for s in selected} + exported = {} + for name, path in files.items(): + parts = Path(name).parts + if name == ".writer.lock": + continue + if parts[0] == ".recovery": + raise ExportRefused("Pending content journal requires reconciliation") + if len(parts) < 3 or parts[0] not in {"spaces", ".revisions"} or parts[1] not in known: + raise ExportRefused("Unattributed content requires owner reconciliation") + if parts[1] in own: + exported[name] = path + for space in selected: + if space.has_content_binding and not any( + name.startswith(f"spaces/{space.slug}/pages/") for name in exported): + raise ExportRefused("External-only content must be captured before export") + return exported + + +def _data(tenant): + members = Member.objects.filter(tenant_id=tenant) + spaces = Space.objects.filter(tenant_id=tenant) + if not members.exists() and not spaces.exists(): + raise ExportRefused("Unknown client tenant") + if (Space.objects.filter(tenant_id=tenant, created_by__isnull=False).exclude( + created_by__tenant_id=tenant).exists() + or Space.objects.filter(created_by__tenant_id=tenant).exclude(tenant_id=tenant).exists() + or SpaceMembership.objects.filter(space__tenant_id=tenant).exclude( + member__tenant_id=tenant).exists() + or SpaceMembership.objects.filter(member__tenant_id=tenant).exclude( + space__tenant_id=tenant).exists()): + raise ExportRefused("Cross-tenant relationship requires reconciliation") + users = User.objects.filter(member__tenant_id=tenant) + if any(user.has_usable_password() for user in users): + raise ExportRefused("Unexpected local credential requires operator review") + memberships = SpaceMembership.objects.filter(space__tenant_id=tenant) + querysets = [users, members, spaces, memberships] + objects = {qs.model._meta.label_lower: {str(pk) for pk in qs.values_list("pk", flat=True)} + for qs in querysets} + logs = [] + for entry in LogEntry.objects.select_related("content_type").order_by("pk"): + label = (f"{entry.content_type.app_label}.{entry.content_type.model}" + if entry.content_type else None) + if entry.object_id in objects.get(label, set()): + logs.append({"id": entry.pk, "action_time": entry.action_time, + "actor_user_id": entry.user_id, "target_model": label, + "object_id": entry.object_id, "object_repr": entry.object_repr, + "action_flag": entry.action_flag, "change_message": entry.change_message}) + elif str(entry.user_id) in objects["members.user"]: + model = apps.get_model(label) if label in objects else None + if model is None or not model.objects.filter(pk=entry.object_id).exists(): + raise ExportRefused("Unattributed client admin history requires reconciliation") + # Known other-client target: its content is not this client's data. + def permissions(qs): + return list(qs.order_by("content_type__app_label", "codename").values( + "content_type__app_label", "content_type__model", "codename", "name")) + access = [{"user_id": user.pk, "direct_permissions": permissions(user.user_permissions.all()), + "groups": [{"name": group.name, "permissions": permissions(group.permissions.all())} + for group in user.groups.order_by("pk")]} for user in users] + return {"records": {qs.model._meta.label_lower: _rows( + qs, exclude=("password",) if qs.model is User else ()) for qs in querysets}, + "user_authorization": access, "admin_history": logs}, list(spaces) + + +def export_client_dataset(tenant, root, destination): + """Write a private portable ZIP; callers must supply a quiesced source pair. + + SQL repeatable-read and before/after file hashes detect ordinary drift, but + cannot establish an external writer fence or recover missing remote content. + """ + inventory = schema_inventory() + root, destination = Path(root).absolute(), Path(destination).absolute() + if (not tenant or not isinstance(tenant, str) or any(p.is_symlink() for p in [ + destination, *destination.parents]) or destination.exists() + or destination.resolve().is_relative_to(root.resolve()) + or destination.parent.stat().st_mode & 0o077): + raise ExportRefused("Fresh output in a private directory outside content required") + with transaction.atomic(): + if connection.vendor == "postgresql": + with connection.cursor() as cursor: + cursor.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + data, selected = _data(tenant) + all_spaces = list(Space.objects.all()) + files = _content_files(root, all_spaces, selected) + before = {name: _hash_file(path) for name, path in files.items()} + manifest = { + "schema": "coulomb.client-dataset-export.v1", "tenant_id": tenant, + "inventory": inventory, "data": data, "files_sha256": before, + "external_bindings": [str(s.pk) for s in selected if s.has_content_binding], + "source_coherence": "Caller must verify externally quiesced source pair.", + "scope": "Client application records and all locally retained content/revisions.", + "excluded": ["password sentinels", "session credentials and OIDC transient state", + "other tenants", "operator process logs", "independent backups"], + "deletion_performed": False, "offboarding_complete": False, + } + fd, temporary = tempfile.mkstemp(prefix=".client-export-", dir=destination.parent) + try: + with os.fdopen(fd, "w+b") as stream: + with zipfile.ZipFile(stream, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("manifest.json", json.dumps( + manifest, cls=DjangoJSONEncoder, sort_keys=True, indent=2)) + for name, path in files.items(): + archive.write(path, "content/" + name) + stream.flush() + os.fsync(stream.fileno()) + after_files = _content_files(root, all_spaces, selected) + after = {name: _hash_file(path) for name, path in after_files.items()} + # Verify the bytes actually archived, not only the source hashes on + # either side of a concurrent read/write. + with zipfile.ZipFile(temporary) as archive: + archived = {} + for name in before: + with archive.open("content/" + name) as stream: + archived[name] = _hash_stream(stream) + if before != after or before != archived: + raise ExportRefused("Content changed during export") + # Link publication is exclusive; replacement could overwrite a + # pre-existing private export created by another operator process. + os.link(temporary, destination) + directory = os.open(destination.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + os.unlink(temporary) + return {"schema": manifest["schema"], "content_files": len(files), + "records": {label: len(rows) for label, rows in data["records"].items()}, + "admin_events": len(data["admin_history"]), + "archive_sha256": _hash_file(destination), + "offboarding_complete": False} diff --git a/coulomb_social/apps/members/management/__init__.py b/coulomb_social/apps/members/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/coulomb_social/apps/members/management/commands/__init__.py b/coulomb_social/apps/members/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/coulomb_social/apps/members/management/commands/export_client_dataset.py b/coulomb_social/apps/members/management/commands/export_client_dataset.py new file mode 100644 index 0000000..94f9764 --- /dev/null +++ b/coulomb_social/apps/members/management/commands/export_client_dataset.py @@ -0,0 +1,38 @@ +"""Private operator export, separate from an individual member's JSON download.""" + +import json +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from coulomb_social.apps.members.client_export import export_client_dataset, schema_inventory + + +class Command(BaseCommand): + help = ("Export a client's local records/content from an externally quiesced source pair. " + "No teardown, backup expiry or remote-content retrieval is performed.") + + def add_arguments(self, parser): + parser.add_argument("--inventory", action="store_true", + help="Print schema-derived inventory only; no record values.") + parser.add_argument("--tenant") + parser.add_argument("--output", type=Path) + parser.add_argument("--quiesced-source", action="store_true", + help="Operator assertion, not verified admission/fence evidence.") + + def handle(self, *args, **options): + try: + if options["inventory"]: + if options["output"] or options["tenant"]: + raise ValueError("Inventory does not accept export destinations") + self.stdout.write(json.dumps(schema_inventory(), indent=2)) + return + output = options["output"] + if (not output or not options["tenant"] or not options["quiesced_source"] + or output.resolve().is_relative_to(Path(settings.BASE_DIR).resolve())): + raise ValueError("Private output outside checkout and quiesced source required") + result = export_client_dataset(options["tenant"], settings.CONTENT_ROOT, output) + self.stdout.write(json.dumps(result, sort_keys=True)) + except (OSError, ValueError): + raise CommandError("Client export refused; inspect private source conditions locally.") diff --git a/docs/release/client-data-export-rehearsal-2026-10-30.json b/docs/release/client-data-export-rehearsal-2026-10-30.json new file mode 100644 index 0000000..a0efdfe --- /dev/null +++ b/docs/release/client-data-export-rehearsal-2026-10-30.json @@ -0,0 +1,41 @@ +{ + "status": "recovered_client_export_verified", + "schema": "coulomb.client-export-rehearsal.v1", + "content_files": 81, + "records": { + "members.user": 1, + "members.member": 1, + "spaces.space": 1, + "spaces.spacemembership": 1 + }, + "admin_events": 0, + "archive_sha256": "eca30e86b1a8783bc6cf70bcc11714a61457a90a624d99080f228ad11e2f76e2", + "offboarding_complete": false, + "all_source_tables_compared": 13, + "all_source_files_compared": 82, + "archive_member_hashes_verified": true, + "selected_client_content_complete_locally": true, + "source_sql_and_content_unchanged": true, + "external_bindings_require_review": 0, + "sessions_and_credentials_excluded": true, + "dataset_schema": "coulomb.client-dataset-export.v1", + "module_sha256": "f558fbb09557a968d754b1b75d257b020bff6ac9ec2e4dd37e31a79be0afdb0b", + "runner_sha256": "4e722a696187d1c32112276ae66a526f4765a30c3ab8891bb22b4bc5727f4aec", + "restore_receipt_sha256": "5f745cdae0a34c4a98f0e4429ceea49475b4e5cd9d2f8f09dea6be000e200258", + "candidate_image_id": "sha256:512819463a1902b43a615beee14be7bf97e6a05e016385082b9380692ad1a8cd", + "source_overlay_used": true, + "private_temporary_export_removed": true, + "network_internal": true, + "published_ports": 0, + "production_access": false, + "production_writes": false, + "full_gate": { + "command": "make check", + "tests": 340, + "failures": 0, + "errors": 0, + "skipped": 0 + }, + "management_command_sha256": "93295fc30a5593ffa82dd1e41aa83e44661e0b48860226b381b0b1906ab65783", + "tests_sha256": "e52749b0e2885bc3ccc278b26e2d37ed7d52e43a4336a14523e04d9933332bd8" +} diff --git a/docs/release/client-data-export-review-2026-10-30.md b/docs/release/client-data-export-review-2026-10-30.md new file mode 100644 index 0000000..06309c8 --- /dev/null +++ b/docs/release/client-data-export-review-2026-10-30.md @@ -0,0 +1,92 @@ +# Client data export and schema inventory + +The operator export covers a selected tenant's application records and locally +retained content. It addresses the data-export part of the business-app service +contract; instance teardown, backup expiry and productive offboarding acceptance +remain separate. This source has not been deployed and does not change the sealed +storage release proposal. + +`export_client_dataset` produces a private ZIP with a JSON manifest and original +content files. It includes members, their local session principals, spaces, +membership grants, assigned group/permission definitions and attributable admin +history. It includes archived spaces, assets, page aliases and immutable revisions, +including revisions of deleted pages. It never follows a body link, fetches remote +resources or exports another tenant's files or record values. + +Passwords, credential-bearing sessions and transient OIDC state are excluded. +The schema inventory lists their storage separately. App-local principals must +have unusable passwords; an unexpected usable credential refuses export rather +than copying or silently discarding it. Independent backups and process logs stay +under operator retention. Exporting data does not delete any of these stores or +revoke sessions. + +The inventory derives field names, database columns, types, nullability and +relations from the loaded Django models. Every application field has an explicit +classification. A new model or unclassified field refuses export until reviewed. +The inventory also describes session/admin/configuration tables and the generated +authorization join tables. Page bodies, metadata, filenames, attachments, revisions +and admin text can contain arbitrary personal data; structured field categories +are not a claim that free text has been inspected or anonymized. + +## Operator use + +Run against an independently verified, externally quiesced database/content pair. +Prefer the isolated retained restore for inspection. The source assertion below +does not establish live admission, a writer fence or snapshot coherence. The SQL +read uses a repeatable-read, read-only PostgreSQL transaction; file inventories +and hashes before/after archiving detect ordinary content drift. Neither replaces +external quiescence. + +```sh +uv run manage.py export_client_dataset --inventory +uv run manage.py export_client_dataset \ + --tenant '' \ + --quiesced-source \ + --output /private/operator/export/client.zip +``` + +The output directory must already exist with private permissions. The command +requires an output outside the checkout and content root, refuses symlinks and +overwrites, and publishes only after validating the archive's actual member +hashes. The archive and staging file use mode 0600. Terminal output contains only +counts, archive digest and incomplete-offboarding status; original record values +and file paths remain inside the private archive. Hashing streams large files. + +Cross-tenant relationships, duplicate storage slugs, unresolved content journals, +unattributed content, links/special files and unattributable deleted-object admin +history refuse. Resolve their ownership before repeating the export; the command +does not repair or delete source data. An external-only content binding refuses. +External references on spaces that also have local content remain listed in the +manifest for owner reconciliation; their presence does not prove remote content +has been captured. Do not call that export complete until those bindings are +resolved for the client. + +## Verification and remaining acceptance + +Synthetic tests cover tenant isolation, assigned permissions, local principal and +membership records, aliases/assets/deleted-page revisions, session/password +exclusion, ambiguous ownership and source refusal, content drift, private outputs, +no overwrite and schema drift. The full `make check` gate includes the required +browser journeys. + +`scripts/rehearse-client-export.py` runs the exact source as an explicit overlay +on the retained candidate, using the internal isolated recovery network and a +read-only content mount. It checks every archive member hash against the selected +tenant's source files, verifies all source SQL tables/files remain unchanged and +removes the temporary private archive. It reports no original record values. +The public proof is `client-data-export-rehearsal-2026-10-30.json`. + +Native result: one User, Member, Space and SpaceMembership, plus all 81 selected +local content/revision files, exported with matching archive hashes. There were +no external bindings or attributable admin events in this recovered client. +All 13 source SQL tables and 82 source files (including the excluded operational +writer lock) remained unchanged. The temporary private archive was removed. +`make check` passed all 340 tests with zero skips, including required browsers. + +Before §6.2 acceptance, release the reviewed implementation and verify a coherent +operator export for the current target; resolve remote/unattributed data; arrange +private delivery to the client; define the separately authorized instance teardown, +session revocation, process-log disposition and primary/mirror/protected-kit backup +expiry. The existing independent-kit no-expiry protection must not be silently +overridden. No client offboarding or productive deletion is authorized by this +implementation or its tests. diff --git a/docs/release/service-contract-conformance-2026-10-27.md b/docs/release/service-contract-conformance-2026-10-27.md index ccd535e..1a2a572 100644 --- a/docs/release/service-contract-conformance-2026-10-27.md +++ b/docs/release/service-contract-conformance-2026-10-27.md @@ -11,7 +11,7 @@ evidence from the returns required to finish WP-0012/0014. | 3: build/release | Forgejo immutable product digest `9a646d8e…`, source `dd83c90`, values/release and smoke in the revision16 receipt; resource-only revision17 in WP-0017. Release-unit declaration in registry/release-units/coulomb-social.yaml. | Guarded storage/content release and matching source/image/values/smoke receipt. Old contract §3.2 names Gitea; current delivery uses Forgejo. Resolve the fleet successor provenance rather than silently treating that as identical. | | 4: operations | Deployment Ready, health endpoint/runbooks. October 29 protected provider-only image/tools/pair recovery proved all tables/files and packaged app views in 30.232s, with independent escrow and retained proof. October 30 read-only native operating preflight identifies missing recurring worker mirror receipt; both timers remain absent/inactive. | Current ops-hub client/environment/endpoint registration evidence; admitted recurring *paired* backup with matching mirrored point/receipt, retention/headroom and received failed/stale signals; post-release stage-1 CRUD/copy/transfer smoke and CSOC-IN-0004 reconciliation. Provider-only recovery excludes operator-account recovery and a fresh Docker daemon. Database-only backup does not cover CONTENT_ROOT. | | 5: cost | Attribution key `resource:tenant:coulomb:coulomb-social`, dated workload/demand/labor records in docs/resource-evidence; WP-0017 records current requests 15m/176Mi. | Demonstrate the key's client × app × instance join in current finance ingestion and per-instance infrastructure cost report. Planning labor rates and CPU requests are not booked costs. | -| 6: protection | Platform instance isolation, explicit read/write denial, sanitized Markdown, private images, account-local JSON export and successor-aware local exit. No credentials copied into these records. | Full client-dataset export, instance teardown plus backup expiry contract (§6.2), and schema-derived personal-data inventory. Personal export is narrower than complete client offboarding. Productive sharing/portability verdicts and retained-data disposition remain WP-0013. | +| 6: protection | Platform instance isolation, explicit read/write denial, sanitized Markdown, private images, account-local JSON export and successor-aware local exit. October 30 source adds a private operator tenant-dataset ZIP export and schema-derived personal-data inventory; review in client-data-export-review-2026-10-30.md. No credentials copied into these records. | Release and verify coherent current-target client export, resolve external/unattributed data, and agree instance teardown/session/log disposition plus primary/mirror/protected-kit backup expiry (§6.2). Source support does not establish productive offboarding. Productive sharing/portability verdicts and retained-data disposition remain WP-0013. | The August service-objectives record is historical. Its RPO ≤60min/RTO ≤4h and Forgejo content durability assumptions predate the persistent CONTENT_ROOT diff --git a/scripts/rehearse-client-export.py b/scripts/rehearse-client-export.py new file mode 100644 index 0000000..9afc09f --- /dev/null +++ b/scripts/rehearse-client-export.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Export private client data from an isolated retained restore, without source writes.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import runpy +import subprocess +import tempfile +import uuid +from pathlib import Path + +CHECKS = r''' +import hashlib, json, os, zipfile, django +from pathlib import Path +django.setup() +from django.db import connection +from coulomb_social.apps.members.client_export import export_client_dataset +from coulomb_social.apps.spaces.models import Space +assert connection.vendor == "postgresql" +root = Path("/recovered-content") +tenant = os.environ["EXPORT_TENANT"] +def files(): + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in root.rglob("*") if p.is_file()} +def sql(): + with connection.cursor() as c: + c.execute("SELECT tablename FROM pg_tables WHERE schemaname='public' ORDER BY 1") + tables = [r[0] for r in c.fetchall()] + rows = {} + for table in tables: + c.execute("SELECT * FROM " + connection.ops.quote_name(table)) + rows[table] = sorted(json.dumps(r, default=str) for r in c.fetchall()) + return rows +before_files, before_sql = files(), sql() +output = Path("/export/client.zip") +result = export_client_dataset(tenant, root, output) +with zipfile.ZipFile(output) as archive: + manifest = json.loads(archive.read("manifest.json")) + slugs = set(Space.objects.filter(tenant_id=tenant).values_list("slug", flat=True)) + expected = {name: digest for name, digest in before_files.items() + if len(Path(name).parts) >= 3 + and Path(name).parts[0] in {"spaces", ".revisions"} + and Path(name).parts[1] in slugs} + assert manifest["tenant_id"] == tenant + assert manifest["files_sha256"] == expected + assert set(archive.namelist()) == {"manifest.json"} | {"content/" + n for n in expected} + assert all(hashlib.sha256(archive.read("content/" + n)).hexdigest() == h + for n, h in expected.items()) + assert all("password" not in row for row in manifest["data"]["records"]["members.user"]) + assert manifest["offboarding_complete"] is False + assert output.stat().st_mode & 0o777 == 0o600 +assert sql() == before_sql and files() == before_files +print(json.dumps({"status": "recovered_client_export_verified", **result, + "all_source_tables_compared": len(before_sql), "all_source_files_compared": len(before_files), + "archive_member_hashes_verified": True, "selected_client_content_complete_locally": True, + "source_sql_and_content_unchanged": True, + "external_bindings_require_review": len(manifest["external_bindings"]), + "sessions_and_credentials_excluded": True})) +''' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--restore-receipt", type=Path, required=True) + parser.add_argument("--tenant-id", required=True) + parser.add_argument("--image", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists() or args.output.is_symlink(): + raise ValueError("Fresh receipt output required") + previous = runpy.run_path(str(Path(__file__).with_name("rehearse-source-date-backfill.py"))) + run, hashes = previous["run"], previous["hashes"] + raw = args.restore_receipt.read_bytes() + state = json.loads(raw) + if (state["status"] != "restored_verified" + or not re.fullmatch(r"csoc-recovery-\d{14}", state["database_container"]) + or state["network"] != state["database_container"] + "-net"): + raise ValueError("Isolated retained restore required") + net = json.loads(run(["docker", "network", "inspect", state["network"]]))[0] + db = json.loads(run(["docker", "inspect", state["database_container"]]))[0] + if (net["Internal"] is not True or not db["State"]["Running"] + or set(net["Containers"]) != {db["Id"]} + or set(db["NetworkSettings"]["Networks"]) != {state["network"]} + or "recovery-db" not in db["NetworkSettings"]["Networks"][state["network"]]["Aliases"] + or any(db["NetworkSettings"]["Ports"].values()) + or db["Image"] != state["postgres_image_id"]): + raise ValueError("Recovery isolation mismatch") + image = run(["docker", "image", "inspect", args.image, "--format", "{{.Id}}"]).decode().strip() + if not re.fullmatch(r"sha256:[a-f0-9]{64}", image): + raise ValueError("Locally retained image required") + root = Path(state["restored_directory"]) / "content" + if any(p.is_symlink() for p in [root, *root.parents]) or not root.resolve().is_relative_to( + Path("/tmp")): + raise ValueError("Real temporary recovered content required") + before = hashes(root) + module = Path(__file__).resolve().parents[1] / "coulomb_social/apps/members/client_export.py" + source = module.read_bytes() + with tempfile.TemporaryDirectory(prefix="csoc-client-export-") as temporary: + work = Path(temporary) + (work / "client_export.py").write_bytes(source) + (work / "export").mkdir(mode=0o700) + name = "csoc-export-" + uuid.uuid4().hex + command = ["docker", "run", "--rm", "--name", name, "--network", state["network"], + "--read-only", "--tmpfs", "/tmp", "--cap-drop=ALL", + "--security-opt=no-new-privileges", "--user", f"{os.getuid()}:{os.getgid()}", + "--entrypoint", "/app/.venv/bin/python", + "-e", "DJANGO_SETTINGS_MODULE=coulomb_social.settings.base", + "-e", "DATABASE_URL=postgres://postgres@recovery-db/coulomb_social_restore", + "-e", "EXPORT_TENANT=" + args.tenant_id, + "-e", "CONTENT_ROOT=/recovered-content", "-e", "FORGEJO_BASE_URL=", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "-v", str(root) + ":/recovered-content:ro", + "-v", str(work / "client_export.py") + + ":/app/coulomb_social/apps/members/client_export.py:ro", + "-v", str(work / "export") + ":/export", image, "-c", CHECKS] + try: + result = subprocess.run(command, capture_output=True, timeout=180) + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True, + timeout=30, check=False) + if result.returncode: + fd, diagnostic = tempfile.mkstemp(prefix="csoc-client-export-private-diagnostics-") + with os.fdopen(fd, "wb") as stream: + stream.write(result.stdout + result.stderr) + print(json.dumps({"private_diagnostics": diagnostic})) + raise ValueError("Private export check failed") + rows = [json.loads(line) for line in result.stdout.splitlines() if line.startswith(b"{")] + if len(rows) != 1 or hashes(root) != before: + raise ValueError("Single result and preserved source files required") + proof = {**rows[0], "dataset_schema": rows[0]["schema"], + "schema": "coulomb.client-export-rehearsal.v1", + "module_sha256": hashlib.sha256(source).hexdigest(), + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "restore_receipt_sha256": hashlib.sha256(raw).hexdigest(), + "candidate_image_id": image, "source_overlay_used": True, + "private_temporary_export_removed": True, "network_internal": True, + "published_ports": 0, "production_access": False, "production_writes": False} + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(proof, stream, indent=2) + stream.write("\n") + print(json.dumps(proof)) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError, KeyError, subprocess.SubprocessError): + raise SystemExit("Client export rehearsal refused; inspect private diagnostics locally.") diff --git a/tests/test_client_export.py b/tests/test_client_export.py new file mode 100644 index 0000000..299652c --- /dev/null +++ b/tests/test_client_export.py @@ -0,0 +1,174 @@ +"""Client offboarding export uses synthetic tenants and never contacts providers.""" + +import copy +import hashlib +import io +import json +import zipfile + +import pytest +from django.contrib.admin.models import ADDITION, LogEntry +from django.contrib.auth.models import Group, Permission +from django.contrib.contenttypes.models import ContentType +from django.contrib.sessions.models import Session +from django.core.management import call_command +from django.core.management.base import CommandError + +from coulomb_social.apps.members import client_export +from coulomb_social.apps.members.client_export import ExportRefused, export_client_dataset +from coulomb_social.apps.members.models import Member, User +from coulomb_social.apps.spaces.models import Space, SpaceMembership + +pytestmark = pytest.mark.django_db + + +def actor(tenant, name): + user = User.objects.create_user(username=name, email=f"{name}@example.test") + return Member.objects.create(user=user, tenant_id=tenant, issuer="https://issuer.example.test", + subject=name, user_engine_user_id=name) + + +@pytest.fixture +def dataset(tmp_path): + own, other = actor("tenant:a", "own"), actor("tenant:b", "other") + selected = Space.objects.create(tenant_id=own.tenant_id, slug="own", created_by=own) + foreign = Space.objects.create(tenant_id=other.tenant_id, slug="other", created_by=other) + SpaceMembership.objects.create(space=selected, member=own, role="owner") + SpaceMembership.objects.create(space=foreign, member=other, role="owner") + root = tmp_path / "content" + root.mkdir() + files = { + "spaces/own/pages/index.md": b"Private client A body", + "spaces/own/assets/image.bin": b"client A image", + "spaces/own/.page-aliases.json": b'{"prior":["own","index"]}', + ".revisions/own/deleted/old.md": b"client A deleted page recovery bytes", + "spaces/other/pages/index.md": b"Other tenant private body", + ".revisions/other/deleted/old.md": b"Other tenant deleted bytes", + ".writer.lock": b"", + } + for name, value in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(value) + private = tmp_path / "private" + private.mkdir(mode=0o700) + return root, private / "export.zip", selected, own, other + + +def test_export_includes_complete_own_local_data_and_revisions_without_other_tenant_or_credentials( + dataset): + root, output, space, own, other = dataset + permission = Permission.objects.get(content_type__app_label="spaces", codename="change_space") + group = Group.objects.create(name="Synthetic client operators") + group.permissions.add(permission) + own.user.groups.add(group) + own.user.user_permissions.add(permission) + LogEntry.objects.create(user=own.user, content_type=ContentType.objects.get_for_model(Space), + object_id=str(space.pk), object_repr="own client title", + action_flag=ADDITION, change_message="own change") + Session.objects.create(session_key="must-not-export", session_data="synthetic-session-secret", + expire_date="2030-01-01T00:00:00Z") + before = {p.relative_to(root).as_posix(): p.read_bytes() + for p in root.rglob("*") if p.is_file()} + result = export_client_dataset("tenant:a", root, output) + assert result["content_files"] == 4 and result["admin_events"] == 1 + assert result["records"] == {"members.user": 1, "members.member": 1, + "spaces.space": 1, "spaces.spacemembership": 1} + assert output.stat().st_mode & 0o777 == 0o600 + with zipfile.ZipFile(output) as archive: + manifest = json.loads(archive.read("manifest.json")) + assert len(archive.namelist()) == 5 + for name, digest in manifest["files_sha256"].items(): + assert hashlib.sha256(archive.read("content/" + name)).hexdigest() == digest + assert archive.read("content/.revisions/own/deleted/old.md") == before[ + ".revisions/own/deleted/old.md"] + serialized = json.dumps(manifest) + assert "other@example.test" not in serialized + assert str(other.pk) not in serialized and str(other.user_id) not in serialized + assert "password" not in manifest["data"]["records"]["members.user"][0] + assert "synthetic-session-secret" not in serialized and "must-not-export" not in serialized + access = manifest["data"]["user_authorization"][0] + assert access["groups"][0]["permissions"][0]["codename"] == "change_space" + assert access["direct_permissions"][0]["codename"] == "change_space" + assert {p.relative_to(root).as_posix(): p.read_bytes() + for p in root.rglob("*") if p.is_file()} == before + assert Member.objects.count() == 2 and Space.objects.count() == 2 + assert not result["offboarding_complete"] + with pytest.raises(ExportRefused): + export_client_dataset("tenant:a", root, output) + + +@pytest.mark.parametrize("problem", ["foreign_grant", "foreign_creator", "pending_journal", + "orphan_file", "symlink", "usable_password", "remote_only"]) +def test_ambiguous_or_incomplete_source_refuses_without_publishing(dataset, problem): + root, output, space, own, other = dataset + if problem == "foreign_grant": + SpaceMembership.objects.create(space=space, member=other) + elif problem == "foreign_creator": + Space.objects.filter(pk=space.pk).update(created_by=other) + elif problem == "pending_journal": + (root / ".recovery").mkdir() + (root / ".recovery/interrupted.json").write_text("{}") + elif problem == "orphan_file": + (root / "unattributed.bin").write_bytes(b"unknown owner") + elif problem == "symlink": + (root / "spaces/own/assets/link").symlink_to(root / "spaces/other/pages/index.md") + elif problem == "usable_password": + own.user.set_password("synthetic-not-an-app-credential") + own.user.save() + else: + Space.objects.create(tenant_id="tenant:a", slug="external", forgejo_owner="synthetic", + forgejo_repo="remote-only") + with pytest.raises(ExportRefused): + export_client_dataset("tenant:a", root, output) + assert not output.exists() and not list(output.parent.glob(".client-export-*")) + + +def test_content_drift_refuses_and_removes_private_partial(dataset, monkeypatch): + root, output, *_ = dataset + write = zipfile.ZipFile.write + + def changed(archive, *args, **kwargs): + result = write(archive, *args, **kwargs) + (root / "spaces/own/pages/index.md").write_bytes(b"later edit") + return result + + monkeypatch.setattr(zipfile.ZipFile, "write", changed) + with pytest.raises(ExportRefused, match="changed during export"): + export_client_dataset("tenant:a", root, output) + assert not output.exists() and not list(output.parent.glob(".client-export-*")) + + +def test_deleted_admin_target_cannot_be_silently_omitted(dataset): + root, output, space, own, _ = dataset + LogEntry.objects.create(user=own.user, content_type=ContentType.objects.get_for_model(Space), + object_id="00000000-0000-0000-0000-000000000001", + object_repr="deleted client data", action_flag=ADDITION) + with pytest.raises(ExportRefused, match="admin history"): + export_client_dataset("tenant:a", root, output) + assert not output.exists() + + +def test_inventory_exposes_schema_classification_and_refuses_unclassified_field(monkeypatch): + inventory = client_export.schema_inventory() + assert len(inventory["models"]) == 4 + fields = copy.deepcopy(client_export.FIELDS) + fields["members.user"]["identity"] = "id username full_name" # new/unclassified email + monkeypatch.setattr(client_export, "FIELDS", fields) + with pytest.raises(ExportRefused, match="Unclassified application field"): + client_export.schema_inventory() + + +def test_command_inventory_is_value_free_and_export_requires_operator_source_assertion( + dataset, settings): + root, output, *_ = dataset + settings.CONTENT_ROOT = str(root) + stream = io.StringIO() + call_command("export_client_dataset", inventory=True, stdout=stream) + assert "own@example.test" not in stream.getvalue() + assert json.loads(stream.getvalue())["schema"] == "coulomb.client-data-inventory.v1" + with pytest.raises(CommandError): + call_command("export_client_dataset", tenant="tenant:a", output=output) + call_command("export_client_dataset", tenant="tenant:a", output=output, + quiesced_source=True, stdout=io.StringIO()) + assert output.exists() diff --git a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md index 82d12bc..7fde34a 100644 --- a/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md +++ b/workplans/CSOC-WP-0012-durable-recovery-service-readiness.md @@ -246,6 +246,22 @@ Review/proof: docs/release/backup-capacity-review-2026-10-30.{md,json}. Limits a still proposed; provider retention, native operating publication/retrieval, received alerts and historical-copy disposition remain open. No timer activated. +2026-10-30 client-offboarding follow-through: added a private operator client +dataset export and schema-derived inventory, separate from personal portability. +It selects tenant records/permissions/admin history and local content, assets, +aliases and all retained revisions; rejects ambiguous ownership, pending recovery, +source drift and unclassified application schema; excludes session credentials +and requires unusable local passwords. No source deletion, teardown or backup +expiry is performed. Review: docs/release/client-data-export-review-2026-10-30.md. +Current-target release/coherent export acceptance and the owner-governed teardown/ +session/log/backup-expiry contract remain open under service-contract §6.2. +The exact module overlaid on the retained candidate exported the recovered client's +four application record sets and all81local content/revision files with matching +archive hashes, zero external bindings and all13source tables/82source files +unchanged. Temporary private archive removed. Proof: +docs/release/client-data-export-rehearsal-2026-10-30.json. Full make check340passes, +zero skips. No deployment, production export, teardown or expiry was performed. + ## Completion Finish when every task above has its stated evidence and acceptance, or hand off any newly discovered residual to a live work record first. Cross-owner dependencies remain their existing canonical records.