From 2dfc837e54937d7e7add5946f37a1c0cd5fe9a0d Mon Sep 17 00:00:00 2001 From: tegwick Date: Tue, 6 Oct 2026 22:37:49 +0200 Subject: [PATCH] feat: prepare verified owner-only ReichelAg content transfer Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a10e2d-a5ef-7e92-b03d-b1694e915680 --- scripts/apply-approved-content-import.py | 181 ++++++++++++++++++ scripts/export-bubble-corpus.mjs | 26 ++- ...-WP-0008-production-transfer-acceptance.md | 24 ++- 3 files changed, 225 insertions(+), 6 deletions(-) create mode 100644 scripts/apply-approved-content-import.py diff --git a/scripts/apply-approved-content-import.py b/scripts/apply-approved-content-import.py new file mode 100644 index 0000000..ecd0efa --- /dev/null +++ b/scripts/apply-approved-content-import.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Node-side reviewed ReichelAg cohort import. Default is preflight only. + +Requires a private staged corpus manifest and matching source-package digest. +Preserves deployment/Secrets/image, pauses writers for actual import, never +replaces existing content, and resumes the app in a finally block. Use a fresh +paired pre-import snapshot and isolated rehearsal before --execute. +""" +from __future__ import annotations + +import argparse +import copy +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import tarfile +import time +from pathlib import Path + +IMAGE = ('forgejo.coulomb.social/coulomb/coulomb-social@sha256:' + 'bf975939e22b9b0a5d9ac05584ae6265ba201169df6dc48e506ec684b30a9c58') +NS = 'coulomb-social' +CLUSTER_UID = 'a553c742-0115-43d4-99a4-a5ca56fe0786' + + +def k(*args: str, payload: bytes | None = None) -> bytes: + p = subprocess.run(['kubectl', *args], input=payload, capture_output=True, timeout=180) + if p.returncode: + raise RuntimeError('command_failed') + return p.stdout + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--execute', action='store_true') + parser.add_argument('--package', type=Path, required=True) + parser.add_argument('--package-sha256', required=True) + parser.add_argument('--staged-source', type=Path, required=True) + parser.add_argument('--snapshot', type=Path, required=True) + args = parser.parse_args() + os.environ['KUBECONFIG'] = '/etc/rancher/k3s/k3s.yaml' + cluster_identity = json.loads(k('get', 'namespace', 'kube-system', '-o', 'json')) + assert cluster_identity['metadata']['uid'] == CLUSTER_UID + assert hashlib.sha256(args.package.read_bytes()).hexdigest() == args.package_sha256 + manifest = json.loads((args.staged_source / 'manifest.json').read_text()) + assert manifest['version'] == 1 and len(manifest['spaces']) == 1 + entry = manifest['spaces'][0] + assert entry['slug'] == 'reichelag' and entry['tenant'] == 'tenant:platform' + assert entry['tree'] == 'trees/reichelag' and len(entry['memberships']) == 1 + assert entry['memberships'][0]['role'] == 'owner' + source = args.staged_source / 'trees/reichelag' + assert len(list((source / 'pages').glob('*.md'))) == 37 + assert sum(p.is_file() for p in (source / 'assets').rglob('*')) == 36 + assert all(not p.is_symlink() for p in args.staged_source.rglob('*')) + paired = json.loads((args.snapshot / 'receipt.json').read_text()) + assert paired['status'] == 'captured' and paired['writers_quiesced'] and paired['app_resumed'] + assert paired['database'] == 'coulomb_social_db' and paired['image'] == IMAGE + for name, key in [('database.dump', 'database_sha256'), ('content.tar.gz', 'content_sha256')]: + assert hashlib.sha256((args.snapshot / name).read_bytes()).hexdigest() == paired[key] + dep = json.loads(k('get', 'deployment', 'coulomb-social', '-n', NS, '-o', 'json')) + assert dep['spec']['replicas'] == 1 + spec = copy.deepcopy(dep['spec']['template']['spec']) + assert spec['containers'][0]['image'] == IMAGE and len(spec['containers']) == 1 + c = spec['containers'][0] + c.pop('livenessProbe', None) + c.pop('readinessProbe', None) + c['command'] = ['python', '-c', 'import time;time.sleep(1800)'] + c.pop('args', None) + spec['restartPolicy'] = 'Never' + spec['automountServiceAccountToken'] = False + name = 'csoc-approved-reichelag-' + time.strftime('%Y%m%d%H%M%S', time.gmtime()) + pod = {'apiVersion': 'v1', 'kind': 'Pod', + 'metadata': {'name': name, 'namespace': NS, 'labels': {'csoc-import': name}}, + 'spec': spec} + payload = json.dumps(pod).encode() + k('create', '--dry-run=server', '-f', '-', payload=payload) + if not args.execute: + print(json.dumps({'status': 'reviewed', 'server_dry_run': True, + 'source_package_matches': True, 'paired_snapshot_matches': True, + 'pages': 37, 'assets': 36, 'owner_grants': 1, 'replace': False})) + return 0 + paused = created = False + started = time.monotonic() + try: + paused = True + k('scale', 'deployment/coulomb-social', '-n', NS, '--replicas=0') + k('wait', '--for=delete', 'pod', '-n', NS, + '-l', 'app.kubernetes.io/instance=coulomb-social', '--timeout=120s') + created = True + k('create', '-f', '-', payload=payload) + k('wait', '--for=condition=Ready', 'pod/' + name, '-n', NS, '--timeout=120s') + # Refuse if any writer changed the metadata/content after the paired point. + module_spec = importlib.util.spec_from_file_location( + 'restore_guard', Path(__file__).with_name('restore-paired-snapshot.py')) + restore_guard = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(restore_guard) + primary = json.loads(k('get', 'cluster', 'apps-pg', '-n', 'databases', + '-o', 'json'))['status']['currentPrimary'] + original_sql = k('exec', '-i', '-n', 'databases', primary, '-c', 'postgres', '--', + 'pg_restore', '--data-only', '-f', '-', + payload=(args.snapshot / 'database.dump').read_bytes()) + current_sql = k('exec', '-n', 'databases', primary, '-c', 'postgres', '--', + 'pg_dump', '-U', 'postgres', '-d', 'coulomb_social_db', + '--data-only', '--no-owner', '--no-acl') + baseline_data = restore_guard.data_fingerprint(original_sql) + assert baseline_data == restore_guard.data_fingerprint(current_sql) + prior_files = {} + with tarfile.open(args.snapshot / 'content.tar.gz', 'r:gz') as archive: + for item in archive.getmembers(): + if item.isfile(): + prior_files[str(Path(item.name).relative_to('content'))] = hashlib.sha256( + archive.extractfile(item).read()).hexdigest() + prior_sha = hashlib.sha256(json.dumps(prior_files, sort_keys=True).encode()).hexdigest() + fingerprint_code = ('import os,json,hashlib;from pathlib import Path; ' + 'root=Path(os.environ["CONTENT_ROOT"]); ' + 'rows={str(p.relative_to(root)): ' + 'hashlib.sha256(p.read_bytes()).hexdigest() ' + 'for p in root.rglob("*") if p.is_file()}; ' + 'print(hashlib.sha256(json.dumps(rows,sort_keys=True).encode()).hexdigest())') + current_sha = k('exec', '-n', NS, name, '--', 'python', '-c', + fingerprint_code).decode().strip() + assert prior_sha == current_sha + k('cp', '--no-preserve', '-n', NS, str(args.staged_source), name + ':/tmp/approved-source') + code = '''import io,json,hashlib +from pathlib import Path +from django.core.management import call_command +from coulomb_social.apps.spaces.models import Space,SpaceMembership +from coulomb_social.apps.spaces import pageops +from coulomb_social.apps.spaces.content import load_space_page +from coulomb_social.apps.spaces.services import spaces_for_member +source=Path('/tmp/approved-source') +manifest=json.loads((source/'manifest.json').read_text()); entry=manifest['spaces'][0] +assert not Space.objects.filter(slug='reichelag').exists() +assert not pageops.space_dir('reichelag').exists() +call_command('import_content_corpus',str(source/'manifest.json'),dry_run=True,stdout=io.StringIO()) +call_command('import_content_corpus',str(source/'manifest.json'),stdout=io.StringIO()) +s=Space.objects.get(slug='reichelag'); grants=SpaceMembership.objects.filter(space=s) +assert grants.count()==1 and grants.get().role=='owner' +m=grants.get().member +assert m.tenant_id==s.tenant_id=='tenant:platform' +assert m.issuer==entry['memberships'][0]['issuer'] and m.subject==entry['memberships'][0]['subject'] +assert spaces_for_member(m).filter(pk=s.pk).exists() +for p in (source/'trees/reichelag').rglob('*'): + if p.is_file(): + dest=pageops.space_dir('reichelag')/p.relative_to(source/'trees/reichelag') + assert hashlib.sha256(dest.read_bytes()).digest()==hashlib.sha256(p.read_bytes()).digest() +for p in (source/'trees/reichelag/pages').glob('*.md'): + rendered=load_space_page(s,p.stem) + assert rendered.source=='content-plane' and rendered.html +print(json.dumps({'status':'imported_verified','pages':37,'assets':36,'owner_grants':1, + 'source_file_hashes_match':True,'all_pages_render':True,'member_can_list_space':True})) +''' + result = k('exec', '-n', NS, name, '--', 'python', 'manage.py', 'shell', '-c', code) + rows = [json.loads(line) for line in result.decode().splitlines() if line.startswith('{')] + assert len(rows) == 1 and rows[0]['status'] == 'imported_verified' + receipt = rows[0] + finally: + try: + if created: + k('delete', 'pod', name, '-n', NS, '--ignore-not-found', + '--wait=true', '--timeout=120s') + finally: + if paused: + k('scale', 'deployment/coulomb-social', '-n', NS, '--replicas=1') + k('rollout', 'status', 'deployment/coulomb-social', '-n', NS, '--timeout=180s') + receipt['app_resumed'] = True + receipt['pre_import_db_and_content_match_paired_snapshot'] = True + receipt['elapsed_seconds'] = round(time.monotonic() - started, 2) + print(json.dumps(receipt)) + return 0 + + +if __name__ == '__main__': + try: + sys.exit(main()) + except Exception as exc: + print(json.dumps({'status': 'failed', 'failure_type': type(exc).__name__})) + sys.exit(1) diff --git a/scripts/export-bubble-corpus.mjs b/scripts/export-bubble-corpus.mjs index 1cdfaa4..74a2aa8 100755 --- a/scripts/export-bubble-corpus.mjs +++ b/scripts/export-bubble-corpus.mjs @@ -207,22 +207,31 @@ async function loadSpacesList() { } let targetRootId = ""; +let targetCaptureStep = "space_list"; if (TARGET_LABEL) { try { await loadSpacesList(); - const link = page.getByText(TARGET_LABEL, { exact: true }); + targetCaptureStep = "space_link"; + const link = page.getByRole("link", { name: TARGET_LABEL, exact: true }) + .and(page.locator('a[href*="/pg_chunk/"]')); if (await link.count() !== 1) throw new Error("target_space_link_not_unique"); const href = await link.getAttribute("href"); if (!href || !new URL(href, BASE).pathname.startsWith("/pg_chunk/")) { throw new Error("target_space_root_not_found"); } targetRootId = new URL(href, BASE).pathname.split("/").pop(); + targetCaptureStep = "protocol_route"; await page.goto(`${BASE}${TARGET_ROUTE}`, { waitUntil: "domcontentloaded", timeout: 60000 }); await page.waitForTimeout(4000); + targetCaptureStep = "filter_control"; const filter = page.locator("button").filter({ has: page.locator("svg text").filter({ hasText: /^\s*filter_list\s*$/ }), }); await filter.first().click(); + targetCaptureStep = "space_filter"; + await page.waitForFunction(label => + [...document.querySelectorAll("select option")] + .some(o => o.textContent.trim() === label), TARGET_LABEL, { timeout: 10000 }); const selects = page.locator("select"); let selected = false; for (let i = 0; i < await selects.count(); i++) { @@ -234,6 +243,7 @@ if (TARGET_LABEL) { break; } if (!selected) throw new Error("target_space_filter_not_found"); + targetCaptureStep = "search_control"; await page.getByRole("button", { name: "Search", exact: true }).click(); await page.waitForTimeout(5000); for (let i = 0; i < 24; i++) { @@ -242,7 +252,7 @@ if (TARGET_LABEL) { } } catch { await browser.close(); - console.error("Target capture failed; no import tree published."); + console.error(`Target capture failed at ${targetCaptureStep}; no import tree published.`); process.exit(3); } } else { @@ -334,6 +344,7 @@ ensure(join(treeRoot, "pages")); ensure(join(treeRoot, "assets")); const used = new Set(); let visuals = 0; +let targetVisual = ""; for (const a of sampleArts) { let pageSlug = slugify(a.nomer_text || a._id); let n = 2; @@ -357,6 +368,7 @@ for (const a of sampleArts) { visuals++; } } + if (normId(a._id) === targetRootId) targetVisual = visualRel; const md = `--- id: bubble:${a._id} title: ${JSON.stringify(a.nomer_text || "")} @@ -364,6 +376,8 @@ abstractor: ${JSON.stringify(a.abstractor_text || "")} visual: ${JSON.stringify(visualRel)} space: ${JSON.stringify(lab.slug)} bubble_type: article +source_created_at: ${JSON.stringify(a["Created Date"] ?? null)} +source_modified_at: ${JSON.stringify(a["Modified Date"] ?? null)} archived: ${Boolean(a.archived_boolean)} parent: ${JSON.stringify(a.context_custom_article ? `bubble:${normId(String(a.context_custom_article))}` : "")} --- @@ -377,13 +391,15 @@ write( `--- id: bubble-space:${sampleId} title: ${JSON.stringify(lab.title)} -abstractor: ${JSON.stringify(`Imported sample (${sampleArts.length} articles)`)} -visual: "" +abstractor: ${JSON.stringify(TARGET_LABEL + ? (articles[targetRootId]?.abstractor_text || "") + : `Imported sample (${sampleArts.length} articles)`)} +visual: ${JSON.stringify(targetVisual)} space: ${JSON.stringify(lab.slug)} bubble_type: space --- -# ${lab.title} +${TARGET_LABEL ? (articles[targetRootId]?.content_text || `# ${lab.title}`) : `# ${lab.title}`} `, ); write( diff --git a/workplans/CSOC-WP-0008-production-transfer-acceptance.md b/workplans/CSOC-WP-0008-production-transfer-acceptance.md index 270d341..48f62ed 100644 --- a/workplans/CSOC-WP-0008-production-transfer-acceptance.md +++ b/workplans/CSOC-WP-0008-production-transfer-acceptance.md @@ -102,7 +102,7 @@ snapshots; no private archive in git or State Hub. ```task id: CSOC-WP-0008-T03 -status: wait +status: progress priority: high state_hub_task_id: "0615eff2-bf2b-503b-8f15-0591d4666446" ``` @@ -427,3 +427,25 @@ UI framework architecture is a separate next phase (CSOC-WP-0011), before broad UX rebuilding. Existing browser session renewal or a dedicated Bubble account with read access can provide the source; never accept password/session contents in chat. User renewal remains the next input needed for private protocol capture. + +Attended source access renewed: founder confirmed signed in and ReichelAg +protocols visible in the local browser. Session saved privately (4cookies, +no expired signature). T03 resumes authenticated capture and import rehearsal; +needs_human login requirement cleared. Browser capture now scopes actual link +roles and waits for native filter options before applying Search. + +Authenticated capture completed: explicit ReichelAG target, no old seed;36source +articles/36assets and37markdown files including the space index. All29August +article IDs present plus7new articles; latest dated protocol2026-10-02. Source +home metadata/visual and source article creation/modification timestamps retained. +Reviewed owner map: only existing tegwick identity in tenant:platform, owner +for new reichelag space; no other membership grants or data replacement. + +Isolated real-content import rehearsal:37pages/36assets, every source file hash +matches, every page renders, owner-only explicit access. Fresh pre-import paired +snapshot started20:21:30Z, app resumed after53.36seconds. Source package +a0ab5085e63a93fd351a1ffb85a092555cb340d72f9f5f3f510e43952388fbbc. +Bounded node import helper scripts/apply-approved-content-import.py server +dry-run passed, checks source/snapshot hashes and refuses replacements. It +compares metadata row/sequence fingerprints and content hashes with the paired +point after pausing writers, before import, then always resumes the deployment.