Record approved retry release, private recovery proof and completed workplan

Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a11538-05ea-7b63-b3c7-a2df5f38898d

deploy/k8s/railiance/deploy-verification-2026-10-07.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Exact founder-approved email-connect verification release, 2026-10-07.
+
+Default is server dry-run. Never prints SQLite bytes, credentials or child errors.
+"""
+import argparse
+import copy
+import hashlib
+import json
+import os
+import signal
+import sqlite3
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+NS = 'email-connect'
+UID = 'a553c742-0115-43d4-99a4-a5ca56fe0786'
+OLD = 'forgejo.coulomb.social/coulomb/email-connect@sha256:e095d7e51c68a29ff6b82d735c734d5d27e0041b937dc958ecee5bcb5ddf5e8b'
+NEW = 'forgejo.coulomb.social/coulomb/email-connect@sha256:1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c'
+DECISION = '8da41538-8482-4c72-b83d-e0fce1b662f0'
+
+
+class Refused(RuntimeError):
+    pass
+
+
+def k(*args, body=None, timeout=180):
+    result = subprocess.run(['kubectl', *args], input=body, capture_output=True, timeout=timeout)
+    if result.returncode:
+        raise Refused('command_failed')
+    return result.stdout
+
+
+def document(*args):
+    return json.loads(k(*args, '-o', 'json'))
+
+
+def image(dep):
+    return dep['spec']['template']['spec']['containers'][0]['image']
+
+
+def private_write(path, data):
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(fd, 'wb') as stream:
+        stream.write(data)
+        stream.flush()
+        os.fsync(stream.fileno())
+
+
+def backup_pod(name):
+    return {'apiVersion': 'v1', 'kind': 'Pod',
+        'metadata': {'name': name, 'namespace': NS, 'labels': {'email-connect-recovery': name}},
+        'spec': {'restartPolicy': 'Never', 'automountServiceAccountToken': False,
+            'securityContext': {'runAsNonRoot': True, 'runAsUser': 10001,
+                'runAsGroup': 10001, 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}},
+            'containers': [{'name': 'backup', 'image': NEW,
+                'command': ['python', '-c', 'import time;time.sleep(600)'],
+                'resources': {'requests': {'cpu': '10m', 'memory': '32Mi'},
+                    'limits': {'cpu': '100m', 'memory': '128Mi'}},
+                'securityContext': {'allowPrivilegeEscalation': False,
+                    'readOnlyRootFilesystem': True, 'capabilities': {'drop': ['ALL']}},
+                'volumeMounts': [{'name': 'data', 'mountPath': '/data', 'readOnly': True},
+                    {'name': 'tmp', 'mountPath': '/tmp'}]}],
+            'volumes': [{'name': 'data', 'persistentVolumeClaim':
+                {'claimName': 'email-connect-data', 'readOnly': True}},
+                {'name': 'tmp', 'emptyDir': {}}]}}
+
+
+def patch():
+    return json.dumps([
+        {'op': 'test', 'path': '/spec/template/spec/containers/0/image', 'value': OLD},
+        {'op': 'replace', 'path': '/spec/template/spec/containers/0/image', 'value': NEW},
+    ])
+
+
+def capture(name, destination):
+    code = '''import os,sqlite3,sys
+src=sqlite3.connect('file:/data/email-connect.db?mode=ro',uri=True)
+dst=sqlite3.connect('/tmp/verification-pre-release.db')
+src.backup(dst)
+assert dst.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
+dst.close();src.close()
+os.chmod('/tmp/verification-pre-release.db',0o600)
+with open('/tmp/verification-pre-release.db','rb') as stream:sys.stdout.buffer.write(stream.read())
+'''
+    data = k('exec', '-n', NS, name, '--', 'python', '-c', code)
+    if not data.startswith(b'SQLite format 3\x00'):
+        raise Refused('invalid_sqlite_backup')
+    path = destination / 'database.sqlite3'
+    private_write(path, data)
+    with sqlite3.connect('file:' + str(path) + '?mode=ro', uri=True) as database:
+        if database.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
+            raise Refused('invalid_sqlite_backup')
+    return {'database_sha256': hashlib.sha256(data).hexdigest(),
+            'database_bytes': len(data), 'sqlite_integrity': 'ok',
+            'source_pvc_read_only': True, 'writers_quiesced': True}
+
+
+def verify():
+    code = '''import json,os,sqlite3,urllib.request,urllib.error
+base='http://127.0.0.1:8080'
+results={}
+for path in ['/healthz','/readyz']:
+    with urllib.request.urlopen(base+path,timeout=5) as response:results[path]=response.status
+for label,path,payload,token,key,expected,reason in [
+ ('unauthenticated','/v1/send',{},'synthetic-invalid',None,401,'unauthorized'),
+ ('template_denial','/v1/send',{'id':'verification-release-denial','source':'user-engine','type':'arbitrary.send'},os.environ['EMAIL_CONNECT_INGEST_TOKEN'].strip(),'verification-release-denial',400,'template_not_allowed'),
+ ('invalid_verification','/v1/registration-verifications',{},os.environ['EMAIL_CONNECT_INGEST_TOKEN'].strip(),None,400,'invalid_verification_request')]:
+    headers={'Content-Type':'application/json','Authorization':'Bearer '+token}
+    if key:headers['Idempotency-Key']=key
+    request=urllib.request.Request(base+path,data=json.dumps(payload).encode(),headers=headers,method='POST')
+    try:urllib.request.urlopen(request,timeout=5);raise RuntimeError('expected_denial')
+    except urllib.error.HTTPError as error:
+        body=json.load(error)
+        assert error.code==expected and body.get('error')==reason
+        results[label]=error.code
+with sqlite3.connect('file:/data/email-connect.db?mode=ro',uri=True) as database:
+    assert database.execute('PRAGMA integrity_check').fetchone()[0]=='ok'
+    assert database.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='verification_attempts'").fetchone()
+results.update(sqlite_integrity='ok',reservation_schema_present=True,mail_sent=False)
+print(json.dumps(results))
+'''
+    return json.loads(k('exec', '-n', NS, 'deployment/email-connect', '--', 'python', '-c', code))
+
+
+def deploy(before, body, destination):
+    destination.mkdir(mode=0o700)
+    receipt = {'status': 'incomplete', 'decision': DECISION, 'old_image': OLD,
+        'new_image': NEW, 'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}
+    started = time.monotonic()
+    paused = False
+    helper = False
+    succeeded = False
+    name = json.loads(body)['metadata']['name']
+    try:
+        paused = True
+        receipt['phase'] = 'pause'
+        k('scale', 'deployment/email-connect', '-n', NS, '--replicas=0')
+        k('wait', '--for=delete', 'pod', '-n', NS,
+            '-l', 'app.kubernetes.io/name=email-connect', '--timeout=120s')
+        if document('get', 'deployment', 'email-connect', '-n', NS)['spec']['replicas'] != 0:
+            raise Refused('writers_not_quiesced')
+        helper = True
+        k('create', '-f', '-', body=body)
+        k('wait', '--for=condition=Ready', 'pod/' + name, '-n', NS, '--timeout=180s', timeout=240)
+        receipt['phase'] = 'capture'
+        receipt.update(capture(name, destination))
+        k('delete', 'pod', name, '-n', NS, '--wait=true', '--timeout=120s')
+        helper = False
+        receipt['phase'] = 'image_patch'
+        k('patch', 'deployment', 'email-connect', '-n', NS, '--type=json', '-p', patch())
+        k('scale', 'deployment/email-connect', '-n', NS, '--replicas=1')
+        k('rollout', 'status', 'deployment/email-connect', '-n', NS, '--timeout=180s', timeout=240)
+        actual = document('get', 'deployment', 'email-connect', '-n', NS)
+        expected = copy.deepcopy(before['spec'])
+        expected['template']['spec']['containers'][0]['image'] = NEW
+        if actual['spec'] != expected or actual['status'].get('readyReplicas') != 1:
+            raise Refused('unexpected_release_spec')
+        receipt['phase'] = 'verification'
+        receipt['checks'] = verify()
+        receipt.update(status='released_verified', image_only_spec_change=True, ready_replicas=1)
+        succeeded = True
+    except BaseException as exc:
+        receipt.update(status='failed', failure_type=type(exc).__name__)
+        raise
+    finally:
+        try:
+            if helper:
+                k('delete', 'pod', name, '-n', NS, '--ignore-not-found=true', '--wait=true', '--timeout=120s')
+        except Exception:
+            receipt['helper_cleanup_failed'] = True
+        if paused and not succeeded:
+            try:
+                current = image(document('get', 'deployment', 'email-connect', '-n', NS))
+                replicas = '1' if current == OLD else '0'
+                k('scale', 'deployment/email-connect', '-n', NS, '--replicas=' + replicas)
+                receipt['failure_replicas'] = int(replicas)
+                if replicas == '1':
+                    k('rollout', 'status', 'deployment/email-connect', '-n', NS, '--timeout=180s', timeout=240)
+                    receipt['old_receiver_resumed'] = True
+            except Exception:
+                receipt['receiver_state_requires_check'] = True
+        receipt['elapsed_seconds'] = round(time.monotonic() - started, 2)
+        private_write(destination / 'release-receipt.json', json.dumps(receipt, indent=2).encode())
+        print(json.dumps(receipt))
+    return receipt
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--execute', action='store_true')
+    parser.add_argument('--destination', type=Path, required=True)
+    args = parser.parse_args()
+    os.environ['KUBECONFIG'] = '/etc/rancher/k3s/k3s.yaml'
+    if document('get', 'namespace', 'kube-system')['metadata']['uid'] != UID:
+        raise Refused('wrong_cluster')
+    before = document('get', 'deployment', 'email-connect', '-n', NS)
+    if image(before) != OLD or before['spec']['replicas'] != 1 or before['status'].get('readyReplicas') != 1:
+        raise Refused('unexpected_release')
+    if document('get', 'pvc', 'email-connect-data', '-n', NS)['status']['phase'] != 'Bound':
+        raise Refused('pvc_not_bound')
+    config = document('get', 'configmap', 'email-connect-config', '-n', NS)['data']
+    if config.get('EMAIL_CONNECT_DATABASE_PATH') != '/data/email-connect.db':
+        raise Refused('unexpected_database_path')
+    name = 'email-connect-verification-backup-' + time.strftime('%Y%m%d%H%M%S', time.gmtime())
+    body = json.dumps(backup_pod(name)).encode()
+    k('create', '--dry-run=server', '-f', '-', body=body)
+    projected = document('patch', 'deployment', 'email-connect', '-n', NS,
+        '--type=json', '-p', patch(), '--dry-run=server')
+    expected = copy.deepcopy(before['spec'])
+    expected['template']['spec']['containers'][0]['image'] = NEW
+    if projected['spec'] != expected:
+        raise Refused('unexpected_dry_run_spec')
+    if not args.execute:
+        print(json.dumps({'status': 'reviewed', 'live_changes': False,
+            'image_only_spec_diff': True, 'readonly_backup_helper': True}))
+        return
+    if not args.destination.is_absolute() or args.destination.exists():
+        raise Refused('destination_must_be_new')
+    def terminate(signum, frame):
+        raise Refused('release_interrupted')
+    signal.signal(signal.SIGTERM, terminate)
+    deploy(before, body, args.destination)
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as exc:
+        print(json.dumps({'status': 'failed', 'failure_type': type(exc).__name__}))
+        sys.exit(1)

docs/verification-retry-review-2026-10-07.md

@@ -1,5 +1,9 @@
 # Verification retry correction — 2026-10-07
 
+Released: digest1e613d43 is live with one Ready replica. Founder approval,
+private backup, deployment and no-send verification are complete; the detailed
+receipt below includes the corrected validation-client failure.
+
 Source contract: POST /v1/registration-verifications keeps its 202 JSON
 {request_id, accepted: true} response. Repeated tenant/client/registration
 operations return the same receipt after provider acceptance without resending.
@@ -39,7 +43,7 @@ be read by the old image, but schema compatibility is not delivery safety.
 
 Source: 725850591754039771031d5fa8ba29562d6d4ce7. Image:
 forgejo.coulomb.social/coulomb/email-connect@sha256:1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c.
-Candidate pin is committed in deployment.yaml; it has not been applied live.
+Owner pin in deployment.yaml is now applied under the recorded founder approval.
 The packaged image passed synthetic retry/restart smoke as UID10001, with
 read-only root, temporary SQLite and network disabled. One provider call,
 unchanged request receipt and digest-only handle persistence were verified.
@@ -52,8 +56,9 @@ proves only the container image changes. Both guarded JSON image patch and
 non-root read-only-PVC backup helper passed server dry-run. No live object changed.
 Existing PVC=email-connect-data, Secret reference=email-connect-runtime.
 
-Founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 is open. Scope:
-ADMINISTER @ realm:kubernetes/railiance01, activation=APPROVED after founder go.
+Founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 was resolved following
+the founder's Ok, go ahead. Scope:
+ADMINISTER @ realm:kubernetes/railiance01, activation=APPROVED for this exact release.
 Do not apply the whole package to make this image correction: preserve live
 runtime ownership and use the reviewed guarded image patch.
 
@@ -73,3 +78,42 @@ Failure before the patch: remove helper and resume the old one replica. Failure
 after new code has served: stop sending and keep the PVC/backup; reconcile before
 any old-image rollback because the old verification code ignores reservations.
 This packet grants no new caller, platform sender or offsite backup consumer.
+
+
+## Completed release receipt
+
+Release started2026-10-07T20:29:29Z; final new-pod Ready20:33:16Z,
+window227 seconds. First capture/update attempt58.27s completed the backup
+and image change, then failed its client-side authenticated check: the probe
+used the raw mounted token while the service normalizes it with strip(). Raw
+header construction raises ValueError; corrected normalization gives the
+expected denial. The safety path held the new receiver at zero replicas during
+repair; it never rolled back to old verification code. Resumption and all
+corrected checks took5.35s; one Ready replica remained healthy afterward.
+
+Validation: healthz/readyz200, bad bearer401, approved bearer + disallowed
+invitation template400, empty verification400. SQLite integrity passes and the
+new verification_attempts table exists. These calls create no delivery/challenge
+records and send no mail. The post-repair spec comparison proves the resumption
+changed only replicas0→1; the pre-release server comparison proves the reviewed
+permanent change is the image. Backup helper pod count is zero.
+
+Private backup: node root0700
+/tmp/email-connect-preverification-20261007T202924Z/database.sqlite3, mode0600,
+40960 bytes, SHA256
+3b4651064fa7dd7bd5073ed8567480ea1f4cb527dbe6c7462cc778182e7da1f8.
+Matching workstation0700 /tmp/csoc-email-private-hkt09g6u/database.sqlite3,
+mode0600. Isolated image rehearsal as UID10001/network disabled adds the
+reservation table and preserves every row fingerprint in the four existing
+tables. No private values enter this receipt or Git; copies remain for the
+retention/custody handoff, not as proof of governed durable backup coverage.
+
+Five synthetic deployment failure cases passed, including capture refusal,
+pre-patch failure and lost patch response/post-patch verification failure.
+Corrected historic execution helper is preserved in
+../deploy/k8s/railiance/deploy-verification-2026-10-07.py. Its old-image guard
+refuses repeating this completed release; it is not a generic rollout tool.
+
+EMAIL-WP-0006 is finished. Residual caller/sender/limit/ledger recovery work is
+live in CSOC-WP-0010-T04, inbox/bounce/retention/copy custody in T05, and
+cross-owner journey/user acceptance in T06. CSOC-WP-0008/0010 remain active.

workplans/EMAIL-WP-0006-verification-retry-safety.md

@@ -4,7 +4,7 @@ type: workplan
 title: "Make registration verification delivery retry safe"
 domain: infotech
 repo: email-connect
-status: active
+status: finished
 owner: codex
 topic_slug: netkingdom
 created: "2026-10-07"
@@ -57,11 +57,10 @@ and retry. Harness container/network were removed after validation.
 
 ```task
 id: EMAIL-WP-0006-T03
-status: wait
+status: done
 priority: high
-needs_human: true
+needs_human: false
 decision_id: "8da41538-8482-4c72-b83d-e0fce1b662f0"
-intervention_note: "Approve exact email-connect retry correction digest1e613d43 with brief receiver pause and private SQLite backup; existing sender/caller/custody unchanged. See release review."
 state_hub_task_id: "fcf55c0a-49b5-53a8-9497-3dfcc18c3828"
 ```
 
@@ -89,4 +88,22 @@ No live custody change was made.
 
 2026-10-07 candidate prepared: source7258505, published digest1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c;
 UID10001 network-disabled packaged smoke and exact live/server dry-run pass.
-Current runtime remains e095d7e5; founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 open.
+2026-10-07 live release completed after founder's Ok, go ahead,
+decision8da41538-8482-4c72-b83d-e0fce1b662f0 resolved. Digest1e613d43 is live,
+one Ready replica, health200, unauth401, authenticated template/invalid request400,
+SQLite integrity and new reservation schema verified. No mail sent by validation.
+Quiesced private SQLite backup40960 bytes/hash3b465106 passes isolated UID10001
+network-disabled migration rehearsal, preserving rows in all four old tables.
+Initial probe omitted service token normalization and raised ValueError locally;
+sending was held stopped during correction. Fixed probe passes; final
+Ready20:33:16Z, release window227s, zero helper pods remain. Private backups
+are retained on node/workstation under0700/0600; receipt in the release review.
+
+Residuals handed off BEFORE finish to existing live CSOC-WP-0010-T04
+(ca41be06-3611-5647-b469-4f89f56f77f9): caller/tenant/template permissions,
+per-caller limits, platform sender custody and reservation-ledger recovery.
+CSOC-WP-0010-T05 (d9e165d9-dd0b-54c2-bda1-bd314b999b5b) owns inbox/reply/bounce
+operations and retention/custody of evidence and temporary private copies.
+CSOC-WP-0010-T06 (97a2da91-debe-5ce2-9b68-1965a5c5abfe) retains cross-owner
+browser/user acceptance. Parent source0dcca9e and its primary consistency sync
+record these live handoffs; neither parent workplan is marked finished.