diff --git a/deploy/k8s/railiance/deploy-verification-2026-10-07.py b/deploy/k8s/railiance/deploy-verification-2026-10-07.py new file mode 100644 index 0000000..2b46944 --- /dev/null +++ b/deploy/k8s/railiance/deploy-verification-2026-10-07.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Exact founder-approved email-connect verification release, 2026-10-07. + +Default is server dry-run. Never prints SQLite bytes, credentials or child errors. +""" +import argparse +import copy +import hashlib +import json +import os +import signal +import sqlite3 +import subprocess +import sys +import time +from pathlib import Path + +NS = 'email-connect' +UID = 'a553c742-0115-43d4-99a4-a5ca56fe0786' +OLD = 'forgejo.coulomb.social/coulomb/email-connect@sha256:e095d7e51c68a29ff6b82d735c734d5d27e0041b937dc958ecee5bcb5ddf5e8b' +NEW = 'forgejo.coulomb.social/coulomb/email-connect@sha256:1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c' +DECISION = '8da41538-8482-4c72-b83d-e0fce1b662f0' + + +class Refused(RuntimeError): + pass + + +def k(*args, body=None, timeout=180): + result = subprocess.run(['kubectl', *args], input=body, capture_output=True, timeout=timeout) + if result.returncode: + raise Refused('command_failed') + return result.stdout + + +def document(*args): + return json.loads(k(*args, '-o', 'json')) + + +def image(dep): + return dep['spec']['template']['spec']['containers'][0]['image'] + + +def private_write(path, data): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'wb') as stream: + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + + +def backup_pod(name): + return {'apiVersion': 'v1', 'kind': 'Pod', + 'metadata': {'name': name, 'namespace': NS, 'labels': {'email-connect-recovery': name}}, + 'spec': {'restartPolicy': 'Never', 'automountServiceAccountToken': False, + 'securityContext': {'runAsNonRoot': True, 'runAsUser': 10001, + 'runAsGroup': 10001, 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}}, + 'containers': [{'name': 'backup', 'image': NEW, + 'command': ['python', '-c', 'import time;time.sleep(600)'], + 'resources': {'requests': {'cpu': '10m', 'memory': '32Mi'}, + 'limits': {'cpu': '100m', 'memory': '128Mi'}}, + 'securityContext': {'allowPrivilegeEscalation': False, + 'readOnlyRootFilesystem': True, 'capabilities': {'drop': ['ALL']}}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/data', 'readOnly': True}, + {'name': 'tmp', 'mountPath': '/tmp'}]}], + 'volumes': [{'name': 'data', 'persistentVolumeClaim': + {'claimName': 'email-connect-data', 'readOnly': True}}, + {'name': 'tmp', 'emptyDir': {}}]}} + + +def patch(): + return json.dumps([ + {'op': 'test', 'path': '/spec/template/spec/containers/0/image', 'value': OLD}, + {'op': 'replace', 'path': '/spec/template/spec/containers/0/image', 'value': NEW}, + ]) + + +def capture(name, destination): + code = '''import os,sqlite3,sys +src=sqlite3.connect('file:/data/email-connect.db?mode=ro',uri=True) +dst=sqlite3.connect('/tmp/verification-pre-release.db') +src.backup(dst) +assert dst.execute('PRAGMA integrity_check').fetchone()[0]=='ok' +dst.close();src.close() +os.chmod('/tmp/verification-pre-release.db',0o600) +with open('/tmp/verification-pre-release.db','rb') as stream:sys.stdout.buffer.write(stream.read()) +''' + data = k('exec', '-n', NS, name, '--', 'python', '-c', code) + if not data.startswith(b'SQLite format 3\x00'): + raise Refused('invalid_sqlite_backup') + path = destination / 'database.sqlite3' + private_write(path, data) + with sqlite3.connect('file:' + str(path) + '?mode=ro', uri=True) as database: + if database.execute('PRAGMA integrity_check').fetchone()[0] != 'ok': + raise Refused('invalid_sqlite_backup') + return {'database_sha256': hashlib.sha256(data).hexdigest(), + 'database_bytes': len(data), 'sqlite_integrity': 'ok', + 'source_pvc_read_only': True, 'writers_quiesced': True} + + +def verify(): + code = '''import json,os,sqlite3,urllib.request,urllib.error +base='http://127.0.0.1:8080' +results={} +for path in ['/healthz','/readyz']: + with urllib.request.urlopen(base+path,timeout=5) as response:results[path]=response.status +for label,path,payload,token,key,expected,reason in [ + ('unauthenticated','/v1/send',{},'synthetic-invalid',None,401,'unauthorized'), + ('template_denial','/v1/send',{'id':'verification-release-denial','source':'user-engine','type':'arbitrary.send'},os.environ['EMAIL_CONNECT_INGEST_TOKEN'].strip(),'verification-release-denial',400,'template_not_allowed'), + ('invalid_verification','/v1/registration-verifications',{},os.environ['EMAIL_CONNECT_INGEST_TOKEN'].strip(),None,400,'invalid_verification_request')]: + headers={'Content-Type':'application/json','Authorization':'Bearer '+token} + if key:headers['Idempotency-Key']=key + request=urllib.request.Request(base+path,data=json.dumps(payload).encode(),headers=headers,method='POST') + try:urllib.request.urlopen(request,timeout=5);raise RuntimeError('expected_denial') + except urllib.error.HTTPError as error: + body=json.load(error) + assert error.code==expected and body.get('error')==reason + results[label]=error.code +with sqlite3.connect('file:/data/email-connect.db?mode=ro',uri=True) as database: + assert database.execute('PRAGMA integrity_check').fetchone()[0]=='ok' + assert database.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='verification_attempts'").fetchone() +results.update(sqlite_integrity='ok',reservation_schema_present=True,mail_sent=False) +print(json.dumps(results)) +''' + return json.loads(k('exec', '-n', NS, 'deployment/email-connect', '--', 'python', '-c', code)) + + +def deploy(before, body, destination): + destination.mkdir(mode=0o700) + receipt = {'status': 'incomplete', 'decision': DECISION, 'old_image': OLD, + 'new_image': NEW, 'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} + started = time.monotonic() + paused = False + helper = False + succeeded = False + name = json.loads(body)['metadata']['name'] + try: + paused = True + receipt['phase'] = 'pause' + k('scale', 'deployment/email-connect', '-n', NS, '--replicas=0') + k('wait', '--for=delete', 'pod', '-n', NS, + '-l', 'app.kubernetes.io/name=email-connect', '--timeout=120s') + if document('get', 'deployment', 'email-connect', '-n', NS)['spec']['replicas'] != 0: + raise Refused('writers_not_quiesced') + helper = True + k('create', '-f', '-', body=body) + k('wait', '--for=condition=Ready', 'pod/' + name, '-n', NS, '--timeout=180s', timeout=240) + receipt['phase'] = 'capture' + receipt.update(capture(name, destination)) + k('delete', 'pod', name, '-n', NS, '--wait=true', '--timeout=120s') + helper = False + receipt['phase'] = 'image_patch' + k('patch', 'deployment', 'email-connect', '-n', NS, '--type=json', '-p', patch()) + k('scale', 'deployment/email-connect', '-n', NS, '--replicas=1') + k('rollout', 'status', 'deployment/email-connect', '-n', NS, '--timeout=180s', timeout=240) + actual = document('get', 'deployment', 'email-connect', '-n', NS) + expected = copy.deepcopy(before['spec']) + expected['template']['spec']['containers'][0]['image'] = NEW + if actual['spec'] != expected or actual['status'].get('readyReplicas') != 1: + raise Refused('unexpected_release_spec') + receipt['phase'] = 'verification' + receipt['checks'] = verify() + receipt.update(status='released_verified', image_only_spec_change=True, ready_replicas=1) + succeeded = True + except BaseException as exc: + receipt.update(status='failed', failure_type=type(exc).__name__) + raise + finally: + try: + if helper: + k('delete', 'pod', name, '-n', NS, '--ignore-not-found=true', '--wait=true', '--timeout=120s') + except Exception: + receipt['helper_cleanup_failed'] = True + if paused and not succeeded: + try: + current = image(document('get', 'deployment', 'email-connect', '-n', NS)) + replicas = '1' if current == OLD else '0' + k('scale', 'deployment/email-connect', '-n', NS, '--replicas=' + replicas) + receipt['failure_replicas'] = int(replicas) + if replicas == '1': + k('rollout', 'status', 'deployment/email-connect', '-n', NS, '--timeout=180s', timeout=240) + receipt['old_receiver_resumed'] = True + except Exception: + receipt['receiver_state_requires_check'] = True + receipt['elapsed_seconds'] = round(time.monotonic() - started, 2) + private_write(destination / 'release-receipt.json', json.dumps(receipt, indent=2).encode()) + print(json.dumps(receipt)) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--execute', action='store_true') + parser.add_argument('--destination', type=Path, required=True) + args = parser.parse_args() + os.environ['KUBECONFIG'] = '/etc/rancher/k3s/k3s.yaml' + if document('get', 'namespace', 'kube-system')['metadata']['uid'] != UID: + raise Refused('wrong_cluster') + before = document('get', 'deployment', 'email-connect', '-n', NS) + if image(before) != OLD or before['spec']['replicas'] != 1 or before['status'].get('readyReplicas') != 1: + raise Refused('unexpected_release') + if document('get', 'pvc', 'email-connect-data', '-n', NS)['status']['phase'] != 'Bound': + raise Refused('pvc_not_bound') + config = document('get', 'configmap', 'email-connect-config', '-n', NS)['data'] + if config.get('EMAIL_CONNECT_DATABASE_PATH') != '/data/email-connect.db': + raise Refused('unexpected_database_path') + name = 'email-connect-verification-backup-' + time.strftime('%Y%m%d%H%M%S', time.gmtime()) + body = json.dumps(backup_pod(name)).encode() + k('create', '--dry-run=server', '-f', '-', body=body) + projected = document('patch', 'deployment', 'email-connect', '-n', NS, + '--type=json', '-p', patch(), '--dry-run=server') + expected = copy.deepcopy(before['spec']) + expected['template']['spec']['containers'][0]['image'] = NEW + if projected['spec'] != expected: + raise Refused('unexpected_dry_run_spec') + if not args.execute: + print(json.dumps({'status': 'reviewed', 'live_changes': False, + 'image_only_spec_diff': True, 'readonly_backup_helper': True})) + return + if not args.destination.is_absolute() or args.destination.exists(): + raise Refused('destination_must_be_new') + def terminate(signum, frame): + raise Refused('release_interrupted') + signal.signal(signal.SIGTERM, terminate) + deploy(before, body, args.destination) + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + print(json.dumps({'status': 'failed', 'failure_type': type(exc).__name__})) + sys.exit(1) diff --git a/docs/verification-retry-review-2026-10-07.md b/docs/verification-retry-review-2026-10-07.md index f8b5fec..3de2ed0 100644 --- a/docs/verification-retry-review-2026-10-07.md +++ b/docs/verification-retry-review-2026-10-07.md @@ -1,5 +1,9 @@ # Verification retry correction — 2026-10-07 +Released: digest1e613d43 is live with one Ready replica. Founder approval, +private backup, deployment and no-send verification are complete; the detailed +receipt below includes the corrected validation-client failure. + Source contract: POST /v1/registration-verifications keeps its 202 JSON {request_id, accepted: true} response. Repeated tenant/client/registration operations return the same receipt after provider acceptance without resending. @@ -39,7 +43,7 @@ be read by the old image, but schema compatibility is not delivery safety. Source: 725850591754039771031d5fa8ba29562d6d4ce7. Image: forgejo.coulomb.social/coulomb/email-connect@sha256:1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c. -Candidate pin is committed in deployment.yaml; it has not been applied live. +Owner pin in deployment.yaml is now applied under the recorded founder approval. The packaged image passed synthetic retry/restart smoke as UID10001, with read-only root, temporary SQLite and network disabled. One provider call, unchanged request receipt and digest-only handle persistence were verified. @@ -52,8 +56,9 @@ proves only the container image changes. Both guarded JSON image patch and non-root read-only-PVC backup helper passed server dry-run. No live object changed. Existing PVC=email-connect-data, Secret reference=email-connect-runtime. -Founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 is open. Scope: -ADMINISTER @ realm:kubernetes/railiance01, activation=APPROVED after founder go. +Founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 was resolved following +the founder's Ok, go ahead. Scope: +ADMINISTER @ realm:kubernetes/railiance01, activation=APPROVED for this exact release. Do not apply the whole package to make this image correction: preserve live runtime ownership and use the reviewed guarded image patch. @@ -73,3 +78,42 @@ Failure before the patch: remove helper and resume the old one replica. Failure after new code has served: stop sending and keep the PVC/backup; reconcile before any old-image rollback because the old verification code ignores reservations. This packet grants no new caller, platform sender or offsite backup consumer. + + +## Completed release receipt + +Release started2026-10-07T20:29:29Z; final new-pod Ready20:33:16Z, +window227 seconds. First capture/update attempt58.27s completed the backup +and image change, then failed its client-side authenticated check: the probe +used the raw mounted token while the service normalizes it with strip(). Raw +header construction raises ValueError; corrected normalization gives the +expected denial. The safety path held the new receiver at zero replicas during +repair; it never rolled back to old verification code. Resumption and all +corrected checks took5.35s; one Ready replica remained healthy afterward. + +Validation: healthz/readyz200, bad bearer401, approved bearer + disallowed +invitation template400, empty verification400. SQLite integrity passes and the +new verification_attempts table exists. These calls create no delivery/challenge +records and send no mail. The post-repair spec comparison proves the resumption +changed only replicas0→1; the pre-release server comparison proves the reviewed +permanent change is the image. Backup helper pod count is zero. + +Private backup: node root0700 +/tmp/email-connect-preverification-20261007T202924Z/database.sqlite3, mode0600, +40960 bytes, SHA256 +3b4651064fa7dd7bd5073ed8567480ea1f4cb527dbe6c7462cc778182e7da1f8. +Matching workstation0700 /tmp/csoc-email-private-hkt09g6u/database.sqlite3, +mode0600. Isolated image rehearsal as UID10001/network disabled adds the +reservation table and preserves every row fingerprint in the four existing +tables. No private values enter this receipt or Git; copies remain for the +retention/custody handoff, not as proof of governed durable backup coverage. + +Five synthetic deployment failure cases passed, including capture refusal, +pre-patch failure and lost patch response/post-patch verification failure. +Corrected historic execution helper is preserved in +../deploy/k8s/railiance/deploy-verification-2026-10-07.py. Its old-image guard +refuses repeating this completed release; it is not a generic rollout tool. + +EMAIL-WP-0006 is finished. Residual caller/sender/limit/ledger recovery work is +live in CSOC-WP-0010-T04, inbox/bounce/retention/copy custody in T05, and +cross-owner journey/user acceptance in T06. CSOC-WP-0008/0010 remain active. diff --git a/workplans/EMAIL-WP-0006-verification-retry-safety.md b/workplans/EMAIL-WP-0006-verification-retry-safety.md index f384600..39ebd6d 100644 --- a/workplans/EMAIL-WP-0006-verification-retry-safety.md +++ b/workplans/EMAIL-WP-0006-verification-retry-safety.md @@ -4,7 +4,7 @@ type: workplan title: "Make registration verification delivery retry safe" domain: infotech repo: email-connect -status: active +status: finished owner: codex topic_slug: netkingdom created: "2026-10-07" @@ -57,11 +57,10 @@ and retry. Harness container/network were removed after validation. ```task id: EMAIL-WP-0006-T03 -status: wait +status: done priority: high -needs_human: true +needs_human: false decision_id: "8da41538-8482-4c72-b83d-e0fce1b662f0" -intervention_note: "Approve exact email-connect retry correction digest1e613d43 with brief receiver pause and private SQLite backup; existing sender/caller/custody unchanged. See release review." state_hub_task_id: "fcf55c0a-49b5-53a8-9497-3dfcc18c3828" ``` @@ -89,4 +88,22 @@ No live custody change was made. 2026-10-07 candidate prepared: source7258505, published digest1e613d43734c44c5cd32c6083fda8b4da5d01ef43d10c6ca9c29a33c788cf75c; UID10001 network-disabled packaged smoke and exact live/server dry-run pass. -Current runtime remains e095d7e5; founder decision 8da41538-8482-4c72-b83d-e0fce1b662f0 open. +2026-10-07 live release completed after founder's Ok, go ahead, +decision8da41538-8482-4c72-b83d-e0fce1b662f0 resolved. Digest1e613d43 is live, +one Ready replica, health200, unauth401, authenticated template/invalid request400, +SQLite integrity and new reservation schema verified. No mail sent by validation. +Quiesced private SQLite backup40960 bytes/hash3b465106 passes isolated UID10001 +network-disabled migration rehearsal, preserving rows in all four old tables. +Initial probe omitted service token normalization and raised ValueError locally; +sending was held stopped during correction. Fixed probe passes; final +Ready20:33:16Z, release window227s, zero helper pods remain. Private backups +are retained on node/workstation under0700/0600; receipt in the release review. + +Residuals handed off BEFORE finish to existing live CSOC-WP-0010-T04 +(ca41be06-3611-5647-b469-4f89f56f77f9): caller/tenant/template permissions, +per-caller limits, platform sender custody and reservation-ledger recovery. +CSOC-WP-0010-T05 (d9e165d9-dd0b-54c2-bda1-bd314b999b5b) owns inbox/reply/bounce +operations and retention/custody of evidence and temporary private copies. +CSOC-WP-0010-T06 (97a2da91-debe-5ce2-9b68-1965a5c5abfe) retains cross-owner +browser/user acceptance. Parent source0dcca9e and its primary consistency sync +record these live handoffs; neither parent workplan is marked finished.