Implement durable recurring backup alert delivery candidate

Assistant: codex
Assistant-Model: gpt-6-astra
Assistant-Session: 01a18e2d-f513-7742-b8e7-12af0b744ff3

docs/release/backup-alert-isolated-smtp-2026-10-31.json

@@ -0,0 +1,23 @@
+{
+  "schema": "coulomb.backup-alert-isolated-smtp.v1",
+  "observed_at": "2026-10-31T09:14:32.318517+00:00",
+  "real_alertmanager_api_and_route_readback": true,
+  "synthetic_smtp_received": 1,
+  "event_id_in_email": true,
+  "loopback_only_smtp": true,
+  "external_email_sent": false,
+  "production_route_activated": false,
+  "transport_handoff": {
+    "status": "active_email_route_verified",
+    "accepted_at": "2026-10-31T09:14:32.318517+00:00",
+    "email_delivery_proven": false
+  },
+  "fixture_overrides": [
+    "loopback SMTP without TLS/auth",
+    "example.invalid addresses",
+    "zero group wait",
+    "one second group interval"
+  ],
+  "rehearsal_source_sha256": "988c0bfc5397fee1093a44a65c30528f9426abd6130fbfdfb178a374b6f2a1eb",
+  "consumer_source_sha256": "07522437d4276b92f0a44149adc6c9e81bb18dbe59ce0f1f9e58a48347700d0e"
+}

docs/release/backup-alert-isolated-smtp-2026-10-31.rehearsal.txt

@@ -0,0 +1,51 @@
+import datetime, importlib.util, json, pathlib, socket, socketserver, subprocess, tempfile, threading, time, urllib.request
+ROOT=pathlib.Path('/home/worsch/coulomb-social')
+spec=importlib.util.spec_from_file_location('alerts',ROOT/'scripts/backup_alerts.py');alerts=importlib.util.module_from_spec(spec);spec.loader.exec_module(alerts)
+received=[]
+class SMTP(socketserver.StreamRequestHandler):
+ def handle(self):
+  self.wfile.write(b'220 local synthetic SMTP\r\n');self.wfile.flush()
+  data=False;body=[]
+  while line:=self.rfile.readline():
+   if data:
+    if line==b'.\r\n':received.append(b''.join(body).decode());data=False;self.wfile.write(b'250 received\r\n')
+    else:body.append(line)
+   elif line.upper().startswith((b'EHLO',b'HELO')):self.wfile.write(b'250 localhost\r\n')
+   elif line.upper().startswith((b'MAIL',b'RCPT',b'RSET',b'NOOP')):self.wfile.write(b'250 OK\r\n')
+   elif line.upper().startswith(b'DATA'):data=True;body=[];self.wfile.write(b'354 send data\r\n')
+   elif line.upper().startswith(b'QUIT'):self.wfile.write(b'221 bye\r\n');self.wfile.flush();return
+   else:self.wfile.write(b'500 unsupported\r\n')
+   self.wfile.flush()
+with tempfile.TemporaryDirectory(prefix='csoc-native-alert-') as temp, socketserver.ThreadingTCPServer(('127.0.0.1',0),SMTP) as smtp:
+ threading.Thread(target=smtp.serve_forever,daemon=True).start()
+ root=pathlib.Path(temp)
+ cid=subprocess.check_output(['docker','create','--network','none','sha256:690c7b525f4367aa91f73e2f91c632206d32e97c6384bdbf2fb7a861b420340d'],text=True).strip()
+ try:subprocess.run(['docker','cp',cid+':/bin/alertmanager',str(root/'alertmanager')],check=True,capture_output=True)
+ finally:subprocess.run(['docker','rm',cid],check=True,capture_output=True)
+ config=json.loads((ROOT/'deploy/backup/alertmanager-config.pending.json').read_text())
+ email=config['receivers'][1]['email_configs'][0]
+ email.pop('auth_username');email.pop('auth_password_file');email['to']='receiver@example.invalid';email['from']='sender@example.invalid';email['smarthost']='127.0.0.1:'+str(smtp.server_address[1]);email['require_tls']=False
+ config['route']['group_wait']='0s';config['route']['group_interval']='1s'
+ (root/'config.json').write_text(json.dumps(config))
+ with socket.socket() as sock:sock.bind(('127.0.0.1',0));port=sock.getsockname()[1]
+ endpoint='http://127.0.0.1:'+str(port)
+ process=subprocess.Popen([str(root/'alertmanager'),'--config.file='+str(root/'config.json'),'--storage.path='+str(root/'data'),'--web.listen-address=127.0.0.1:'+str(port),'--cluster.listen-address='],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ try:
+  for i in range(100):
+   if process.poll() is not None:raise RuntimeError('isolated_alertmanager_stopped')
+   try:urllib.request.urlopen(endpoint+'/-/ready',timeout=1).close();break
+   except Exception:time.sleep(.1)
+  else:raise RuntimeError('isolated_alertmanager_not_ready')
+  now=datetime.datetime.now(datetime.timezone.utc)
+  event={'event_id':'8f002752-1b8d-4aba-a67e-a76bf5768d41','condition':'backup_stale','test':True,'observed_at':now.isoformat()}
+  try:result=alerts.deliver_alertmanager(endpoint,event,now)
+  except Exception:
+   print(json.dumps(alerts.request_json(endpoint+"/api/v2/alerts")));raise
+  for i in range(100):
+   if received:break
+   time.sleep(.1)
+  assert len(received)==1 and event['event_id'] in received[0] and 'receiver@example.invalid' in received[0]
+  proof={'schema':'coulomb.backup-alert-isolated-smtp.v1','observed_at':now.isoformat(),'real_alertmanager_api_and_route_readback':True,'synthetic_smtp_received':1,'event_id_in_email':True,'loopback_only_smtp':True,'external_email_sent':False,'production_route_activated':False,'transport_handoff':result,'fixture_overrides':['loopback SMTP without TLS/auth','example.invalid addresses','zero group wait','one second group interval']}
+  (ROOT/'docs/release/backup-alert-isolated-smtp-2026-10-31.json').write_text(json.dumps(proof,indent=2)+'\n');print(json.dumps(proof))
+ finally:
+  process.terminate();process.wait(timeout=10);smtp.shutdown()

docs/release/backup-alert-recurring-integration-2026-10-31.md

@@ -0,0 +1,107 @@
+# Recurring backup alert integration — source candidate
+
+Both destinations are accepted and their one-off tests received: platform-root's
+State Hub inbox and bernd.worsch@gmail.com. The founder confirmed Gmail inbox
+receipt; the exact proof Job and SMTP receipt were independently verified.
+That does not activate recurring delivery.
+
+The consumer now has a durable outbox and source wiring for recurring delivery.
+It reuses State Hub and the existing Alertmanager/SMTP projection; it adds no
+mail service, mailbox credential or public endpoint. Deployment remains pending.
+
+## Consumer behavior
+
+`scripts/backup_alerts.py` locks a private journal, writes and fsyncs each event
+before a send, and checkpoints each channel separately. A failed State Hub call
+does not prevent the email handoff, and vice versa. Replays retain the UUID.
+State Hub delivery requires the exact central message receipt; a queue receipt
+is insufficient. A lost HTTP reply can produce a duplicate inbox message with
+the same event ID: this is at-least-once delivery, not exactly-once delivery.
+
+Alertmanager submission uses its [v2 API](https://raw.githubusercontent.com/prometheus/alertmanager/main/api/v2/openapi.yaml).
+The consumer reads back exact labels/times and the expected email receiver for
+active events. Millisecond timestamps match the native API's serialization.
+Acceptance is an email-transport handoff, never a recipient acknowledgment.
+Alertmanager owns SMTP retry/grouping/silencing. No SMTP password enters this
+consumer process, State Hub, output or repository.
+
+`observe-backup-alerts.py` calls the existing complete health check. Pending and
+stale reasons create their respective alert episodes; other evidence/capacity
+failures create `backup_failed`. Repeated unhealthy observations refresh the same
+episode; a later recurrence after recovery gets a new UUID. The existing hourly
+freshness unit drives observation and replay. Its exit status measures whether
+monitoring/delivery succeeded; backup health remains explicit in its JSON.
+
+Both the paired-backup and freshness units have an OnFailure template handler.
+The handler validates systemd's MONITOR_UNIT, MONITOR_SERVICE_RESULT and
+MONITOR_INVOCATION_ID, and uses the failed invocation UUID for replay. A later
+healthy report clears explicit failed-run events only if its fully verified
+paired capture is newer than the failure. An old healthy backup cannot erase a
+new failed-run alert. No automatic restore, capture replay or deletion is added.
+
+Active alerts have a three-hour lease, refreshed hourly. Resolved emails explicitly
+say that lease expiry or an explicit clear is not recovery proof. Independent
+producer/notification-debt monitoring remains an operating requirement: a dead
+producer cannot monitor itself. Native delivery and receipt tests must include
+producer interruption, notification backlog and channel outage cases.
+
+The separate private state directory is `/var/lib/coulomb-social-backup-alerts`.
+Records are limited to 16 KiB each and 1,024 events; pressure at 75% produces a
+health alert. Confirmed records are retained, not automatically deleted. This
+bounds committed record count, not filesystem allocation or abandoned temporary
+files after a crash. Reviewed archival/capacity handling is still required. A
+full journal refuses new events while explicit replay of existing events remains
+available. Processing rotates attempts to prevent bounded service executions
+from permanently starving later records.
+
+## Email route and ownership
+
+`deploy/backup/alertmanager-config.pending.json` is a consumer route proposal,
+not a live Helm values file. It selects only this application's three backup
+conditions and platform-root owner, leaving unrelated alerts on the existing
+pending receiver. SMTP uses the existing `telemetry-alert-smtp/password`
+projection, sender platform@coulomb.social, verified TLS and the chosen Gmail
+recipient. It includes resolved notifications so short incidents can still be
+reported. The pending generic telemetry acknowledgment service is not required
+or activated by this backup route; recipient receipt is not audit acknowledgment.
+
+The observed live service is `telemetry-alertmanager`, ClusterIP 10.43.37.119,
+port 9093; GET /api/v2/receivers returned only pending-operator-channel.
+The route is therefore **not active**. `alerting.env.example` records candidate
+private endpoints and contains no credential. Recheck their identities before
+installation. rapp-telemetry owns the exact Helm merge, existing-secret mount,
+current release diff, route activation and rollback. Preserve all other admitted
+routes and mounted secrets when preparing that exact owner operation.
+
+## Verification and remaining execution
+
+Full `make check`: 460 passed, zero failures/errors/skips, including the mandatory
+browser journeys. The 14 new outbox/producer checks cover independent failures,
+queue receipts, replay, recovery, journal pressure and systemd invocation binding.
+`backup-alert-recurring-sources-2026-10-31.json` binds the checked sources.
+
+Native amtool validated the exact route configuration and all three positive
+conditions plus four unrelated labelsets. Its container had no network.
+`backup-alert-route-validation-2026-10-31.json` binds the image and config hash.
+The actual target systemd 255 verified all five candidate units from temporary
+files, then removed those files. Return code was zero; warnings concerned two
+unrelated unreadable identity timers. Nothing was installed or enabled.
+`backup-alert-unit-validation-2026-10-31.json` binds the exact unit files.
+
+An isolated real Alertmanager process accepted/read back a synthetic event and
+delivered an email to a loopback-only SMTP fixture. Its overrides replaced
+recipient/sender with example.invalid addresses, SMTP with a local listener
+without authentication/TLS, and grouping delays with short fixture values.
+This proves API, routing, template and message correlation, not live TLS/custody.
+The previous founder test proves the live SMTP projection separately. Receipt
+and reproduction source are `backup-alert-isolated-smtp-2026-10-31.json` and
+`.rehearsal.txt`. The local process, copied binary and state were removed.
+
+Before activation, prepare and review the exact owner route/mount change and
+rebuilt operating bundle, install consumer units disabled, and verify live
+failed/pending/stale test events in both channels. Include channel failure,
+restart/replay and producer-death detection evidence. Preserve stable test IDs
+and do not replay the already completed one-off email Job. The existing capacity,
+rolling retention, historical-copy disposition and service-contract obligations
+remain required before the already approved schedule is enabled. WP-0012-T03
+remains in progress; this candidate does not complete WP-0013/14/15 acceptance.

docs/release/backup-alert-recurring-sources-2026-10-31.json

@@ -0,0 +1,26 @@
+{
+  "schema": "coulomb.backup-alert-source-validation.v1",
+  "full_gate": "make check",
+  "results": {
+    "tests": 460,
+    "errors": 0,
+    "failures": 0,
+    "skipped": 0
+  },
+  "sources": {
+    "scripts/backup_alerts.py": "07522437d4276b92f0a44149adc6c9e81bb18dbe59ce0f1f9e58a48347700d0e",
+    "scripts/notify-backup-alert.py": "0bd53bc5ad22d8bcb0326fb18a17737744b06416fe785dad077308e45f051d5a",
+    "scripts/observe-backup-alerts.py": "17caf4fa65283242f618b039ff1d018801c9890ea2a9178912cbdd4b50ee2664",
+    "scripts/check-paired-backup-health.py": "2d40ec6e4c59ea7fe1d46065d197b9a87ca6811f023cc6f3e20e4184a3490566",
+    "tests/test_backup_alert_outbox.py": "eb1c3263999592d7bed70a6b092a65edc660558d1036409d3473ceb60d36cdf5",
+    "deploy/backup/alertmanager-config.pending.json": "03622039baa7ba953a72263df666bd81b9cf9469ccc6977c21fcfd23d4bc90c3",
+    "deploy/backup/alerting.env.example": "509844b5c8ef618b812d52f32e19c3c559623636dca62f7b16f437aa90cc0458",
+    "deploy/backup/coulomb-social-backup-alert@.service": "4bfa019d29b1d0de278f03553602a5e824de50ed99ebcf3b5035cf4d5b06fe98",
+    "deploy/backup/coulomb-social-backup-freshness.service": "8d6278c2cbca7f48b659c239c1d1e181fa4b9cf023fe7429850e294c6cd84d0e",
+    "deploy/backup/coulomb-social-paired-backup.service": "8b01b788bb08f453fdd4807104185c89411e0b0b4dedee9240f14f110ceb5513",
+    "deploy/backup/coulomb-social-backup-freshness.timer": "75d1d0721c07fdd0703910c06bb8a678396440e7d95411a37ee7de4938f3ac31",
+    "deploy/backup/coulomb-social-paired-backup.timer": "5e87235e66f89a64a972562d6f163f62420628fc44fb7fd8d603a96c347dfbb9"
+  },
+  "installed": false,
+  "recurring_alerts_activated": false
+}

docs/release/backup-alert-route-validation-2026-10-31.json

@@ -0,0 +1,12 @@
+{
+  "schema": "coulomb.backup-alert-route-validation.v1",
+  "observed_at": "2026-10-31T09:12:50.612789+00:00",
+  "validator_image_id": "sha256:690c7b525f4367aa91f73e2f91c632206d32e97c6384bdbf2fb7a861b420340d",
+  "config_sha256": "03622039baa7ba953a72263df666bd81b9cf9469ccc6977c21fcfd23d4bc90c3",
+  "native_config_check": true,
+  "native_positive_routes": 3,
+  "native_unrelated_routes_excluded": 4,
+  "network": "none",
+  "live_configuration_changed": false,
+  "notification_sent": false
+}

docs/release/backup-alert-unit-validation-2026-10-31.json

@@ -0,0 +1,15 @@
+{
+  "returncode": 0,
+  "output": "net-kingdom-identity-backup.timer: Failed to open /etc/systemd/system/net-kingdom-identity-backup.timer: Permission denied\nnet-kingdom-identity-backup-health.timer: Failed to open /etc/systemd/system/net-kingdom-identity-backup-health.timer: Permission denied\n",
+  "units": 5,
+  "installed": false,
+  "schema": "coulomb.backup-alert-unit-validation.v1",
+  "observed_at": "2026-10-31T09:13:11.864616+00:00",
+  "source_sha256": {
+    "deploy/backup/coulomb-social-backup-alert@.service": "4bfa019d29b1d0de278f03553602a5e824de50ed99ebcf3b5035cf4d5b06fe98",
+    "deploy/backup/coulomb-social-backup-freshness.service": "8d6278c2cbca7f48b659c239c1d1e181fa4b9cf023fe7429850e294c6cd84d0e",
+    "deploy/backup/coulomb-social-paired-backup.service": "8b01b788bb08f453fdd4807104185c89411e0b0b4dedee9240f14f110ceb5513",
+    "deploy/backup/coulomb-social-backup-freshness.timer": "75d1d0721c07fdd0703910c06bb8a678396440e7d95411a37ee7de4938f3ac31",
+    "deploy/backup/coulomb-social-paired-backup.timer": "5e87235e66f89a64a972562d6f163f62420628fc44fb7fd8d603a96c347dfbb9"
+  }
+}

docs/release/backup-operating-delivery-2026-10-31.md

@@ -1,5 +1,10 @@
 # Backup operating source delivery review
 
+**Recurring-alert update:** source integration is now prepared in
+`backup-alert-recurring-integration-2026-10-31.md`. Both sealed bundles below
+exclude the new outbox, observer, failure handler and route proposal. Neither is
+a complete current installation bundle; rebuild and review before installation.
+
 **Superseded after verified revision19:** the bundle below is historical and must
 not be installed. Current preparation is
 `backup-operating-delivery-revision19-2026-10-31.json`: archive060620da,