Retry one dropped OpenBao request and report the last intake in health.

The bridge forward was down 2026-10-05 20:30 to about 00:20 and dropped
the first request after reconnecting; 13 timer runs reported
credential_unavailable without health showing it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 7780@bnt-lap001
Assistant-Session: 622cd705-cb31-4322-ba08-dd42e44fa583
This commit is contained in:
tegwick 2026-10-06 00:26:09 +02:00
parent 65887ef637
commit ac8227eead
4 changed files with 43 additions and 3 deletions

View file

@ -2,6 +2,7 @@
import argparse
import fcntl
import json
from datetime import datetime, timezone
import sys
from pathlib import Path
@ -46,6 +47,8 @@ def run_due(root, config, mailbox=None, post=notify.post_message):
except (OSError, ValueError) as exc:
report['intake'] = {'ok': False, 'status': 'error', 'error': str(exc)}
report['ok'] = report['intake']['ok']
store.atomic(state / 'last-intake.json', json.dumps(
{**report['intake'], 'at': datetime.now(timezone.utc).isoformat()}, indent=2).encode() + b'\n')
report['requeued'] = len(intake.requeue_quarantine(root, registry))
report['notices'] = len(notify.match(root, registry))
hub = config.get('hub', {})

View file

@ -172,7 +172,10 @@ def health(root, now=None, registry=None):
cursor_path = Path(root) / 'data' / 'state' / 'imap-cursor.json'
cursor = json.loads(cursor_path.read_text()) if cursor_path.exists() else {}
pending, _ = pending_hub(root, registry)
return {'last_scan': cursor.get('scanned_at'), 'items_per_source': counts,
last_path = Path(root) / 'data' / 'state' / 'last-intake.json'
last_intake = json.loads(last_path.read_text()) if last_path.exists() else None
return {'last_scan': cursor.get('scanned_at'), 'last_intake': last_intake,
'items_per_source': counts,
'silent_sources': silent, 'quarantined_items': len(quarantine),
'pending_hub_notices': sum(len(v) for v in pending.values()),
'integrity_problems': store.verify(root)}

View file

@ -15,7 +15,22 @@ class CredentialUnavailable(Exception):
"""The credential could not be obtained. The message never contains a secret."""
def _call(addr, method, path, token=None, body=None, timeout=15):
def _call(addr, method, path, token=None, body=None, timeout=15, attempts=2):
"""One OpenBao HTTP call. A connection error is retried once, because the
SSH bridge forward often drops the first request after it reconnects."""
for attempt in range(attempts):
try:
return _once(addr, method, path, token, body, timeout)
except _Unreachable as exc:
if attempt + 1 == attempts:
raise CredentialUnavailable(str(exc)) from None
class _Unreachable(Exception):
pass
def _once(addr, method, path, token, body, timeout):
request = urllib.request.Request(
addr.rstrip('/') + '/v1/' + path.lstrip('/'), method=method,
data=None if body is None else json.dumps(body).encode(),
@ -28,7 +43,7 @@ def _call(addr, method, path, token=None, body=None, timeout=15):
# Status only. The response body could echo request data.
raise CredentialUnavailable(f'OpenBao {method} {path.split("/")[0]}/… returned {exc.code}') from None
except (urllib.error.URLError, OSError) as exc:
raise CredentialUnavailable(f'OpenBao unreachable at {addr}: {type(exc).__name__}') from None
raise _Unreachable(f'OpenBao unreachable at {addr}: {type(exc).__name__}') from None
def kv_v2_data_path(kv_path):

View file

@ -52,6 +52,25 @@ class Vault(unittest.TestCase):
'auth/approle/login', 'platform/data/workloads/inbound-library/imap',
'auth/token/revoke-self'])
def test_dropped_first_request_is_retried_once(self):
attempts = []
def flaky(addr, method, path, token, body, timeout):
attempts.append(path)
if len(attempts) == 1:
raise vault._Unreachable('OpenBao unreachable at x: URLError')
return {'ok': True}
original = vault._once
vault._once = flaky
try:
self.assertEqual(vault._call('http://bao', 'GET', 'sys/health'), {'ok': True})
attempts.clear()
vault._once = lambda *a: (_ for _ in ()).throw(vault._Unreachable('down'))
with self.assertRaises(vault.CredentialUnavailable):
vault._call('http://bao', 'GET', 'sys/health')
finally:
vault._once = original
def test_denied_read_still_revokes_and_hides_secret(self):
bao = FakeBao(deny_read=True)
with self.assertRaises(vault.CredentialUnavailable) as caught: