Retry one dropped OpenBao request and report the last intake in health.
The bridge forward was down 2026-10-05 20:30 to about 00:20 and dropped the first request after reconnecting; 13 timer runs reported credential_unavailable without health showing it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Assistant: claude-code Assistant-Model: opus Assistant-Process: 7780@bnt-lap001 Assistant-Session: 622cd705-cb31-4322-ba08-dd42e44fa583
This commit is contained in:
parent
65887ef637
commit
ac8227eead
4 changed files with 43 additions and 3 deletions
|
|
@ -2,6 +2,7 @@
|
|||
import argparse
|
||||
import fcntl
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -46,6 +47,8 @@ def run_due(root, config, mailbox=None, post=notify.post_message):
|
|||
except (OSError, ValueError) as exc:
|
||||
report['intake'] = {'ok': False, 'status': 'error', 'error': str(exc)}
|
||||
report['ok'] = report['intake']['ok']
|
||||
store.atomic(state / 'last-intake.json', json.dumps(
|
||||
{**report['intake'], 'at': datetime.now(timezone.utc).isoformat()}, indent=2).encode() + b'\n')
|
||||
report['requeued'] = len(intake.requeue_quarantine(root, registry))
|
||||
report['notices'] = len(notify.match(root, registry))
|
||||
hub = config.get('hub', {})
|
||||
|
|
|
|||
|
|
@ -172,7 +172,10 @@ def health(root, now=None, registry=None):
|
|||
cursor_path = Path(root) / 'data' / 'state' / 'imap-cursor.json'
|
||||
cursor = json.loads(cursor_path.read_text()) if cursor_path.exists() else {}
|
||||
pending, _ = pending_hub(root, registry)
|
||||
return {'last_scan': cursor.get('scanned_at'), 'items_per_source': counts,
|
||||
last_path = Path(root) / 'data' / 'state' / 'last-intake.json'
|
||||
last_intake = json.loads(last_path.read_text()) if last_path.exists() else None
|
||||
return {'last_scan': cursor.get('scanned_at'), 'last_intake': last_intake,
|
||||
'items_per_source': counts,
|
||||
'silent_sources': silent, 'quarantined_items': len(quarantine),
|
||||
'pending_hub_notices': sum(len(v) for v in pending.values()),
|
||||
'integrity_problems': store.verify(root)}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,22 @@ class CredentialUnavailable(Exception):
|
|||
"""The credential could not be obtained. The message never contains a secret."""
|
||||
|
||||
|
||||
def _call(addr, method, path, token=None, body=None, timeout=15):
|
||||
def _call(addr, method, path, token=None, body=None, timeout=15, attempts=2):
|
||||
"""One OpenBao HTTP call. A connection error is retried once, because the
|
||||
SSH bridge forward often drops the first request after it reconnects."""
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return _once(addr, method, path, token, body, timeout)
|
||||
except _Unreachable as exc:
|
||||
if attempt + 1 == attempts:
|
||||
raise CredentialUnavailable(str(exc)) from None
|
||||
|
||||
|
||||
class _Unreachable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _once(addr, method, path, token, body, timeout):
|
||||
request = urllib.request.Request(
|
||||
addr.rstrip('/') + '/v1/' + path.lstrip('/'), method=method,
|
||||
data=None if body is None else json.dumps(body).encode(),
|
||||
|
|
@ -28,7 +43,7 @@ def _call(addr, method, path, token=None, body=None, timeout=15):
|
|||
# Status only. The response body could echo request data.
|
||||
raise CredentialUnavailable(f'OpenBao {method} {path.split("/")[0]}/… returned {exc.code}') from None
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
raise CredentialUnavailable(f'OpenBao unreachable at {addr}: {type(exc).__name__}') from None
|
||||
raise _Unreachable(f'OpenBao unreachable at {addr}: {type(exc).__name__}') from None
|
||||
|
||||
|
||||
def kv_v2_data_path(kv_path):
|
||||
|
|
|
|||
|
|
@ -52,6 +52,25 @@ class Vault(unittest.TestCase):
|
|||
'auth/approle/login', 'platform/data/workloads/inbound-library/imap',
|
||||
'auth/token/revoke-self'])
|
||||
|
||||
def test_dropped_first_request_is_retried_once(self):
|
||||
attempts = []
|
||||
|
||||
def flaky(addr, method, path, token, body, timeout):
|
||||
attempts.append(path)
|
||||
if len(attempts) == 1:
|
||||
raise vault._Unreachable('OpenBao unreachable at x: URLError')
|
||||
return {'ok': True}
|
||||
original = vault._once
|
||||
vault._once = flaky
|
||||
try:
|
||||
self.assertEqual(vault._call('http://bao', 'GET', 'sys/health'), {'ok': True})
|
||||
attempts.clear()
|
||||
vault._once = lambda *a: (_ for _ in ()).throw(vault._Unreachable('down'))
|
||||
with self.assertRaises(vault.CredentialUnavailable):
|
||||
vault._call('http://bao', 'GET', 'sys/health')
|
||||
finally:
|
||||
vault._once = original
|
||||
|
||||
def test_denied_read_still_revokes_and_hides_secret(self):
|
||||
bao = FakeBao(deny_read=True)
|
||||
with self.assertRaises(vault.CredentialUnavailable) as caught:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue