canned-prompts/service/tests/test_health.py
tegwick f6e20b5e0c Survive credential rotation: re-read the lease for every connection
The deployment ran for one lease window and then sat unready for eight hours.
Platform credentials are 30-minute leases, not passwords: the service read the
mounted URL once at start-up, so External Secrets kept the file current while
the engine held the URL it booted with, and every reconnection after the first
expiry used a credential the database had already revoked.

make_engine now takes an optional refresh callable, invoked by a do_connect
hook each time the pool opens a connection, and pool_recycle is 900s so a
pooled connection is retired well inside the lease. Only username and password
are taken from the refreshed URL — host, port and database come from the engine,
so a malformed refresh cannot silently redirect the service somewhere else.

Two things behaved correctly and are worth keeping. /readyz reported the real
cause, "database unreachable: OperationalError", rather than a generic failure.
And liveness stayed independent of the database, so the pod was never
restart-looped: it was alive, unable to serve, and said so. Pointing liveness at
a database-dependent path would have masked this as a crash loop.

Service tests 47 -> 49, including one asserting pool_recycle stays inside the
shortest lease the platform issues.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 388925@bnt-lap001
Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502
2026-09-08 10:11:35 +02:00

166 lines
6.5 KiB
Python

"""Health surface.
The point of these tests is the negative cases. An endpoint that returns 200
under every condition tells an orchestrator nothing, and the failure is silent
exactly when it matters.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from alembic import command
from alembic.config import Config
from alembic.script import ScriptDirectory
from fastapi.testclient import TestClient
from canned_prompts_service.api import create_app
from canned_prompts_service.db import check_readiness, make_engine
from canned_prompts_service.settings import Settings
ROOT = Path(__file__).resolve().parents[1]
def expected_head() -> str:
"""Computed, not hardcoded: a new migration must not fail these tests."""
config = Config(str(ROOT / "alembic.ini"))
config.set_main_option("script_location", str(ROOT / "migrations"))
return ScriptDirectory.from_config(config).get_current_head()
def migrated_url(tmp_path: Path) -> str:
url = f"sqlite:///{tmp_path / 'svc.db'}"
config = Config(str(ROOT / "alembic.ini"))
config.set_main_option("script_location", str(ROOT / "migrations"))
config.set_main_option("sqlalchemy.url", url)
command.upgrade(config, "head")
return url
@pytest.fixture()
def ready_client(tmp_path: Path) -> TestClient:
url = migrated_url(tmp_path)
return TestClient(create_app(Settings(database_url=url), make_engine(url)))
def test_healthz_is_up_without_a_database() -> None:
"""Liveness must not depend on the database, or a DB blip restarts pods."""
client = TestClient(create_app(Settings(), None))
assert client.get("/healthz").json() == {"status": "ok"}
def test_readyz_fails_without_a_database() -> None:
client = TestClient(create_app(Settings(), None))
response = client.get("/readyz")
assert response.status_code == 503
assert response.json()["ready"] is False
assert "no database" in response.json()["detail"]
def test_readyz_fails_when_the_database_is_unreachable(tmp_path: Path) -> None:
engine = make_engine("sqlite:////nonexistent/dir/does-not-exist.db")
client = TestClient(create_app(Settings(database_url="x"), engine))
response = client.get("/readyz")
assert response.status_code == 503
assert response.json()["ready"] is False
def test_readyz_fails_when_the_schema_is_not_migrated(tmp_path: Path) -> None:
"""Reachable but unmigrated is not ready — it would 500 on the first query."""
engine = make_engine(f"sqlite:///{tmp_path / 'empty.db'}")
client = TestClient(create_app(Settings(database_url="x"), engine))
response = client.get("/readyz")
assert response.status_code == 503
assert response.json()["detail"] == "schema not migrated"
def test_readyz_reports_the_migration_when_ready(ready_client: TestClient) -> None:
body = ready_client.get("/readyz").json()
assert body["ready"] is True
assert body["migration"] == expected_head()
def test_state_health_matches_the_fleet_shape(ready_client: TestClient) -> None:
body = ready_client.get("/state/health").json()
assert body["status"] == "ok"
assert body["service"] == "canned-prompts"
assert body["db"] == "connected"
assert body["migration"] == expected_head()
def test_state_health_degrades_rather_than_lying() -> None:
client = TestClient(create_app(Settings(), None))
response = client.get("/state/health")
assert response.status_code == 503
assert response.json()["status"] == "degraded"
def test_settings_have_no_database_fallback() -> None:
"""Falling back to a local database when misconfigured hides the mistake."""
assert Settings().database_url == ""
assert Settings().configured is False
def test_database_url_may_come_from_a_file(tmp_path: Path) -> None:
"""A mounted secret should stay a file, not become an env var."""
secret = tmp_path / "url"
secret.write_text("sqlite:///from-file.db\n", encoding="utf-8")
settings = Settings(database_url_file=str(secret))
assert settings.configured is True
assert settings.resolved_database_url == "sqlite:///from-file.db"
def test_file_wins_over_env_when_both_are_set(tmp_path: Path) -> None:
"""A rotated secret must take effect, not be shadowed by a stale env var."""
secret = tmp_path / "url"
secret.write_text("sqlite:///from-file.db", encoding="utf-8")
settings = Settings(database_url="sqlite:///from-env.db", database_url_file=str(secret))
assert settings.resolved_database_url == "sqlite:///from-file.db"
def test_unreadable_secret_file_fails_loudly(tmp_path: Path) -> None:
settings = Settings(database_url_file=str(tmp_path / "missing"))
with pytest.raises(RuntimeError, match="cannot read secret file"):
_ = settings.resolved_database_url
def test_absent_publish_token_file_means_read_only_not_broken(tmp_path: Path) -> None:
"""The token secret is mounted optional and deliberately not issued. Its
absence is the documented read-only posture, not a fault — treating it as
one returned 500 from /packages instead of a 503 explaining why."""
settings = Settings(publish_token_file=str(tmp_path / "absent"))
assert settings.resolved_publish_token == ""
def test_absent_database_file_still_fails_loudly(tmp_path: Path) -> None:
"""The database URL is required; silence there would hide a real fault."""
settings = Settings(database_url_file=str(tmp_path / "absent"))
with pytest.raises(RuntimeError, match="cannot read secret file"):
_ = settings.resolved_database_url
def test_engine_rereads_credentials_on_each_connection(tmp_path: Path) -> None:
"""Platform credentials are 30-minute leases, not passwords. Reading the
file once at start-up worked for one lease window and then failed
permanently — External Secrets kept the file current while the engine kept
the URL it booted with."""
secret = tmp_path / "url"
secret.write_text(f"sqlite:///{tmp_path / 'a.db'}", encoding="utf-8")
settings = Settings(database_url_file=str(secret))
seen: list[str] = []
engine = make_engine(
settings.resolved_database_url,
refresh=lambda: (seen.append(settings.resolved_database_url) or settings.resolved_database_url),
)
with engine.connect():
pass
assert seen, "the refresh hook must run when the pool opens a connection"
def test_pool_recycle_is_shorter_than_the_shortest_lease() -> None:
"""30-minute runtime lease; a pooled connection must be retired first."""
from canned_prompts_service.db import POOL_RECYCLE_SECONDS
assert POOL_RECYCLE_SECONDS < 30 * 60