Give the runtime real readiness, graceful shutdown and stated limits

Closes gap G08. /healthz returned a constant without probing anything, the server
called ListenAndServe with no signal handling, and the operational limits of
in-memory state, startup-loaded keys and local-only logout lived in code comments
rather than anywhere an operator would look.

/readyz probes LLDAP, Authelia and privacyIDEA; /healthz stays liveness and
probes nothing. Keeping them distinct matters: wiring liveness to dependency
health means an orchestrator restarts KeyCape when a dependency blinks, and a
restart also discards every in-flight login, so the reaction is worse than the
condition it reacts to.

LLDAP is probed with a bind rather than a dial, since a rotated or revoked
service password leaves the port open and every lookup failing -- exactly what
readiness should catch and exactly what a dial would miss. The response names the
failing check but never the reason: the endpoint is unauthenticated and upstream
error text carries hostnames and sometimes credentials-in-URLs. Results are
cached for 2s so an unauthenticated endpoint cannot be used to drive unbounded
upstream traffic, and probes run concurrently under a 3s bound so a hung
dependency makes the endpoint answer rather than hang with it.

SIGTERM and SIGINT now drain in-flight requests for 15s, under the 30s read/write
timeouts so a stuck request cannot outlive the window before SIGKILL.

docs/operations.md states the single-replica topology and why, and three limits
easy to get wrong: the constant key-1 key ID makes same-kid rotation a trap for
consumers caching JWKS, removing a client does not revoke its issued tokens, and
/logout is local only. No throughput figures are given, since nothing here
benchmarks KeyCape. Shared storage and refresh tokens stay excluded, as G08
allows.

Verified in the running executable: 503 naming all three checks failed while
/healthz returned 200, the LLDAP check flipping to ok once started, and 40/40
requests succeeding across a SIGTERM.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NV9oijZukGyGbRQGGKnK4P

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 713576@bnt-lap001
Assistant-Session: 384c511d-9bce-4cb8-a676-2aef6c0c8df6

src/internal/adapters/lldap/adapter.go

@@ -435,3 +435,39 @@ var (
 	_ domain.UserRepository = (*LDAPAdapter)(nil)
 	_ domain.GroupLister    = (*LDAPAdapter)(nil)
 )
+
+// Ping reports whether the directory is reachable and the configured service
+// credentials still bind. It is the readiness probe for LLDAP (KEY-WP-0025).
+//
+// A bind rather than a bare dial: a rotated or revoked service password leaves
+// the port open and every lookup failing, which is exactly the state readiness
+// exists to catch. dial already binds, so this opens and closes one connection.
+func (a *LDAPAdapter) Ping(ctx context.Context) error {
+	type dialResult struct {
+		conn LDAPConn
+		err  error
+	}
+	// dial is synchronous and has no context, so race it against the caller's
+	// deadline rather than letting a hung directory outlive the probe budget.
+	results := make(chan dialResult, 1)
+	go func() {
+		conn, err := a.dial()
+		results <- dialResult{conn, err}
+	}()
+
+	select {
+	case <-ctx.Done():
+		go func() {
+			if result := <-results; result.err == nil {
+				result.conn.Close()
+			}
+		}()
+		return ctx.Err()
+	case result := <-results:
+		if result.err != nil {
+			return result.err
+		}
+		result.conn.Close()
+		return nil
+	}
+}

src/internal/server/readiness/readiness.go

@@ -0,0 +1,129 @@
+// Package readiness answers whether this instance can currently serve, as
+// distinct from whether the process is alive.
+//
+// /healthz says the process is running and is the right liveness signal: an
+// orchestrator restarting KeyCape because LLDAP is briefly down would turn a
+// dependency blip into an outage of its own. /readyz says the dependencies
+// KeyCape needs to complete a login are reachable, which is what should gate
+// traffic (KEY-WP-0025).
+//
+// Probes are reachability and credential checks, not functional tests. A ready
+// response means the dependency answered, not that every operation against it
+// will succeed.
+package readiness
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Check is one dependency probe. Probe must respect the context deadline.
+type Check struct {
+	Name  string
+	Probe func(ctx context.Context) error
+}
+
+// Reporter serves the readiness endpoint over a set of checks.
+type Reporter struct {
+	checks  []Check
+	ttl     time.Duration
+	timeout time.Duration
+
+	// now is swappable so tests can advance time without sleeping.
+	now func() time.Time
+
+	mu       sync.Mutex
+	cached   *result
+	cachedAt time.Time
+}
+
+type result struct {
+	Status string        `json:"status"`
+	Checks []checkStatus `json:"checks"`
+}
+
+type checkStatus struct {
+	Name   string `json:"name"`
+	Status string `json:"status"`
+}
+
+// New returns a Reporter.
+//
+// ttl caches the last result. The endpoint is necessarily unauthenticated, so
+// without a cache anyone able to reach it could drive one upstream request per
+// dependency per probe; the cache bounds that regardless of how often it is
+// called. timeout bounds each probe, so a hung dependency makes the endpoint
+// answer "not ready" rather than hang with it.
+func New(ttl, timeout time.Duration, checks ...Check) *Reporter {
+	return &Reporter{checks: checks, ttl: ttl, timeout: timeout, now: time.Now}
+}
+
+// ServeHTTP reports readiness: 200 when every check passes, 503 otherwise.
+//
+// The body names each check and whether it passed, but never why. The endpoint
+// is unauthenticated, and upstream error text tends to carry hostnames, versions
+// and occasionally credentials-in-URLs. Detail belongs in the server's logs.
+func (r *Reporter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	res := r.evaluate(req.Context())
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	if res.Status != "ready" {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	} else {
+		w.WriteHeader(http.StatusOK)
+	}
+	_ = json.NewEncoder(w).Encode(res)
+}
+
+func (r *Reporter) evaluate(ctx context.Context) result {
+	r.mu.Lock()
+	if r.cached != nil && r.now().Sub(r.cachedAt) < r.ttl {
+		cached := *r.cached
+		r.mu.Unlock()
+		return cached
+	}
+	r.mu.Unlock()
+
+	res := r.runChecks(ctx)
+
+	r.mu.Lock()
+	r.cached, r.cachedAt = &res, r.now()
+	r.mu.Unlock()
+	return res
+}
+
+// runChecks probes every dependency concurrently: readiness should cost about
+// as long as the slowest probe, not the sum of them.
+func (r *Reporter) runChecks(ctx context.Context) result {
+	statuses := make([]checkStatus, len(r.checks))
+	var wg sync.WaitGroup
+	for i, check := range r.checks {
+		wg.Add(1)
+		go func(i int, check Check) {
+			defer wg.Done()
+			probeCtx, cancel := context.WithTimeout(ctx, r.timeout)
+			defer cancel()
+			status := "ok"
+			if check.Probe == nil || check.Probe(probeCtx) != nil {
+				status = "failed"
+			}
+			statuses[i] = checkStatus{Name: check.Name, Status: status}
+		}(i, check)
+	}
+	wg.Wait()
+
+	sort.Slice(statuses, func(i, j int) bool { return statuses[i].Name < statuses[j].Name })
+	res := result{Status: "ready", Checks: statuses}
+	for _, status := range statuses {
+		if status.Status != "ok" {
+			res.Status = "not_ready"
+			break
+		}
+	}
+	return res
+}

src/internal/server/readiness/readiness_test.go

@@ -0,0 +1,136 @@
+package readiness_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"keycape/internal/server/readiness"
+)
+
+func probe(err error) func(context.Context) error {
+	return func(context.Context) error { return err }
+}
+
+func serve(t *testing.T, r *readiness.Reporter) (int, map[string]interface{}) {
+	t.Helper()
+	w := httptest.NewRecorder()
+	r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/readyz", nil))
+	var body map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
+		t.Fatalf("body is not JSON: %v (%s)", err, w.Body.String())
+	}
+	return w.Code, body
+}
+
+func TestReadyWhenEveryCheckPasses(t *testing.T) {
+	r := readiness.New(0, time.Second,
+		readiness.Check{Name: "lldap", Probe: probe(nil)},
+		readiness.Check{Name: "authelia", Probe: probe(nil)},
+	)
+	code, body := serve(t, r)
+	if code != http.StatusOK || body["status"] != "ready" {
+		t.Fatalf("status %d body %v", code, body)
+	}
+}
+
+// A dependency KeyCape needs to complete a login being down must take this
+// instance out of rotation, not be reported as healthy.
+func TestNotReadyWhenAnyCheckFails(t *testing.T) {
+	r := readiness.New(0, time.Second,
+		readiness.Check{Name: "lldap", Probe: probe(errors.New("connection refused"))},
+		readiness.Check{Name: "authelia", Probe: probe(nil)},
+	)
+	code, body := serve(t, r)
+	if code != http.StatusServiceUnavailable || body["status"] != "not_ready" {
+		t.Fatalf("status %d body %v", code, body)
+	}
+}
+
+// A nil probe is a wiring mistake; treating it as passing would report readiness
+// nobody checked.
+func TestNilProbeIsNotReady(t *testing.T) {
+	r := readiness.New(0, time.Second, readiness.Check{Name: "misconfigured"})
+	if code, _ := serve(t, r); code != http.StatusServiceUnavailable {
+		t.Fatalf("status %d, want 503", code)
+	}
+}
+
+// The endpoint is unauthenticated, so upstream error text must not reach it:
+// that text carries hostnames and sometimes credentials-in-URLs.
+func TestFailureDetailIsNotDisclosed(t *testing.T) {
+	secret := "ldap://admin:hunter2@lldap.internal:3890"
+	r := readiness.New(0, time.Second,
+		readiness.Check{Name: "lldap", Probe: probe(errors.New("dial " + secret + ": refused"))})
+	w := httptest.NewRecorder()
+	r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/readyz", nil))
+	for _, leaked := range []string{"hunter2", "lldap.internal", "refused"} {
+		if strings.Contains(w.Body.String(), leaked) {
+			t.Fatalf("response discloses %q: %s", leaked, w.Body.String())
+		}
+	}
+}
+
+// Unauthenticated and uncached, each call would drive one upstream request per
+// dependency. The TTL bounds that however often the endpoint is polled.
+func TestResultsAreCachedWithinTheTTL(t *testing.T) {
+	var probes int64
+	r := readiness.New(time.Minute, time.Second, readiness.Check{
+		Name:  "lldap",
+		Probe: func(context.Context) error { atomic.AddInt64(&probes, 1); return nil },
+	})
+	for i := 0; i < 5; i++ {
+		if code, _ := serve(t, r); code != http.StatusOK {
+			t.Fatalf("call %d: status %d", i, code)
+		}
+	}
+	if got := atomic.LoadInt64(&probes); got != 1 {
+		t.Fatalf("probed %d times, want 1 within the TTL", got)
+	}
+}
+
+// A hung dependency must make the endpoint answer, not hang alongside it.
+func TestSlowProbeTimesOutRatherThanBlocking(t *testing.T) {
+	r := readiness.New(0, 50*time.Millisecond, readiness.Check{
+		Name: "slow",
+		Probe: func(ctx context.Context) error {
+			<-ctx.Done()
+			return ctx.Err()
+		},
+	})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		if code, _ := serve(t, r); code != http.StatusServiceUnavailable {
+			t.Errorf("status %d, want 503", code)
+		}
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("readiness endpoint blocked on a hung dependency")
+	}
+}
+
+// Checks run concurrently: readiness should cost about the slowest probe.
+func TestChecksRunConcurrently(t *testing.T) {
+	slow := func(context.Context) error { time.Sleep(150 * time.Millisecond); return nil }
+	r := readiness.New(0, time.Second,
+		readiness.Check{Name: "a", Probe: slow},
+		readiness.Check{Name: "b", Probe: slow},
+		readiness.Check{Name: "c", Probe: slow},
+	)
+	start := time.Now()
+	if code, _ := serve(t, r); code != http.StatusOK {
+		t.Fatalf("status %d", code)
+	}
+	if elapsed := time.Since(start); elapsed > 400*time.Millisecond {
+		t.Fatalf("checks appear to run serially: %s for three 150ms probes", elapsed)
+	}
+}