Implement local decision delivery and reconcile blocked workplan requirements
All checks were successful
CI Smoke / host-smoke (push) Successful in 0s
CI Smoke / container-smoke (push) Successful in 2s
Build and Publish Container Image / build-and-push (push) Successful in 1m1s

Assistant: codex
Assistant-Model: gpt-6.1-sol
Assistant-Session: 01a1046e-fcad-7ea3-bf36-361fc8d679e8
This commit is contained in:
tegwick 2026-10-04 11:40:37 +02:00
parent 257a83e997
commit 3cb1cfaedf
25 changed files with 1258 additions and 50 deletions

View file

@ -0,0 +1,233 @@
package emission
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
const HeartbeatType = "audit-core.heartbeat"
// RareClasses is the source-owned inventory in cadence.yaml.
func RareClasses() []string {
return []string{ClassPrefix + "deny", ClassPrefix + "redact", ClassPrefix + "not_applicable", ClassPrefix + "audit_only"}
}
type heartbeatData struct {
Class string `json:"class"`
Assertion string `json:"assertion"`
Committed uint64 `json:"committed"`
ReleasedUncommitted uint64 `json:"released_uncommitted"`
}
func (o *Outbox) noteHeartbeat(event Event) {
if event.Type != HeartbeatType {
return
}
b, err := json.Marshal(event.Data)
if err != nil {
return
}
var data heartbeatData
if json.Unmarshal(b, &data) != nil {
return
}
at, err := time.Parse(time.RFC3339Nano, event.OccurredAt)
if err == nil {
o.heartbeats[data.Class] = at
}
}
// AppendDueHeartbeats commits one heartbeat per rare class at most every 24h.
// The timestamp is rebuilt from the log on restart, never from a process timer.
func (o *Outbox) AppendDueHeartbeats() error {
o.mu.Lock()
defer o.mu.Unlock()
now := o.clock().UTC()
var events []Event
for _, class := range RareClasses() {
if at, ok := o.heartbeats[class]; ok && now.Sub(at) < 24*time.Hour {
continue
}
id, err := eventID()
if err != nil {
return err
}
assertion := "nothing-to-report"
if o.uncommitted[class] > 0 {
assertion = "released-uncommitted-finding"
}
events = append(events, Event{ID: id, Type: HeartbeatType, Source: o.source, Subject: o.source,
Tenant: PlatformTenant, CorrelationID: id, OccurredAt: now.Format(time.RFC3339Nano),
Data: heartbeatData{Class: class, Assertion: assertion, Committed: o.committed[class], ReleasedUncommitted: o.uncommitted[class]}})
}
if len(events) == 0 {
return nil
}
return o.appendEvents(now.Format(time.RFC3339Nano), events)
}
type drainCursor struct {
Seq uint64 `json:"seq"`
Offset int64 `json:"offset"`
}
func (o *Outbox) loadCursor() error {
b, err := os.ReadFile(filepath.Join(filepath.Dir(o.path), "drain.json"))
if errors.Is(err, os.ErrNotExist) {
return nil
}
if err != nil {
return fmt.Errorf("read drain cursor: %w", err)
}
if err := json.Unmarshal(b, &o.cursor); err != nil {
return fmt.Errorf("decode drain cursor: %w", err)
}
if o.cursor.Offset < 0 || (o.cursor.Seq == 0) != (o.cursor.Offset == 0) {
return errors.New("invalid drain cursor")
}
return nil
}
// nextEntry reads only the committed prefix; mu protects the size snapshot.
func (o *Outbox) nextEntry() (entry, int64, error) {
f, err := os.Open(o.path)
if err != nil {
return entry{}, 0, err
}
defer f.Close()
r := bufio.NewReader(io.NewSectionReader(f, o.cursor.Offset, o.size-o.cursor.Offset))
line, err := r.ReadBytes('\n')
if err != nil {
return entry{}, 0, err
}
var e entry
if err := json.Unmarshal(line, &e); err != nil {
return entry{}, 0, err
}
if e.Seq != o.cursor.Seq+1 || e.Event.Source != o.source {
return entry{}, 0, errors.New("inconsistent next outbox entry")
}
return e, o.cursor.Offset + int64(len(line)), nil
}
func (o *Outbox) saveCursor(cursor drainCursor) error {
dir := filepath.Dir(o.path)
f, err := os.CreateTemp(dir, ".drain-*")
if err != nil {
return err
}
defer os.Remove(f.Name())
b, _ := json.Marshal(cursor)
if _, err = f.Write(b); err == nil {
err = f.Sync()
}
err = errors.Join(err, f.Close())
if err != nil {
return err
}
if err = os.Rename(f.Name(), filepath.Join(dir, "drain.json")); err != nil {
return err
}
d, err := os.Open(dir)
if err != nil {
return err
}
defer d.Close()
return d.Sync()
}
// DeliveryConfig is supplied only after sender admission. Tokens are read from
// a mounted file for each event to support rotation; they never enter logs.
type DeliveryConfig struct {
URL string
TokenFile string
}
func (c DeliveryConfig) Validate() error {
u, err := url.Parse(c.URL)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "/v1/events" {
return errors.New("audit URL must be an explicit http(s) /v1/events endpoint without credentials, query or fragment")
}
if c.TokenFile == "" {
return errors.New("audit delivery requires a token file")
}
return nil
}
// Drain sends a bounded batch sequentially. Only 200/202 advance the durable
// cursor. Failed or ambiguous requests replay the same event/idempotency key.
// No receiver error payload or token is included in an error.
func (o *Outbox) Drain(ctx context.Context, config DeliveryConfig, limit int) (int, error) {
if err := config.Validate(); err != nil {
return 0, err
}
if limit < 1 {
return 0, errors.New("drain limit must be positive")
}
o.drainMu.Lock()
defer o.drainMu.Unlock()
client := &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }}
delivered := 0
for delivered < limit {
o.mu.Lock()
if o.cursor.Seq == o.seq {
o.mu.Unlock()
return delivered, nil
}
e, offset, err := o.nextEntry()
o.mu.Unlock()
if err != nil {
return delivered, fmt.Errorf("read pending event: %w", err)
}
token, err := os.ReadFile(config.TokenFile)
if err != nil {
return delivered, errors.New("audit token file unavailable")
}
value := strings.TrimSpace(string(token))
if value == "" || strings.ContainsAny(value, "\r\n") {
return delivered, errors.New("audit token file is empty or invalid")
}
body, err := json.Marshal(e.Event)
if err != nil {
return delivered, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, config.URL, bytes.NewReader(body))
if err != nil {
return delivered, errors.New("construct audit request failed")
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+value)
req.Header.Set("Idempotency-Key", e.Event.ID)
response, err := client.Do(req)
if err != nil {
return delivered, errors.New("audit delivery failed; event retained")
}
response.Body.Close()
if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusAccepted {
return delivered, fmt.Errorf("audit delivery HTTP %d; event retained", response.StatusCode)
}
o.mu.Lock()
cursor := drainCursor{Seq: e.Seq, Offset: offset}
err = o.saveCursor(cursor)
if err == nil {
o.cursor = cursor
}
o.mu.Unlock()
if err != nil {
return delivered, fmt.Errorf("persist drain cursor: %w", err)
}
delivered++
}
return delivered, nil
}

View file

@ -0,0 +1,246 @@
package emission_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/netkingdom/flex-auth/internal/emission"
"github.com/netkingdom/flex-auth/pkg/api"
)
func TestHeartbeatScheduleSurvivesRestart(t *testing.T) {
dir := t.TempDir()
now := time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)
o, err := emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
o.SetClock(func() time.Time { return now })
if err := o.AppendDueHeartbeats(); err != nil {
t.Fatal(err)
}
if err := o.AppendDueHeartbeats(); err != nil {
t.Fatal(err)
}
o.Close()
o, err = emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
defer o.Close()
o.SetClock(func() time.Time { return now })
if err := o.AppendDueHeartbeats(); err != nil {
t.Fatal(err)
}
if got := len(readEvents(t, dir)); got != 4 {
t.Fatalf("heartbeats after restart = %d", got)
}
seen := map[string]bool{}
for _, e := range readEvents(t, dir) {
if e["type"] != emission.HeartbeatType || e["id"] != e["correlation_id"] {
t.Fatalf("invalid heartbeat: %v", e)
}
if _, err := time.Parse(time.RFC3339Nano, e["occurred_at"].(string)); err != nil {
t.Fatal(err)
}
data := e["data"].(map[string]any)
seen[data["class"].(string)] = true
}
for _, class := range emission.RareClasses() {
if !seen[class] {
t.Fatalf("no heartbeat for %s", class)
}
}
now = now.Add(24 * time.Hour)
o.NoteReleasedUncommitted(api.DecisionEffectDeny)
if err := o.AppendDueHeartbeats(); err != nil {
t.Fatal(err)
}
if got := len(readEvents(t, dir)); got != 8 {
t.Fatalf("daily heartbeats = %d", got)
}
for _, event := range readEvents(t, dir)[4:] {
data := event["data"].(map[string]any)
if data["class"] == "flex-auth.decision.deny" && data["assertion"] != "released-uncommitted-finding" {
t.Fatal("known emission failure concealed by nothing-to-report heartbeat")
}
}
}
func TestDrainRetainsRejectionsAndResumesWithStableIDs(t *testing.T) {
for _, rejection := range []int{400, 401, 403, 500, 302, 204} {
t.Run(http.StatusText(rejection), func(t *testing.T) {
dir := t.TempDir()
tokenFile := filepath.Join(dir, "sender-token")
if err := os.WriteFile(tokenFile, []byte("test-token"), 0600); err != nil {
t.Fatal(err)
}
o, err := emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
if err := o.AppendBatch([]api.DecisionEnvelope{deny("decision:1", "tenant:a"), deny("decision:2", "tenant:a")}); err != nil {
t.Fatal(err)
}
var ids []string
status := rejection
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var event emission.Event
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
t.Error(err)
}
if r.URL.Path != "/v1/events" || r.Header.Get("Idempotency-Key") != event.ID || r.Header.Get("Authorization") != "Bearer test-token" {
t.Errorf("bad delivery headers/path")
}
ids = append(ids, event.ID)
w.Header().Set("Location", "/redirect-must-not-be-followed")
w.WriteHeader(status)
w.Write([]byte("sensitive receiver payload"))
}))
defer server.Close()
cfg := emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}
if n, err := o.Drain(context.Background(), cfg, 100); n != 0 || err == nil || strings.Contains(err.Error(), "sensitive") {
t.Fatalf("rejection advanced or exposed response: %d %v", n, err)
}
if o.Status().Pending != 2 {
t.Fatal("rejected event lost")
}
status = 202
if n, err := o.Drain(context.Background(), cfg, 1); n != 1 || err != nil {
t.Fatalf("accepted: %d %v", n, err)
}
o.Close()
o, err = emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
defer o.Close()
if o.Status().Pending != 1 {
t.Fatal("cursor did not survive restart")
}
status = 200
if n, err := o.Drain(context.Background(), cfg, 100); n != 1 || err != nil {
t.Fatalf("duplicate: %d %v", n, err)
}
if len(ids) != 3 || ids[0] != ids[1] || ids[1] == ids[2] {
t.Fatalf("unstable/reused event IDs: %v", ids)
}
if o.Status().Pending != 0 {
t.Fatal("pending after drain")
}
})
}
}
func TestDrainAmbiguousAcceptanceReplaysAndTokenRotates(t *testing.T) {
dir := t.TempDir()
tokenFile := filepath.Join(dir, "token")
os.WriteFile(tokenFile, []byte("first"), 0600)
o, err := emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
defer o.Close()
o.Append(deny("decision:1", "tenant:a"))
var ids, tokens []string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ids = append(ids, r.Header.Get("Idempotency-Key"))
tokens = append(tokens, r.Header.Get("Authorization"))
w.WriteHeader(202)
}))
defer server.Close()
cfg := emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}
// Receiver accepts, but persisting the cursor fails. Retain and retry.
os.Mkdir(filepath.Join(dir, "drain.json"), 0700)
if n, err := o.Drain(context.Background(), cfg, 1); n != 0 || err == nil {
t.Fatalf("cursor failure: %d %v", n, err)
}
os.Remove(filepath.Join(dir, "drain.json"))
os.WriteFile(tokenFile, []byte("rotated"), 0600)
if n, err := o.Drain(context.Background(), cfg, 1); n != 1 || err != nil {
t.Fatalf("retry: %d %v", n, err)
}
if len(ids) != 2 || ids[0] != ids[1] || tokens[1] != "Bearer rotated" {
t.Fatalf("retry/rotation: %v %v", ids, tokens)
}
}
func TestOutboxReportsLagAndRefusesSourceReuse(t *testing.T) {
dir := t.TempDir()
o, err := emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
now := time.Now().UTC()
o.SetClock(func() time.Time { return now })
o.Append(deny("decision:1", "tenant:a"))
if o.Status().LagExceeded {
t.Fatal("fresh event treated as lag finding")
}
now = now.Add(2 * time.Hour)
if !o.Status().LagExceeded {
t.Fatal("overdue event not reported")
}
o.Close()
if other, err := emission.Open(dir, "flex-auth.other"); err == nil {
other.Close()
t.Fatal("source changed on existing outbox")
}
}
func TestSlowReceiverDoesNotBlockDecisionCommit(t *testing.T) {
dir := t.TempDir()
tokenFile := filepath.Join(dir, "token")
if err := os.WriteFile(tokenFile, []byte("local-test-token"), 0600); err != nil {
t.Fatal(err)
}
o, err := emission.Open(dir, "flex-auth.test")
if err != nil {
t.Fatal(err)
}
defer o.Close()
if err := o.Append(deny("decision:1", "tenant:a")); err != nil {
t.Fatal(err)
}
arrived, release := make(chan struct{}), make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
close(arrived)
<-release
w.WriteHeader(202)
}))
defer server.Close()
drained := make(chan error, 1)
go func() {
_, err := o.Drain(context.Background(), emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}, 1)
drained <- err
}()
defer func() {
close(release)
if err := <-drained; err != nil {
t.Error(err)
}
}()
<-arrived
committed := make(chan error, 1)
go func() { committed <- o.Append(deny("decision:2", "tenant:a")) }()
select {
case err := <-committed:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("remote HTTP holds the decision commit lock")
}
// Deferred release lets the drain finish even on test failure. Assert the
// pending count while HTTP is still blocked rather than waiting for it.
if o.Status().Pending != 2 {
t.Fatal("new commit lost during delivery")
}
}

View file

@ -61,7 +61,10 @@ type Status struct {
ReleasedUncommitted map[string]uint64 `json:"released_uncommitted"`
// ReleasedUncommitted is held in memory: a restart during an outbox outage
// loses it (FLEX-DEC-2026-018, consequences).
Means string `json:"means"`
Means string `json:"means"`
Pending uint64 `json:"pending"`
OldestPendingAt string `json:"oldest_pending_at,omitempty"`
LagExceeded bool `json:"lag_exceeded"`
}
// Outbox is a durable, append-only, fsync-per-commit event log.
@ -76,6 +79,9 @@ type Outbox struct {
size int64 // bytes committed; a failed commit is cut back to it
committed map[string]uint64
uncommitted map[string]uint64
drainMu sync.Mutex
cursor drainCursor
heartbeats map[string]time.Time
}
// Open opens or creates the outbox in dir and rebuilds committed counts from it.
@ -92,10 +98,17 @@ func Open(dir, source string) (*Outbox, error) {
clock: time.Now,
committed: map[string]uint64{},
uncommitted: map[string]uint64{},
heartbeats: map[string]time.Time{},
}
if err := o.loadCursor(); err != nil {
return nil, err
}
if err := o.load(); err != nil {
return nil, err
}
if o.cursor.Seq > o.seq || o.cursor.Offset > o.size {
return nil, errors.New("drain cursor is ahead of committed outbox")
}
file, err := os.OpenFile(o.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
if err != nil {
return nil, fmt.Errorf("open outbox: %w", err)
@ -126,11 +139,21 @@ func (o *Outbox) load() error {
if err := json.Unmarshal(data[good:good+end], &e); err != nil {
return fmt.Errorf("outbox line %d is corrupt: %w", line, err)
}
if e.Seq != o.seq+1 || e.Event.Source != o.source {
return fmt.Errorf("outbox line %d has inconsistent sequence or source", line)
}
o.seq = e.Seq
o.committed[e.Event.Type]++
good += end + 1
if e.Seq == o.cursor.Seq && int64(good) != o.cursor.Offset {
return errors.New("drain cursor does not match outbox")
}
o.noteHeartbeat(e.Event)
}
o.size = int64(good)
if o.cursor.Seq > o.seq {
return errors.New("drain cursor is ahead of outbox")
}
return nil
}
@ -159,15 +182,24 @@ func (o *Outbox) AppendBatch(decisions []api.DecisionEnvelope) error {
o.mu.Lock()
defer o.mu.Unlock()
now := o.clock().UTC().Format(time.RFC3339Nano)
var buf []byte
seq := o.seq
var events []Event
for _, decision := range decisions {
id, err := eventID()
if err != nil {
return err
}
events = append(events, o.event(id, now, decision))
}
return o.appendEvents(now, events)
}
// appendEvents requires mu; heartbeats use the same durable commit as decisions.
func (o *Outbox) appendEvents(now string, events []Event) error {
var buf []byte
seq := o.seq
for _, event := range events {
seq++
line, err := json.Marshal(entry{Seq: seq, CommittedAt: now, Event: o.event(id, now, decision)})
line, err := json.Marshal(entry{Seq: seq, CommittedAt: now, Event: event})
if err != nil {
return fmt.Errorf("marshal outbox entry: %w", err)
}
@ -181,8 +213,9 @@ func (o *Outbox) AppendBatch(decisions []api.DecisionEnvelope) error {
}
o.seq = seq
o.size += int64(len(buf))
for _, decision := range decisions {
o.committed[Class(decision.Effect)]++
for _, event := range events {
o.committed[event.Type]++
o.noteHeartbeat(event)
}
return nil
}
@ -217,6 +250,16 @@ func (o *Outbox) Status() Status {
"released_uncommitted is decisions released without a record " +
"since this process started, and any non-zero value is a finding",
}
status.Pending = o.seq - o.cursor.Seq
if status.Pending > 0 {
if e, _, err := o.nextEntry(); err == nil {
status.OldestPendingAt = e.CommittedAt
at, _ := time.Parse(time.RFC3339Nano, e.CommittedAt)
status.LagExceeded = status.Pending > 100 || o.clock().Sub(at) > time.Hour
} else {
status.LagExceeded = true
}
}
for _, effect := range api.DecisionEffects() {
class := Class(effect)
status.Committed[class] = o.committed[class]

View file

@ -4,6 +4,7 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
@ -150,6 +151,11 @@ func TestCadenceClassifiesEveryEffect(t *testing.T) {
Action string `yaml:"action"`
EvidenceClass string `yaml:"evidence_class"`
} `yaml:"classes"`
Heartbeat struct {
Classes []struct {
Class string `yaml:"class"`
} `yaml:"classes"`
} `yaml:"heartbeat"`
}
if err := yaml.Unmarshal(data, &cadence); err != nil {
t.Fatal(err)
@ -177,4 +183,17 @@ func TestCadenceClassifiesEveryEffect(t *testing.T) {
t.Fatalf("cadence.yaml load-bearing classes %v; want %v", declared, want)
}
}
var heartbeatClasses []string
for _, class := range cadence.Heartbeat.Classes {
heartbeatClasses = append(heartbeatClasses, class.Class)
}
rare := emission.RareClasses()
sort.Strings(heartbeatClasses)
sort.Strings(rare)
if strings.Join(heartbeatClasses, ",") != strings.Join(rare, ",") {
t.Fatalf("runtime heartbeat inventory %v differs from declared %v", rare, heartbeatClasses)
}
if cadence.Classes["heartbeat"].Action != emission.HeartbeatType {
t.Fatal("heartbeat wire class differs from declaration")
}
}

View file

@ -95,14 +95,16 @@ func TestInformedDecisionAttributeAdmission(t *testing.T) {
}
}
func TestSittingImagePinUnchanged(t *testing.T) {
func TestSittingImagePinMatchesRecordedBuild(t *testing.T) {
raw, err := os.ReadFile(filepath.Join(repoRoot(t), "values", "informed-decision-sitting.yaml"))
if err != nil {
t.Fatal(err)
}
text := string(raw)
if !strings.Contains(text, "sha256:f6fa2062f7095e66c7a91f20cd130914b5481d46d9d909339e028f7ecdf0065d") {
t.Fatal("sitting image digest changed")
// values/informed-decision-sitting.yaml records the v9 image of 103d8b6,
// pinned in 257a83e. The earlier v7 pin is no longer the baseline.
if !strings.Contains(text, "sha256:aa7ccbafb2a8f884c97119e295fd168fa4e67e974682a8c30b35149201469eff") {
t.Fatal("sitting image digest differs from the recorded v9 build")
}
if !strings.Contains(text, "/opt/flex-auth/examples/informed-decision-sitting/policy.md") {
t.Fatal("sitting policy path changed")