Implement local decision delivery and reconcile blocked workplan requirements
Assistant: codex Assistant-Model: gpt-6.1-sol Assistant-Session: 01a1046e-fcad-7ea3-bf36-361fc8d679e8
This commit is contained in:
parent
257a83e997
commit
3cb1cfaedf
25 changed files with 1258 additions and 50 deletions
233
internal/emission/delivery.go
Normal file
233
internal/emission/delivery.go
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
package emission
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const HeartbeatType = "audit-core.heartbeat"
|
||||
|
||||
// RareClasses is the source-owned inventory in cadence.yaml.
|
||||
func RareClasses() []string {
|
||||
return []string{ClassPrefix + "deny", ClassPrefix + "redact", ClassPrefix + "not_applicable", ClassPrefix + "audit_only"}
|
||||
}
|
||||
|
||||
type heartbeatData struct {
|
||||
Class string `json:"class"`
|
||||
Assertion string `json:"assertion"`
|
||||
Committed uint64 `json:"committed"`
|
||||
ReleasedUncommitted uint64 `json:"released_uncommitted"`
|
||||
}
|
||||
|
||||
func (o *Outbox) noteHeartbeat(event Event) {
|
||||
if event.Type != HeartbeatType {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(event.Data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var data heartbeatData
|
||||
if json.Unmarshal(b, &data) != nil {
|
||||
return
|
||||
}
|
||||
at, err := time.Parse(time.RFC3339Nano, event.OccurredAt)
|
||||
if err == nil {
|
||||
o.heartbeats[data.Class] = at
|
||||
}
|
||||
}
|
||||
|
||||
// AppendDueHeartbeats commits one heartbeat per rare class at most every 24h.
|
||||
// The timestamp is rebuilt from the log on restart, never from a process timer.
|
||||
func (o *Outbox) AppendDueHeartbeats() error {
|
||||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
now := o.clock().UTC()
|
||||
var events []Event
|
||||
for _, class := range RareClasses() {
|
||||
if at, ok := o.heartbeats[class]; ok && now.Sub(at) < 24*time.Hour {
|
||||
continue
|
||||
}
|
||||
id, err := eventID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
assertion := "nothing-to-report"
|
||||
if o.uncommitted[class] > 0 {
|
||||
assertion = "released-uncommitted-finding"
|
||||
}
|
||||
events = append(events, Event{ID: id, Type: HeartbeatType, Source: o.source, Subject: o.source,
|
||||
Tenant: PlatformTenant, CorrelationID: id, OccurredAt: now.Format(time.RFC3339Nano),
|
||||
Data: heartbeatData{Class: class, Assertion: assertion, Committed: o.committed[class], ReleasedUncommitted: o.uncommitted[class]}})
|
||||
}
|
||||
if len(events) == 0 {
|
||||
return nil
|
||||
}
|
||||
return o.appendEvents(now.Format(time.RFC3339Nano), events)
|
||||
}
|
||||
|
||||
type drainCursor struct {
|
||||
Seq uint64 `json:"seq"`
|
||||
Offset int64 `json:"offset"`
|
||||
}
|
||||
|
||||
func (o *Outbox) loadCursor() error {
|
||||
b, err := os.ReadFile(filepath.Join(filepath.Dir(o.path), "drain.json"))
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("read drain cursor: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(b, &o.cursor); err != nil {
|
||||
return fmt.Errorf("decode drain cursor: %w", err)
|
||||
}
|
||||
if o.cursor.Offset < 0 || (o.cursor.Seq == 0) != (o.cursor.Offset == 0) {
|
||||
return errors.New("invalid drain cursor")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// nextEntry reads only the committed prefix; mu protects the size snapshot.
|
||||
func (o *Outbox) nextEntry() (entry, int64, error) {
|
||||
f, err := os.Open(o.path)
|
||||
if err != nil {
|
||||
return entry{}, 0, err
|
||||
}
|
||||
defer f.Close()
|
||||
r := bufio.NewReader(io.NewSectionReader(f, o.cursor.Offset, o.size-o.cursor.Offset))
|
||||
line, err := r.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return entry{}, 0, err
|
||||
}
|
||||
var e entry
|
||||
if err := json.Unmarshal(line, &e); err != nil {
|
||||
return entry{}, 0, err
|
||||
}
|
||||
if e.Seq != o.cursor.Seq+1 || e.Event.Source != o.source {
|
||||
return entry{}, 0, errors.New("inconsistent next outbox entry")
|
||||
}
|
||||
return e, o.cursor.Offset + int64(len(line)), nil
|
||||
}
|
||||
|
||||
func (o *Outbox) saveCursor(cursor drainCursor) error {
|
||||
dir := filepath.Dir(o.path)
|
||||
f, err := os.CreateTemp(dir, ".drain-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
b, _ := json.Marshal(cursor)
|
||||
if _, err = f.Write(b); err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
err = errors.Join(err, f.Close())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = os.Rename(f.Name(), filepath.Join(dir, "drain.json")); err != nil {
|
||||
return err
|
||||
}
|
||||
d, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer d.Close()
|
||||
return d.Sync()
|
||||
}
|
||||
|
||||
// DeliveryConfig is supplied only after sender admission. Tokens are read from
|
||||
// a mounted file for each event to support rotation; they never enter logs.
|
||||
type DeliveryConfig struct {
|
||||
URL string
|
||||
TokenFile string
|
||||
}
|
||||
|
||||
func (c DeliveryConfig) Validate() error {
|
||||
u, err := url.Parse(c.URL)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil || u.RawQuery != "" || u.Fragment != "" || u.Path != "/v1/events" {
|
||||
return errors.New("audit URL must be an explicit http(s) /v1/events endpoint without credentials, query or fragment")
|
||||
}
|
||||
if c.TokenFile == "" {
|
||||
return errors.New("audit delivery requires a token file")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Drain sends a bounded batch sequentially. Only 200/202 advance the durable
|
||||
// cursor. Failed or ambiguous requests replay the same event/idempotency key.
|
||||
// No receiver error payload or token is included in an error.
|
||||
func (o *Outbox) Drain(ctx context.Context, config DeliveryConfig, limit int) (int, error) {
|
||||
if err := config.Validate(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if limit < 1 {
|
||||
return 0, errors.New("drain limit must be positive")
|
||||
}
|
||||
o.drainMu.Lock()
|
||||
defer o.drainMu.Unlock()
|
||||
client := &http.Client{Timeout: 10 * time.Second, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
delivered := 0
|
||||
for delivered < limit {
|
||||
o.mu.Lock()
|
||||
if o.cursor.Seq == o.seq {
|
||||
o.mu.Unlock()
|
||||
return delivered, nil
|
||||
}
|
||||
e, offset, err := o.nextEntry()
|
||||
o.mu.Unlock()
|
||||
if err != nil {
|
||||
return delivered, fmt.Errorf("read pending event: %w", err)
|
||||
}
|
||||
token, err := os.ReadFile(config.TokenFile)
|
||||
if err != nil {
|
||||
return delivered, errors.New("audit token file unavailable")
|
||||
}
|
||||
value := strings.TrimSpace(string(token))
|
||||
if value == "" || strings.ContainsAny(value, "\r\n") {
|
||||
return delivered, errors.New("audit token file is empty or invalid")
|
||||
}
|
||||
body, err := json.Marshal(e.Event)
|
||||
if err != nil {
|
||||
return delivered, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, config.URL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return delivered, errors.New("construct audit request failed")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+value)
|
||||
req.Header.Set("Idempotency-Key", e.Event.ID)
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return delivered, errors.New("audit delivery failed; event retained")
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusAccepted {
|
||||
return delivered, fmt.Errorf("audit delivery HTTP %d; event retained", response.StatusCode)
|
||||
}
|
||||
o.mu.Lock()
|
||||
cursor := drainCursor{Seq: e.Seq, Offset: offset}
|
||||
err = o.saveCursor(cursor)
|
||||
if err == nil {
|
||||
o.cursor = cursor
|
||||
}
|
||||
o.mu.Unlock()
|
||||
if err != nil {
|
||||
return delivered, fmt.Errorf("persist drain cursor: %w", err)
|
||||
}
|
||||
delivered++
|
||||
}
|
||||
return delivered, nil
|
||||
}
|
||||
246
internal/emission/delivery_test.go
Normal file
246
internal/emission/delivery_test.go
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
package emission_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/netkingdom/flex-auth/internal/emission"
|
||||
"github.com/netkingdom/flex-auth/pkg/api"
|
||||
)
|
||||
|
||||
func TestHeartbeatScheduleSurvivesRestart(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
now := time.Date(2026, 10, 4, 0, 0, 0, 0, time.UTC)
|
||||
o, err := emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o.SetClock(func() time.Time { return now })
|
||||
if err := o.AppendDueHeartbeats(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := o.AppendDueHeartbeats(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o.Close()
|
||||
o, err = emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer o.Close()
|
||||
o.SetClock(func() time.Time { return now })
|
||||
if err := o.AppendDueHeartbeats(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(readEvents(t, dir)); got != 4 {
|
||||
t.Fatalf("heartbeats after restart = %d", got)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, e := range readEvents(t, dir) {
|
||||
if e["type"] != emission.HeartbeatType || e["id"] != e["correlation_id"] {
|
||||
t.Fatalf("invalid heartbeat: %v", e)
|
||||
}
|
||||
if _, err := time.Parse(time.RFC3339Nano, e["occurred_at"].(string)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := e["data"].(map[string]any)
|
||||
seen[data["class"].(string)] = true
|
||||
}
|
||||
for _, class := range emission.RareClasses() {
|
||||
if !seen[class] {
|
||||
t.Fatalf("no heartbeat for %s", class)
|
||||
}
|
||||
}
|
||||
now = now.Add(24 * time.Hour)
|
||||
o.NoteReleasedUncommitted(api.DecisionEffectDeny)
|
||||
if err := o.AppendDueHeartbeats(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(readEvents(t, dir)); got != 8 {
|
||||
t.Fatalf("daily heartbeats = %d", got)
|
||||
}
|
||||
for _, event := range readEvents(t, dir)[4:] {
|
||||
data := event["data"].(map[string]any)
|
||||
if data["class"] == "flex-auth.decision.deny" && data["assertion"] != "released-uncommitted-finding" {
|
||||
t.Fatal("known emission failure concealed by nothing-to-report heartbeat")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDrainRetainsRejectionsAndResumesWithStableIDs(t *testing.T) {
|
||||
for _, rejection := range []int{400, 401, 403, 500, 302, 204} {
|
||||
t.Run(http.StatusText(rejection), func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tokenFile := filepath.Join(dir, "sender-token")
|
||||
if err := os.WriteFile(tokenFile, []byte("test-token"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o, err := emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := o.AppendBatch([]api.DecisionEnvelope{deny("decision:1", "tenant:a"), deny("decision:2", "tenant:a")}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ids []string
|
||||
status := rejection
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var event emission.Event
|
||||
if err := json.NewDecoder(r.Body).Decode(&event); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if r.URL.Path != "/v1/events" || r.Header.Get("Idempotency-Key") != event.ID || r.Header.Get("Authorization") != "Bearer test-token" {
|
||||
t.Errorf("bad delivery headers/path")
|
||||
}
|
||||
ids = append(ids, event.ID)
|
||||
w.Header().Set("Location", "/redirect-must-not-be-followed")
|
||||
w.WriteHeader(status)
|
||||
w.Write([]byte("sensitive receiver payload"))
|
||||
}))
|
||||
defer server.Close()
|
||||
cfg := emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}
|
||||
if n, err := o.Drain(context.Background(), cfg, 100); n != 0 || err == nil || strings.Contains(err.Error(), "sensitive") {
|
||||
t.Fatalf("rejection advanced or exposed response: %d %v", n, err)
|
||||
}
|
||||
if o.Status().Pending != 2 {
|
||||
t.Fatal("rejected event lost")
|
||||
}
|
||||
status = 202
|
||||
if n, err := o.Drain(context.Background(), cfg, 1); n != 1 || err != nil {
|
||||
t.Fatalf("accepted: %d %v", n, err)
|
||||
}
|
||||
o.Close()
|
||||
o, err = emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer o.Close()
|
||||
if o.Status().Pending != 1 {
|
||||
t.Fatal("cursor did not survive restart")
|
||||
}
|
||||
status = 200
|
||||
if n, err := o.Drain(context.Background(), cfg, 100); n != 1 || err != nil {
|
||||
t.Fatalf("duplicate: %d %v", n, err)
|
||||
}
|
||||
if len(ids) != 3 || ids[0] != ids[1] || ids[1] == ids[2] {
|
||||
t.Fatalf("unstable/reused event IDs: %v", ids)
|
||||
}
|
||||
if o.Status().Pending != 0 {
|
||||
t.Fatal("pending after drain")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDrainAmbiguousAcceptanceReplaysAndTokenRotates(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tokenFile := filepath.Join(dir, "token")
|
||||
os.WriteFile(tokenFile, []byte("first"), 0600)
|
||||
o, err := emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer o.Close()
|
||||
o.Append(deny("decision:1", "tenant:a"))
|
||||
var ids, tokens []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ids = append(ids, r.Header.Get("Idempotency-Key"))
|
||||
tokens = append(tokens, r.Header.Get("Authorization"))
|
||||
w.WriteHeader(202)
|
||||
}))
|
||||
defer server.Close()
|
||||
cfg := emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}
|
||||
// Receiver accepts, but persisting the cursor fails. Retain and retry.
|
||||
os.Mkdir(filepath.Join(dir, "drain.json"), 0700)
|
||||
if n, err := o.Drain(context.Background(), cfg, 1); n != 0 || err == nil {
|
||||
t.Fatalf("cursor failure: %d %v", n, err)
|
||||
}
|
||||
os.Remove(filepath.Join(dir, "drain.json"))
|
||||
os.WriteFile(tokenFile, []byte("rotated"), 0600)
|
||||
if n, err := o.Drain(context.Background(), cfg, 1); n != 1 || err != nil {
|
||||
t.Fatalf("retry: %d %v", n, err)
|
||||
}
|
||||
if len(ids) != 2 || ids[0] != ids[1] || tokens[1] != "Bearer rotated" {
|
||||
t.Fatalf("retry/rotation: %v %v", ids, tokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutboxReportsLagAndRefusesSourceReuse(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
o, err := emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
o.SetClock(func() time.Time { return now })
|
||||
o.Append(deny("decision:1", "tenant:a"))
|
||||
if o.Status().LagExceeded {
|
||||
t.Fatal("fresh event treated as lag finding")
|
||||
}
|
||||
now = now.Add(2 * time.Hour)
|
||||
if !o.Status().LagExceeded {
|
||||
t.Fatal("overdue event not reported")
|
||||
}
|
||||
o.Close()
|
||||
if other, err := emission.Open(dir, "flex-auth.other"); err == nil {
|
||||
other.Close()
|
||||
t.Fatal("source changed on existing outbox")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSlowReceiverDoesNotBlockDecisionCommit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
tokenFile := filepath.Join(dir, "token")
|
||||
if err := os.WriteFile(tokenFile, []byte("local-test-token"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
o, err := emission.Open(dir, "flex-auth.test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer o.Close()
|
||||
if err := o.Append(deny("decision:1", "tenant:a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
arrived, release := make(chan struct{}), make(chan struct{})
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
close(arrived)
|
||||
<-release
|
||||
w.WriteHeader(202)
|
||||
}))
|
||||
defer server.Close()
|
||||
drained := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := o.Drain(context.Background(), emission.DeliveryConfig{URL: server.URL + "/v1/events", TokenFile: tokenFile}, 1)
|
||||
drained <- err
|
||||
}()
|
||||
defer func() {
|
||||
close(release)
|
||||
if err := <-drained; err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
<-arrived
|
||||
committed := make(chan error, 1)
|
||||
go func() { committed <- o.Append(deny("decision:2", "tenant:a")) }()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("remote HTTP holds the decision commit lock")
|
||||
}
|
||||
// Deferred release lets the drain finish even on test failure. Assert the
|
||||
// pending count while HTTP is still blocked rather than waiting for it.
|
||||
if o.Status().Pending != 2 {
|
||||
t.Fatal("new commit lost during delivery")
|
||||
}
|
||||
}
|
||||
|
|
@ -61,7 +61,10 @@ type Status struct {
|
|||
ReleasedUncommitted map[string]uint64 `json:"released_uncommitted"`
|
||||
// ReleasedUncommitted is held in memory: a restart during an outbox outage
|
||||
// loses it (FLEX-DEC-2026-018, consequences).
|
||||
Means string `json:"means"`
|
||||
Means string `json:"means"`
|
||||
Pending uint64 `json:"pending"`
|
||||
OldestPendingAt string `json:"oldest_pending_at,omitempty"`
|
||||
LagExceeded bool `json:"lag_exceeded"`
|
||||
}
|
||||
|
||||
// Outbox is a durable, append-only, fsync-per-commit event log.
|
||||
|
|
@ -76,6 +79,9 @@ type Outbox struct {
|
|||
size int64 // bytes committed; a failed commit is cut back to it
|
||||
committed map[string]uint64
|
||||
uncommitted map[string]uint64
|
||||
drainMu sync.Mutex
|
||||
cursor drainCursor
|
||||
heartbeats map[string]time.Time
|
||||
}
|
||||
|
||||
// Open opens or creates the outbox in dir and rebuilds committed counts from it.
|
||||
|
|
@ -92,10 +98,17 @@ func Open(dir, source string) (*Outbox, error) {
|
|||
clock: time.Now,
|
||||
committed: map[string]uint64{},
|
||||
uncommitted: map[string]uint64{},
|
||||
heartbeats: map[string]time.Time{},
|
||||
}
|
||||
if err := o.loadCursor(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := o.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if o.cursor.Seq > o.seq || o.cursor.Offset > o.size {
|
||||
return nil, errors.New("drain cursor is ahead of committed outbox")
|
||||
}
|
||||
file, err := os.OpenFile(o.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o640)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open outbox: %w", err)
|
||||
|
|
@ -126,11 +139,21 @@ func (o *Outbox) load() error {
|
|||
if err := json.Unmarshal(data[good:good+end], &e); err != nil {
|
||||
return fmt.Errorf("outbox line %d is corrupt: %w", line, err)
|
||||
}
|
||||
if e.Seq != o.seq+1 || e.Event.Source != o.source {
|
||||
return fmt.Errorf("outbox line %d has inconsistent sequence or source", line)
|
||||
}
|
||||
o.seq = e.Seq
|
||||
o.committed[e.Event.Type]++
|
||||
good += end + 1
|
||||
if e.Seq == o.cursor.Seq && int64(good) != o.cursor.Offset {
|
||||
return errors.New("drain cursor does not match outbox")
|
||||
}
|
||||
o.noteHeartbeat(e.Event)
|
||||
}
|
||||
o.size = int64(good)
|
||||
if o.cursor.Seq > o.seq {
|
||||
return errors.New("drain cursor is ahead of outbox")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -159,15 +182,24 @@ func (o *Outbox) AppendBatch(decisions []api.DecisionEnvelope) error {
|
|||
o.mu.Lock()
|
||||
defer o.mu.Unlock()
|
||||
now := o.clock().UTC().Format(time.RFC3339Nano)
|
||||
var buf []byte
|
||||
seq := o.seq
|
||||
var events []Event
|
||||
for _, decision := range decisions {
|
||||
id, err := eventID()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
events = append(events, o.event(id, now, decision))
|
||||
}
|
||||
return o.appendEvents(now, events)
|
||||
}
|
||||
|
||||
// appendEvents requires mu; heartbeats use the same durable commit as decisions.
|
||||
func (o *Outbox) appendEvents(now string, events []Event) error {
|
||||
var buf []byte
|
||||
seq := o.seq
|
||||
for _, event := range events {
|
||||
seq++
|
||||
line, err := json.Marshal(entry{Seq: seq, CommittedAt: now, Event: o.event(id, now, decision)})
|
||||
line, err := json.Marshal(entry{Seq: seq, CommittedAt: now, Event: event})
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal outbox entry: %w", err)
|
||||
}
|
||||
|
|
@ -181,8 +213,9 @@ func (o *Outbox) AppendBatch(decisions []api.DecisionEnvelope) error {
|
|||
}
|
||||
o.seq = seq
|
||||
o.size += int64(len(buf))
|
||||
for _, decision := range decisions {
|
||||
o.committed[Class(decision.Effect)]++
|
||||
for _, event := range events {
|
||||
o.committed[event.Type]++
|
||||
o.noteHeartbeat(event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -217,6 +250,16 @@ func (o *Outbox) Status() Status {
|
|||
"released_uncommitted is decisions released without a record " +
|
||||
"since this process started, and any non-zero value is a finding",
|
||||
}
|
||||
status.Pending = o.seq - o.cursor.Seq
|
||||
if status.Pending > 0 {
|
||||
if e, _, err := o.nextEntry(); err == nil {
|
||||
status.OldestPendingAt = e.CommittedAt
|
||||
at, _ := time.Parse(time.RFC3339Nano, e.CommittedAt)
|
||||
status.LagExceeded = status.Pending > 100 || o.clock().Sub(at) > time.Hour
|
||||
} else {
|
||||
status.LagExceeded = true
|
||||
}
|
||||
}
|
||||
for _, effect := range api.DecisionEffects() {
|
||||
class := Class(effect)
|
||||
status.Committed[class] = o.committed[class]
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
|
@ -150,6 +151,11 @@ func TestCadenceClassifiesEveryEffect(t *testing.T) {
|
|||
Action string `yaml:"action"`
|
||||
EvidenceClass string `yaml:"evidence_class"`
|
||||
} `yaml:"classes"`
|
||||
Heartbeat struct {
|
||||
Classes []struct {
|
||||
Class string `yaml:"class"`
|
||||
} `yaml:"classes"`
|
||||
} `yaml:"heartbeat"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &cadence); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -177,4 +183,17 @@ func TestCadenceClassifiesEveryEffect(t *testing.T) {
|
|||
t.Fatalf("cadence.yaml load-bearing classes %v; want %v", declared, want)
|
||||
}
|
||||
}
|
||||
var heartbeatClasses []string
|
||||
for _, class := range cadence.Heartbeat.Classes {
|
||||
heartbeatClasses = append(heartbeatClasses, class.Class)
|
||||
}
|
||||
rare := emission.RareClasses()
|
||||
sort.Strings(heartbeatClasses)
|
||||
sort.Strings(rare)
|
||||
if strings.Join(heartbeatClasses, ",") != strings.Join(rare, ",") {
|
||||
t.Fatalf("runtime heartbeat inventory %v differs from declared %v", rare, heartbeatClasses)
|
||||
}
|
||||
if cadence.Classes["heartbeat"].Action != emission.HeartbeatType {
|
||||
t.Fatal("heartbeat wire class differs from declaration")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,14 +95,16 @@ func TestInformedDecisionAttributeAdmission(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestSittingImagePinUnchanged(t *testing.T) {
|
||||
func TestSittingImagePinMatchesRecordedBuild(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join(repoRoot(t), "values", "informed-decision-sitting.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
if !strings.Contains(text, "sha256:f6fa2062f7095e66c7a91f20cd130914b5481d46d9d909339e028f7ecdf0065d") {
|
||||
t.Fatal("sitting image digest changed")
|
||||
// values/informed-decision-sitting.yaml records the v9 image of 103d8b6,
|
||||
// pinned in 257a83e. The earlier v7 pin is no longer the baseline.
|
||||
if !strings.Contains(text, "sha256:aa7ccbafb2a8f884c97119e295fd168fa4e67e974682a8c30b35149201469eff") {
|
||||
t.Fatal("sitting image digest differs from the recorded v9 build")
|
||||
}
|
||||
if !strings.Contains(text, "/opt/flex-auth/examples/informed-decision-sitting/policy.md") {
|
||||
t.Fatal("sitting policy path changed")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue