package fitness import ( "fmt" "testing" "time" "github.com/tegwick/fluid-core/internal/contract" ) func f64(v float64) *float64 { return &v } func obs(metric string, rev contract.RevisionID, value float64, samples int) Observation { return Observation{Metric: metric, Revision: rev, Value: value, Samples: samples} } // specsFromBlueprint mirrors the section 33 worked example: requests per task // must drop, latency must not regress past its guardrail. func specsFromBlueprint() []MetricSpec { return []MetricSpec{ {Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower, Target: f64(1.2)}, {Name: MetricP95LatencyMS, Role: RoleGuardrail, Direction: Lower, Threshold: f64(315)}, {Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.01)}, } } func TestSucceedsWhenTargetsMetAndGuardrailsHold(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{Start: time.Now().Add(-time.Hour)}, specsFromBlueprint(), []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), obs(MetricRequestsPerTask, "R-2", 1.15, 400), obs(MetricP95LatencyMS, "R-1", 300, 400), obs(MetricP95LatencyMS, "R-2", 302, 400), obs(MetricErrorRate, "R-1", 0.008, 400), obs(MetricErrorRate, "R-2", 0.007, 400), }) if eval.Verdict != VerdictSucceeded { t.Fatalf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons) } primary := eval.PrimaryResults() if len(primary) != 1 { t.Fatalf("primary metrics = %d, want 1", len(primary)) } if primary[0].TargetMet == nil || !*primary[0].TargetMet { t.Error("primary target not marked as met") } if primary[0].Delta >= 0 { t.Errorf("delta = %v, expected a reduction", primary[0].Delta) } } // TestGuardrailDominatesPrimarySuccess is the section 48.5 protection: a // candidate that hit its target while regressing a guardrail has traded // something it was told not to. func TestGuardrailDominatesPrimarySuccess(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), obs(MetricRequestsPerTask, "R-2", 1.05, 400), // target smashed obs(MetricP95LatencyMS, "R-1", 300, 400), obs(MetricP95LatencyMS, "R-2", 980, 400), // and latency ruined obs(MetricErrorRate, "R-1", 0.008, 400), obs(MetricErrorRate, "R-2", 0.009, 400), }) if eval.Verdict != VerdictGuardrailBreached { t.Fatalf("verdict = %s, want GUARDRAIL_BREACHED; reasons %v", eval.Verdict, eval.Reasons) } if len(eval.Reasons) == 0 { t.Error("a breach was reported with no reason") } var latency *MetricResult for i, m := range eval.GuardrailResults() { if m.Name == MetricP95LatencyMS { latency = &eval.GuardrailResults()[i] } } if latency == nil || latency.GuardrailBreached == nil || !*latency.GuardrailBreached { t.Error("the breached guardrail is not marked as breached") } } func TestFailsWhenPrimaryTargetMissed(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), obs(MetricRequestsPerTask, "R-2", 2.6, 400), // barely moved obs(MetricP95LatencyMS, "R-1", 300, 400), obs(MetricP95LatencyMS, "R-2", 301, 400), obs(MetricErrorRate, "R-1", 0.008, 400), obs(MetricErrorRate, "R-2", 0.008, 400), }) if eval.Verdict != VerdictFailed { t.Fatalf("verdict = %s, want FAILED", eval.Verdict) } if len(eval.Reasons) == 0 || eval.Reasons[0] == "" { t.Error("failure reported without saying which target was missed") } } // TestUnderpoweredIsInconclusiveNotSuccessful: promoting on three requests is // promoting on noise. func TestUnderpoweredIsInconclusiveNotSuccessful(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 3), obs(MetricRequestsPerTask, "R-2", 1.0, 2), obs(MetricP95LatencyMS, "R-1", 300, 3), obs(MetricP95LatencyMS, "R-2", 290, 2), obs(MetricErrorRate, "R-1", 0.0, 3), obs(MetricErrorRate, "R-2", 0.0, 2), }) if eval.Verdict != VerdictInconclusive { t.Fatalf("verdict = %s, want INCONCLUSIVE on tiny samples", eval.Verdict) } for _, m := range eval.PrimaryResults() { if !m.Underpowered { t.Error("a two-sample comparison was not marked underpowered") } } } // TestMissingMeasurementIsNotZero: defaulting an absent metric to zero would // read as a dramatic change that never happened. func TestMissingMeasurementIsNotZero(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), // no candidate measurement at all obs(MetricP95LatencyMS, "R-1", 300, 400), obs(MetricP95LatencyMS, "R-2", 300, 400), obs(MetricErrorRate, "R-1", 0.008, 400), obs(MetricErrorRate, "R-2", 0.008, 400), }) if eval.Verdict == VerdictSucceeded { t.Fatal("a missing primary measurement produced a success verdict") } for _, m := range eval.PrimaryResults() { if m.Current != 0 && m.Baseline != 0 { continue } if !m.Underpowered { t.Error("an absent measurement was not flagged") } } } func TestNoPrimaryMetricIsInconclusive(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ {Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.05)}, }, []Observation{ obs(MetricErrorRate, "R-1", 0.01, 400), obs(MetricErrorRate, "R-2", 0.01, 400), }) if eval.Verdict != VerdictInconclusive { t.Errorf("verdict = %s, want INCONCLUSIVE with no primary metric", eval.Verdict) } } func TestPrimaryWithoutTargetCannotPass(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ {Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower}, // no target }, []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), obs(MetricRequestsPerTask, "R-2", 1.0, 400), }) if eval.Verdict == VerdictSucceeded { t.Error("a primary metric with no declared target was treated as met") } } func TestHigherIsBetterDirection(t *testing.T) { e := NewEvaluator() eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ {Name: MetricSuccessRate, Role: RolePrimary, Direction: Higher, Target: f64(0.99)}, {Name: MetricSuccessRate + "_guard", Role: RoleGuardrail, Direction: Higher, Threshold: f64(0.95)}, }, []Observation{ obs(MetricSuccessRate, "R-1", 0.97, 400), obs(MetricSuccessRate, "R-2", 0.995, 400), obs(MetricSuccessRate+"_guard", "R-1", 0.97, 400), obs(MetricSuccessRate+"_guard", "R-2", 0.96, 400), }) if eval.Verdict != VerdictSucceeded { t.Errorf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons) } } func TestEvaluationIsDeterministic(t *testing.T) { e := NewEvaluator() specs := specsFromBlueprint() observations := []Observation{ obs(MetricRequestsPerTask, "R-1", 2.7, 400), obs(MetricRequestsPerTask, "R-2", 2.6, 400), obs(MetricP95LatencyMS, "R-1", 300, 400), obs(MetricP95LatencyMS, "R-2", 999, 400), obs(MetricErrorRate, "R-1", 0.008, 400), obs(MetricErrorRate, "R-2", 0.5, 400), } first := e.Evaluate("R-1", "R-2", Window{}, specs, observations) for i := 0; i < 50; i++ { again := e.Evaluate("R-1", "R-2", Window{}, specs, observations) if again.Verdict != first.Verdict || len(again.Reasons) != len(first.Reasons) { t.Fatal("evaluation varied between runs") } for j := range first.Reasons { if again.Reasons[j] != first.Reasons[j] { t.Fatalf("reason order varied: %q vs %q", first.Reasons[j], again.Reasons[j]) } } for j := range first.Metrics { if again.Metrics[j].Name != first.Metrics[j].Name { t.Fatal("metric order varied between runs") } } } } func TestMeasurerDerivesRequestsPerTask(t *testing.T) { m := NewMeasurer() now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) var events []contract.FluidTelemetry // R-1: three calls per task. R-2: one. for c := 0; c < 10; c++ { chain := fmt.Sprintf("chain-%d", c) for i := 0; i < 3; i++ { events = append(events, telemetry("R-1", fmt.Sprintf("c-%d", c), chain, now, 120, false)) } events = append(events, telemetry("R-2", fmt.Sprintf("c-%d", c), chain+"-b", now, 130, false)) } observations := m.Measure(events, Window{Start: now.Add(-time.Hour)}) got := map[contract.RevisionID]float64{} for _, o := range observations { if o.Metric == MetricRequestsPerTask { got[o.Revision] = o.Value } } if got["R-1"] != 3 { t.Errorf("R-1 requests per task = %v, want 3", got["R-1"]) } if got["R-2"] != 1 { t.Errorf("R-2 requests per task = %v, want 1", got["R-2"]) } } func TestMeasurerRespectsWindow(t *testing.T) { m := NewMeasurer() now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) events := []contract.FluidTelemetry{ telemetry("R-1", "c-1", "old", now.AddDate(0, 0, -10), 100, false), telemetry("R-1", "c-1", "new", now, 100, false), } // Only events inside the window may count, or the comparison is not // reproducible from its recorded period. observations := m.Measure(events, Window{Start: now.Add(-time.Hour)}) for _, o := range observations { if o.Metric == MetricErrorRate && o.Samples != 1 { t.Errorf("samples = %d, want 1; events outside the window were counted", o.Samples) } } } func TestMeasurerComputesErrorRate(t *testing.T) { m := NewMeasurer() now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) var events []contract.FluidTelemetry for i := 0; i < 8; i++ { events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("ch-%d", i), now, 100, false)) } for i := 0; i < 2; i++ { events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("er-%d", i), now, 100, true)) } for _, o := range m.Measure(events, Window{}) { if o.Metric == MetricErrorRate && o.Value != 0.2 { t.Errorf("error rate = %v, want 0.2", o.Value) } if o.Metric == MetricSuccessRate && o.Value != 0.8 { t.Errorf("success rate = %v, want 0.8", o.Value) } } } func telemetry(rev contract.RevisionID, consumer, chain string, at time.Time, latency float64, isError bool) contract.FluidTelemetry { r := rev l := latency status := int64(200) ev := contract.FluidTelemetry{ ID: fmt.Sprintf("tl-%s-%s", consumer, chain), OccurredAt: at, Kind: contract.FluidTelemetryKindRequest, ConsumerRef: consumer, Revision: &r, Sequence: &contract.FluidTelemetrySequence{ChainID: chain}, Request: &contract.FluidTelemetryRequest{Route: "/v1/x", Method: "GET", Status: &status, LatencyMS: &l}, } if isError { ev.Kind = contract.FluidTelemetryKindError ev.Error = &contract.FluidTelemetryError{Class: contract.FluidTelemetryErrorClassBackendFailure} } return ev }