diff --git a/internal/fitness/fitness.go b/internal/fitness/fitness.go new file mode 100644 index 0000000..c75e052 --- /dev/null +++ b/internal/fitness/fitness.go @@ -0,0 +1,357 @@ +// Package fitness evaluates how well an interface revision fulfils its purpose. +// +// ArchitectureBlueprint.md section 18 requires four kinds of metric to stay +// distinct: primary metrics the hypothesis predicts, guardrails that must not +// regress, secondary observations, and learning signals. Section 48.5 names the +// failure mode this prevents — a single fitness number that hides the +// dimensions and guardrails underneath it. +// +// There is deliberately no universal scalar. FluidAPIStandards.md section 20 +// declines to define one, and a framework that invented one anyway would be +// making the interface's most important trade-offs on the operator's behalf. +package fitness + +import ( + "fmt" + "math" + "sort" + "time" + + "github.com/tegwick/fluid-core/internal/contract" +) + +// MetricRole distinguishes what a measurement is for. +type MetricRole string + +const ( + // RolePrimary metrics are the outcomes the hypothesis predicted. + RolePrimary MetricRole = "primary" + // RoleGuardrail metrics must not regress beyond a threshold, whatever the + // primary metrics do. + RoleGuardrail MetricRole = "guardrail" + // RoleSecondary metrics are useful context, not decision inputs. + RoleSecondary MetricRole = "secondary" + // RoleLearning metrics improve future hypothesis formation. + RoleLearning MetricRole = "learning" +) + +// Direction says which way is better for a metric. +type Direction string + +const ( + Lower Direction = "lower" + Higher Direction = "higher" + Unchanged Direction = "unchanged" +) + +// MetricSpec declares how one metric is judged. +type MetricSpec struct { + Name string `json:"name"` + Role MetricRole `json:"role"` + Direction Direction `json:"direction"` + // Target is the value a primary metric must reach. + Target *float64 `json:"target,omitempty"` + // Threshold is the limit a guardrail must not cross. + Threshold *float64 `json:"threshold,omitempty"` +} + +// Window is a measurement period. +// +// Retaining it is not bookkeeping: Blueprint section 18 requires the evaluator +// to keep the baseline and the window, because a comparison whose period is +// unknown cannot be reproduced or challenged. +type Window struct { + Start time.Time `json:"start"` + End *time.Time `json:"end,omitempty"` +} + +// Observation is a metric measured over one revision. +type Observation struct { + Metric string `json:"metric"` + Revision contract.RevisionID `json:"revision"` + Value float64 `json:"value"` + Samples int `json:"samples"` +} + +// MetricResult is one metric compared between control and candidate. +type MetricResult struct { + Name string `json:"name"` + Role MetricRole `json:"role"` + Direction Direction `json:"direction"` + + Baseline float64 `json:"baseline"` + Current float64 `json:"current"` + Delta float64 `json:"delta"` + Target *float64 `json:"target,omitempty"` + Threshold *float64 `json:"threshold,omitempty"` + + // TargetMet applies to primary metrics only. + TargetMet *bool `json:"target_met,omitempty"` + // GuardrailBreached applies to guardrails only. + GuardrailBreached *bool `json:"guardrail_breached,omitempty"` + + BaselineSamples int `json:"baseline_samples"` + CurrentSamples int `json:"current_samples"` + // Underpowered marks a comparison with too little data to lean on. + Underpowered bool `json:"underpowered"` +} + +// Verdict is the evaluator's overall reading. +type Verdict string + +const ( + // VerdictSucceeded: every primary target met, no guardrail breached. + VerdictSucceeded Verdict = "SUCCEEDED" + // VerdictFailed: a primary target was missed without a guardrail breach. + VerdictFailed Verdict = "FAILED" + // VerdictGuardrailBreached: a guardrail regressed, whatever else happened. + VerdictGuardrailBreached Verdict = "GUARDRAIL_BREACHED" + // VerdictInconclusive: not enough evidence to say either way. + VerdictInconclusive Verdict = "INCONCLUSIVE" +) + +// Evaluation is a complete fitness comparison. +type Evaluation struct { + Experiment contract.ExperimentID `json:"experiment,omitempty"` + Control contract.RevisionID `json:"control"` + Candidate contract.RevisionID `json:"candidate"` + Window Window `json:"window"` + + Verdict Verdict `json:"verdict"` + Reasons []string `json:"reasons,omitempty"` + Metrics []MetricResult `json:"metrics"` +} + +// PrimaryResults returns just the primary metrics. +func (e Evaluation) PrimaryResults() []MetricResult { return e.byRole(RolePrimary) } + +// GuardrailResults returns just the guardrails. +func (e Evaluation) GuardrailResults() []MetricResult { return e.byRole(RoleGuardrail) } + +func (e Evaluation) byRole(role MetricRole) []MetricResult { + var out []MetricResult + for _, m := range e.Metrics { + if m.Role == role { + out = append(out, m) + } + } + return out +} + +// Evaluator compares a candidate revision against its control. +type Evaluator struct { + // MinSamples is the per-side sample count below which a comparison is + // marked underpowered. A difference measured on three requests is noise + // wearing a result's clothes. + MinSamples int +} + +// NewEvaluator returns an evaluator with a workable default. +func NewEvaluator() *Evaluator { return &Evaluator{MinSamples: 30} } + +// Evaluate compares observations against the declared specs. +// +// Specs are an input, not something derived from the data. Blueprint section 18 +// says success criteria must not be changed after results are visible without +// recording the amendment, and an evaluator that inferred its own criteria +// would make that impossible to enforce. +func (e *Evaluator) Evaluate( + control, candidate contract.RevisionID, + window Window, + specs []MetricSpec, + observations []Observation, +) Evaluation { + byMetric := map[string]map[contract.RevisionID]Observation{} + for _, o := range observations { + if byMetric[o.Metric] == nil { + byMetric[o.Metric] = map[contract.RevisionID]Observation{} + } + byMetric[o.Metric][o.Revision] = o + } + + eval := Evaluation{Control: control, Candidate: candidate, Window: window} + + ordered := make([]MetricSpec, len(specs)) + copy(ordered, specs) + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].Role != ordered[j].Role { + return roleRank(ordered[i].Role) < roleRank(ordered[j].Role) + } + return ordered[i].Name < ordered[j].Name + }) + + var ( + missingPrimary []string + breached []string + underpowered []string + primaryCount int + primaryMetCount int + haveAnyPrimary bool + ) + + for _, spec := range ordered { + base, hasBase := byMetric[spec.Name][control] + cur, hasCur := byMetric[spec.Name][candidate] + + if !hasBase || !hasCur { + // A metric with no measurement on one side is reported as absent + // rather than defaulted to zero, which would read as a dramatic + // improvement or regression that never happened. + eval.Metrics = append(eval.Metrics, MetricResult{ + Name: spec.Name, + Role: spec.Role, + Direction: spec.Direction, + Target: spec.Target, + Threshold: spec.Threshold, + Underpowered: true, + }) + if spec.Role == RolePrimary { + primaryCount++ + underpowered = append(underpowered, + fmt.Sprintf("%s has no measurement on both sides", spec.Name)) + } + continue + } + + r := MetricResult{ + Name: spec.Name, + Role: spec.Role, + Direction: spec.Direction, + Baseline: base.Value, + Current: cur.Value, + Delta: round4(cur.Value - base.Value), + Target: spec.Target, + Threshold: spec.Threshold, + BaselineSamples: base.Samples, + CurrentSamples: cur.Samples, + } + + if base.Samples < e.MinSamples || cur.Samples < e.MinSamples { + r.Underpowered = true + underpowered = append(underpowered, fmt.Sprintf( + "%s has %d control and %d candidate samples, below the %d needed", + spec.Name, base.Samples, cur.Samples, e.MinSamples)) + } + + switch spec.Role { + case RolePrimary: + primaryCount++ + haveAnyPrimary = true + met := meetsTarget(spec, cur.Value) + r.TargetMet = &met + if met { + primaryMetCount++ + } else { + missingPrimary = append(missingPrimary, describeMiss(spec, cur.Value)) + } + + case RoleGuardrail: + crossed := breachesGuardrail(spec, cur.Value) + r.GuardrailBreached = &crossed + if crossed { + breached = append(breached, describeBreach(spec, cur.Value)) + } + } + + eval.Metrics = append(eval.Metrics, r) + } + + // Guardrails dominate. A candidate that hit every target while regressing a + // guardrail has not succeeded; it has traded something it was told not to. + switch { + case len(breached) > 0: + eval.Verdict = VerdictGuardrailBreached + eval.Reasons = breached + + case !haveAnyPrimary || primaryCount == 0: + eval.Verdict = VerdictInconclusive + eval.Reasons = []string{"no primary metric was declared, so there is nothing to conclude"} + + case len(underpowered) > 0 && len(missingPrimary) == 0: + // Every target appears met, but on too little data to act on. Reporting + // success here is how an experiment gets promoted on noise. + eval.Verdict = VerdictInconclusive + eval.Reasons = underpowered + + case len(missingPrimary) > 0: + eval.Verdict = VerdictFailed + eval.Reasons = missingPrimary + + default: + eval.Verdict = VerdictSucceeded + eval.Reasons = []string{fmt.Sprintf("%d of %d primary targets met with no guardrail breach", + primaryMetCount, primaryCount)} + } + + sort.Strings(eval.Reasons) + return eval +} + +func meetsTarget(spec MetricSpec, value float64) bool { + if spec.Target == nil { + // A primary metric with no target cannot be judged, so it is not met. + return false + } + switch spec.Direction { + case Lower: + return value <= *spec.Target + case Higher: + return value >= *spec.Target + case Unchanged: + return math.Abs(value-*spec.Target) < 1e-9 + } + return false +} + +func breachesGuardrail(spec MetricSpec, value float64) bool { + if spec.Threshold == nil { + return false + } + switch spec.Direction { + case Lower: + // Lower is better, so exceeding the threshold is the breach. + return value > *spec.Threshold + case Higher: + return value < *spec.Threshold + case Unchanged: + return math.Abs(value-*spec.Threshold) > 1e-9 + } + return false +} + +func describeMiss(spec MetricSpec, value float64) string { + if spec.Target == nil { + return fmt.Sprintf("%s is a primary metric with no declared target", spec.Name) + } + return fmt.Sprintf("%s reached %.4g, target was %s %.4g", + spec.Name, value, comparator(spec.Direction), *spec.Target) +} + +func describeBreach(spec MetricSpec, value float64) string { + return fmt.Sprintf("guardrail %s at %.4g breached its threshold of %.4g", + spec.Name, value, *spec.Threshold) +} + +func comparator(d Direction) string { + switch d { + case Lower: + return "at most" + case Higher: + return "at least" + } + return "exactly" +} + +func roleRank(r MetricRole) int { + switch r { + case RolePrimary: + return 0 + case RoleGuardrail: + return 1 + case RoleSecondary: + return 2 + } + return 3 +} + +func round4(v float64) float64 { return math.Round(v*10000) / 10000 } diff --git a/internal/fitness/fitness_test.go b/internal/fitness/fitness_test.go new file mode 100644 index 0000000..d10cec0 --- /dev/null +++ b/internal/fitness/fitness_test.go @@ -0,0 +1,325 @@ +package fitness + +import ( + "fmt" + "testing" + "time" + + "github.com/tegwick/fluid-core/internal/contract" +) + +func f64(v float64) *float64 { return &v } + +func obs(metric string, rev contract.RevisionID, value float64, samples int) Observation { + return Observation{Metric: metric, Revision: rev, Value: value, Samples: samples} +} + +// specsFromBlueprint mirrors the section 33 worked example: requests per task +// must drop, latency must not regress past its guardrail. +func specsFromBlueprint() []MetricSpec { + return []MetricSpec{ + {Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower, Target: f64(1.2)}, + {Name: MetricP95LatencyMS, Role: RoleGuardrail, Direction: Lower, Threshold: f64(315)}, + {Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.01)}, + } +} + +func TestSucceedsWhenTargetsMetAndGuardrailsHold(t *testing.T) { + e := NewEvaluator() + + eval := e.Evaluate("R-1", "R-2", Window{Start: time.Now().Add(-time.Hour)}, specsFromBlueprint(), + []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + obs(MetricRequestsPerTask, "R-2", 1.15, 400), + obs(MetricP95LatencyMS, "R-1", 300, 400), + obs(MetricP95LatencyMS, "R-2", 302, 400), + obs(MetricErrorRate, "R-1", 0.008, 400), + obs(MetricErrorRate, "R-2", 0.007, 400), + }) + + if eval.Verdict != VerdictSucceeded { + t.Fatalf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons) + } + + primary := eval.PrimaryResults() + if len(primary) != 1 { + t.Fatalf("primary metrics = %d, want 1", len(primary)) + } + if primary[0].TargetMet == nil || !*primary[0].TargetMet { + t.Error("primary target not marked as met") + } + if primary[0].Delta >= 0 { + t.Errorf("delta = %v, expected a reduction", primary[0].Delta) + } +} + +// TestGuardrailDominatesPrimarySuccess is the section 48.5 protection: a +// candidate that hit its target while regressing a guardrail has traded +// something it was told not to. +func TestGuardrailDominatesPrimarySuccess(t *testing.T) { + e := NewEvaluator() + + eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + obs(MetricRequestsPerTask, "R-2", 1.05, 400), // target smashed + obs(MetricP95LatencyMS, "R-1", 300, 400), + obs(MetricP95LatencyMS, "R-2", 980, 400), // and latency ruined + obs(MetricErrorRate, "R-1", 0.008, 400), + obs(MetricErrorRate, "R-2", 0.009, 400), + }) + + if eval.Verdict != VerdictGuardrailBreached { + t.Fatalf("verdict = %s, want GUARDRAIL_BREACHED; reasons %v", eval.Verdict, eval.Reasons) + } + if len(eval.Reasons) == 0 { + t.Error("a breach was reported with no reason") + } + + var latency *MetricResult + for i, m := range eval.GuardrailResults() { + if m.Name == MetricP95LatencyMS { + latency = &eval.GuardrailResults()[i] + } + } + if latency == nil || latency.GuardrailBreached == nil || !*latency.GuardrailBreached { + t.Error("the breached guardrail is not marked as breached") + } +} + +func TestFailsWhenPrimaryTargetMissed(t *testing.T) { + e := NewEvaluator() + + eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + obs(MetricRequestsPerTask, "R-2", 2.6, 400), // barely moved + obs(MetricP95LatencyMS, "R-1", 300, 400), + obs(MetricP95LatencyMS, "R-2", 301, 400), + obs(MetricErrorRate, "R-1", 0.008, 400), + obs(MetricErrorRate, "R-2", 0.008, 400), + }) + + if eval.Verdict != VerdictFailed { + t.Fatalf("verdict = %s, want FAILED", eval.Verdict) + } + if len(eval.Reasons) == 0 || eval.Reasons[0] == "" { + t.Error("failure reported without saying which target was missed") + } +} + +// TestUnderpoweredIsInconclusiveNotSuccessful: promoting on three requests is +// promoting on noise. +func TestUnderpoweredIsInconclusiveNotSuccessful(t *testing.T) { + e := NewEvaluator() + + eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 3), + obs(MetricRequestsPerTask, "R-2", 1.0, 2), + obs(MetricP95LatencyMS, "R-1", 300, 3), + obs(MetricP95LatencyMS, "R-2", 290, 2), + obs(MetricErrorRate, "R-1", 0.0, 3), + obs(MetricErrorRate, "R-2", 0.0, 2), + }) + + if eval.Verdict != VerdictInconclusive { + t.Fatalf("verdict = %s, want INCONCLUSIVE on tiny samples", eval.Verdict) + } + for _, m := range eval.PrimaryResults() { + if !m.Underpowered { + t.Error("a two-sample comparison was not marked underpowered") + } + } +} + +// TestMissingMeasurementIsNotZero: defaulting an absent metric to zero would +// read as a dramatic change that never happened. +func TestMissingMeasurementIsNotZero(t *testing.T) { + e := NewEvaluator() + + eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + // no candidate measurement at all + obs(MetricP95LatencyMS, "R-1", 300, 400), + obs(MetricP95LatencyMS, "R-2", 300, 400), + obs(MetricErrorRate, "R-1", 0.008, 400), + obs(MetricErrorRate, "R-2", 0.008, 400), + }) + + if eval.Verdict == VerdictSucceeded { + t.Fatal("a missing primary measurement produced a success verdict") + } + for _, m := range eval.PrimaryResults() { + if m.Current != 0 && m.Baseline != 0 { + continue + } + if !m.Underpowered { + t.Error("an absent measurement was not flagged") + } + } +} + +func TestNoPrimaryMetricIsInconclusive(t *testing.T) { + e := NewEvaluator() + eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ + {Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.05)}, + }, []Observation{ + obs(MetricErrorRate, "R-1", 0.01, 400), + obs(MetricErrorRate, "R-2", 0.01, 400), + }) + + if eval.Verdict != VerdictInconclusive { + t.Errorf("verdict = %s, want INCONCLUSIVE with no primary metric", eval.Verdict) + } +} + +func TestPrimaryWithoutTargetCannotPass(t *testing.T) { + e := NewEvaluator() + eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ + {Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower}, // no target + }, []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + obs(MetricRequestsPerTask, "R-2", 1.0, 400), + }) + + if eval.Verdict == VerdictSucceeded { + t.Error("a primary metric with no declared target was treated as met") + } +} + +func TestHigherIsBetterDirection(t *testing.T) { + e := NewEvaluator() + eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{ + {Name: MetricSuccessRate, Role: RolePrimary, Direction: Higher, Target: f64(0.99)}, + {Name: MetricSuccessRate + "_guard", Role: RoleGuardrail, Direction: Higher, Threshold: f64(0.95)}, + }, []Observation{ + obs(MetricSuccessRate, "R-1", 0.97, 400), + obs(MetricSuccessRate, "R-2", 0.995, 400), + obs(MetricSuccessRate+"_guard", "R-1", 0.97, 400), + obs(MetricSuccessRate+"_guard", "R-2", 0.96, 400), + }) + + if eval.Verdict != VerdictSucceeded { + t.Errorf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons) + } +} + +func TestEvaluationIsDeterministic(t *testing.T) { + e := NewEvaluator() + specs := specsFromBlueprint() + observations := []Observation{ + obs(MetricRequestsPerTask, "R-1", 2.7, 400), + obs(MetricRequestsPerTask, "R-2", 2.6, 400), + obs(MetricP95LatencyMS, "R-1", 300, 400), + obs(MetricP95LatencyMS, "R-2", 999, 400), + obs(MetricErrorRate, "R-1", 0.008, 400), + obs(MetricErrorRate, "R-2", 0.5, 400), + } + + first := e.Evaluate("R-1", "R-2", Window{}, specs, observations) + for i := 0; i < 50; i++ { + again := e.Evaluate("R-1", "R-2", Window{}, specs, observations) + if again.Verdict != first.Verdict || len(again.Reasons) != len(first.Reasons) { + t.Fatal("evaluation varied between runs") + } + for j := range first.Reasons { + if again.Reasons[j] != first.Reasons[j] { + t.Fatalf("reason order varied: %q vs %q", first.Reasons[j], again.Reasons[j]) + } + } + for j := range first.Metrics { + if again.Metrics[j].Name != first.Metrics[j].Name { + t.Fatal("metric order varied between runs") + } + } + } +} + +func TestMeasurerDerivesRequestsPerTask(t *testing.T) { + m := NewMeasurer() + now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) + + var events []contract.FluidTelemetry + // R-1: three calls per task. R-2: one. + for c := 0; c < 10; c++ { + chain := fmt.Sprintf("chain-%d", c) + for i := 0; i < 3; i++ { + events = append(events, telemetry("R-1", fmt.Sprintf("c-%d", c), chain, now, 120, false)) + } + events = append(events, telemetry("R-2", fmt.Sprintf("c-%d", c), chain+"-b", now, 130, false)) + } + + observations := m.Measure(events, Window{Start: now.Add(-time.Hour)}) + + got := map[contract.RevisionID]float64{} + for _, o := range observations { + if o.Metric == MetricRequestsPerTask { + got[o.Revision] = o.Value + } + } + if got["R-1"] != 3 { + t.Errorf("R-1 requests per task = %v, want 3", got["R-1"]) + } + if got["R-2"] != 1 { + t.Errorf("R-2 requests per task = %v, want 1", got["R-2"]) + } +} + +func TestMeasurerRespectsWindow(t *testing.T) { + m := NewMeasurer() + now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) + + events := []contract.FluidTelemetry{ + telemetry("R-1", "c-1", "old", now.AddDate(0, 0, -10), 100, false), + telemetry("R-1", "c-1", "new", now, 100, false), + } + + // Only events inside the window may count, or the comparison is not + // reproducible from its recorded period. + observations := m.Measure(events, Window{Start: now.Add(-time.Hour)}) + for _, o := range observations { + if o.Metric == MetricErrorRate && o.Samples != 1 { + t.Errorf("samples = %d, want 1; events outside the window were counted", o.Samples) + } + } +} + +func TestMeasurerComputesErrorRate(t *testing.T) { + m := NewMeasurer() + now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC) + + var events []contract.FluidTelemetry + for i := 0; i < 8; i++ { + events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("ch-%d", i), now, 100, false)) + } + for i := 0; i < 2; i++ { + events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("er-%d", i), now, 100, true)) + } + + for _, o := range m.Measure(events, Window{}) { + if o.Metric == MetricErrorRate && o.Value != 0.2 { + t.Errorf("error rate = %v, want 0.2", o.Value) + } + if o.Metric == MetricSuccessRate && o.Value != 0.8 { + t.Errorf("success rate = %v, want 0.8", o.Value) + } + } +} + +func telemetry(rev contract.RevisionID, consumer, chain string, at time.Time, latency float64, isError bool) contract.FluidTelemetry { + r := rev + l := latency + status := int64(200) + ev := contract.FluidTelemetry{ + ID: fmt.Sprintf("tl-%s-%s", consumer, chain), + OccurredAt: at, + Kind: contract.FluidTelemetryKindRequest, + ConsumerRef: consumer, + Revision: &r, + Sequence: &contract.FluidTelemetrySequence{ChainID: chain}, + Request: &contract.FluidTelemetryRequest{Route: "/v1/x", Method: "GET", Status: &status, LatencyMS: &l}, + } + if isError { + ev.Kind = contract.FluidTelemetryKindError + ev.Error = &contract.FluidTelemetryError{Class: contract.FluidTelemetryErrorClassBackendFailure} + } + return ev +} diff --git a/internal/fitness/measure.go b/internal/fitness/measure.go new file mode 100644 index 0000000..bc39537 --- /dev/null +++ b/internal/fitness/measure.go @@ -0,0 +1,164 @@ +package fitness + +import ( + "sort" + "time" + + "github.com/tegwick/fluid-core/internal/contract" +) + +// Standard metric names the measurer derives from telemetry. +// +// These are the metrics the Blueprint's worked example turns on, and having +// them named in one place stops a hypothesis and an evaluation from measuring +// subtly different things under the same word. +const ( + MetricRequestsPerTask = "requests_per_completed_task" + MetricP95LatencyMS = "p95_latency_ms" + MetricErrorRate = "error_rate" + MetricSuccessRate = "success_rate" +) + +// Measurer derives metric observations from raw telemetry. +type Measurer struct { + // ChainGap bounds one completed task when the consumer supplies no chain + // id, matching the topology analyzer's grouping. + ChainGap time.Duration +} + +// NewMeasurer returns a measurer with the default grouping window. +func NewMeasurer() *Measurer { return &Measurer{ChainGap: 30 * time.Second} } + +// Measure computes the standard metrics per revision over a window. +// +// Only events inside the window count. Blueprint section 18 requires the +// measurement window to be retained, and quietly including events outside it +// would make a comparison irreproducible. +func (m *Measurer) Measure(events []contract.FluidTelemetry, window Window) []Observation { + type acc struct { + requests int + errors int + tasks map[string]int + latency []float64 + } + byRevision := map[contract.RevisionID]*acc{} + + for _, ev := range events { + if !inWindow(ev.OccurredAt, window) { + continue + } + if ev.Revision == nil { + continue + } + rev := *ev.Revision + + a, ok := byRevision[rev] + if !ok { + a = &acc{tasks: map[string]int{}} + byRevision[rev] = a + } + + a.requests++ + if ev.Error != nil { + a.errors++ + } + if ev.Request != nil && ev.Request.LatencyMS != nil { + a.latency = append(a.latency, *ev.Request.LatencyMS) + } + a.tasks[taskKey(ev, m.ChainGap)]++ + } + + revisions := make([]contract.RevisionID, 0, len(byRevision)) + for r := range byRevision { + revisions = append(revisions, r) + } + sort.Slice(revisions, func(i, j int) bool { return revisions[i] < revisions[j] }) + + var out []Observation + for _, rev := range revisions { + a := byRevision[rev] + if a.requests == 0 { + continue + } + + // Requests per completed task is the metric that catches an interface + // making consumers assemble what it could have handed them. + tasks := len(a.tasks) + if tasks > 0 { + out = append(out, Observation{ + Metric: MetricRequestsPerTask, + Revision: rev, + Value: round4(float64(a.requests) / float64(tasks)), + Samples: tasks, + }) + } + + out = append(out, Observation{ + Metric: MetricErrorRate, + Revision: rev, + Value: round4(float64(a.errors) / float64(a.requests)), + Samples: a.requests, + }) + out = append(out, Observation{ + Metric: MetricSuccessRate, + Revision: rev, + Value: round4(float64(a.requests-a.errors) / float64(a.requests)), + Samples: a.requests, + }) + + if len(a.latency) > 0 { + out = append(out, Observation{ + Metric: MetricP95LatencyMS, + Revision: rev, + Value: round4(percentile(a.latency, 0.95)), + Samples: len(a.latency), + }) + } + } + return out +} + +// taskKey groups events into completed tasks. +func taskKey(ev contract.FluidTelemetry, gap time.Duration) string { + consumer := ev.ConsumerRef + if consumer == "" { + consumer = ev.CorrelationID + } + if ev.Sequence != nil && ev.Sequence.ChainID != "" { + return consumer + "/" + ev.Sequence.ChainID + } + // Without a chain id, bucket by consumer and elapsed gap. This is a + // heuristic, which is the reason to prefer chain ids from agentic + // consumers where they can supply them. + bucket := ev.OccurredAt.Truncate(gap).UTC().Format(time.RFC3339) + return consumer + "/" + bucket +} + +func inWindow(t time.Time, w Window) bool { + if !w.Start.IsZero() && t.Before(w.Start) { + return false + } + if w.End != nil && t.After(*w.End) { + return false + } + return true +} + +// percentile returns the nearest-rank percentile of values. +func percentile(values []float64, p float64) float64 { + sorted := make([]float64, len(values)) + copy(sorted, values) + sort.Float64s(sorted) + + if len(sorted) == 1 { + return sorted[0] + } + rank := int(p*float64(len(sorted)-1) + 0.5) + if rank < 0 { + rank = 0 + } + if rank >= len(sorted) { + rank = len(sorted) - 1 + } + return sorted[rank] +}