Add fitness evaluator and telemetry measurer
FLUID-WP-0005 T07. Primary metrics, guardrails, secondary observations and learning signals stay in separate roles, and there is no universal scalar: the standards decline to define one, and inventing one would make the interface's most important trade-offs on the operator's behalf. Three verdict rules carry the weight. A guardrail breach dominates primary success, because a candidate that hit its target while regressing a guardrail traded something it was told not to. Apparent success on too few samples is inconclusive rather than successful, since promoting on three requests is promoting on noise. A metric missing from one side is reported absent rather than defaulted to zero, which would read as a dramatic change that never happened. Specs are an input, never derived from the data, so that changing success criteria after seeing results requires a recorded amendment rather than being indistinguishable from normal operation. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014KmVxhJ35tCo7rE7UnLwWu Assistant: claude-code Assistant-Model: opus Assistant-Process: 1116572@bnt-lap001 Assistant-Session: 8ba9bb93-a72a-4883-b189-2499cce5c400
This commit is contained in:
parent
0ac35892a5
commit
6e705aa0af
3 changed files with 846 additions and 0 deletions
357
internal/fitness/fitness.go
Normal file
357
internal/fitness/fitness.go
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
// Package fitness evaluates how well an interface revision fulfils its purpose.
|
||||
//
|
||||
// ArchitectureBlueprint.md section 18 requires four kinds of metric to stay
|
||||
// distinct: primary metrics the hypothesis predicts, guardrails that must not
|
||||
// regress, secondary observations, and learning signals. Section 48.5 names the
|
||||
// failure mode this prevents — a single fitness number that hides the
|
||||
// dimensions and guardrails underneath it.
|
||||
//
|
||||
// There is deliberately no universal scalar. FluidAPIStandards.md section 20
|
||||
// declines to define one, and a framework that invented one anyway would be
|
||||
// making the interface's most important trade-offs on the operator's behalf.
|
||||
package fitness
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/tegwick/fluid-core/internal/contract"
|
||||
)
|
||||
|
||||
// MetricRole distinguishes what a measurement is for.
|
||||
type MetricRole string
|
||||
|
||||
const (
|
||||
// RolePrimary metrics are the outcomes the hypothesis predicted.
|
||||
RolePrimary MetricRole = "primary"
|
||||
// RoleGuardrail metrics must not regress beyond a threshold, whatever the
|
||||
// primary metrics do.
|
||||
RoleGuardrail MetricRole = "guardrail"
|
||||
// RoleSecondary metrics are useful context, not decision inputs.
|
||||
RoleSecondary MetricRole = "secondary"
|
||||
// RoleLearning metrics improve future hypothesis formation.
|
||||
RoleLearning MetricRole = "learning"
|
||||
)
|
||||
|
||||
// Direction says which way is better for a metric.
|
||||
type Direction string
|
||||
|
||||
const (
|
||||
Lower Direction = "lower"
|
||||
Higher Direction = "higher"
|
||||
Unchanged Direction = "unchanged"
|
||||
)
|
||||
|
||||
// MetricSpec declares how one metric is judged.
|
||||
type MetricSpec struct {
|
||||
Name string `json:"name"`
|
||||
Role MetricRole `json:"role"`
|
||||
Direction Direction `json:"direction"`
|
||||
// Target is the value a primary metric must reach.
|
||||
Target *float64 `json:"target,omitempty"`
|
||||
// Threshold is the limit a guardrail must not cross.
|
||||
Threshold *float64 `json:"threshold,omitempty"`
|
||||
}
|
||||
|
||||
// Window is a measurement period.
|
||||
//
|
||||
// Retaining it is not bookkeeping: Blueprint section 18 requires the evaluator
|
||||
// to keep the baseline and the window, because a comparison whose period is
|
||||
// unknown cannot be reproduced or challenged.
|
||||
type Window struct {
|
||||
Start time.Time `json:"start"`
|
||||
End *time.Time `json:"end,omitempty"`
|
||||
}
|
||||
|
||||
// Observation is a metric measured over one revision.
|
||||
type Observation struct {
|
||||
Metric string `json:"metric"`
|
||||
Revision contract.RevisionID `json:"revision"`
|
||||
Value float64 `json:"value"`
|
||||
Samples int `json:"samples"`
|
||||
}
|
||||
|
||||
// MetricResult is one metric compared between control and candidate.
|
||||
type MetricResult struct {
|
||||
Name string `json:"name"`
|
||||
Role MetricRole `json:"role"`
|
||||
Direction Direction `json:"direction"`
|
||||
|
||||
Baseline float64 `json:"baseline"`
|
||||
Current float64 `json:"current"`
|
||||
Delta float64 `json:"delta"`
|
||||
Target *float64 `json:"target,omitempty"`
|
||||
Threshold *float64 `json:"threshold,omitempty"`
|
||||
|
||||
// TargetMet applies to primary metrics only.
|
||||
TargetMet *bool `json:"target_met,omitempty"`
|
||||
// GuardrailBreached applies to guardrails only.
|
||||
GuardrailBreached *bool `json:"guardrail_breached,omitempty"`
|
||||
|
||||
BaselineSamples int `json:"baseline_samples"`
|
||||
CurrentSamples int `json:"current_samples"`
|
||||
// Underpowered marks a comparison with too little data to lean on.
|
||||
Underpowered bool `json:"underpowered"`
|
||||
}
|
||||
|
||||
// Verdict is the evaluator's overall reading.
|
||||
type Verdict string
|
||||
|
||||
const (
|
||||
// VerdictSucceeded: every primary target met, no guardrail breached.
|
||||
VerdictSucceeded Verdict = "SUCCEEDED"
|
||||
// VerdictFailed: a primary target was missed without a guardrail breach.
|
||||
VerdictFailed Verdict = "FAILED"
|
||||
// VerdictGuardrailBreached: a guardrail regressed, whatever else happened.
|
||||
VerdictGuardrailBreached Verdict = "GUARDRAIL_BREACHED"
|
||||
// VerdictInconclusive: not enough evidence to say either way.
|
||||
VerdictInconclusive Verdict = "INCONCLUSIVE"
|
||||
)
|
||||
|
||||
// Evaluation is a complete fitness comparison.
|
||||
type Evaluation struct {
|
||||
Experiment contract.ExperimentID `json:"experiment,omitempty"`
|
||||
Control contract.RevisionID `json:"control"`
|
||||
Candidate contract.RevisionID `json:"candidate"`
|
||||
Window Window `json:"window"`
|
||||
|
||||
Verdict Verdict `json:"verdict"`
|
||||
Reasons []string `json:"reasons,omitempty"`
|
||||
Metrics []MetricResult `json:"metrics"`
|
||||
}
|
||||
|
||||
// PrimaryResults returns just the primary metrics.
|
||||
func (e Evaluation) PrimaryResults() []MetricResult { return e.byRole(RolePrimary) }
|
||||
|
||||
// GuardrailResults returns just the guardrails.
|
||||
func (e Evaluation) GuardrailResults() []MetricResult { return e.byRole(RoleGuardrail) }
|
||||
|
||||
func (e Evaluation) byRole(role MetricRole) []MetricResult {
|
||||
var out []MetricResult
|
||||
for _, m := range e.Metrics {
|
||||
if m.Role == role {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Evaluator compares a candidate revision against its control.
|
||||
type Evaluator struct {
|
||||
// MinSamples is the per-side sample count below which a comparison is
|
||||
// marked underpowered. A difference measured on three requests is noise
|
||||
// wearing a result's clothes.
|
||||
MinSamples int
|
||||
}
|
||||
|
||||
// NewEvaluator returns an evaluator with a workable default.
|
||||
func NewEvaluator() *Evaluator { return &Evaluator{MinSamples: 30} }
|
||||
|
||||
// Evaluate compares observations against the declared specs.
|
||||
//
|
||||
// Specs are an input, not something derived from the data. Blueprint section 18
|
||||
// says success criteria must not be changed after results are visible without
|
||||
// recording the amendment, and an evaluator that inferred its own criteria
|
||||
// would make that impossible to enforce.
|
||||
func (e *Evaluator) Evaluate(
|
||||
control, candidate contract.RevisionID,
|
||||
window Window,
|
||||
specs []MetricSpec,
|
||||
observations []Observation,
|
||||
) Evaluation {
|
||||
byMetric := map[string]map[contract.RevisionID]Observation{}
|
||||
for _, o := range observations {
|
||||
if byMetric[o.Metric] == nil {
|
||||
byMetric[o.Metric] = map[contract.RevisionID]Observation{}
|
||||
}
|
||||
byMetric[o.Metric][o.Revision] = o
|
||||
}
|
||||
|
||||
eval := Evaluation{Control: control, Candidate: candidate, Window: window}
|
||||
|
||||
ordered := make([]MetricSpec, len(specs))
|
||||
copy(ordered, specs)
|
||||
sort.SliceStable(ordered, func(i, j int) bool {
|
||||
if ordered[i].Role != ordered[j].Role {
|
||||
return roleRank(ordered[i].Role) < roleRank(ordered[j].Role)
|
||||
}
|
||||
return ordered[i].Name < ordered[j].Name
|
||||
})
|
||||
|
||||
var (
|
||||
missingPrimary []string
|
||||
breached []string
|
||||
underpowered []string
|
||||
primaryCount int
|
||||
primaryMetCount int
|
||||
haveAnyPrimary bool
|
||||
)
|
||||
|
||||
for _, spec := range ordered {
|
||||
base, hasBase := byMetric[spec.Name][control]
|
||||
cur, hasCur := byMetric[spec.Name][candidate]
|
||||
|
||||
if !hasBase || !hasCur {
|
||||
// A metric with no measurement on one side is reported as absent
|
||||
// rather than defaulted to zero, which would read as a dramatic
|
||||
// improvement or regression that never happened.
|
||||
eval.Metrics = append(eval.Metrics, MetricResult{
|
||||
Name: spec.Name,
|
||||
Role: spec.Role,
|
||||
Direction: spec.Direction,
|
||||
Target: spec.Target,
|
||||
Threshold: spec.Threshold,
|
||||
Underpowered: true,
|
||||
})
|
||||
if spec.Role == RolePrimary {
|
||||
primaryCount++
|
||||
underpowered = append(underpowered,
|
||||
fmt.Sprintf("%s has no measurement on both sides", spec.Name))
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
r := MetricResult{
|
||||
Name: spec.Name,
|
||||
Role: spec.Role,
|
||||
Direction: spec.Direction,
|
||||
Baseline: base.Value,
|
||||
Current: cur.Value,
|
||||
Delta: round4(cur.Value - base.Value),
|
||||
Target: spec.Target,
|
||||
Threshold: spec.Threshold,
|
||||
BaselineSamples: base.Samples,
|
||||
CurrentSamples: cur.Samples,
|
||||
}
|
||||
|
||||
if base.Samples < e.MinSamples || cur.Samples < e.MinSamples {
|
||||
r.Underpowered = true
|
||||
underpowered = append(underpowered, fmt.Sprintf(
|
||||
"%s has %d control and %d candidate samples, below the %d needed",
|
||||
spec.Name, base.Samples, cur.Samples, e.MinSamples))
|
||||
}
|
||||
|
||||
switch spec.Role {
|
||||
case RolePrimary:
|
||||
primaryCount++
|
||||
haveAnyPrimary = true
|
||||
met := meetsTarget(spec, cur.Value)
|
||||
r.TargetMet = &met
|
||||
if met {
|
||||
primaryMetCount++
|
||||
} else {
|
||||
missingPrimary = append(missingPrimary, describeMiss(spec, cur.Value))
|
||||
}
|
||||
|
||||
case RoleGuardrail:
|
||||
crossed := breachesGuardrail(spec, cur.Value)
|
||||
r.GuardrailBreached = &crossed
|
||||
if crossed {
|
||||
breached = append(breached, describeBreach(spec, cur.Value))
|
||||
}
|
||||
}
|
||||
|
||||
eval.Metrics = append(eval.Metrics, r)
|
||||
}
|
||||
|
||||
// Guardrails dominate. A candidate that hit every target while regressing a
|
||||
// guardrail has not succeeded; it has traded something it was told not to.
|
||||
switch {
|
||||
case len(breached) > 0:
|
||||
eval.Verdict = VerdictGuardrailBreached
|
||||
eval.Reasons = breached
|
||||
|
||||
case !haveAnyPrimary || primaryCount == 0:
|
||||
eval.Verdict = VerdictInconclusive
|
||||
eval.Reasons = []string{"no primary metric was declared, so there is nothing to conclude"}
|
||||
|
||||
case len(underpowered) > 0 && len(missingPrimary) == 0:
|
||||
// Every target appears met, but on too little data to act on. Reporting
|
||||
// success here is how an experiment gets promoted on noise.
|
||||
eval.Verdict = VerdictInconclusive
|
||||
eval.Reasons = underpowered
|
||||
|
||||
case len(missingPrimary) > 0:
|
||||
eval.Verdict = VerdictFailed
|
||||
eval.Reasons = missingPrimary
|
||||
|
||||
default:
|
||||
eval.Verdict = VerdictSucceeded
|
||||
eval.Reasons = []string{fmt.Sprintf("%d of %d primary targets met with no guardrail breach",
|
||||
primaryMetCount, primaryCount)}
|
||||
}
|
||||
|
||||
sort.Strings(eval.Reasons)
|
||||
return eval
|
||||
}
|
||||
|
||||
func meetsTarget(spec MetricSpec, value float64) bool {
|
||||
if spec.Target == nil {
|
||||
// A primary metric with no target cannot be judged, so it is not met.
|
||||
return false
|
||||
}
|
||||
switch spec.Direction {
|
||||
case Lower:
|
||||
return value <= *spec.Target
|
||||
case Higher:
|
||||
return value >= *spec.Target
|
||||
case Unchanged:
|
||||
return math.Abs(value-*spec.Target) < 1e-9
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func breachesGuardrail(spec MetricSpec, value float64) bool {
|
||||
if spec.Threshold == nil {
|
||||
return false
|
||||
}
|
||||
switch spec.Direction {
|
||||
case Lower:
|
||||
// Lower is better, so exceeding the threshold is the breach.
|
||||
return value > *spec.Threshold
|
||||
case Higher:
|
||||
return value < *spec.Threshold
|
||||
case Unchanged:
|
||||
return math.Abs(value-*spec.Threshold) > 1e-9
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func describeMiss(spec MetricSpec, value float64) string {
|
||||
if spec.Target == nil {
|
||||
return fmt.Sprintf("%s is a primary metric with no declared target", spec.Name)
|
||||
}
|
||||
return fmt.Sprintf("%s reached %.4g, target was %s %.4g",
|
||||
spec.Name, value, comparator(spec.Direction), *spec.Target)
|
||||
}
|
||||
|
||||
func describeBreach(spec MetricSpec, value float64) string {
|
||||
return fmt.Sprintf("guardrail %s at %.4g breached its threshold of %.4g",
|
||||
spec.Name, value, *spec.Threshold)
|
||||
}
|
||||
|
||||
func comparator(d Direction) string {
|
||||
switch d {
|
||||
case Lower:
|
||||
return "at most"
|
||||
case Higher:
|
||||
return "at least"
|
||||
}
|
||||
return "exactly"
|
||||
}
|
||||
|
||||
func roleRank(r MetricRole) int {
|
||||
switch r {
|
||||
case RolePrimary:
|
||||
return 0
|
||||
case RoleGuardrail:
|
||||
return 1
|
||||
case RoleSecondary:
|
||||
return 2
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
func round4(v float64) float64 { return math.Round(v*10000) / 10000 }
|
||||
325
internal/fitness/fitness_test.go
Normal file
325
internal/fitness/fitness_test.go
Normal file
|
|
@ -0,0 +1,325 @@
|
|||
package fitness
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tegwick/fluid-core/internal/contract"
|
||||
)
|
||||
|
||||
func f64(v float64) *float64 { return &v }
|
||||
|
||||
func obs(metric string, rev contract.RevisionID, value float64, samples int) Observation {
|
||||
return Observation{Metric: metric, Revision: rev, Value: value, Samples: samples}
|
||||
}
|
||||
|
||||
// specsFromBlueprint mirrors the section 33 worked example: requests per task
|
||||
// must drop, latency must not regress past its guardrail.
|
||||
func specsFromBlueprint() []MetricSpec {
|
||||
return []MetricSpec{
|
||||
{Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower, Target: f64(1.2)},
|
||||
{Name: MetricP95LatencyMS, Role: RoleGuardrail, Direction: Lower, Threshold: f64(315)},
|
||||
{Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.01)},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSucceedsWhenTargetsMetAndGuardrailsHold(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
|
||||
eval := e.Evaluate("R-1", "R-2", Window{Start: time.Now().Add(-time.Hour)}, specsFromBlueprint(),
|
||||
[]Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
obs(MetricRequestsPerTask, "R-2", 1.15, 400),
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 400),
|
||||
obs(MetricP95LatencyMS, "R-2", 302, 400),
|
||||
obs(MetricErrorRate, "R-1", 0.008, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.007, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictSucceeded {
|
||||
t.Fatalf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons)
|
||||
}
|
||||
|
||||
primary := eval.PrimaryResults()
|
||||
if len(primary) != 1 {
|
||||
t.Fatalf("primary metrics = %d, want 1", len(primary))
|
||||
}
|
||||
if primary[0].TargetMet == nil || !*primary[0].TargetMet {
|
||||
t.Error("primary target not marked as met")
|
||||
}
|
||||
if primary[0].Delta >= 0 {
|
||||
t.Errorf("delta = %v, expected a reduction", primary[0].Delta)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardrailDominatesPrimarySuccess is the section 48.5 protection: a
|
||||
// candidate that hit its target while regressing a guardrail has traded
|
||||
// something it was told not to.
|
||||
func TestGuardrailDominatesPrimarySuccess(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
obs(MetricRequestsPerTask, "R-2", 1.05, 400), // target smashed
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 400),
|
||||
obs(MetricP95LatencyMS, "R-2", 980, 400), // and latency ruined
|
||||
obs(MetricErrorRate, "R-1", 0.008, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.009, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictGuardrailBreached {
|
||||
t.Fatalf("verdict = %s, want GUARDRAIL_BREACHED; reasons %v", eval.Verdict, eval.Reasons)
|
||||
}
|
||||
if len(eval.Reasons) == 0 {
|
||||
t.Error("a breach was reported with no reason")
|
||||
}
|
||||
|
||||
var latency *MetricResult
|
||||
for i, m := range eval.GuardrailResults() {
|
||||
if m.Name == MetricP95LatencyMS {
|
||||
latency = &eval.GuardrailResults()[i]
|
||||
}
|
||||
}
|
||||
if latency == nil || latency.GuardrailBreached == nil || !*latency.GuardrailBreached {
|
||||
t.Error("the breached guardrail is not marked as breached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailsWhenPrimaryTargetMissed(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
obs(MetricRequestsPerTask, "R-2", 2.6, 400), // barely moved
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 400),
|
||||
obs(MetricP95LatencyMS, "R-2", 301, 400),
|
||||
obs(MetricErrorRate, "R-1", 0.008, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.008, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictFailed {
|
||||
t.Fatalf("verdict = %s, want FAILED", eval.Verdict)
|
||||
}
|
||||
if len(eval.Reasons) == 0 || eval.Reasons[0] == "" {
|
||||
t.Error("failure reported without saying which target was missed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUnderpoweredIsInconclusiveNotSuccessful: promoting on three requests is
|
||||
// promoting on noise.
|
||||
func TestUnderpoweredIsInconclusiveNotSuccessful(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 3),
|
||||
obs(MetricRequestsPerTask, "R-2", 1.0, 2),
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 3),
|
||||
obs(MetricP95LatencyMS, "R-2", 290, 2),
|
||||
obs(MetricErrorRate, "R-1", 0.0, 3),
|
||||
obs(MetricErrorRate, "R-2", 0.0, 2),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictInconclusive {
|
||||
t.Fatalf("verdict = %s, want INCONCLUSIVE on tiny samples", eval.Verdict)
|
||||
}
|
||||
for _, m := range eval.PrimaryResults() {
|
||||
if !m.Underpowered {
|
||||
t.Error("a two-sample comparison was not marked underpowered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestMissingMeasurementIsNotZero: defaulting an absent metric to zero would
|
||||
// read as a dramatic change that never happened.
|
||||
func TestMissingMeasurementIsNotZero(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, specsFromBlueprint(), []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
// no candidate measurement at all
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 400),
|
||||
obs(MetricP95LatencyMS, "R-2", 300, 400),
|
||||
obs(MetricErrorRate, "R-1", 0.008, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.008, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict == VerdictSucceeded {
|
||||
t.Fatal("a missing primary measurement produced a success verdict")
|
||||
}
|
||||
for _, m := range eval.PrimaryResults() {
|
||||
if m.Current != 0 && m.Baseline != 0 {
|
||||
continue
|
||||
}
|
||||
if !m.Underpowered {
|
||||
t.Error("an absent measurement was not flagged")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoPrimaryMetricIsInconclusive(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{
|
||||
{Name: MetricErrorRate, Role: RoleGuardrail, Direction: Lower, Threshold: f64(0.05)},
|
||||
}, []Observation{
|
||||
obs(MetricErrorRate, "R-1", 0.01, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.01, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictInconclusive {
|
||||
t.Errorf("verdict = %s, want INCONCLUSIVE with no primary metric", eval.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrimaryWithoutTargetCannotPass(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{
|
||||
{Name: MetricRequestsPerTask, Role: RolePrimary, Direction: Lower}, // no target
|
||||
}, []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
obs(MetricRequestsPerTask, "R-2", 1.0, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict == VerdictSucceeded {
|
||||
t.Error("a primary metric with no declared target was treated as met")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHigherIsBetterDirection(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
eval := e.Evaluate("R-1", "R-2", Window{}, []MetricSpec{
|
||||
{Name: MetricSuccessRate, Role: RolePrimary, Direction: Higher, Target: f64(0.99)},
|
||||
{Name: MetricSuccessRate + "_guard", Role: RoleGuardrail, Direction: Higher, Threshold: f64(0.95)},
|
||||
}, []Observation{
|
||||
obs(MetricSuccessRate, "R-1", 0.97, 400),
|
||||
obs(MetricSuccessRate, "R-2", 0.995, 400),
|
||||
obs(MetricSuccessRate+"_guard", "R-1", 0.97, 400),
|
||||
obs(MetricSuccessRate+"_guard", "R-2", 0.96, 400),
|
||||
})
|
||||
|
||||
if eval.Verdict != VerdictSucceeded {
|
||||
t.Errorf("verdict = %s, reasons %v", eval.Verdict, eval.Reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluationIsDeterministic(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
specs := specsFromBlueprint()
|
||||
observations := []Observation{
|
||||
obs(MetricRequestsPerTask, "R-1", 2.7, 400),
|
||||
obs(MetricRequestsPerTask, "R-2", 2.6, 400),
|
||||
obs(MetricP95LatencyMS, "R-1", 300, 400),
|
||||
obs(MetricP95LatencyMS, "R-2", 999, 400),
|
||||
obs(MetricErrorRate, "R-1", 0.008, 400),
|
||||
obs(MetricErrorRate, "R-2", 0.5, 400),
|
||||
}
|
||||
|
||||
first := e.Evaluate("R-1", "R-2", Window{}, specs, observations)
|
||||
for i := 0; i < 50; i++ {
|
||||
again := e.Evaluate("R-1", "R-2", Window{}, specs, observations)
|
||||
if again.Verdict != first.Verdict || len(again.Reasons) != len(first.Reasons) {
|
||||
t.Fatal("evaluation varied between runs")
|
||||
}
|
||||
for j := range first.Reasons {
|
||||
if again.Reasons[j] != first.Reasons[j] {
|
||||
t.Fatalf("reason order varied: %q vs %q", first.Reasons[j], again.Reasons[j])
|
||||
}
|
||||
}
|
||||
for j := range first.Metrics {
|
||||
if again.Metrics[j].Name != first.Metrics[j].Name {
|
||||
t.Fatal("metric order varied between runs")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasurerDerivesRequestsPerTask(t *testing.T) {
|
||||
m := NewMeasurer()
|
||||
now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
var events []contract.FluidTelemetry
|
||||
// R-1: three calls per task. R-2: one.
|
||||
for c := 0; c < 10; c++ {
|
||||
chain := fmt.Sprintf("chain-%d", c)
|
||||
for i := 0; i < 3; i++ {
|
||||
events = append(events, telemetry("R-1", fmt.Sprintf("c-%d", c), chain, now, 120, false))
|
||||
}
|
||||
events = append(events, telemetry("R-2", fmt.Sprintf("c-%d", c), chain+"-b", now, 130, false))
|
||||
}
|
||||
|
||||
observations := m.Measure(events, Window{Start: now.Add(-time.Hour)})
|
||||
|
||||
got := map[contract.RevisionID]float64{}
|
||||
for _, o := range observations {
|
||||
if o.Metric == MetricRequestsPerTask {
|
||||
got[o.Revision] = o.Value
|
||||
}
|
||||
}
|
||||
if got["R-1"] != 3 {
|
||||
t.Errorf("R-1 requests per task = %v, want 3", got["R-1"])
|
||||
}
|
||||
if got["R-2"] != 1 {
|
||||
t.Errorf("R-2 requests per task = %v, want 1", got["R-2"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasurerRespectsWindow(t *testing.T) {
|
||||
m := NewMeasurer()
|
||||
now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
events := []contract.FluidTelemetry{
|
||||
telemetry("R-1", "c-1", "old", now.AddDate(0, 0, -10), 100, false),
|
||||
telemetry("R-1", "c-1", "new", now, 100, false),
|
||||
}
|
||||
|
||||
// Only events inside the window may count, or the comparison is not
|
||||
// reproducible from its recorded period.
|
||||
observations := m.Measure(events, Window{Start: now.Add(-time.Hour)})
|
||||
for _, o := range observations {
|
||||
if o.Metric == MetricErrorRate && o.Samples != 1 {
|
||||
t.Errorf("samples = %d, want 1; events outside the window were counted", o.Samples)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeasurerComputesErrorRate(t *testing.T) {
|
||||
m := NewMeasurer()
|
||||
now := time.Date(2026, 9, 5, 8, 0, 0, 0, time.UTC)
|
||||
|
||||
var events []contract.FluidTelemetry
|
||||
for i := 0; i < 8; i++ {
|
||||
events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("ch-%d", i), now, 100, false))
|
||||
}
|
||||
for i := 0; i < 2; i++ {
|
||||
events = append(events, telemetry("R-1", "c-1", fmt.Sprintf("er-%d", i), now, 100, true))
|
||||
}
|
||||
|
||||
for _, o := range m.Measure(events, Window{}) {
|
||||
if o.Metric == MetricErrorRate && o.Value != 0.2 {
|
||||
t.Errorf("error rate = %v, want 0.2", o.Value)
|
||||
}
|
||||
if o.Metric == MetricSuccessRate && o.Value != 0.8 {
|
||||
t.Errorf("success rate = %v, want 0.8", o.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func telemetry(rev contract.RevisionID, consumer, chain string, at time.Time, latency float64, isError bool) contract.FluidTelemetry {
|
||||
r := rev
|
||||
l := latency
|
||||
status := int64(200)
|
||||
ev := contract.FluidTelemetry{
|
||||
ID: fmt.Sprintf("tl-%s-%s", consumer, chain),
|
||||
OccurredAt: at,
|
||||
Kind: contract.FluidTelemetryKindRequest,
|
||||
ConsumerRef: consumer,
|
||||
Revision: &r,
|
||||
Sequence: &contract.FluidTelemetrySequence{ChainID: chain},
|
||||
Request: &contract.FluidTelemetryRequest{Route: "/v1/x", Method: "GET", Status: &status, LatencyMS: &l},
|
||||
}
|
||||
if isError {
|
||||
ev.Kind = contract.FluidTelemetryKindError
|
||||
ev.Error = &contract.FluidTelemetryError{Class: contract.FluidTelemetryErrorClassBackendFailure}
|
||||
}
|
||||
return ev
|
||||
}
|
||||
164
internal/fitness/measure.go
Normal file
164
internal/fitness/measure.go
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
package fitness
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/tegwick/fluid-core/internal/contract"
|
||||
)
|
||||
|
||||
// Standard metric names the measurer derives from telemetry.
|
||||
//
|
||||
// These are the metrics the Blueprint's worked example turns on, and having
|
||||
// them named in one place stops a hypothesis and an evaluation from measuring
|
||||
// subtly different things under the same word.
|
||||
const (
|
||||
MetricRequestsPerTask = "requests_per_completed_task"
|
||||
MetricP95LatencyMS = "p95_latency_ms"
|
||||
MetricErrorRate = "error_rate"
|
||||
MetricSuccessRate = "success_rate"
|
||||
)
|
||||
|
||||
// Measurer derives metric observations from raw telemetry.
|
||||
type Measurer struct {
|
||||
// ChainGap bounds one completed task when the consumer supplies no chain
|
||||
// id, matching the topology analyzer's grouping.
|
||||
ChainGap time.Duration
|
||||
}
|
||||
|
||||
// NewMeasurer returns a measurer with the default grouping window.
|
||||
func NewMeasurer() *Measurer { return &Measurer{ChainGap: 30 * time.Second} }
|
||||
|
||||
// Measure computes the standard metrics per revision over a window.
|
||||
//
|
||||
// Only events inside the window count. Blueprint section 18 requires the
|
||||
// measurement window to be retained, and quietly including events outside it
|
||||
// would make a comparison irreproducible.
|
||||
func (m *Measurer) Measure(events []contract.FluidTelemetry, window Window) []Observation {
|
||||
type acc struct {
|
||||
requests int
|
||||
errors int
|
||||
tasks map[string]int
|
||||
latency []float64
|
||||
}
|
||||
byRevision := map[contract.RevisionID]*acc{}
|
||||
|
||||
for _, ev := range events {
|
||||
if !inWindow(ev.OccurredAt, window) {
|
||||
continue
|
||||
}
|
||||
if ev.Revision == nil {
|
||||
continue
|
||||
}
|
||||
rev := *ev.Revision
|
||||
|
||||
a, ok := byRevision[rev]
|
||||
if !ok {
|
||||
a = &acc{tasks: map[string]int{}}
|
||||
byRevision[rev] = a
|
||||
}
|
||||
|
||||
a.requests++
|
||||
if ev.Error != nil {
|
||||
a.errors++
|
||||
}
|
||||
if ev.Request != nil && ev.Request.LatencyMS != nil {
|
||||
a.latency = append(a.latency, *ev.Request.LatencyMS)
|
||||
}
|
||||
a.tasks[taskKey(ev, m.ChainGap)]++
|
||||
}
|
||||
|
||||
revisions := make([]contract.RevisionID, 0, len(byRevision))
|
||||
for r := range byRevision {
|
||||
revisions = append(revisions, r)
|
||||
}
|
||||
sort.Slice(revisions, func(i, j int) bool { return revisions[i] < revisions[j] })
|
||||
|
||||
var out []Observation
|
||||
for _, rev := range revisions {
|
||||
a := byRevision[rev]
|
||||
if a.requests == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Requests per completed task is the metric that catches an interface
|
||||
// making consumers assemble what it could have handed them.
|
||||
tasks := len(a.tasks)
|
||||
if tasks > 0 {
|
||||
out = append(out, Observation{
|
||||
Metric: MetricRequestsPerTask,
|
||||
Revision: rev,
|
||||
Value: round4(float64(a.requests) / float64(tasks)),
|
||||
Samples: tasks,
|
||||
})
|
||||
}
|
||||
|
||||
out = append(out, Observation{
|
||||
Metric: MetricErrorRate,
|
||||
Revision: rev,
|
||||
Value: round4(float64(a.errors) / float64(a.requests)),
|
||||
Samples: a.requests,
|
||||
})
|
||||
out = append(out, Observation{
|
||||
Metric: MetricSuccessRate,
|
||||
Revision: rev,
|
||||
Value: round4(float64(a.requests-a.errors) / float64(a.requests)),
|
||||
Samples: a.requests,
|
||||
})
|
||||
|
||||
if len(a.latency) > 0 {
|
||||
out = append(out, Observation{
|
||||
Metric: MetricP95LatencyMS,
|
||||
Revision: rev,
|
||||
Value: round4(percentile(a.latency, 0.95)),
|
||||
Samples: len(a.latency),
|
||||
})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// taskKey groups events into completed tasks.
|
||||
func taskKey(ev contract.FluidTelemetry, gap time.Duration) string {
|
||||
consumer := ev.ConsumerRef
|
||||
if consumer == "" {
|
||||
consumer = ev.CorrelationID
|
||||
}
|
||||
if ev.Sequence != nil && ev.Sequence.ChainID != "" {
|
||||
return consumer + "/" + ev.Sequence.ChainID
|
||||
}
|
||||
// Without a chain id, bucket by consumer and elapsed gap. This is a
|
||||
// heuristic, which is the reason to prefer chain ids from agentic
|
||||
// consumers where they can supply them.
|
||||
bucket := ev.OccurredAt.Truncate(gap).UTC().Format(time.RFC3339)
|
||||
return consumer + "/" + bucket
|
||||
}
|
||||
|
||||
func inWindow(t time.Time, w Window) bool {
|
||||
if !w.Start.IsZero() && t.Before(w.Start) {
|
||||
return false
|
||||
}
|
||||
if w.End != nil && t.After(*w.End) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// percentile returns the nearest-rank percentile of values.
|
||||
func percentile(values []float64, p float64) float64 {
|
||||
sorted := make([]float64, len(values))
|
||||
copy(sorted, values)
|
||||
sort.Float64s(sorted)
|
||||
|
||||
if len(sorted) == 1 {
|
||||
return sorted[0]
|
||||
}
|
||||
rank := int(p*float64(len(sorted)-1) + 0.5)
|
||||
if rank < 0 {
|
||||
rank = 0
|
||||
}
|
||||
if rank >= len(sorted) {
|
||||
rank = len(sorted) - 1
|
||||
}
|
||||
return sorted[rank]
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue