// Package fitness evaluates how well an interface revision fulfils its purpose. // // ArchitectureBlueprint.md section 18 requires four kinds of metric to stay // distinct: primary metrics the hypothesis predicts, guardrails that must not // regress, secondary observations, and learning signals. Section 48.5 names the // failure mode this prevents — a single fitness number that hides the // dimensions and guardrails underneath it. // // There is deliberately no universal scalar. FluidAPIStandards.md section 20 // declines to define one, and a framework that invented one anyway would be // making the interface's most important trade-offs on the operator's behalf. package fitness import ( "fmt" "math" "sort" "time" "github.com/tegwick/fluid-core/internal/contract" ) // MetricRole distinguishes what a measurement is for. type MetricRole string const ( // RolePrimary metrics are the outcomes the hypothesis predicted. RolePrimary MetricRole = "primary" // RoleGuardrail metrics must not regress beyond a threshold, whatever the // primary metrics do. RoleGuardrail MetricRole = "guardrail" // RoleSecondary metrics are useful context, not decision inputs. RoleSecondary MetricRole = "secondary" // RoleLearning metrics improve future hypothesis formation. RoleLearning MetricRole = "learning" ) // Direction says which way is better for a metric. type Direction string const ( Lower Direction = "lower" Higher Direction = "higher" Unchanged Direction = "unchanged" ) // MetricSpec declares how one metric is judged. type MetricSpec struct { Name string `json:"name"` Role MetricRole `json:"role"` Direction Direction `json:"direction"` // Target is the value a primary metric must reach. Target *float64 `json:"target,omitempty"` // Threshold is the limit a guardrail must not cross. Threshold *float64 `json:"threshold,omitempty"` } // Window is a measurement period. // // Retaining it is not bookkeeping: Blueprint section 18 requires the evaluator // to keep the baseline and the window, because a comparison whose period is // unknown cannot be reproduced or challenged. type Window struct { Start time.Time `json:"start"` End *time.Time `json:"end,omitempty"` } // Observation is a metric measured over one revision. type Observation struct { Metric string `json:"metric"` Revision contract.RevisionID `json:"revision"` Value float64 `json:"value"` Samples int `json:"samples"` } // MetricResult is one metric compared between control and candidate. type MetricResult struct { Name string `json:"name"` Role MetricRole `json:"role"` Direction Direction `json:"direction"` Baseline float64 `json:"baseline"` Current float64 `json:"current"` Delta float64 `json:"delta"` Target *float64 `json:"target,omitempty"` Threshold *float64 `json:"threshold,omitempty"` // TargetMet applies to primary metrics only. TargetMet *bool `json:"target_met,omitempty"` // GuardrailBreached applies to guardrails only. GuardrailBreached *bool `json:"guardrail_breached,omitempty"` BaselineSamples int `json:"baseline_samples"` CurrentSamples int `json:"current_samples"` // Underpowered marks a comparison with too little data to lean on. Underpowered bool `json:"underpowered"` } // Verdict is the evaluator's overall reading. type Verdict string const ( // VerdictSucceeded: every primary target met, no guardrail breached. VerdictSucceeded Verdict = "SUCCEEDED" // VerdictFailed: a primary target was missed without a guardrail breach. VerdictFailed Verdict = "FAILED" // VerdictGuardrailBreached: a guardrail regressed, whatever else happened. VerdictGuardrailBreached Verdict = "GUARDRAIL_BREACHED" // VerdictInconclusive: not enough evidence to say either way. VerdictInconclusive Verdict = "INCONCLUSIVE" ) // Evaluation is a complete fitness comparison. type Evaluation struct { Experiment contract.ExperimentID `json:"experiment,omitempty"` Control contract.RevisionID `json:"control"` Candidate contract.RevisionID `json:"candidate"` Window Window `json:"window"` Verdict Verdict `json:"verdict"` Reasons []string `json:"reasons,omitempty"` Metrics []MetricResult `json:"metrics"` } // PrimaryResults returns just the primary metrics. func (e Evaluation) PrimaryResults() []MetricResult { return e.byRole(RolePrimary) } // GuardrailResults returns just the guardrails. func (e Evaluation) GuardrailResults() []MetricResult { return e.byRole(RoleGuardrail) } func (e Evaluation) byRole(role MetricRole) []MetricResult { var out []MetricResult for _, m := range e.Metrics { if m.Role == role { out = append(out, m) } } return out } // Evaluator compares a candidate revision against its control. type Evaluator struct { // MinSamples is the per-side sample count below which a comparison is // marked underpowered. A difference measured on three requests is noise // wearing a result's clothes. MinSamples int } // NewEvaluator returns an evaluator with a workable default. func NewEvaluator() *Evaluator { return &Evaluator{MinSamples: 30} } // Evaluate compares observations against the declared specs. // // Specs are an input, not something derived from the data. Blueprint section 18 // says success criteria must not be changed after results are visible without // recording the amendment, and an evaluator that inferred its own criteria // would make that impossible to enforce. func (e *Evaluator) Evaluate( control, candidate contract.RevisionID, window Window, specs []MetricSpec, observations []Observation, ) Evaluation { byMetric := map[string]map[contract.RevisionID]Observation{} for _, o := range observations { if byMetric[o.Metric] == nil { byMetric[o.Metric] = map[contract.RevisionID]Observation{} } byMetric[o.Metric][o.Revision] = o } eval := Evaluation{Control: control, Candidate: candidate, Window: window} ordered := make([]MetricSpec, len(specs)) copy(ordered, specs) sort.SliceStable(ordered, func(i, j int) bool { if ordered[i].Role != ordered[j].Role { return roleRank(ordered[i].Role) < roleRank(ordered[j].Role) } return ordered[i].Name < ordered[j].Name }) var ( missingPrimary []string breached []string underpowered []string primaryCount int primaryMetCount int haveAnyPrimary bool ) for _, spec := range ordered { base, hasBase := byMetric[spec.Name][control] cur, hasCur := byMetric[spec.Name][candidate] if !hasBase || !hasCur { // A metric with no measurement on one side is reported as absent // rather than defaulted to zero, which would read as a dramatic // improvement or regression that never happened. eval.Metrics = append(eval.Metrics, MetricResult{ Name: spec.Name, Role: spec.Role, Direction: spec.Direction, Target: spec.Target, Threshold: spec.Threshold, Underpowered: true, }) if spec.Role == RolePrimary { primaryCount++ underpowered = append(underpowered, fmt.Sprintf("%s has no measurement on both sides", spec.Name)) } continue } r := MetricResult{ Name: spec.Name, Role: spec.Role, Direction: spec.Direction, Baseline: base.Value, Current: cur.Value, Delta: round4(cur.Value - base.Value), Target: spec.Target, Threshold: spec.Threshold, BaselineSamples: base.Samples, CurrentSamples: cur.Samples, } if base.Samples < e.MinSamples || cur.Samples < e.MinSamples { r.Underpowered = true underpowered = append(underpowered, fmt.Sprintf( "%s has %d control and %d candidate samples, below the %d needed", spec.Name, base.Samples, cur.Samples, e.MinSamples)) } switch spec.Role { case RolePrimary: primaryCount++ haveAnyPrimary = true met := meetsTarget(spec, cur.Value) r.TargetMet = &met if met { primaryMetCount++ } else { missingPrimary = append(missingPrimary, describeMiss(spec, cur.Value)) } case RoleGuardrail: crossed := breachesGuardrail(spec, cur.Value) r.GuardrailBreached = &crossed if crossed { breached = append(breached, describeBreach(spec, cur.Value)) } } eval.Metrics = append(eval.Metrics, r) } // Guardrails dominate. A candidate that hit every target while regressing a // guardrail has not succeeded; it has traded something it was told not to. switch { case len(breached) > 0: eval.Verdict = VerdictGuardrailBreached eval.Reasons = breached case !haveAnyPrimary || primaryCount == 0: eval.Verdict = VerdictInconclusive eval.Reasons = []string{"no primary metric was declared, so there is nothing to conclude"} case len(underpowered) > 0 && len(missingPrimary) == 0: // Every target appears met, but on too little data to act on. Reporting // success here is how an experiment gets promoted on noise. eval.Verdict = VerdictInconclusive eval.Reasons = underpowered case len(missingPrimary) > 0: eval.Verdict = VerdictFailed eval.Reasons = missingPrimary default: eval.Verdict = VerdictSucceeded eval.Reasons = []string{fmt.Sprintf("%d of %d primary targets met with no guardrail breach", primaryMetCount, primaryCount)} } sort.Strings(eval.Reasons) return eval } func meetsTarget(spec MetricSpec, value float64) bool { if spec.Target == nil { // A primary metric with no target cannot be judged, so it is not met. return false } switch spec.Direction { case Lower: return value <= *spec.Target case Higher: return value >= *spec.Target case Unchanged: return math.Abs(value-*spec.Target) < 1e-9 } return false } func breachesGuardrail(spec MetricSpec, value float64) bool { if spec.Threshold == nil { return false } switch spec.Direction { case Lower: // Lower is better, so exceeding the threshold is the breach. return value > *spec.Threshold case Higher: return value < *spec.Threshold case Unchanged: return math.Abs(value-*spec.Threshold) > 1e-9 } return false } func describeMiss(spec MetricSpec, value float64) string { if spec.Target == nil { return fmt.Sprintf("%s is a primary metric with no declared target", spec.Name) } return fmt.Sprintf("%s reached %.4g, target was %s %.4g", spec.Name, value, comparator(spec.Direction), *spec.Target) } func describeBreach(spec MetricSpec, value float64) string { return fmt.Sprintf("guardrail %s at %.4g breached its threshold of %.4g", spec.Name, value, *spec.Threshold) } func comparator(d Direction) string { switch d { case Lower: return "at most" case Higher: return "at least" } return "exactly" } func roleRank(r MetricRole) int { switch r { case RolePrimary: return 0 case RoleGuardrail: return 1 case RoleSecondary: return 2 } return 3 } func round4(v float64) float64 { return math.Round(v*10000) / 10000 }