fluid-core/internal/fitness/fitness.go

// Package fitness evaluates how well an interface revision fulfils its purpose.
//
// ArchitectureBlueprint.md section 18 requires four kinds of metric to stay
// distinct: primary metrics the hypothesis predicts, guardrails that must not
// regress, secondary observations, and learning signals. Section 48.5 names the
// failure mode this prevents — a single fitness number that hides the
// dimensions and guardrails underneath it.
//
// There is deliberately no universal scalar. FluidAPIStandards.md section 20
// declines to define one, and a framework that invented one anyway would be
// making the interface's most important trade-offs on the operator's behalf.
package fitness

import (
	"fmt"
	"math"
	"sort"
	"time"

	"github.com/tegwick/fluid-core/internal/contract"
)

// MetricRole distinguishes what a measurement is for.
type MetricRole string

const (
	// RolePrimary metrics are the outcomes the hypothesis predicted.
	RolePrimary MetricRole = "primary"
	// RoleGuardrail metrics must not regress beyond a threshold, whatever the
	// primary metrics do.
	RoleGuardrail MetricRole = "guardrail"
	// RoleSecondary metrics are useful context, not decision inputs.
	RoleSecondary MetricRole = "secondary"
	// RoleLearning metrics improve future hypothesis formation.
	RoleLearning MetricRole = "learning"
)

// Direction says which way is better for a metric.
type Direction string

const (
	Lower     Direction = "lower"
	Higher    Direction = "higher"
	Unchanged Direction = "unchanged"
)

// MetricSpec declares how one metric is judged.
type MetricSpec struct {
	Name      string     `json:"name"`
	Role      MetricRole `json:"role"`
	Direction Direction  `json:"direction"`
	// Target is the value a primary metric must reach.
	Target *float64 `json:"target,omitempty"`
	// Threshold is the limit a guardrail must not cross.
	Threshold *float64 `json:"threshold,omitempty"`
}

// Window is a measurement period.
//
// Retaining it is not bookkeeping: Blueprint section 18 requires the evaluator
// to keep the baseline and the window, because a comparison whose period is
// unknown cannot be reproduced or challenged.
type Window struct {
	Start time.Time  `json:"start"`
	End   *time.Time `json:"end,omitempty"`
}

// Observation is a metric measured over one revision.
type Observation struct {
	Metric   string              `json:"metric"`
	Revision contract.RevisionID `json:"revision"`
	Value    float64             `json:"value"`
	Samples  int                 `json:"samples"`
}

// MetricResult is one metric compared between control and candidate.
type MetricResult struct {
	Name      string     `json:"name"`
	Role      MetricRole `json:"role"`
	Direction Direction  `json:"direction"`

	Baseline  float64  `json:"baseline"`
	Current   float64  `json:"current"`
	Delta     float64  `json:"delta"`
	Target    *float64 `json:"target,omitempty"`
	Threshold *float64 `json:"threshold,omitempty"`

	// TargetMet applies to primary metrics only.
	TargetMet *bool `json:"target_met,omitempty"`
	// GuardrailBreached applies to guardrails only.
	GuardrailBreached *bool `json:"guardrail_breached,omitempty"`

	BaselineSamples int `json:"baseline_samples"`
	CurrentSamples  int `json:"current_samples"`
	// Underpowered marks a comparison with too little data to lean on.
	Underpowered bool `json:"underpowered"`
}

// Verdict is the evaluator's overall reading.
type Verdict string

const (
	// VerdictSucceeded: every primary target met, no guardrail breached.
	VerdictSucceeded Verdict = "SUCCEEDED"
	// VerdictFailed: a primary target was missed without a guardrail breach.
	VerdictFailed Verdict = "FAILED"
	// VerdictGuardrailBreached: a guardrail regressed, whatever else happened.
	VerdictGuardrailBreached Verdict = "GUARDRAIL_BREACHED"
	// VerdictInconclusive: not enough evidence to say either way.
	VerdictInconclusive Verdict = "INCONCLUSIVE"
)

// Evaluation is a complete fitness comparison.
type Evaluation struct {
	Experiment contract.ExperimentID `json:"experiment,omitempty"`
	Control    contract.RevisionID   `json:"control"`
	Candidate  contract.RevisionID   `json:"candidate"`
	Window     Window                `json:"window"`

	Verdict Verdict        `json:"verdict"`
	Reasons []string       `json:"reasons,omitempty"`
	Metrics []MetricResult `json:"metrics"`
}

// PrimaryResults returns just the primary metrics.
func (e Evaluation) PrimaryResults() []MetricResult { return e.byRole(RolePrimary) }

// GuardrailResults returns just the guardrails.
func (e Evaluation) GuardrailResults() []MetricResult { return e.byRole(RoleGuardrail) }

func (e Evaluation) byRole(role MetricRole) []MetricResult {
	var out []MetricResult
	for _, m := range e.Metrics {
		if m.Role == role {
			out = append(out, m)
		}
	}
	return out
}

// Evaluator compares a candidate revision against its control.
type Evaluator struct {
	// MinSamples is the per-side sample count below which a comparison is
	// marked underpowered. A difference measured on three requests is noise
	// wearing a result's clothes.
	MinSamples int
}

// NewEvaluator returns an evaluator with a workable default.
func NewEvaluator() *Evaluator { return &Evaluator{MinSamples: 30} }

// Evaluate compares observations against the declared specs.
//
// Specs are an input, not something derived from the data. Blueprint section 18
// says success criteria must not be changed after results are visible without
// recording the amendment, and an evaluator that inferred its own criteria
// would make that impossible to enforce.
func (e *Evaluator) Evaluate(
	control, candidate contract.RevisionID,
	window Window,
	specs []MetricSpec,
	observations []Observation,
) Evaluation {
	byMetric := map[string]map[contract.RevisionID]Observation{}
	for _, o := range observations {
		if byMetric[o.Metric] == nil {
			byMetric[o.Metric] = map[contract.RevisionID]Observation{}
		}
		byMetric[o.Metric][o.Revision] = o
	}

	eval := Evaluation{Control: control, Candidate: candidate, Window: window}

	ordered := make([]MetricSpec, len(specs))
	copy(ordered, specs)
	sort.SliceStable(ordered, func(i, j int) bool {
		if ordered[i].Role != ordered[j].Role {
			return roleRank(ordered[i].Role) < roleRank(ordered[j].Role)
		}
		return ordered[i].Name < ordered[j].Name
	})

	var (
		missingPrimary  []string
		breached        []string
		underpowered    []string
		primaryCount    int
		primaryMetCount int
		haveAnyPrimary  bool
	)

	for _, spec := range ordered {
		base, hasBase := byMetric[spec.Name][control]
		cur, hasCur := byMetric[spec.Name][candidate]

		if !hasBase || !hasCur {
			// A metric with no measurement on one side is reported as absent
			// rather than defaulted to zero, which would read as a dramatic
			// improvement or regression that never happened.
			eval.Metrics = append(eval.Metrics, MetricResult{
				Name:         spec.Name,
				Role:         spec.Role,
				Direction:    spec.Direction,
				Target:       spec.Target,
				Threshold:    spec.Threshold,
				Underpowered: true,
			})
			if spec.Role == RolePrimary {
				primaryCount++
				underpowered = append(underpowered,
					fmt.Sprintf("%s has no measurement on both sides", spec.Name))
			}
			continue
		}

		r := MetricResult{
			Name:            spec.Name,
			Role:            spec.Role,
			Direction:       spec.Direction,
			Baseline:        base.Value,
			Current:         cur.Value,
			Delta:           round4(cur.Value - base.Value),
			Target:          spec.Target,
			Threshold:       spec.Threshold,
			BaselineSamples: base.Samples,
			CurrentSamples:  cur.Samples,
		}

		if base.Samples < e.MinSamples || cur.Samples < e.MinSamples {
			r.Underpowered = true
			underpowered = append(underpowered, fmt.Sprintf(
				"%s has %d control and %d candidate samples, below the %d needed",
				spec.Name, base.Samples, cur.Samples, e.MinSamples))
		}

		switch spec.Role {
		case RolePrimary:
			primaryCount++
			haveAnyPrimary = true
			met := meetsTarget(spec, cur.Value)
			r.TargetMet = &met
			if met {
				primaryMetCount++
			} else {
				missingPrimary = append(missingPrimary, describeMiss(spec, cur.Value))
			}

		case RoleGuardrail:
			crossed := breachesGuardrail(spec, cur.Value)
			r.GuardrailBreached = &crossed
			if crossed {
				breached = append(breached, describeBreach(spec, cur.Value))
			}
		}

		eval.Metrics = append(eval.Metrics, r)
	}

	// Guardrails dominate. A candidate that hit every target while regressing a
	// guardrail has not succeeded; it has traded something it was told not to.
	switch {
	case len(breached) > 0:
		eval.Verdict = VerdictGuardrailBreached
		eval.Reasons = breached

	case !haveAnyPrimary || primaryCount == 0:
		eval.Verdict = VerdictInconclusive
		eval.Reasons = []string{"no primary metric was declared, so there is nothing to conclude"}

	case len(underpowered) > 0 && len(missingPrimary) == 0:
		// Every target appears met, but on too little data to act on. Reporting
		// success here is how an experiment gets promoted on noise.
		eval.Verdict = VerdictInconclusive
		eval.Reasons = underpowered

	case len(missingPrimary) > 0:
		eval.Verdict = VerdictFailed
		eval.Reasons = missingPrimary

	default:
		eval.Verdict = VerdictSucceeded
		eval.Reasons = []string{fmt.Sprintf("%d of %d primary targets met with no guardrail breach",
			primaryMetCount, primaryCount)}
	}

	sort.Strings(eval.Reasons)
	return eval
}

func meetsTarget(spec MetricSpec, value float64) bool {
	if spec.Target == nil {
		// A primary metric with no target cannot be judged, so it is not met.
		return false
	}
	switch spec.Direction {
	case Lower:
		return value <= *spec.Target
	case Higher:
		return value >= *spec.Target
	case Unchanged:
		return math.Abs(value-*spec.Target) < 1e-9
	}
	return false
}

func breachesGuardrail(spec MetricSpec, value float64) bool {
	if spec.Threshold == nil {
		return false
	}
	switch spec.Direction {
	case Lower:
		// Lower is better, so exceeding the threshold is the breach.
		return value > *spec.Threshold
	case Higher:
		return value < *spec.Threshold
	case Unchanged:
		return math.Abs(value-*spec.Threshold) > 1e-9
	}
	return false
}

func describeMiss(spec MetricSpec, value float64) string {
	if spec.Target == nil {
		return fmt.Sprintf("%s is a primary metric with no declared target", spec.Name)
	}
	return fmt.Sprintf("%s reached %.4g, target was %s %.4g",
		spec.Name, value, comparator(spec.Direction), *spec.Target)
}

func describeBreach(spec MetricSpec, value float64) string {
	return fmt.Sprintf("guardrail %s at %.4g breached its threshold of %.4g",
		spec.Name, value, *spec.Threshold)
}

func comparator(d Direction) string {
	switch d {
	case Lower:
		return "at most"
	case Higher:
		return "at least"
	}
	return "exactly"
}

func roleRank(r MetricRole) int {
	switch r {
	case RolePrimary:
		return 0
	case RoleGuardrail:
		return 1
	case RoleSecondary:
		return 2
	}
	return 3
}

func round4(v float64) float64 { return math.Round(v*10000) / 10000 }