fluid-core/internal/science/experiment_test.go

package science

import (
	"context"
	"errors"
	"testing"

	"github.com/tegwick/fluid-core/internal/contract"
)

func plannedExperiment(id contract.ExperimentID) contract.FluidExperiment {
	return contract.FluidExperiment{
		ID:                 id,
		HypothesisRefs:     []contract.HypothesisID{"H-1"},
		ControlRevision:    "R-1",
		CandidateRevisions: []contract.RevisionID{"R-2"},
		Cohorts:            []contract.CohortID{"coding-agents"},
		Allocation:         map[string]contract.UnitInterval{"control": 0.9, "candidate": 0.1},
		Metrics: contract.FluidExperimentMetrics{
			Primary:    []string{"requests_per_completed_task"},
			Guardrails: []string{"p95_latency_ms", "error_rate"},
		},
		StopConditions: []string{"hard_guardrail_violation", "manual_stop"},
	}
}

func withHypothesis(t *testing.T, hs *HypothesisStore) {
	t.Helper()
	ctx := context.Background()
	if _, err := hs.Create(ctx, complete("H-1", "latest entry"), operator); err != nil {
		t.Fatal(err)
	}
	for _, s := range []State{
		contract.FluidHypothesisStateREADY,
		contract.FluidHypothesisStatePRIORITIZED,
		contract.FluidHypothesisStateDESIGNING,
	} {
		if _, err := hs.Transition(ctx, "H-1", s, operator, "advancing"); err != nil {
			t.Fatal(err)
		}
	}
}

func TestDesignValidatesTheBlueprintRequirements(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	for _, tc := range []struct {
		name   string
		mutate func(*contract.FluidExperiment)
	}{
		{"no hypothesis", func(e *contract.FluidExperiment) { e.HypothesisRefs = nil }},
		{"no stop condition", func(e *contract.FluidExperiment) { e.StopConditions = nil }},
		{"no primary metric", func(e *contract.FluidExperiment) { e.Metrics.Primary = nil }},
		{"no candidate", func(e *contract.FluidExperiment) { e.CandidateRevisions = nil }},
	} {
		e := plannedExperiment("E-1")
		tc.mutate(&e)
		if _, err := ec.Design(ctx, e, operator); err == nil {
			t.Errorf("%s: accepted", tc.name)
		}
	}
}

// TestAllocationMustSumToOne: shares summing to less would leave matching
// traffic falling through to the default, quietly contaminating the control.
func TestAllocationMustSumToOne(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	e := plannedExperiment("E-1")
	e.Allocation = map[string]contract.UnitInterval{"control": 0.5, "candidate": 0.1}
	if _, err := ec.Design(ctx, e, operator); err == nil {
		t.Error("an allocation summing to 0.6 was accepted")
	}

	e.Allocation = map[string]contract.UnitInterval{"control": 0.9, "candidate": 0.1}
	if _, err := ec.Design(ctx, e, operator); err != nil {
		t.Errorf("a valid allocation was refused: %v", err)
	}
}

// TestStartProducesRoutingPolicyRatherThanTouchingTraffic is the section 17
// separation: the controller writes policy the router consumes.
func TestStartProducesRoutingPolicy(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}

	e, policy, err := ec.Start(ctx, "E-1", 7, "R-1", operator)
	if err != nil {
		t.Fatal(err)
	}
	if e.Result.State != contract.FluidExperimentResultStateRUNNING {
		t.Errorf("state = %s", e.Result.State)
	}
	if policy.Generation != 7 || policy.DefaultRevision != "R-1" {
		t.Errorf("policy = %+v", policy)
	}
	if len(policy.Rules) != 1 {
		t.Fatalf("policy has %d rules, want 1", len(policy.Rules))
	}

	// Arm names must be resolved to revision ids: the router only understands
	// those, and a rule naming "candidate" would match nothing.
	rule := policy.Rules[0]
	if _, ok := rule.Allocation["R-1"]; !ok {
		t.Errorf("control arm not resolved to a revision id: %+v", rule.Allocation)
	}
	if got := rule.Allocation["R-2"]; got != 0.1 {
		t.Errorf("candidate share = %v, want 0.1", got)
	}
	if rule.Experiment == nil || *rule.Experiment != "E-1" {
		t.Error("rule does not name its experiment")
	}

	// The hypothesis follows its experiment.
	h, err := hs.Get(ctx, "H-1")
	if err != nil {
		t.Fatal(err)
	}
	if h.State != contract.FluidHypothesisStateEXPERIMENTING {
		t.Errorf("hypothesis state = %s, want EXPERIMENTING", h.State)
	}
}

// TestStopReturnsToAKnownGoodState: experiments must be interruptible, and
// stopping should be a document replacement, not an unwind.
func TestStopReturnsToAKnownGoodState(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-1", 1, "R-1", operator); err != nil {
		t.Fatal(err)
	}

	e, policy, err := ec.Stop(ctx, "E-1", 2, "R-1", operator, "p95 latency breached its guardrail")
	if err != nil {
		t.Fatal(err)
	}
	if e.Result.State != contract.FluidExperimentResultStateSTOPPED {
		t.Errorf("state = %s", e.Result.State)
	}
	if len(policy.Rules) != 0 {
		t.Errorf("the stop policy still carries %d rules", len(policy.Rules))
	}
	if policy.DefaultRevision != "R-1" || policy.Generation != 2 {
		t.Errorf("stop policy does not return traffic to the default: %+v", policy)
	}

	if _, _, err := ec.Stop(ctx, "E-1", 3, "R-1", operator, "again"); !errors.Is(err, ErrNotRunning) {
		t.Errorf("stopping a stopped experiment returned %v", err)
	}
}

func TestStopRequiresAReason(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-1", 1, "R-1", operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Stop(ctx, "E-1", 2, "R-1", operator, ""); err == nil {
		t.Error("an experiment was stopped with no reason recorded")
	}
}

// TestConcurrencyIsBounded: every running experiment splits the traffic the
// others are measuring.
func TestConcurrencyIsBounded(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)
	ec.SetMaxParallel(2)

	for _, id := range []contract.ExperimentID{"E-1", "E-2", "E-3"} {
		if _, err := ec.Design(ctx, plannedExperiment(id), operator); err != nil {
			t.Fatal(err)
		}
	}

	if _, _, err := ec.Start(ctx, "E-1", 1, "R-1", operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-2", 2, "R-1", operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-3", 3, "R-1", operator); !errors.Is(err, ErrTooManyExperiments) {
		t.Errorf("a third experiment started past the limit: %v", err)
	}

	// Stopping one frees a slot.
	if _, _, err := ec.Stop(ctx, "E-1", 4, "R-1", operator, "done"); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-3", 5, "R-1", operator); err != nil {
		t.Errorf("a slot did not free after stopping: %v", err)
	}
}

// TestAmendmentIsRecorded makes changing the terms distinguishable from
// quietly editing the record.
func TestAmendmentIsRecorded(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}

	if _, err := ec.Amend(ctx, "E-1", "", "no change given", operator); err == nil {
		t.Error("an empty amendment was accepted")
	}

	e, err := ec.Amend(ctx, "E-1", "extended the window by 14 days",
		"weekly publishing cadence gives too few samples in 7 days", operator)
	if err != nil {
		t.Fatal(err)
	}
	if len(e.Amendments) != 1 {
		t.Fatalf("amendments = %d, want 1", len(e.Amendments))
	}
	if e.Amendments[0].Actor.ID != operator.ID || e.Amendments[0].Reason == "" {
		t.Errorf("amendment does not record who and why: %+v", e.Amendments[0])
	}
}

func TestFinalizeMovesHypothesisToEvaluating(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Start(ctx, "E-1", 1, "R-1", operator); err != nil {
		t.Fatal(err)
	}
	if _, _, err := ec.Stop(ctx, "E-1", 2, "R-1", operator, "window elapsed"); err != nil {
		t.Fatal(err)
	}

	e, err := ec.Finalize(ctx, "E-1", "R-2", operator, "candidate met its target",
		[]contract.EvidenceRef{"metrics:E-1/window-1"})
	if err != nil {
		t.Fatal(err)
	}
	if e.Result.State != contract.FluidExperimentResultStateCOMPLETED {
		t.Errorf("state = %s", e.Result.State)
	}
	if e.Result.PreferredRevision == nil || *e.Result.PreferredRevision != "R-2" {
		t.Error("preferred revision not recorded")
	}

	h, err := hs.Get(ctx, "H-1")
	if err != nil {
		t.Fatal(err)
	}
	if h.State != contract.FluidHypothesisStateEVALUATING {
		t.Errorf("hypothesis state = %s, want EVALUATING", h.State)
	}
}

func TestFinalizeRefusesAnExperimentThatNeverRan(t *testing.T) {
	ctx := context.Background()
	hs, ec, _ := newStores(t)
	withHypothesis(t, hs)

	if _, err := ec.Design(ctx, plannedExperiment("E-1"), operator); err != nil {
		t.Fatal(err)
	}
	if _, err := ec.Finalize(ctx, "E-1", "R-2", operator, "it would have worked", nil); err == nil {
		t.Error("an experiment that never ran was finalized")
	}
}