fluid-core/internal/science/science_test.go

package science

import (
	"context"
	"errors"
	"path/filepath"
	"strings"
	"testing"

	_ "modernc.org/sqlite"

	"github.com/tegwick/fluid-core/internal/contract"
	"github.com/tegwick/fluid-core/internal/evidence"
)

const iface contract.InterfaceID = "hall-publishing"

var operator = contract.Actor{Type: contract.ActorTypeHuman, ID: "worsch"}

func ptr[T any](v T) *T { return &v }

func newStores(t *testing.T) (*HypothesisStore, *ExperimentController, *evidence.SQLStore) {
	t.Helper()
	store, err := evidence.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "e.db"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = store.Close() })

	hs := NewHypothesisStore(store, iface)
	return hs, NewExperimentController(store, hs, iface), store
}

// complete returns a hypothesis with everything READY requires.
func complete(id contract.HypothesisID, title string) contract.FluidHypothesis {
	return contract.FluidHypothesis{
		ID:    id,
		Title: title,
		Observation: contract.FluidHypothesisObservation{
			Summary:      "consumers list the collection repeatedly to find one entry",
			EvidenceRefs: []contract.EvidenceRef{"topology:GET /entries"},
		},
		Pressure: contract.FluidHypothesisPressure{
			Classes: []contract.PressureClass{contract.PressureClassSuccessfulButInefficientUsage},
		},
		Explanation: contract.FluidHypothesisExplanation{
			Claim: "latest entry is a first-class consumer concept the interface does not expose",
		},
		ProposedAdaptation: contract.FluidHypothesisProposedAdaptation{
			Class:   contract.AdaptationClassContract,
			Summary: "add an explicit latest-entry capability",
		},
		ExpectedOutcomes: []contract.ExpectedOutcome{
			{Metric: "requests_per_completed_task", Target: 1.2, Direction: contract.ExpectedOutcomeDirectionLower},
		},
		SuccessCriteria: contract.FluidHypothesisSuccessCriteria{
			Expression: "requests_per_completed_task <= 1.2 AND no guardrail violation",
		},
		Complexity: contract.FluidHypothesisComplexity{
			ExpectedDelta: contract.ComplexityDelta{OperationCount: ptr(1.0)},
		},
		Risk: contract.FluidHypothesisRisk{Level: contract.FluidHypothesisRiskLevelLOW},
	}
}

func TestDraftMayBeIncompleteButReadyMayNot(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	// Drafting stays cheap: a half-formed idea is allowed to be half-formed.
	sparse := contract.FluidHypothesis{ID: "H-1", Title: "maybe the list endpoint is wrong"}
	if _, err := hs.Create(ctx, sparse, operator); err != nil {
		t.Fatalf("an incomplete draft was refused: %v", err)
	}

	_, err := hs.Transition(ctx, "H-1", contract.FluidHypothesisStateREADY, operator, "let us look at this")
	if !errors.Is(err, ErrIncomplete) {
		t.Fatalf("an incomplete hypothesis reached READY: %v", err)
	}
	// The error must say what is missing, or it is not actionable.
	for _, want := range []string{"evidence_refs", "expected_outcomes", "success_criteria"} {
		if !strings.Contains(err.Error(), want) {
			t.Errorf("error does not name missing %s: %v", want, err)
		}
	}
}

// TestCannotSkipToAccepted: a hypothesis jumping straight to ACCEPTED would
// claim evidence it never gathered.
func TestCannotSkipToAccepted(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	if _, err := hs.Create(ctx, complete("H-1", "latest entry"), operator); err != nil {
		t.Fatal(err)
	}

	_, err := hs.Transition(ctx, "H-1", contract.FluidHypothesisStateACCEPTED, operator, "looks right to me")
	if !errors.Is(err, ErrInvalidTransition) {
		t.Errorf("DRAFT jumped to ACCEPTED: %v", err)
	}
}

func TestLifecycleHappyPath(t *testing.T) {
	ctx := context.Background()
	hs, _, store := newStores(t)

	if _, err := hs.Create(ctx, complete("H-1", "latest entry"), operator); err != nil {
		t.Fatal(err)
	}
	for _, to := range []State{
		contract.FluidHypothesisStateREADY,
		contract.FluidHypothesisStatePRIORITIZED,
		contract.FluidHypothesisStateDESIGNING,
		contract.FluidHypothesisStateEXPERIMENTING,
		contract.FluidHypothesisStateEVALUATING,
	} {
		if _, err := hs.Transition(ctx, "H-1", to, operator, "advancing"); err != nil {
			t.Fatalf("transition to %s failed: %v", to, err)
		}
	}

	got, err := hs.RecordOutcome(ctx, "H-1", contract.FluidHypothesisOutcomeStatusCONFIRMED,
		"requests per task fell from 2.7 to 1.15", []contract.EvidenceRef{"metrics:E-1"}, operator)
	if err != nil {
		t.Fatal(err)
	}
	if got.State != contract.FluidHypothesisStateACCEPTED {
		t.Errorf("state = %s, want ACCEPTED", got.State)
	}

	events, err := store.Events(ctx, evidence.EventFilter{EntityID: "H-1"})
	if err != nil {
		t.Fatal(err)
	}
	if len(events) < 7 {
		t.Errorf("lifecycle left %d events; every transition should be observable", len(events))
	}
}

// TestOutcomeOnlyWhileEvaluating: recording a result while still running would
// assume what the experiment set out to find.
func TestOutcomeOnlyWhileEvaluating(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	if _, err := hs.Create(ctx, complete("H-1", "latest entry"), operator); err != nil {
		t.Fatal(err)
	}
	if _, err := hs.RecordOutcome(ctx, "H-1", contract.FluidHypothesisOutcomeStatusCONFIRMED,
		"it worked", nil, operator); err == nil {
		t.Error("an outcome was recorded on a DRAFT hypothesis")
	}
}

// TestInconclusiveDefersRatherThanRejects: inconclusive is not refuted, and
// burying a possibly-good idea is the wrong default.
func TestInconclusiveDefersRatherThanRejects(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	if _, err := hs.Create(ctx, complete("H-1", "latest entry"), operator); err != nil {
		t.Fatal(err)
	}
	advance(t, hs, "H-1", contract.FluidHypothesisStateEVALUATING)

	got, err := hs.RecordOutcome(ctx, "H-1", contract.FluidHypothesisOutcomeStatusINCONCLUSIVE,
		"only 12 samples per arm", nil, operator)
	if err != nil {
		t.Fatal(err)
	}
	if got.State != contract.FluidHypothesisStateDEFERRED {
		t.Errorf("state = %s, want DEFERRED", got.State)
	}
}

func TestCompetitionIsSymmetric(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	for _, id := range []contract.HypothesisID{"H-1", "H-2", "H-3"} {
		if _, err := hs.Create(ctx, complete(id, string(id)), operator); err != nil {
			t.Fatal(err)
		}
	}

	group, err := hs.Compete(ctx, "CG-1", []contract.HypothesisID{"H-1", "H-2", "H-3"}, operator)
	if err != nil {
		t.Fatal(err)
	}
	if len(group.Members) != 3 {
		t.Fatalf("group has %d members", len(group.Members))
	}

	// Every member must know about every rival, or a reader looking at one
	// would miss that alternatives exist.
	for _, id := range group.Members {
		h, err := hs.Get(ctx, id)
		if err != nil {
			t.Fatal(err)
		}
		if h.Competition == nil || len(h.Competition.Alternatives) != 2 {
			t.Errorf("%s does not list both rivals: %+v", id, h.Competition)
		}
	}

	if _, err := hs.Compete(ctx, "CG-2", []contract.HypothesisID{"H-1"}, operator); err == nil {
		t.Error("a competition group of one was accepted")
	}
}

// TestResolveSupersedesRatherThanRejects: rejection says the explanation was
// wrong; superseded says a better one won.
func TestResolveSupersedesRatherThanRejects(t *testing.T) {
	ctx := context.Background()
	hs, _, _ := newStores(t)

	for _, id := range []contract.HypothesisID{"H-1", "H-2"} {
		if _, err := hs.Create(ctx, complete(id, string(id)), operator); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := hs.Compete(ctx, "CG-1", []contract.HypothesisID{"H-1", "H-2"}, operator); err != nil {
		t.Fatal(err)
	}

	advance(t, hs, "H-1", contract.FluidHypothesisStateEVALUATING)
	if _, err := hs.RecordOutcome(ctx, "H-1", contract.FluidHypothesisOutcomeStatusCONFIRMED,
		"teaser plus link won on read-through", nil, operator); err != nil {
		t.Fatal(err)
	}

	group, err := hs.Resolve(ctx, "CG-1", "H-1", operator, "H-1 measured better on the primary metric")
	if err != nil {
		t.Fatal(err)
	}
	if !group.Resolved || group.Preferred != "H-1" {
		t.Errorf("group not resolved to H-1: %+v", group)
	}

	loser, err := hs.Get(ctx, "H-2")
	if err != nil {
		t.Fatal(err)
	}
	if loser.State != contract.FluidHypothesisStateSUPERSEDED {
		t.Errorf("loser state = %s, want SUPERSEDED", loser.State)
	}

	if _, err := hs.Resolve(ctx, "CG-1", "H-99", operator, "reason"); !errors.Is(err, ErrNotCompeting) {
		t.Errorf("a non-member won a competition: %v", err)
	}
}

func advance(t *testing.T, hs *HypothesisStore, id contract.HypothesisID, to State) {
	t.Helper()
	ctx := context.Background()
	path := []State{
		contract.FluidHypothesisStateREADY,
		contract.FluidHypothesisStatePRIORITIZED,
		contract.FluidHypothesisStateDESIGNING,
		contract.FluidHypothesisStateEXPERIMENTING,
		contract.FluidHypothesisStateEVALUATING,
	}
	for _, s := range path {
		if _, err := hs.Transition(ctx, id, s, operator, "advancing"); err != nil {
			t.Fatalf("advance to %s: %v", s, err)
		}
		if s == to {
			return
		}
	}
}