fluid-core/conformance/suite/containment_test.go

package suite

import (
	"context"
	"errors"
	"net/http"
	"testing"
	"time"

	"github.com/tegwick/fluid-core/internal/contract"
	"github.com/tegwick/fluid-core/internal/publish"
	"github.com/tegwick/fluid-core/internal/runtime"
)

// ---------------------------------------------------------------------------
// ArchitectureBlueprint.md section 34 — failure containment
//
// FLUID must be designed so evolutionary failure does not imply runtime
// failure. Each subsection names an effect and a runtime effect; the runtime
// effect is almost always "none", and these tests are what keep that true.
// ---------------------------------------------------------------------------

// 34.1 Daimon failure: no new hypotheses or adaptations. Runtime effect: none.
//
// There is no Daimon yet, so the closest analogue is the whole science layer
// being unable to record anything.
func TestContainment_ScienceLayerFailure(t *testing.T) {
	h := New(t)
	ctx := context.Background()

	h.Drain()
	if err := h.Store.Close(); err != nil {
		t.Fatal(err)
	}

	// Adaptation is now impossible.
	if _, err := h.Hypotheses.List(ctx, ""); err == nil {
		t.Log("note: hypothesis listing still succeeded; the store may be cached")
	}

	// The runtime is unaffected.
	for i := 0; i < 10; i++ {
		mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
			"serving with the science layer dead")
	}
}

// 34.2 Telemetry pipeline failure: reduced learning. Runtime effect: none.
//
// Telemetry backpressure MUST NOT block normal API requests.
func TestContainment_TelemetryBackpressure(t *testing.T) {
	stalled := &stallingSink{}
	h := New(t)

	// Replace the emitter with one writing into a sink that never completes.
	h.Emitter.Close()
	h.Emitter = runtime.NewEmitter(stalled, runtime.EmitterOptions{
		Buffer: 4, Workers: 1, WriteTimeout: time.Hour,
	})

	gw, err := runtime.NewGateway(runtime.GatewayOptions{
		Interface: Interface,
		Registry:  h.Registry,
		Resolver:  runtime.NewResolver(h.Registry, true),
		Connector: runtime.NewConnector(),
		Emitter:   h.Emitter,
		Cohorts:   runtime.StaticCohort("agents"),
		Validator: h.Validator,
	})
	if err != nil {
		t.Fatal(err)
	}
	h.Gateway = gw

	// Far more requests than the buffer can hold, against a sink that never
	// drains. Latency must stay flat.
	start := time.Now()
	for i := 0; i < 200; i++ {
		mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
			"serving under telemetry backpressure")
	}
	elapsed := time.Since(start)

	if elapsed > 10*time.Second {
		t.Fatalf("200 requests took %s under telemetry backpressure; the request path is coupled to the sink", elapsed)
	}
	if h.Emitter.Stats().Dropped == 0 {
		t.Error("expected telemetry to be dropped rather than queued without bound")
	}
}

// 34.3 Builder failure: candidate not produced. Runtime effect: none.
func TestContainment_BuilderFailure(t *testing.T) {
	h := New(t)

	// A candidate that fails verification is never published, and the running
	// interface does not notice.
	bad := contract.Revision{
		SchemaVersion: "0.1", ID: "R-bad", Interface: Interface,
		State:    contract.RevisionStateStable,
		Contract: contract.RevisionContract{Type: contract.RevisionContractTypeOpenapi, Digest: contract.Digest("sha256:" + repeat("3", 64))},
		Runtime:  contract.RevisionRuntime{Upstream: h.AdapterURL("R-1")},
		Intent:   contract.RevisionIntent{Version: "IEI-1"},
		Policy: contract.RevisionPolicy{
			Compatibility: contract.RevisionPolicyCompatibilityBreaking,
			SecurityCheck: contract.RevisionPolicySecurityCheckFailed,
		},
	}
	if _, _, err := h.Pipeline.Run(context.Background(),
		publish.NewCandidate(bad, Operator), publish.PromotionRequest{
			AdaptationClasses:     []contract.AdaptationClass{contract.AdaptationClassPresentation},
			RequestedTrafficShare: 0.1,
			Approved:              true,
			ApprovedBy:            &Operator,
		}); err == nil {
		t.Fatal("a candidate failing security passed verification")
	}

	for i := 0; i < 5; i++ {
		mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
			"serving after a failed build")
	}
}

// 34.5 Experiment failure: candidate traffic removed. Runtime response: route
// to a known-good revision.
func TestContainment_ExperimentFailure(t *testing.T) {
	h := New(t)
	ctx := context.Background()

	loop := SeedHypothesis(t, h)
	exp := DesignExperiment(t, h, loop.Hypothesis)

	_, startPolicy, err := h.Experiments.Start(ctx, exp, 2, "R-1", Operator)
	if err != nil {
		t.Fatal(err)
	}
	if err := h.Registry.PutPolicy(startPolicy); err != nil {
		t.Fatal(err)
	}

	// The candidate's adapter dies mid-experiment.
	h.StopAdapter("R-2")

	// Stopping the experiment restores service for everyone, including
	// consumers that had been allocated to the broken arm.
	_, stopPolicy, err := h.Experiments.Stop(ctx, exp, 3, "R-1", Operator, "candidate adapter unreachable")
	if err != nil {
		t.Fatal(err)
	}
	if err := h.Registry.PutPolicy(stopPolicy); err != nil {
		t.Fatal(err)
	}

	for i := 0; i < 20; i++ {
		rec := h.Call(consumerName(i), http.MethodGet, "/v1/entries")
		mustStatus(t, rec, http.StatusOK, "serving after experiment rollback")
		if got := rec.Header().Get("X-FLUID-Revision"); got != "R-1" {
			t.Fatalf("consumer %d still routed to %q after rollback", i, got)
		}
	}
}

// 34.6 Evolution store failure: runtime MUST continue using cached published
// revision configuration.
func TestContainment_EvidenceStoreFailure(t *testing.T) {
	h := New(t)

	h.Drain()
	if err := h.Store.Close(); err != nil {
		t.Fatal(err)
	}

	// Reads and writes to the store now fail.
	if _, err := h.Store.Records(context.Background(), contract.KindRevision); err == nil {
		t.Error("the evidence store still answers after being closed")
	}

	// The registry holds its own cached view, so routing is unaffected.
	for i := 0; i < 10; i++ {
		rec := h.Call("c-1", http.MethodGet, "/v1/entries")
		mustStatus(t, rec, http.StatusOK, "serving from cached configuration")
		if rec.Header().Get("X-FLUID-Revision") != "R-1" {
			t.Fatal("routing changed after the evidence store died")
		}
	}
}

// 34.7 AI budget exhaustion: adaptive processes pause. Runtime effect: none.
//
// Covered by TestInvariant16_AdaptiveBudgetDoesNotAffectRuntime; this asserts
// the complementary direction, that a saturated adaptive layer cannot make the
// data plane refuse work.
func TestContainment_AdaptiveSaturation(t *testing.T) {
	h := New(t)
	ctx := context.Background()

	h.Experiments.SetMaxParallel(1)
	loop := SeedHypothesis(t, h)
	first := DesignExperiment(t, h, loop.Hypothesis)
	if _, _, err := h.Experiments.Start(ctx, first, 2, "R-1", Operator); err != nil {
		t.Fatal(err)
	}

	second := DesignExperimentNamed(t, h, loop.Hypothesis, "E-blocked")
	_, _, err := h.Experiments.Start(ctx, second, 3, "R-1", Operator)
	if err == nil {
		t.Fatal("the adaptive limit did not bind")
	}

	for i := 0; i < 10; i++ {
		mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
			"serving with adaptation saturated")
	}
	_ = errors.Is
}

// stallingSink never returns, standing in for a wedged evidence store.
type stallingSink struct{}

func (s *stallingSink) Write(ctx context.Context, _ contract.FluidTelemetry) error {
	<-ctx.Done()
	return ctx.Err()
}

func repeat(s string, n int) string {
	out := make([]byte, 0, n*len(s))
	for i := 0; i < n; i++ {
		out = append(out, s...)
	}
	return string(out)
}

func consumerName(i int) string {
	return "consumer-" + string(rune('a'+i%26))
}