package suite import ( "context" "errors" "net/http" "testing" "time" "github.com/tegwick/fluid-core/internal/contract" "github.com/tegwick/fluid-core/internal/publish" "github.com/tegwick/fluid-core/internal/runtime" ) // --------------------------------------------------------------------------- // ArchitectureBlueprint.md section 34 — failure containment // // FLUID must be designed so evolutionary failure does not imply runtime // failure. Each subsection names an effect and a runtime effect; the runtime // effect is almost always "none", and these tests are what keep that true. // --------------------------------------------------------------------------- // 34.1 Daimon failure: no new hypotheses or adaptations. Runtime effect: none. // // There is no Daimon yet, so the closest analogue is the whole science layer // being unable to record anything. func TestContainment_ScienceLayerFailure(t *testing.T) { h := New(t) ctx := context.Background() h.Drain() if err := h.Store.Close(); err != nil { t.Fatal(err) } // Adaptation is now impossible. if _, err := h.Hypotheses.List(ctx, ""); err == nil { t.Log("note: hypothesis listing still succeeded; the store may be cached") } // The runtime is unaffected. for i := 0; i < 10; i++ { mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK, "serving with the science layer dead") } } // 34.2 Telemetry pipeline failure: reduced learning. Runtime effect: none. // // Telemetry backpressure MUST NOT block normal API requests. func TestContainment_TelemetryBackpressure(t *testing.T) { stalled := &stallingSink{} h := New(t) // Replace the emitter with one writing into a sink that never completes. h.Emitter.Close() h.Emitter = runtime.NewEmitter(stalled, runtime.EmitterOptions{ Buffer: 4, Workers: 1, WriteTimeout: time.Hour, }) gw, err := runtime.NewGateway(runtime.GatewayOptions{ Interface: Interface, Registry: h.Registry, Resolver: runtime.NewResolver(h.Registry, true), Connector: runtime.NewConnector(), Emitter: h.Emitter, Cohorts: runtime.StaticCohort("agents"), Validator: h.Validator, }) if err != nil { t.Fatal(err) } h.Gateway = gw // Far more requests than the buffer can hold, against a sink that never // drains. Latency must stay flat. start := time.Now() for i := 0; i < 200; i++ { mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK, "serving under telemetry backpressure") } elapsed := time.Since(start) if elapsed > 10*time.Second { t.Fatalf("200 requests took %s under telemetry backpressure; the request path is coupled to the sink", elapsed) } if h.Emitter.Stats().Dropped == 0 { t.Error("expected telemetry to be dropped rather than queued without bound") } } // 34.3 Builder failure: candidate not produced. Runtime effect: none. func TestContainment_BuilderFailure(t *testing.T) { h := New(t) // A candidate that fails verification is never published, and the running // interface does not notice. bad := contract.Revision{ SchemaVersion: "0.1", ID: "R-bad", Interface: Interface, State: contract.RevisionStateStable, Contract: contract.RevisionContract{Type: contract.RevisionContractTypeOpenapi, Digest: contract.Digest("sha256:" + repeat("3", 64))}, Runtime: contract.RevisionRuntime{Upstream: h.AdapterURL("R-1")}, Intent: contract.RevisionIntent{Version: "IEI-1"}, Policy: contract.RevisionPolicy{ Compatibility: contract.RevisionPolicyCompatibilityBreaking, SecurityCheck: contract.RevisionPolicySecurityCheckFailed, }, } if _, _, err := h.Pipeline.Run(context.Background(), publish.NewCandidate(bad, Operator), publish.PromotionRequest{ AdaptationClasses: []contract.AdaptationClass{contract.AdaptationClassPresentation}, RequestedTrafficShare: 0.1, Approved: true, ApprovedBy: &Operator, }); err == nil { t.Fatal("a candidate failing security passed verification") } for i := 0; i < 5; i++ { mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK, "serving after a failed build") } } // 34.5 Experiment failure: candidate traffic removed. Runtime response: route // to a known-good revision. func TestContainment_ExperimentFailure(t *testing.T) { h := New(t) ctx := context.Background() loop := SeedHypothesis(t, h) exp := DesignExperiment(t, h, loop.Hypothesis) _, startPolicy, err := h.Experiments.Start(ctx, exp, 2, "R-1", Operator) if err != nil { t.Fatal(err) } if err := h.Registry.PutPolicy(startPolicy); err != nil { t.Fatal(err) } // The candidate's adapter dies mid-experiment. h.StopAdapter("R-2") // Stopping the experiment restores service for everyone, including // consumers that had been allocated to the broken arm. _, stopPolicy, err := h.Experiments.Stop(ctx, exp, 3, "R-1", Operator, "candidate adapter unreachable") if err != nil { t.Fatal(err) } if err := h.Registry.PutPolicy(stopPolicy); err != nil { t.Fatal(err) } for i := 0; i < 20; i++ { rec := h.Call(consumerName(i), http.MethodGet, "/v1/entries") mustStatus(t, rec, http.StatusOK, "serving after experiment rollback") if got := rec.Header().Get("X-FLUID-Revision"); got != "R-1" { t.Fatalf("consumer %d still routed to %q after rollback", i, got) } } } // 34.6 Evolution store failure: runtime MUST continue using cached published // revision configuration. func TestContainment_EvidenceStoreFailure(t *testing.T) { h := New(t) h.Drain() if err := h.Store.Close(); err != nil { t.Fatal(err) } // Reads and writes to the store now fail. if _, err := h.Store.Records(context.Background(), contract.KindRevision); err == nil { t.Error("the evidence store still answers after being closed") } // The registry holds its own cached view, so routing is unaffected. for i := 0; i < 10; i++ { rec := h.Call("c-1", http.MethodGet, "/v1/entries") mustStatus(t, rec, http.StatusOK, "serving from cached configuration") if rec.Header().Get("X-FLUID-Revision") != "R-1" { t.Fatal("routing changed after the evidence store died") } } } // 34.7 AI budget exhaustion: adaptive processes pause. Runtime effect: none. // // Covered by TestInvariant16_AdaptiveBudgetDoesNotAffectRuntime; this asserts // the complementary direction, that a saturated adaptive layer cannot make the // data plane refuse work. func TestContainment_AdaptiveSaturation(t *testing.T) { h := New(t) ctx := context.Background() h.Experiments.SetMaxParallel(1) loop := SeedHypothesis(t, h) first := DesignExperiment(t, h, loop.Hypothesis) if _, _, err := h.Experiments.Start(ctx, first, 2, "R-1", Operator); err != nil { t.Fatal(err) } second := DesignExperimentNamed(t, h, loop.Hypothesis, "E-blocked") _, _, err := h.Experiments.Start(ctx, second, 3, "R-1", Operator) if err == nil { t.Fatal("the adaptive limit did not bind") } for i := 0; i < 10; i++ { mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK, "serving with adaptation saturated") } _ = errors.Is } // stallingSink never returns, standing in for a wedged evidence store. type stallingSink struct{} func (s *stallingSink) Write(ctx context.Context, _ contract.FluidTelemetry) error { <-ctx.Done() return ctx.Err() } func repeat(s string, n int) string { out := make([]byte, 0, n*len(s)) for i := 0; i < n; i++ { out = append(out, s...) } return string(out) } func consumerName(i int) string { return "consumer-" + string(rune('a'+i%26)) }