Add the conformance suite, echo fixture and integration guide
Some checks failed
ci / build (push) Has been cancelled
Some checks failed
ci / build (push) Has been cancelled
Completes FLUID-WP-0007. The seven minimal-conformance requirements and the mechanically checkable architectural invariants are asserted as tests rather than claimed in a README, because a conformance claim nobody re-checks is one that quietly stops being true. Only the checkable subset of the invariants is asserted; pretending a test can settle the rest would be worse than leaving them to review. TestFirstVerticalSlice runs all eleven steps of Blueprint 50 with no human steps: two revisions, explicit routing, telemetry, a cohort dimension, detected pressure, a hypothesis, a candidate, a 90/10 experiment, fitness comparison, promotion, and a complete audit trail. Requests per completed task fall from 5.65 to 1.00 against a 1.20 target. A companion test runs the loop twice and requires the same verdict, since a loop whose conclusion depended on run order would be measuring the harness rather than the interface. The failure-containment matrix covers Blueprint 34 directly: the data plane keeps serving with the evidence store closed, with telemetry wedged against a sink that never returns, after a failed build, after an experiment rollback, and with the adaptive concurrency limit saturated. Fixes a real bug the suite exposed. Drain closed the emitter outright, so every request after the first flush emitted into a dead emitter and was silently lost -- the kind of fault that makes a later measurement quietly wrong rather than loudly broken. Emitter.Flush now waits for delivery without stopping it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014KmVxhJ35tCo7rE7UnLwWu Assistant: claude-code Assistant-Model: opus Assistant-Process: 1116572@bnt-lap001 Assistant-Session: 8ba9bb93-a72a-4883-b189-2499cce5c400
This commit is contained in:
parent
61d8d8cabe
commit
55363905bc
16 changed files with 1885 additions and 23 deletions
236
conformance/suite/containment_test.go
Normal file
236
conformance/suite/containment_test.go
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
package suite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tegwick/fluid-core/internal/contract"
|
||||
"github.com/tegwick/fluid-core/internal/publish"
|
||||
"github.com/tegwick/fluid-core/internal/runtime"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ArchitectureBlueprint.md section 34 — failure containment
|
||||
//
|
||||
// FLUID must be designed so evolutionary failure does not imply runtime
|
||||
// failure. Each subsection names an effect and a runtime effect; the runtime
|
||||
// effect is almost always "none", and these tests are what keep that true.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// 34.1 Daimon failure: no new hypotheses or adaptations. Runtime effect: none.
|
||||
//
|
||||
// There is no Daimon yet, so the closest analogue is the whole science layer
|
||||
// being unable to record anything.
|
||||
func TestContainment_ScienceLayerFailure(t *testing.T) {
|
||||
h := New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
h.Drain()
|
||||
if err := h.Store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Adaptation is now impossible.
|
||||
if _, err := h.Hypotheses.List(ctx, ""); err == nil {
|
||||
t.Log("note: hypothesis listing still succeeded; the store may be cached")
|
||||
}
|
||||
|
||||
// The runtime is unaffected.
|
||||
for i := 0; i < 10; i++ {
|
||||
mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
|
||||
"serving with the science layer dead")
|
||||
}
|
||||
}
|
||||
|
||||
// 34.2 Telemetry pipeline failure: reduced learning. Runtime effect: none.
|
||||
//
|
||||
// Telemetry backpressure MUST NOT block normal API requests.
|
||||
func TestContainment_TelemetryBackpressure(t *testing.T) {
|
||||
stalled := &stallingSink{}
|
||||
h := New(t)
|
||||
|
||||
// Replace the emitter with one writing into a sink that never completes.
|
||||
h.Emitter.Close()
|
||||
h.Emitter = runtime.NewEmitter(stalled, runtime.EmitterOptions{
|
||||
Buffer: 4, Workers: 1, WriteTimeout: time.Hour,
|
||||
})
|
||||
|
||||
gw, err := runtime.NewGateway(runtime.GatewayOptions{
|
||||
Interface: Interface,
|
||||
Registry: h.Registry,
|
||||
Resolver: runtime.NewResolver(h.Registry, true),
|
||||
Connector: runtime.NewConnector(),
|
||||
Emitter: h.Emitter,
|
||||
Cohorts: runtime.StaticCohort("agents"),
|
||||
Validator: h.Validator,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.Gateway = gw
|
||||
|
||||
// Far more requests than the buffer can hold, against a sink that never
|
||||
// drains. Latency must stay flat.
|
||||
start := time.Now()
|
||||
for i := 0; i < 200; i++ {
|
||||
mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
|
||||
"serving under telemetry backpressure")
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if elapsed > 10*time.Second {
|
||||
t.Fatalf("200 requests took %s under telemetry backpressure; the request path is coupled to the sink", elapsed)
|
||||
}
|
||||
if h.Emitter.Stats().Dropped == 0 {
|
||||
t.Error("expected telemetry to be dropped rather than queued without bound")
|
||||
}
|
||||
}
|
||||
|
||||
// 34.3 Builder failure: candidate not produced. Runtime effect: none.
|
||||
func TestContainment_BuilderFailure(t *testing.T) {
|
||||
h := New(t)
|
||||
|
||||
// A candidate that fails verification is never published, and the running
|
||||
// interface does not notice.
|
||||
bad := contract.Revision{
|
||||
SchemaVersion: "0.1", ID: "R-bad", Interface: Interface,
|
||||
State: contract.RevisionStateStable,
|
||||
Contract: contract.RevisionContract{Type: contract.RevisionContractTypeOpenapi, Digest: contract.Digest("sha256:" + repeat("3", 64))},
|
||||
Runtime: contract.RevisionRuntime{Upstream: h.AdapterURL("R-1")},
|
||||
Intent: contract.RevisionIntent{Version: "IEI-1"},
|
||||
Policy: contract.RevisionPolicy{
|
||||
Compatibility: contract.RevisionPolicyCompatibilityBreaking,
|
||||
SecurityCheck: contract.RevisionPolicySecurityCheckFailed,
|
||||
},
|
||||
}
|
||||
if _, _, err := h.Pipeline.Run(context.Background(),
|
||||
publish.NewCandidate(bad, Operator), publish.PromotionRequest{
|
||||
AdaptationClasses: []contract.AdaptationClass{contract.AdaptationClassPresentation},
|
||||
RequestedTrafficShare: 0.1,
|
||||
Approved: true,
|
||||
ApprovedBy: &Operator,
|
||||
}); err == nil {
|
||||
t.Fatal("a candidate failing security passed verification")
|
||||
}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
|
||||
"serving after a failed build")
|
||||
}
|
||||
}
|
||||
|
||||
// 34.5 Experiment failure: candidate traffic removed. Runtime response: route
|
||||
// to a known-good revision.
|
||||
func TestContainment_ExperimentFailure(t *testing.T) {
|
||||
h := New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
loop := SeedHypothesis(t, h)
|
||||
exp := DesignExperiment(t, h, loop.Hypothesis)
|
||||
|
||||
_, startPolicy, err := h.Experiments.Start(ctx, exp, 2, "R-1", Operator)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Registry.PutPolicy(startPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The candidate's adapter dies mid-experiment.
|
||||
h.StopAdapter("R-2")
|
||||
|
||||
// Stopping the experiment restores service for everyone, including
|
||||
// consumers that had been allocated to the broken arm.
|
||||
_, stopPolicy, err := h.Experiments.Stop(ctx, exp, 3, "R-1", Operator, "candidate adapter unreachable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Registry.PutPolicy(stopPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for i := 0; i < 20; i++ {
|
||||
rec := h.Call(consumerName(i), http.MethodGet, "/v1/entries")
|
||||
mustStatus(t, rec, http.StatusOK, "serving after experiment rollback")
|
||||
if got := rec.Header().Get("X-FLUID-Revision"); got != "R-1" {
|
||||
t.Fatalf("consumer %d still routed to %q after rollback", i, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 34.6 Evolution store failure: runtime MUST continue using cached published
|
||||
// revision configuration.
|
||||
func TestContainment_EvidenceStoreFailure(t *testing.T) {
|
||||
h := New(t)
|
||||
|
||||
h.Drain()
|
||||
if err := h.Store.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Reads and writes to the store now fail.
|
||||
if _, err := h.Store.Records(context.Background(), contract.KindRevision); err == nil {
|
||||
t.Error("the evidence store still answers after being closed")
|
||||
}
|
||||
|
||||
// The registry holds its own cached view, so routing is unaffected.
|
||||
for i := 0; i < 10; i++ {
|
||||
rec := h.Call("c-1", http.MethodGet, "/v1/entries")
|
||||
mustStatus(t, rec, http.StatusOK, "serving from cached configuration")
|
||||
if rec.Header().Get("X-FLUID-Revision") != "R-1" {
|
||||
t.Fatal("routing changed after the evidence store died")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 34.7 AI budget exhaustion: adaptive processes pause. Runtime effect: none.
|
||||
//
|
||||
// Covered by TestInvariant16_AdaptiveBudgetDoesNotAffectRuntime; this asserts
|
||||
// the complementary direction, that a saturated adaptive layer cannot make the
|
||||
// data plane refuse work.
|
||||
func TestContainment_AdaptiveSaturation(t *testing.T) {
|
||||
h := New(t)
|
||||
ctx := context.Background()
|
||||
|
||||
h.Experiments.SetMaxParallel(1)
|
||||
loop := SeedHypothesis(t, h)
|
||||
first := DesignExperiment(t, h, loop.Hypothesis)
|
||||
if _, _, err := h.Experiments.Start(ctx, first, 2, "R-1", Operator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
second := DesignExperimentNamed(t, h, loop.Hypothesis, "E-blocked")
|
||||
_, _, err := h.Experiments.Start(ctx, second, 3, "R-1", Operator)
|
||||
if err == nil {
|
||||
t.Fatal("the adaptive limit did not bind")
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
mustStatus(t, h.Call("c-1", http.MethodGet, "/v1/entries"), http.StatusOK,
|
||||
"serving with adaptation saturated")
|
||||
}
|
||||
_ = errors.Is
|
||||
}
|
||||
|
||||
// stallingSink never returns, standing in for a wedged evidence store.
|
||||
type stallingSink struct{}
|
||||
|
||||
func (s *stallingSink) Write(ctx context.Context, _ contract.FluidTelemetry) error {
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func repeat(s string, n int) string {
|
||||
out := make([]byte, 0, n*len(s))
|
||||
for i := 0; i < n; i++ {
|
||||
out = append(out, s...)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
func consumerName(i int) string {
|
||||
return "consumer-" + string(rune('a'+i%26))
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue