Add the conformance suite, echo fixture and integration guide
Some checks failed
ci / build (push) Has been cancelled
Some checks failed
ci / build (push) Has been cancelled
Completes FLUID-WP-0007. The seven minimal-conformance requirements and the mechanically checkable architectural invariants are asserted as tests rather than claimed in a README, because a conformance claim nobody re-checks is one that quietly stops being true. Only the checkable subset of the invariants is asserted; pretending a test can settle the rest would be worse than leaving them to review. TestFirstVerticalSlice runs all eleven steps of Blueprint 50 with no human steps: two revisions, explicit routing, telemetry, a cohort dimension, detected pressure, a hypothesis, a candidate, a 90/10 experiment, fitness comparison, promotion, and a complete audit trail. Requests per completed task fall from 5.65 to 1.00 against a 1.20 target. A companion test runs the loop twice and requires the same verdict, since a loop whose conclusion depended on run order would be measuring the harness rather than the interface. The failure-containment matrix covers Blueprint 34 directly: the data plane keeps serving with the evidence store closed, with telemetry wedged against a sink that never returns, after a failed build, after an experiment rollback, and with the adaptive concurrency limit saturated. Fixes a real bug the suite exposed. Drain closed the emitter outright, so every request after the first flush emitted into a dead emitter and was silently lost -- the kind of fault that makes a later measurement quietly wrong rather than loudly broken. Emitter.Flush now waits for delivery without stopping it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014KmVxhJ35tCo7rE7UnLwWu Assistant: claude-code Assistant-Model: opus Assistant-Process: 1116572@bnt-lap001 Assistant-Session: 8ba9bb93-a72a-4883-b189-2499cce5c400
This commit is contained in:
parent
61d8d8cabe
commit
55363905bc
16 changed files with 1885 additions and 23 deletions
321
conformance/suite/loop.go
Normal file
321
conformance/suite/loop.go
Normal file
|
|
@ -0,0 +1,321 @@
|
|||
package suite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/tegwick/fluid-core/internal/contract"
|
||||
"github.com/tegwick/fluid-core/internal/fitness"
|
||||
"github.com/tegwick/fluid-core/internal/intent"
|
||||
"github.com/tegwick/fluid-core/internal/observation"
|
||||
"github.com/tegwick/fluid-core/internal/policy"
|
||||
"github.com/tegwick/fluid-core/internal/promotion"
|
||||
"github.com/tegwick/fluid-core/internal/science"
|
||||
)
|
||||
|
||||
// Loop records the artifacts one pass of the adaptation loop produced.
|
||||
type Loop struct {
|
||||
Pressure contract.PressureID
|
||||
Hypothesis contract.HypothesisID
|
||||
Experiment contract.ExperimentID
|
||||
Candidate contract.RevisionID
|
||||
Evaluation fitness.Evaluation
|
||||
Decision promotion.Decision
|
||||
}
|
||||
|
||||
// GeneratePressure drives the traffic that makes R-1's shortcoming visible.
|
||||
//
|
||||
// Several independent consumers each fetch the whole collection repeatedly
|
||||
// within one task, which is the Blueprint section 33 shape: the interface is
|
||||
// making them assemble something it could have handed them.
|
||||
func GeneratePressure(t *testing.T, h *Harness) {
|
||||
t.Helper()
|
||||
for c := 0; c < 5; c++ {
|
||||
consumer := fmt.Sprintf("agent-%d", c)
|
||||
for chain := 0; chain < 4; chain++ {
|
||||
for i := 0; i < 3; i++ {
|
||||
rec := h.Call(consumer, http.MethodGet, "/v1/entries")
|
||||
mustStatus(t, rec, http.StatusOK, "pressure traffic")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DetectPressure classifies recorded telemetry and records what it finds.
|
||||
func DetectPressure(t *testing.T, h *Harness) contract.PressureID {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
classifier := observation.NewClassifier(
|
||||
observation.DefaultClassifierOptions(), observation.NewTopologyAnalyzer())
|
||||
findings := classifier.Classify(h.Telemetry())
|
||||
if len(findings) == 0 {
|
||||
t.Fatal("no pressure detected from traffic that plainly shows it")
|
||||
}
|
||||
|
||||
registry := observation.NewPressureRegistry(h.Store, Interface)
|
||||
recorded, err := registry.RecordAll(ctx, findings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for _, p := range recorded {
|
||||
if p.Class == contract.PressureClassSuccessfulButInefficientUsage {
|
||||
return p.ID
|
||||
}
|
||||
}
|
||||
t.Fatalf("inefficient usage was not among the findings: %v", recorded)
|
||||
return ""
|
||||
}
|
||||
|
||||
// SeedHypothesis creates and prepares a hypothesis explaining the pressure.
|
||||
func SeedHypothesis(t *testing.T, h *Harness) Loop {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
hypothesis := contract.FluidHypothesis{
|
||||
ID: "H-1",
|
||||
Title: "latest entry is a first-class consumer concept",
|
||||
Observation: contract.FluidHypothesisObservation{
|
||||
Summary: "consumers fetch the whole collection repeatedly to find the newest entry",
|
||||
EvidenceRefs: []contract.EvidenceRef{"topology:GET /v1/entries"},
|
||||
},
|
||||
Pressure: contract.FluidHypothesisPressure{
|
||||
Classes: []contract.PressureClass{contract.PressureClassSuccessfulButInefficientUsage},
|
||||
},
|
||||
Explanation: contract.FluidHypothesisExplanation{
|
||||
Claim: "the collection resource does not name a concept consumers hold, so they assemble it themselves",
|
||||
},
|
||||
ProposedAdaptation: contract.FluidHypothesisProposedAdaptation{
|
||||
Class: contract.AdaptationClassContract,
|
||||
Summary: "add an explicit latest-entry resource",
|
||||
},
|
||||
ExpectedOutcomes: []contract.ExpectedOutcome{{
|
||||
Metric: fitness.MetricRequestsPerTask,
|
||||
Target: 1.2,
|
||||
Direction: contract.ExpectedOutcomeDirectionLower,
|
||||
}},
|
||||
Guardrails: []contract.Guardrail{{
|
||||
Metric: fitness.MetricErrorRate, Operator: contract.GuardrailOperatorLte, Threshold: 0.01,
|
||||
}},
|
||||
SuccessCriteria: contract.FluidHypothesisSuccessCriteria{
|
||||
Expression: "requests_per_completed_task <= 1.2 with no guardrail violation",
|
||||
},
|
||||
Complexity: contract.FluidHypothesisComplexity{
|
||||
ExpectedDelta: contract.ComplexityDelta{OperationCount: ptr(1.0)},
|
||||
},
|
||||
Risk: contract.FluidHypothesisRisk{Level: contract.FluidHypothesisRiskLevelLOW},
|
||||
}
|
||||
|
||||
if _, err := h.Hypotheses.Create(ctx, hypothesis, Operator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range []contract.FluidHypothesisState{
|
||||
contract.FluidHypothesisStateREADY,
|
||||
contract.FluidHypothesisStatePRIORITIZED,
|
||||
contract.FluidHypothesisStateDESIGNING,
|
||||
} {
|
||||
if _, err := h.Hypotheses.Transition(ctx, "H-1", s, Operator, "advancing the loop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := h.Hypotheses.AttachRevision(ctx, "H-1", "R-2", Operator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
return Loop{Hypothesis: "H-1", Candidate: "R-2"}
|
||||
}
|
||||
|
||||
// DesignExperiment creates the 90/10 experiment the Blueprint slice calls for.
|
||||
func DesignExperiment(t *testing.T, h *Harness, hypothesis contract.HypothesisID) contract.ExperimentID {
|
||||
t.Helper()
|
||||
return DesignExperimentNamed(t, h, hypothesis, "E-1")
|
||||
}
|
||||
|
||||
// DesignExperimentNamed creates an experiment with a chosen id.
|
||||
func DesignExperimentNamed(t *testing.T, h *Harness, hypothesis contract.HypothesisID, id contract.ExperimentID) contract.ExperimentID {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
e := contract.FluidExperiment{
|
||||
ID: id,
|
||||
HypothesisRefs: []contract.HypothesisID{hypothesis},
|
||||
ControlRevision: "R-1",
|
||||
CandidateRevisions: []contract.RevisionID{"R-2"},
|
||||
Cohorts: []contract.CohortID{"agents"},
|
||||
Allocation: map[string]contract.UnitInterval{"control": 0.9, "candidate": 0.1},
|
||||
Metrics: contract.FluidExperimentMetrics{
|
||||
Primary: []string{fitness.MetricRequestsPerTask},
|
||||
Guardrails: []string{fitness.MetricErrorRate},
|
||||
},
|
||||
StopConditions: []string{"hard_guardrail_violation", "manual_stop"},
|
||||
}
|
||||
if _, err := h.Experiments.Design(ctx, e, Operator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
// RunFullLoop executes the complete section 50 vertical slice.
|
||||
//
|
||||
// Two deterministic revisions, explicit routing, telemetry, one cohort
|
||||
// dimension, pressure detection, a hypothesis, a candidate revision, a bounded
|
||||
// experiment, fitness comparison, promotion, and a complete audit trail — with
|
||||
// no human steps.
|
||||
func RunFullLoop(t *testing.T, h *Harness) Loop {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
// 1-3. Traffic, telemetry, pressure.
|
||||
GeneratePressure(t, h)
|
||||
h.Drain()
|
||||
pressureID := DetectPressure(t, h)
|
||||
|
||||
// 4. Hypothesis explaining it.
|
||||
loop := SeedHypothesis(t, h)
|
||||
loop.Pressure = pressureID
|
||||
|
||||
// 5. Link the evidence to the explanation.
|
||||
registry := observation.NewPressureRegistry(h.Store, Interface)
|
||||
if err := registry.LinkHypothesis(ctx, pressureID, loop.Hypothesis); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 6. A bounded experiment, enacted through routing policy.
|
||||
expID := DesignExperiment(t, h, loop.Hypothesis)
|
||||
loop.Experiment = expID
|
||||
|
||||
_, startPolicy, err := h.Experiments.Start(ctx, expID, 2, "R-1", Operator)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Registry.PutPolicy(startPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 7. Measure both arms. Consumers are pinned so each arm gets the shape it
|
||||
// is meant to demonstrate; the allocation itself is exercised separately in
|
||||
// the resolver's own tests.
|
||||
observed := measureBothArms(t, h)
|
||||
|
||||
// 8. Fitness comparison against the declared criteria.
|
||||
specs := []fitness.MetricSpec{
|
||||
{Name: fitness.MetricRequestsPerTask, Role: fitness.RolePrimary,
|
||||
Direction: fitness.Lower, Target: ptr(1.2)},
|
||||
{Name: fitness.MetricErrorRate, Role: fitness.RoleGuardrail,
|
||||
Direction: fitness.Lower, Threshold: ptr(0.01)},
|
||||
}
|
||||
evaluator := fitness.NewEvaluator()
|
||||
evaluator.MinSamples = 10
|
||||
loop.Evaluation = evaluator.Evaluate("R-1", "R-2", observed.window, specs, observed.observations)
|
||||
|
||||
// 9. Stop the experiment and record its conclusion.
|
||||
_, stopPolicy, err := h.Experiments.Stop(ctx, expID, 3, "R-1", Operator, "measurement window elapsed")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := h.Registry.PutPolicy(stopPolicy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := h.Experiments.Finalize(ctx, expID, "R-2", Operator,
|
||||
fmt.Sprintf("fitness verdict %s", loop.Evaluation.Verdict), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 10. Record the hypothesis outcome.
|
||||
status := contract.FluidHypothesisOutcomeStatusREFUTED
|
||||
if loop.Evaluation.Verdict == fitness.VerdictSucceeded {
|
||||
status = contract.FluidHypothesisOutcomeStatusCONFIRMED
|
||||
}
|
||||
if _, err := h.Hypotheses.RecordOutcome(ctx, loop.Hypothesis, status,
|
||||
fmt.Sprintf("verdict %s: %v", loop.Evaluation.Verdict, loop.Evaluation.Reasons), nil, Operator); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 11. Promotion, through the deterministic gate.
|
||||
loop.Decision = decide(t, h, loop)
|
||||
return loop
|
||||
}
|
||||
|
||||
type measurement struct {
|
||||
observations []fitness.Observation
|
||||
window fitness.Window
|
||||
}
|
||||
|
||||
// measureBothArms drives the two revisions and derives their metrics.
|
||||
func measureBothArms(t *testing.T, h *Harness) measurement {
|
||||
t.Helper()
|
||||
start := time.Now().Add(-time.Hour)
|
||||
|
||||
// R-1: three calls per task, the shape the pressure described.
|
||||
for c := 0; c < 12; c++ {
|
||||
consumer := fmt.Sprintf("control-%d", c)
|
||||
for i := 0; i < 3; i++ {
|
||||
mustStatus(t, h.CallPinned(consumer, "R-1", http.MethodGet, "/v1/entries"),
|
||||
http.StatusOK, "control arm")
|
||||
}
|
||||
}
|
||||
// R-2: one call per task, using the resource the hypothesis proposed.
|
||||
for c := 0; c < 12; c++ {
|
||||
consumer := fmt.Sprintf("candidate-%d", c)
|
||||
mustStatus(t, h.CallPinned(consumer, "R-2", http.MethodGet, "/v1/entries/latest"),
|
||||
http.StatusOK, "candidate arm")
|
||||
}
|
||||
h.Drain()
|
||||
|
||||
window := fitness.Window{Start: start}
|
||||
return measurement{
|
||||
observations: fitness.NewMeasurer().Measure(h.Telemetry(), window),
|
||||
window: window,
|
||||
}
|
||||
}
|
||||
|
||||
func decide(t *testing.T, h *Harness, loop Loop) promotion.Decision {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
|
||||
descriptor, err := h.Registry.Revision(loop.Candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
governing, err := h.Intents.GoverningIntent(ctx, loop.Candidate)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
limits := policy.DefaultLimits()
|
||||
limits.AllowedAdaptationClasses = append(limits.AllowedAdaptationClasses,
|
||||
contract.AdaptationClassContract)
|
||||
limits.RequiredMode = intent.ModeExperimental
|
||||
|
||||
controller := promotion.NewController(h.Store, policy.NewGate(limits))
|
||||
d, err := controller.Decide(ctx, promotion.Request{
|
||||
Revision: loop.Candidate,
|
||||
Outcome: promotion.Promote,
|
||||
Reason: "the candidate met its primary target with no guardrail breach",
|
||||
Actor: Operator,
|
||||
Experiment: loop.Experiment,
|
||||
Hypotheses: []contract.HypothesisID{loop.Hypothesis},
|
||||
Evaluation: &loop.Evaluation,
|
||||
GateInput: &policy.Input{
|
||||
Descriptor: descriptor,
|
||||
GoverningMode: governing.Mode,
|
||||
AdaptationClasses: []contract.AdaptationClass{contract.AdaptationClassContract},
|
||||
ComplexityDelta: 0.2,
|
||||
RequestedTrafficShare: 0.2,
|
||||
Approved: true,
|
||||
ApprovedBy: &Operator,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("promotion refused: %v", err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T { return &v }
|
||||
|
||||
var _ = science.CanTransition
|
||||
Loading…
Add table
Add a link
Reference in a new issue