feat(wp-0002): complete T07 — control loop on the live backup resource
The backup is procured and proven, so the loop runs on real evidence. - data/actuals/2026-08.json: first real observation. database 0.6365 GB, stored 0.0066 GB over 8 objects, backup success 1/1, restore RTO 1.08 min. Five proxies null, each with a named owner in measurement_gaps. - data/thresholds/platform-audit-storage.json + tools/thresholds.py: budget variance, abnormal growth, stale backup, unused commitment. Fail-closed — an unmeasured value is reported as unmeasured, never as within. - financial_exchange.py gains a usage mode emitting technical_usage records to fin-hub, with measurement gaps carried through and no infrastructure amount: fin-hub owns the booked fact and a null is never sent as 0.00. - observation schema 0.2 allows null cost and usage proxies; variance.py fails closed rather than reporting a 100% favourable variance on a missing amount. - platform-audit-storage: ordered -> active, commissioned 2026-08-14, on operational fact rather than on the purchase. The optimization case is now approved by the founder. That needed a schema change: Host Europe never supplied written terms, so options gained excluded/exclusion_reason. Previously an unevaluable alternative blocked its case forever, leaving the record claiming no decision while the bucket was in production. An excluded option keeps its unknowns and must say what would bring it back. August produces no variance and should not: the decision forecast starts at 2026-09, so August is a commissioning baseline. Threshold run is 2 within, 1 not applicable, 6 unmeasured, 0 breaches. Also fixes a pre-existing test failure: reef-storage consumers_actual is now rapp-postgres, which the assertion still expected to be empty. 136 tests pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
2704292d45
commit
10b988fa1c
17 changed files with 1312 additions and 122 deletions
|
|
@ -214,11 +214,13 @@ class RegisteredCasesTest(unittest.TestCase):
|
|||
self.assertIn("opt:platform-audit-storage:2026-08", types)
|
||||
self.assertIn("opt:reef-railiance-k3s:2026-08", types)
|
||||
|
||||
def test_storage_case_computes_hetzner_and_blocks_host_europe(self):
|
||||
def test_storage_case_computes_hetzner_and_excludes_host_europe(self):
|
||||
report = evaluate(self.cases["platform-audit-storage-2026-08"])
|
||||
verdicts = {c["option_id"]: c["verdict"] for c in report["comparisons"]}
|
||||
self.assertEqual("reject", verdicts["hetzner-object-storage"])
|
||||
self.assertEqual("blocked_on_evidence", verdicts["host-europe-cloud-storage"])
|
||||
# Host Europe never supplied written terms and was excluded at decision
|
||||
# time rather than left blocking the case indefinitely.
|
||||
self.assertEqual("excluded", verdicts["host-europe-cloud-storage"])
|
||||
hetzner = next(c for c in report["comparisons"] if c["option_id"] == "hetzner-object-storage")
|
||||
self.assertEqual(29.14, hetzner["monthly_delta_eur"])
|
||||
|
||||
|
|
@ -244,5 +246,69 @@ class RegisteredCasesTest(unittest.TestCase):
|
|||
self.assertEqual(before, record, name)
|
||||
|
||||
|
||||
class ExcludedOptionTest(unittest.TestCase):
|
||||
"""An alternative the deciding authority set aside must not block forever."""
|
||||
|
||||
def test_excluded_option_does_not_block_the_case(self):
|
||||
alternative = option("alt", one_time_eur=None, excluded=True,
|
||||
exclusion_reason="no written terms within the decision window")
|
||||
result = compare_option(option("baseline"), alternative)
|
||||
self.assertEqual("excluded", result["verdict"])
|
||||
self.assertEqual([], result["blocking_evidence"])
|
||||
|
||||
def test_exclusion_reason_travels_with_the_verdict(self):
|
||||
alternative = option("alt", excluded=True, exclusion_reason="supplier never replied")
|
||||
self.assertEqual("supplier never replied", compare_option(option("baseline"), alternative)["exclusion_reason"])
|
||||
|
||||
def test_excluding_without_a_reason_is_rejected(self):
|
||||
record = case(alternatives=[option("alt", one_time_eur=None, excluded=True)])
|
||||
with self.assertRaisesRegex(ValueError, "exclusion_reason"):
|
||||
validate_case(record)
|
||||
|
||||
def test_case_with_only_excluded_alternatives_can_be_approved(self):
|
||||
record = case(alternatives=[option("alt", one_time_eur=None, excluded=True,
|
||||
exclusion_reason="unevaluable")])
|
||||
record["decision"].update({"state": "approved", "recommended_option_id": "baseline",
|
||||
"approver": "founder", "approved_on": "2026-08-14"})
|
||||
validate_case(record)
|
||||
|
||||
def test_excluded_option_is_never_recommended(self):
|
||||
record = case(alternatives=[option("alt", recurring_infrastructure_eur_month=0, excluded=True,
|
||||
exclusion_reason="unevaluable")])
|
||||
record["decision"].update({"state": "approved", "recommended_option_id": "baseline",
|
||||
"approver": "founder", "approved_on": "2026-08-14"})
|
||||
self.assertIsNone(evaluate(record)["best_option_id"])
|
||||
|
||||
|
||||
class ApprovedStorageCaseTest(unittest.TestCase):
|
||||
"""The backup decision was made and executed on 2026-08-14."""
|
||||
|
||||
def setUp(self):
|
||||
path = CASE_DIR / "platform-audit-storage-2026-08.json"
|
||||
self.case = json.loads(path.read_text())
|
||||
|
||||
def test_case_is_approved_by_a_named_authority(self):
|
||||
decision = self.case["decision"]
|
||||
self.assertEqual("approved", decision["state"])
|
||||
self.assertEqual("scaleway-standard-multi-az", decision["recommended_option_id"])
|
||||
self.assertEqual("2026-08-14", decision["approved_on"])
|
||||
self.assertTrue(decision["approver"])
|
||||
|
||||
def test_host_europe_is_excluded_with_a_reason_not_silently_dropped(self):
|
||||
host_europe = next(a for a in self.case["alternatives"] if a["option_id"] == "host-europe-cloud-storage")
|
||||
self.assertTrue(host_europe["excluded"])
|
||||
self.assertIn("RESOURCE-WP-0002-T02", host_europe["exclusion_reason"])
|
||||
self.assertTrue(host_europe["unknowns"], "the unknowns stay recorded after exclusion")
|
||||
|
||||
def test_hetzner_stays_a_costed_rejection(self):
|
||||
verdicts = {c["option_id"]: c["verdict"] for c in evaluate(self.case)["comparisons"]}
|
||||
self.assertEqual("reject", verdicts["hetzner-object-storage"])
|
||||
self.assertEqual("excluded", verdicts["host-europe-cloud-storage"])
|
||||
|
||||
def test_outcome_points_at_the_real_actuals(self):
|
||||
self.assertIn("data/actuals/2026-08.json", self.case["outcome"]["actual_refs"])
|
||||
self.assertTrue(self.case["financial_handoff"]["sent"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -113,7 +113,10 @@ class ReefViewTest(unittest.TestCase):
|
|||
[row["resource_id"] for row in view["resources"]],
|
||||
)
|
||||
self.assertEqual(["rapp-postgres"], view["consumers_potential"])
|
||||
self.assertEqual([], view["consumers_actual"])
|
||||
# rapp-postgres became an actual consumer on 2026-08-14: continuous WAL
|
||||
# archiving and a proven restore make it a realised consumer, not a
|
||||
# potential one.
|
||||
self.assertEqual(["rapp-postgres"], view["consumers_actual"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -146,9 +146,15 @@ class LivePortfolioTest(unittest.TestCase):
|
|||
def test_missing_contract_dates_are_surfaced_rather_than_read_as_no_commitments(self):
|
||||
self.assertTrue(self.report["renewals"]["undated"])
|
||||
|
||||
def test_open_optimization_cases_are_listed_with_their_blockers(self):
|
||||
self.assertEqual(2, len(self.report["optimization"]["cases"]))
|
||||
self.assertEqual(2, len(self.report["optimization"]["undecided"]))
|
||||
def test_optimization_cases_are_listed_with_their_state(self):
|
||||
cases = self.report["optimization"]["cases"]
|
||||
self.assertEqual(2, len(cases))
|
||||
states = {c["case_id"]: c["state"] for c in cases}
|
||||
# The storage case was decided and executed on 2026-08-14; the cluster
|
||||
# rightsizing case is still blocked on delegated evidence.
|
||||
self.assertEqual("approved", states["opt:platform-audit-storage:2026-08"])
|
||||
self.assertEqual("blocked_on_evidence", states["opt:reef-railiance-k3s:2026-08"])
|
||||
self.assertEqual(["opt:reef-railiance-k3s:2026-08"], self.report["optimization"]["undecided"])
|
||||
self.assertTrue(self.report["optimization"]["blocked_on_evidence"])
|
||||
|
||||
def test_next_actions_name_an_owning_repository_for_each_gap(self):
|
||||
|
|
|
|||
128
tests/test_thresholds.py
Normal file
128
tests/test_thresholds.py
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import json
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parents[1] / "tools"))
|
||||
from thresholds import evaluate, evaluate_threshold
|
||||
|
||||
ROOT = Path(__file__).parents[1]
|
||||
CONFIG = json.loads((ROOT / "data" / "thresholds" / "platform-audit-storage.json").read_text())
|
||||
OBSERVATION = json.loads((ROOT / "data" / "actuals" / "2026-08.json").read_text())
|
||||
FORECAST = json.loads(
|
||||
(ROOT / "data" / "forecasts" / "platform-audit-storage-scaleway-base-2026-08.json").read_text()
|
||||
)
|
||||
|
||||
|
||||
def threshold(**overrides):
|
||||
result = {
|
||||
"id": "t", "kind": "budget_variance", "metric": "infrastructure_eur",
|
||||
"comparison": "absolute_percentage_error", "limit": 10, "action": "investigate",
|
||||
}
|
||||
result.update(overrides)
|
||||
return result
|
||||
|
||||
|
||||
def observed(**overrides):
|
||||
result = {"period": "2026-09", "infrastructure_eur": 3.0, "backup_success_pct": 100, "measurement_gaps": []}
|
||||
result.update(overrides)
|
||||
return result
|
||||
|
||||
|
||||
class ThresholdVerdictTest(unittest.TestCase):
|
||||
def test_variance_within_limit_passes(self):
|
||||
compared = {"infrastructure_eur": {"absolute_percentage_error": 4.0, "error": 0.11}}
|
||||
self.assertEqual("within", evaluate_threshold(threshold(), observed(), compared)["verdict"])
|
||||
|
||||
def test_variance_above_limit_breaches(self):
|
||||
compared = {"infrastructure_eur": {"absolute_percentage_error": 42.0, "error": 1.2}}
|
||||
result = evaluate_threshold(threshold(), observed(), compared)
|
||||
self.assertEqual("breach", result["verdict"])
|
||||
self.assertEqual(42.0, result["measured"])
|
||||
|
||||
def test_absolute_error_comparison_uses_magnitude_not_sign(self):
|
||||
spec = threshold(metric="internal_labor_hours", comparison="absolute_error", limit=1)
|
||||
compared = {"internal_labor_hours": {"error": -2.5, "absolute_percentage_error": 250.0}}
|
||||
result = evaluate_threshold(spec, observed(internal_labor_hours=1.5), compared)
|
||||
self.assertEqual("breach", result["verdict"])
|
||||
self.assertEqual(2.5, result["measured"])
|
||||
|
||||
def test_minimum_comparison_breaches_below_the_limit(self):
|
||||
spec = threshold(id="backup", metric="backup_success_pct", comparison="minimum", limit=100)
|
||||
self.assertEqual("breach", evaluate_threshold(spec, observed(backup_success_pct=99), None)["verdict"])
|
||||
self.assertEqual("within", evaluate_threshold(spec, observed(), None)["verdict"])
|
||||
|
||||
def test_maximum_comparison_breaches_above_the_limit(self):
|
||||
spec = threshold(id="rto", metric="restore_rto_minutes", comparison="maximum", limit=3.24)
|
||||
self.assertEqual("breach", evaluate_threshold(spec, observed(restore_rto_minutes=9.0), None)["verdict"])
|
||||
self.assertEqual("within", evaluate_threshold(spec, observed(restore_rto_minutes=1.08), None)["verdict"])
|
||||
|
||||
|
||||
class FailClosedTest(unittest.TestCase):
|
||||
"""An unmeasured value must never pass a threshold."""
|
||||
|
||||
def test_missing_value_is_unmeasured_not_within(self):
|
||||
result = evaluate_threshold(threshold(), observed(infrastructure_eur=None), None)
|
||||
self.assertEqual("unmeasured", result["verdict"])
|
||||
|
||||
def test_unmeasured_result_carries_the_named_gap(self):
|
||||
row = observed(
|
||||
infrastructure_eur=None,
|
||||
measurement_gaps=["infrastructure_eur: no invoice yet (owner: fin-hub)"],
|
||||
)
|
||||
result = evaluate_threshold(threshold(), row, None)
|
||||
self.assertIn("owner: fin-hub", result["detail"])
|
||||
|
||||
def test_missing_forecast_row_is_unmeasured_not_within(self):
|
||||
result = evaluate_threshold(threshold(), observed(), None)
|
||||
self.assertEqual("unmeasured", result["verdict"])
|
||||
self.assertIn("no forecast row", result["detail"])
|
||||
|
||||
def test_unknown_variance_entry_is_unmeasured(self):
|
||||
compared = {"infrastructure_eur": {"forecast": 2.89, "actual": None, "status": "unknown"}}
|
||||
self.assertEqual("unmeasured", evaluate_threshold(threshold(), observed(), compared)["verdict"])
|
||||
|
||||
def test_zero_forecast_does_not_pass_on_undefined_percentage(self):
|
||||
compared = {"infrastructure_eur": {"absolute_percentage_error": None, "error": 3.0}}
|
||||
result = evaluate_threshold(threshold(), observed(), compared)
|
||||
self.assertEqual("unmeasured", result["verdict"])
|
||||
self.assertIn("percentage error is undefined", result["detail"])
|
||||
|
||||
def test_not_applicable_threshold_is_distinct_from_passing(self):
|
||||
spec = threshold(id="commitment", metric="commitment_utilization", comparison="minimum",
|
||||
limit=None, status="not_applicable")
|
||||
self.assertEqual("not_applicable", evaluate_threshold(spec, observed(), None)["verdict"])
|
||||
|
||||
|
||||
class AugustObservationTest(unittest.TestCase):
|
||||
"""The real first period: live for four hours, no invoice, no request metrics."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.report = evaluate(CONFIG, OBSERVATION, FORECAST, "2026-08")
|
||||
|
||||
def test_no_breach_and_no_false_pass(self):
|
||||
self.assertEqual([], self.report["breaches"])
|
||||
self.assertEqual(2, self.report["summary"]["within"])
|
||||
self.assertEqual(6, self.report["summary"]["unmeasured"])
|
||||
self.assertEqual(1, self.report["summary"]["not_applicable"])
|
||||
|
||||
def test_august_has_no_forecast_so_variance_thresholds_cannot_pass(self):
|
||||
self.assertFalse(self.report["forecast_available"])
|
||||
self.assertIn("abnormal-growth-stored", self.report["unmeasured"])
|
||||
|
||||
def test_what_was_actually_proven_reads_as_within(self):
|
||||
within = [r["id"] for r in self.report["results"] if r["verdict"] == "within"]
|
||||
self.assertEqual(["stale-backup", "restore-rto-regression"], within)
|
||||
|
||||
def test_commitment_threshold_is_not_applicable_without_a_commitment(self):
|
||||
commitment = next(r for r in self.report["results"] if r["id"] == "unused-commitment")
|
||||
self.assertEqual("not_applicable", commitment["verdict"])
|
||||
|
||||
def test_missing_period_is_an_error_not_an_empty_pass(self):
|
||||
with self.assertRaises(ValueError):
|
||||
evaluate(CONFIG, OBSERVATION, FORECAST, "2026-07")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue