import json import sys import unittest from pathlib import Path sys.path.insert(0, str(Path(__file__).parents[1] / "tools")) from thresholds import evaluate, evaluate_threshold ROOT = Path(__file__).parents[1] CONFIG = json.loads((ROOT / "data" / "thresholds" / "platform-audit-storage.json").read_text()) OBSERVATION = json.loads((ROOT / "data" / "actuals" / "2026-08.json").read_text()) FORECAST = json.loads( (ROOT / "data" / "forecasts" / "platform-audit-storage-scaleway-base-2026-08.json").read_text() ) def threshold(**overrides): result = { "id": "t", "kind": "budget_variance", "metric": "infrastructure_eur", "comparison": "absolute_percentage_error", "limit": 10, "action": "investigate", } result.update(overrides) return result def observed(**overrides): result = {"period": "2026-09", "infrastructure_eur": 3.0, "backup_success_pct": 100, "measurement_gaps": []} result.update(overrides) return result class ThresholdVerdictTest(unittest.TestCase): def test_variance_within_limit_passes(self): compared = {"infrastructure_eur": {"absolute_percentage_error": 4.0, "error": 0.11}} self.assertEqual("within", evaluate_threshold(threshold(), observed(), compared)["verdict"]) def test_variance_above_limit_breaches(self): compared = {"infrastructure_eur": {"absolute_percentage_error": 42.0, "error": 1.2}} result = evaluate_threshold(threshold(), observed(), compared) self.assertEqual("breach", result["verdict"]) self.assertEqual(42.0, result["measured"]) def test_absolute_error_comparison_uses_magnitude_not_sign(self): spec = threshold(metric="internal_labor_hours", comparison="absolute_error", limit=1) compared = {"internal_labor_hours": {"error": -2.5, "absolute_percentage_error": 250.0}} result = evaluate_threshold(spec, observed(internal_labor_hours=1.5), compared) self.assertEqual("breach", result["verdict"]) self.assertEqual(2.5, result["measured"]) def test_minimum_comparison_breaches_below_the_limit(self): spec = threshold(id="backup", metric="backup_success_pct", comparison="minimum", limit=100) self.assertEqual("breach", evaluate_threshold(spec, observed(backup_success_pct=99), None)["verdict"]) self.assertEqual("within", evaluate_threshold(spec, observed(), None)["verdict"]) def test_maximum_comparison_breaches_above_the_limit(self): spec = threshold(id="rto", metric="restore_rto_minutes", comparison="maximum", limit=3.24) self.assertEqual("breach", evaluate_threshold(spec, observed(restore_rto_minutes=9.0), None)["verdict"]) self.assertEqual("within", evaluate_threshold(spec, observed(restore_rto_minutes=1.08), None)["verdict"]) class FailClosedTest(unittest.TestCase): """An unmeasured value must never pass a threshold.""" def test_missing_value_is_unmeasured_not_within(self): result = evaluate_threshold(threshold(), observed(infrastructure_eur=None), None) self.assertEqual("unmeasured", result["verdict"]) def test_unmeasured_result_carries_the_named_gap(self): row = observed( infrastructure_eur=None, measurement_gaps=["infrastructure_eur: no invoice yet (owner: fin-hub)"], ) result = evaluate_threshold(threshold(), row, None) self.assertIn("owner: fin-hub", result["detail"]) def test_missing_forecast_row_is_unmeasured_not_within(self): result = evaluate_threshold(threshold(), observed(), None) self.assertEqual("unmeasured", result["verdict"]) self.assertIn("no forecast row", result["detail"]) def test_unknown_variance_entry_is_unmeasured(self): compared = {"infrastructure_eur": {"forecast": 2.89, "actual": None, "status": "unknown"}} self.assertEqual("unmeasured", evaluate_threshold(threshold(), observed(), compared)["verdict"]) def test_zero_forecast_does_not_pass_on_undefined_percentage(self): compared = {"infrastructure_eur": {"absolute_percentage_error": None, "error": 3.0}} result = evaluate_threshold(threshold(), observed(), compared) self.assertEqual("unmeasured", result["verdict"]) self.assertIn("percentage error is undefined", result["detail"]) def test_not_applicable_threshold_is_distinct_from_passing(self): spec = threshold(id="commitment", metric="commitment_utilization", comparison="minimum", limit=None, status="not_applicable") self.assertEqual("not_applicable", evaluate_threshold(spec, observed(), None)["verdict"]) class AugustObservationTest(unittest.TestCase): """The real first period: live for four hours, no invoice, no request metrics.""" @classmethod def setUpClass(cls): cls.report = evaluate(CONFIG, OBSERVATION, FORECAST, "2026-08") def test_no_breach_and_no_false_pass(self): self.assertEqual([], self.report["breaches"]) self.assertEqual(2, self.report["summary"]["within"]) self.assertEqual(6, self.report["summary"]["unmeasured"]) self.assertEqual(1, self.report["summary"]["not_applicable"]) def test_august_has_no_forecast_so_variance_thresholds_cannot_pass(self): self.assertFalse(self.report["forecast_available"]) self.assertIn("abnormal-growth-stored", self.report["unmeasured"]) def test_what_was_actually_proven_reads_as_within(self): within = [r["id"] for r in self.report["results"] if r["verdict"] == "within"] self.assertEqual(["stale-backup", "restore-rto-regression"], within) def test_commitment_threshold_is_not_applicable_without_a_commitment(self): commitment = next(r for r in self.report["results"] if r["id"] == "unused-commitment") self.assertEqual("not_applicable", commitment["verdict"]) def test_missing_period_is_an_error_not_an_empty_pass(self): with self.assertRaises(ValueError): evaluate(CONFIG, OBSERVATION, FORECAST, "2026-07") if __name__ == "__main__": unittest.main()