resource-control/tests/test_thresholds.py

import json
import sys
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parents[1] / "tools"))
from thresholds import evaluate, evaluate_threshold

ROOT = Path(__file__).parents[1]
CONFIG = json.loads((ROOT / "data" / "thresholds" / "platform-audit-storage.json").read_text())
OBSERVATION = json.loads((ROOT / "data" / "actuals" / "2026-08.json").read_text())
FORECAST = json.loads(
    (ROOT / "data" / "forecasts" / "platform-audit-storage-scaleway-base-2026-08.json").read_text()
)


def threshold(**overrides):
    result = {
        "id": "t", "kind": "budget_variance", "metric": "infrastructure_eur",
        "comparison": "absolute_percentage_error", "limit": 10, "action": "investigate",
    }
    result.update(overrides)
    return result


def observed(**overrides):
    result = {"period": "2026-09", "infrastructure_eur": 3.0, "backup_success_pct": 100, "measurement_gaps": []}
    result.update(overrides)
    return result


class ThresholdVerdictTest(unittest.TestCase):
    def test_variance_within_limit_passes(self):
        compared = {"infrastructure_eur": {"absolute_percentage_error": 4.0, "error": 0.11}}
        self.assertEqual("within", evaluate_threshold(threshold(), observed(), compared)["verdict"])

    def test_variance_above_limit_breaches(self):
        compared = {"infrastructure_eur": {"absolute_percentage_error": 42.0, "error": 1.2}}
        result = evaluate_threshold(threshold(), observed(), compared)
        self.assertEqual("breach", result["verdict"])
        self.assertEqual(42.0, result["measured"])

    def test_absolute_error_comparison_uses_magnitude_not_sign(self):
        spec = threshold(metric="internal_labor_hours", comparison="absolute_error", limit=1)
        compared = {"internal_labor_hours": {"error": -2.5, "absolute_percentage_error": 250.0}}
        result = evaluate_threshold(spec, observed(internal_labor_hours=1.5), compared)
        self.assertEqual("breach", result["verdict"])
        self.assertEqual(2.5, result["measured"])

    def test_minimum_comparison_breaches_below_the_limit(self):
        spec = threshold(id="backup", metric="backup_success_pct", comparison="minimum", limit=100)
        self.assertEqual("breach", evaluate_threshold(spec, observed(backup_success_pct=99), None)["verdict"])
        self.assertEqual("within", evaluate_threshold(spec, observed(), None)["verdict"])

    def test_maximum_comparison_breaches_above_the_limit(self):
        spec = threshold(id="rto", metric="restore_rto_minutes", comparison="maximum", limit=3.24)
        self.assertEqual("breach", evaluate_threshold(spec, observed(restore_rto_minutes=9.0), None)["verdict"])
        self.assertEqual("within", evaluate_threshold(spec, observed(restore_rto_minutes=1.08), None)["verdict"])


class FailClosedTest(unittest.TestCase):
    """An unmeasured value must never pass a threshold."""

    def test_missing_value_is_unmeasured_not_within(self):
        result = evaluate_threshold(threshold(), observed(infrastructure_eur=None), None)
        self.assertEqual("unmeasured", result["verdict"])

    def test_unmeasured_result_carries_the_named_gap(self):
        row = observed(
            infrastructure_eur=None,
            measurement_gaps=["infrastructure_eur: no invoice yet (owner: fin-hub)"],
        )
        result = evaluate_threshold(threshold(), row, None)
        self.assertIn("owner: fin-hub", result["detail"])

    def test_missing_forecast_row_is_unmeasured_not_within(self):
        result = evaluate_threshold(threshold(), observed(), None)
        self.assertEqual("unmeasured", result["verdict"])
        self.assertIn("no forecast row", result["detail"])

    def test_unknown_variance_entry_is_unmeasured(self):
        compared = {"infrastructure_eur": {"forecast": 2.89, "actual": None, "status": "unknown"}}
        self.assertEqual("unmeasured", evaluate_threshold(threshold(), observed(), compared)["verdict"])

    def test_zero_forecast_does_not_pass_on_undefined_percentage(self):
        compared = {"infrastructure_eur": {"absolute_percentage_error": None, "error": 3.0}}
        result = evaluate_threshold(threshold(), observed(), compared)
        self.assertEqual("unmeasured", result["verdict"])
        self.assertIn("percentage error is undefined", result["detail"])

    def test_not_applicable_threshold_is_distinct_from_passing(self):
        spec = threshold(id="commitment", metric="commitment_utilization", comparison="minimum",
                         limit=None, status="not_applicable")
        self.assertEqual("not_applicable", evaluate_threshold(spec, observed(), None)["verdict"])


class AugustObservationTest(unittest.TestCase):
    """The real first period: live for four hours, no invoice, no request metrics."""

    @classmethod
    def setUpClass(cls):
        cls.report = evaluate(CONFIG, OBSERVATION, FORECAST, "2026-08")

    def test_no_breach_and_no_false_pass(self):
        self.assertEqual([], self.report["breaches"])
        self.assertEqual(2, self.report["summary"]["within"])
        self.assertEqual(6, self.report["summary"]["unmeasured"])
        self.assertEqual(1, self.report["summary"]["not_applicable"])

    def test_august_has_no_forecast_so_variance_thresholds_cannot_pass(self):
        self.assertFalse(self.report["forecast_available"])
        self.assertIn("abnormal-growth-stored", self.report["unmeasured"])

    def test_what_was_actually_proven_reads_as_within(self):
        within = [r["id"] for r in self.report["results"] if r["verdict"] == "within"]
        self.assertEqual(["stale-backup", "restore-rto-regression"], within)

    def test_commitment_threshold_is_not_applicable_without_a_commitment(self):
        commitment = next(r for r in self.report["results"] if r["id"] == "unused-commitment")
        self.assertEqual("not_applicable", commitment["verdict"])

    def test_missing_period_is_an_error_not_an_empty_pass(self):
        with self.assertRaises(ValueError):
            evaluate(CONFIG, OBSERVATION, FORECAST, "2026-07")


if __name__ == "__main__":
    unittest.main()