From 10b988fa1c0a6f38a95253b767b141d2a9078322 Mon Sep 17 00:00:00 2001 From: tegwick Date: Fri, 14 Aug 2026 20:53:12 +0200 Subject: [PATCH] =?UTF-8?q?feat(wp-0002):=20complete=20T07=20=E2=80=94=20c?= =?UTF-8?q?ontrol=20loop=20on=20the=20live=20backup=20resource?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backup is procured and proven, so the loop runs on real evidence. - data/actuals/2026-08.json: first real observation. database 0.6365 GB, stored 0.0066 GB over 8 objects, backup success 1/1, restore RTO 1.08 min. Five proxies null, each with a named owner in measurement_gaps. - data/thresholds/platform-audit-storage.json + tools/thresholds.py: budget variance, abnormal growth, stale backup, unused commitment. Fail-closed — an unmeasured value is reported as unmeasured, never as within. - financial_exchange.py gains a usage mode emitting technical_usage records to fin-hub, with measurement gaps carried through and no infrastructure amount: fin-hub owns the booked fact and a null is never sent as 0.00. - observation schema 0.2 allows null cost and usage proxies; variance.py fails closed rather than reporting a 100% favourable variance on a missing amount. - platform-audit-storage: ordered -> active, commissioned 2026-08-14, on operational fact rather than on the purchase. The optimization case is now approved by the founder. That needed a schema change: Host Europe never supplied written terms, so options gained excluded/exclusion_reason. Previously an unevaluable alternative blocked its case forever, leaving the record claiming no decision while the bucket was in production. An excluded option keeps its unknowns and must say what would bring it back. August produces no variance and should not: the decision forecast starts at 2026-09, so August is a commissioning baseline. Threshold run is 2 within, 1 not applicable, 6 unmeasured, 0 breaches. Also fixes a pre-existing test failure: reef-storage consumers_actual is now rapp-postgres, which the assertion still expected to be empty. 136 tests pass. Co-Authored-By: Claude Opus 5 --- Makefile | 12 +- data/actuals/2026-08.json | 46 +++ .../platform-audit-storage-2026-08.json | 100 ++++-- data/resources/platform-audit-storage.json | 75 ++++- data/thresholds/platform-audit-storage.json | 93 ++++++ ...RCE-WP-0002-T07-control-loop-2026-08-14.md | 117 +++++++ .../monthly-resource-observation.schema.json | 220 +++++++++++-- schemas/optimization-case.schema.json | 294 +++++++++++++++--- tests/test_optimization.py | 70 ++++- tests/test_portfolio.py | 5 +- tests/test_portfolio_report.py | 12 +- tests/test_thresholds.py | 128 ++++++++ tools/financial_exchange.py | 75 ++++- tools/optimization.py | 18 ++ tools/thresholds.py | 137 ++++++++ tools/variance.py | 10 + ...WP-0002-procure-postgres-backup-storage.md | 22 +- 17 files changed, 1312 insertions(+), 122 deletions(-) create mode 100644 data/actuals/2026-08.json create mode 100644 data/thresholds/platform-audit-storage.json create mode 100644 docs/evidence/RESOURCE-WP-0002-T07-control-loop-2026-08-14.md create mode 100644 tests/test_thresholds.py create mode 100644 tools/thresholds.py diff --git a/Makefile b/Makefile index 3cef382..1b0ebea 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test forecast variance control-cycle exchange-forecast optimization portfolio-report settlement +.PHONY: test forecast variance control-cycle exchange-forecast exchange-usage optimization portfolio-report settlement thresholds test: python3 -m unittest discover -s tests -p 'test_*.py' @@ -18,6 +18,16 @@ control-cycle: exchange-forecast: python3 tools/financial_exchange.py forecast data/forecasts/platform-audit-storage-scaleway-base-2026-08.json +exchange-usage: + @test -n "$(ACTUAL)" || { echo 'ACTUAL=data/actuals/YYYY-MM.json is required' >&2; exit 2; } + python3 tools/financial_exchange.py usage $(ACTUAL) resource:platform:audit-storage + +thresholds: + @test -n "$(PERIOD)" || { echo 'PERIOD=YYYY-MM is required' >&2; exit 2; } + python3 tools/thresholds.py data/thresholds/platform-audit-storage.json \ + data/actuals/$(PERIOD).json $(PERIOD) \ + data/forecasts/platform-audit-storage-scaleway-base-2026-08.json + optimization: python3 tools/optimization.py $(CASE) diff --git a/data/actuals/2026-08.json b/data/actuals/2026-08.json new file mode 100644 index 0000000..1d06b8f --- /dev/null +++ b/data/actuals/2026-08.json @@ -0,0 +1,46 @@ +{ + "schema_version": "0.2", + "record_type": "usage_observation", + "workload": "rapp-postgres/platform-pg", + "cost_attribution_key": "platform:audit-storage", + "provider_id": "scaleway-standard-multi-az", + "created_at": "2026-08-14T18:24:00Z", + "scenario": "observed", + "forecast_ref": "data/forecasts/platform-audit-storage-scaleway-base-2026-08.json", + "rows": [ + { + "period": "2026-08", + "database_gb": 0.6365, + "stored_gb": 0.0066, + "wal_gb": null, + "restore_egress_gb": null, + "write_requests": null, + "read_requests": null, + "infrastructure_eur": null, + "internal_labor_hours": null, + "internal_labor_eur": null, + "total_eur": null, + "backup_success_pct": 100, + "restore_rto_minutes": 1.08, + "measurement_gaps": [ + "wal_gb: the 8 objects in prefix platform-pg/ hold the first base backup and its WAL together; the T05 evidence does not separate them", + "restore_egress_gb: two scratch restores read from the bucket but bytes transferred were not recorded (owner: resource-control, add to the next drill)", + "write_requests, read_requests: no request metric is exposed to the scoped backup key (owner: resource-control, needs a Scaleway usage export)", + "infrastructure_eur: no invoice for a resource commissioned 2026-08-14; the Scaleway billing API is denied to this key by design (owner: fin-hub, FIN-WP-0004)", + "internal_labor_hours: procurement, credential custody, and two restore drills were performed in this period but no time record was kept (owner: resource-control)" + ] + } + ], + "evidence": [ + "docs/evidence/RESOURCE-WP-0002-T05-restore-2026-08-14.json", + "data/resources/platform-audit-storage.json", + "reef:storage/substrate/object-stores/platform-audit-storage.yaml" + ], + "notes": [ + "First real observation for this resource. It went live mid-period: bucket created 2026-08-14T16:21:56Z, first backup 2026-08-14T18:14:21Z, so 2026-08 covers roughly four hours of operation, not a month.", + "Backup success is 1 of 1 completed backups, not a month of scheduled runs.", + "restore_rto_minutes is the full-restore wall time of 65 seconds; PITR measured the same 65 seconds on the second attempt after the recovery target was corrected.", + "Observed RPO: archive_timeout is 5 minutes and a WAL segment archived about 2 seconds after pg_switch_wal, so the measured RPO is well inside the 5-minute requirement — but on four hours of evidence.", + "Comparing this period against the 12-month decision forecast will show a very large negative stored_gb error. That is a model artifact, not underuse: the forecast assumes 30 days of retained base backups and WAL at steady state, and this period holds one backup." + ] +} diff --git a/data/optimization/platform-audit-storage-2026-08.json b/data/optimization/platform-audit-storage-2026-08.json index 6f53b02..a241fea 100644 --- a/data/optimization/platform-audit-storage-2026-08.json +++ b/data/optimization/platform-audit-storage-2026-08.json @@ -18,8 +18,14 @@ "recurring_external_labor_eur_month": 0, "utilization": { "stored_gb": { - "provisioned": { "value": 457.968, "unit": "GB" }, - "used": { "value": 457.968, "unit": "GB" } + "provisioned": { + "value": 457.968, + "unit": "GB" + }, + "used": { + "value": 457.968, + "unit": "GB" + } } }, "uncertainty": { @@ -31,8 +37,14 @@ ] }, "service_constraints": { - "retention_days": { "value": 30, "unit": "days" }, - "target_rpo_minutes": { "value": 5, "unit": "minutes" } + "retention_days": { + "value": 30, + "unit": "days" + }, + "target_rpo_minutes": { + "value": 5, + "unit": "minutes" + } }, "failure_domains": [ "provider:scaleway", @@ -51,8 +63,14 @@ "recurring_external_labor_eur_month": 0, "utilization": { "stored_gb": { - "provisioned": { "value": 1000, "unit": "GB" }, - "used": { "value": 457.968, "unit": "GB" } + "provisioned": { + "value": 1000, + "unit": "GB" + }, + "used": { + "value": 457.968, + "unit": "GB" + } } }, "uncertainty": { @@ -63,8 +81,14 @@ ] }, "service_constraints": { - "retention_days": { "value": 30, "unit": "days" }, - "target_rpo_minutes": { "value": 5, "unit": "minutes" } + "retention_days": { + "value": 30, + "unit": "days" + }, + "target_rpo_minutes": { + "value": 5, + "unit": "minutes" + } }, "failure_domains": [ "provider:hetzner", @@ -82,8 +106,14 @@ "recurring_external_labor_eur_month": 0, "utilization": { "stored_gb": { - "provisioned": { "value": null, "unit": "GB" }, - "used": { "value": 457.968, "unit": "GB" } + "provisioned": { + "value": null, + "unit": "GB" + }, + "used": { + "value": 457.968, + "unit": "GB" + } } }, "uncertainty": { @@ -94,8 +124,14 @@ ] }, "service_constraints": { - "retention_days": { "value": 30, "unit": "days" }, - "target_rpo_minutes": { "value": 5, "unit": "minutes" } + "retention_days": { + "value": 30, + "unit": "days" + }, + "target_rpo_minutes": { + "value": 5, + "unit": "minutes" + } }, "failure_domains": [ "provider:host-europe" @@ -106,27 +142,35 @@ "storage_eur_per_gb_month, monthly_minimum_eur, egress_eur_per_gb, operations_eur_per_month, support_eur_per_month", "cancellation terms and minimum contract period", "correlated-failure acceptance for same-provider placement (owner: human financial and risk authority)" - ] + ], + "excluded": true, + "exclusion_reason": "No written account confirmation of current S3 orderability, price, or cancellation terms was obtained within the decision window, and platform-pg needed off-host backup. The deciding authority excluded the option rather than delay recovery capability further. It returns to the comparison if Host Europe supplies written commercial terms; RESOURCE-WP-0002-T02 keeps that gap recorded. Note that even fully priced it would place the backup in the same provider:host-europe failure domain as the host it protects." } ], "decision": { - "state": "blocked_on_evidence", - "recommended_option_id": null, - "rationale": "This case exercises the optimization process against real cost-model output; it is not the procurement decision, which RESOURCE-WP-0002-T03 owns and human financial authority approves. Hetzner computes fully and is EUR 29.14 per month more expensive than the Scaleway baseline at month-12 base demand, driven by operator labour rather than storage price. Host Europe cannot be compared at all: its price, cancellation terms, and current orderability are unknown, and it shares a failure domain with the compute host it is meant to protect. No provider is recommended until Host Europe returns written account evidence.", - "approver": null, - "approved_on": null, - "delegated_to": [] + "state": "approved", + "recommended_option_id": "scaleway-standard-multi-az", + "rationale": "Approved 2026-08-14 and executed. Scaleway Standard Multi-AZ in nl-ams is the primary; private bucket railiance-platform-pg-backup exists with versioning on, a 30-day current and noncurrent lifecycle, and owner-only ACL. Hetzner was fully costed and rejected: at month-12 base demand it is EUR 29.14 per month dearer, on operator labour rather than storage price, and its published quota pricing cannot evaluate the high scenario. Host Europe was excluded for lack of written commercial terms, not judged inferior on price. The choice is proven, not just purchased: first backup 48s, full restore 65s with audit_core.events 30=30, PITR 65s to a corrected target, 0 failures (RESOURCE-WP-0002-T05). Accepted risks recorded at decision time: Scaleway has no euro hard-stop, so the founder's email billing alert is the backstop; and the independent second copy required by the correlated-failure rule is still outstanding as RESOURCE-WP-0002-T06.", + "approver": "founder (human financial authority)", + "approved_on": "2026-08-14", + "delegated_to": [ + "railiance-platform", + "rapp-postgres" + ] }, "financial_handoff": { "cost_attribution_key": "platform:audit-storage", - "sent": false, - "reference": null + "sent": true, + "reference": "tools/financial_exchange.py usage data/actuals/2026-08.json resource:platform:audit-storage" }, "outcome": { "feeds_forecast": [ "data/forecasts/platform-audit-storage-scaleway-base-2026-08.json" ], - "actual_refs": [] + "actual_refs": [ + "data/actuals/2026-08.json", + "docs/evidence/RESOURCE-WP-0002-T05-restore-2026-08-14.json" + ] }, "evidence": [ { @@ -152,6 +196,18 @@ "ref": "RESOURCE-WP-0002-T03", "authority": "resource-control", "observed_at": null + }, + { + "kind": "decision", + "ref": "docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md", + "authority": "founder (human financial authority)", + "observed_at": "2026-08-14" + }, + { + "kind": "telemetry", + "ref": "docs/evidence/RESOURCE-WP-0002-T05-restore-2026-08-14.json", + "authority": "reef-railiance CNPG / Scaleway object store", + "observed_at": "2026-08-14" } ] } diff --git a/data/resources/platform-audit-storage.json b/data/resources/platform-audit-storage.json index bd63c3d..21ec359 100644 --- a/data/resources/platform-audit-storage.json +++ b/data/resources/platform-audit-storage.json @@ -32,11 +32,15 @@ "secret:railiance-platform/backup" ], "consumers": { - "potential": ["rapp-postgres"], - "actual": ["rapp-postgres"] + "potential": [ + "rapp-postgres" + ], + "actual": [ + "rapp-postgres" + ] }, "resource_class": "storage", - "status": "ordered", + "status": "active", "management_model": "provider_managed", "provider": { "name": "Scaleway", @@ -53,16 +57,28 @@ "location": { "region": "nl-ams", "country": "NL", - "failure_domains": ["provider:scaleway", "region:nl-ams", "reef:storage"], + "failure_domains": [ + "provider:scaleway", + "region:nl-ams", + "reef:storage" + ], "residency": "European Union" }, "capacity": [ - {"metric": "stored_data", "value": null, "unit": "GB", "kind": "unknown", "observed_at": null} + { + "metric": "stored_data", + "value": null, + "unit": "GB", + "kind": "unknown", + "observed_at": null + } ], "ownership": { "owner": "resource-control", "environment": "production", - "workload_ids": ["rapp-postgres/platform-pg"], + "workload_ids": [ + "rapp-postgres/platform-pg" + ], "tenant_id": null, "allocation": { "mode": "dedicated", @@ -81,7 +97,7 @@ "lifecycle": { "proposed_on": "2026-08-10", "ordered_on": "2026-08-14", - "commissioned_on": null, + "commissioned_on": "2026-08-14", "renews_on": null, "cancel_by": null, "retired_on": null, @@ -89,14 +105,45 @@ }, "relationships": [], "requirements": [ - {"kind": "recovery", "ref": "workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md#acceptance-requirements"}, - {"kind": "retention", "ref": "data/demand/platform-audit-storage.json"} + { + "kind": "recovery", + "ref": "workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md#acceptance-requirements" + }, + { + "kind": "retention", + "ref": "data/demand/platform-audit-storage.json" + } ], "evidence": [ - {"kind": "provider", "ref": "https://www.scaleway.com/en/pricing/storage/", "observed_at": "2026-08-10", "authority": "Scaleway"}, - {"kind": "provider", "ref": "https://www.scaleway.com/en/object-storage/", "observed_at": "2026-08-10", "authority": "Scaleway"}, - {"kind": "decision", "ref": "docs/evidence/RESOURCE-WP-0002-provider-due-diligence-2026-08-10.md", "observed_at": "2026-08-10", "authority": "resource-control"}, - {"kind": "decision", "ref": "docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md", "observed_at": "2026-08-14", "authority": "resource-control"}, - {"kind": "provider", "ref": "reef:storage/substrate/object-stores/platform-audit-storage.yaml", "observed_at": "2026-08-14", "authority": "reef-storage"} + { + "kind": "provider", + "ref": "https://www.scaleway.com/en/pricing/storage/", + "observed_at": "2026-08-10", + "authority": "Scaleway" + }, + { + "kind": "provider", + "ref": "https://www.scaleway.com/en/object-storage/", + "observed_at": "2026-08-10", + "authority": "Scaleway" + }, + { + "kind": "decision", + "ref": "docs/evidence/RESOURCE-WP-0002-provider-due-diligence-2026-08-10.md", + "observed_at": "2026-08-10", + "authority": "resource-control" + }, + { + "kind": "decision", + "ref": "docs/evidence/RESOURCE-WP-0002-primary-selection-2026-08-14.md", + "observed_at": "2026-08-14", + "authority": "resource-control" + }, + { + "kind": "provider", + "ref": "reef:storage/substrate/object-stores/platform-audit-storage.yaml", + "observed_at": "2026-08-14", + "authority": "reef-storage" + } ] } diff --git a/data/thresholds/platform-audit-storage.json b/data/thresholds/platform-audit-storage.json new file mode 100644 index 0000000..e92957e --- /dev/null +++ b/data/thresholds/platform-audit-storage.json @@ -0,0 +1,93 @@ +{ + "schema_version": "0.1", + "resource_id": "resource:platform:audit-storage", + "cost_attribution_key": "platform:audit-storage", + "effective_from": "2026-08-14", + "review": "quarterly, and before any contract or retention change", + "basis": "docs/forecast-actual-control.md error thresholds, plus the operating facts proven in RESOURCE-WP-0002-T05", + "thresholds": [ + { + "id": "budget-variance-infrastructure", + "kind": "budget_variance", + "metric": "infrastructure_eur", + "comparison": "absolute_percentage_error", + "limit": 10, + "action": "investigate before accepting the period; separate usage error from rate, tax, or SKU change" + }, + { + "id": "budget-variance-labor", + "kind": "budget_variance", + "metric": "internal_labor_hours", + "comparison": "absolute_error", + "limit": 1, + "unit": "hours", + "action": "investigate; recurring operator time above forecast is the usual reason a cheap provider is not cheap" + }, + { + "id": "abnormal-growth-stored", + "kind": "abnormal_growth", + "metric": "stored_gb", + "comparison": "absolute_percentage_error", + "limit": 20, + "action": "check retention behaviour, incomplete multipart uploads, versioning, and base-backup frequency before assuming demand grew" + }, + { + "id": "abnormal-growth-wal", + "kind": "abnormal_growth", + "metric": "wal_gb", + "comparison": "absolute_percentage_error", + "limit": 20, + "action": "check for a write-amplifying change in platform-pg before re-forecasting" + }, + { + "id": "unplanned-egress", + "kind": "abnormal_growth", + "metric": "restore_egress_gb", + "comparison": "unplanned", + "limit": 0, + "action": "any egress not explained by a recorded restore drill is investigated as an access or exfiltration question, not only a cost question" + }, + { + "id": "stale-backup", + "kind": "stale_backup", + "metric": "backup_success_pct", + "comparison": "minimum", + "limit": 100, + "action": "a single failed scheduled backup is investigated; backup success is not averaged across a month" + }, + { + "id": "stale-archive", + "kind": "stale_backup", + "metric": "wal_archive_gap_minutes", + "comparison": "maximum", + "limit": 15, + "unit": "minutes", + "action": "three times the 5-minute archive_timeout; beyond this the 5-minute RPO requirement is no longer met", + "measured_baseline": "about 2 seconds after pg_switch_wal, observed 2026-08-14" + }, + { + "id": "restore-rto-regression", + "kind": "stale_backup", + "metric": "restore_rto_minutes", + "comparison": "maximum", + "limit": 3.24, + "unit": "minutes", + "action": "three times the measured 1.08-minute full-restore baseline; a regression is investigated before the next drill is accepted", + "measured_baseline": "65 seconds full restore and 65 seconds PITR, observed 2026-08-14 on 636 MB of PGDATA" + }, + { + "id": "unused-commitment", + "kind": "unused_commitment", + "metric": "commitment_utilization", + "comparison": "minimum", + "limit": null, + "status": "not_applicable", + "action": "Scaleway Multi-AZ is usage-based with no commitment (commitment_ref is null), so there is no unused commitment to detect. This threshold activates only if a term or reserved commitment is ever taken." + } + ], + "known_gaps": [ + "No euro budget ceiling exists for this resource. Scaleway has no hard spend stop and the founder's email alert is the accepted backstop; an authoritative ceiling belongs to fin-hub (FIN-WP-0004).", + "wal_archive_gap_minutes is not yet emitted by any monthly observation; it requires an archive-health metric from rapp-postgres.", + "restore_rto_minutes is populated only in periods that contain a drill; absence is not a pass." + ] +} diff --git a/docs/evidence/RESOURCE-WP-0002-T07-control-loop-2026-08-14.md b/docs/evidence/RESOURCE-WP-0002-T07-control-loop-2026-08-14.md new file mode 100644 index 0000000..7a167f8 --- /dev/null +++ b/docs/evidence/RESOURCE-WP-0002-T07-control-loop-2026-08-14.md @@ -0,0 +1,117 @@ +# RESOURCE-WP-0002-T07 — fin-hub feed and optimization loop — 2026-08-14 + +## Result + +The backup resource is live and proven, so the loop it exists to feed can now +run on real evidence instead of a model. T07 closes the cycle: observation → +thresholds → normalized fin-hub evidence → decided optimization case. + +| Artifact | Purpose | +|---|---| +| `data/actuals/2026-08.json` | First real monthly observation for `platform:audit-storage` | +| `data/thresholds/platform-audit-storage.json` | Budget variance, abnormal growth, stale backup, and unused-commitment thresholds | +| `tools/thresholds.py`, `make thresholds PERIOD=…` | Fail-closed threshold evaluation | +| `tools/financial_exchange.py usage`, `make exchange-usage` | Normalized technical-usage evidence to fin-hub | +| `data/optimization/platform-audit-storage-2026-08.json` | The A/B/C case, now `approved` and executed | +| `data/resources/platform-audit-storage.json` | `ordered` → `active`, commissioned 2026-08-14 | + +## The first period is honest about being four hours long + +The bucket was created 2026-08-14T16:21:56Z and the first backup completed at +18:14:21Z. The 2026-08 row therefore covers about four hours of operation, and +says so. What is measured: + +| Proxy | Value | Source | +|---|---|---| +| `database_gb` | 0.6365 | PGDATA 636 485 632 B | +| `stored_gb` | 0.0066 | 8 objects, 6 604 031 B in `platform-pg/` | +| `backup_success_pct` | 100 | 1 of 1 completed backups | +| `restore_rto_minutes` | 1.08 | 65 s full restore; PITR the same | + +Five proxies are `null`, each with a named owner in `measurement_gaps`: WAL is +not separable from the base backup in the archive, restore egress bytes were +not recorded, no request metric is exposed to the scoped key, no invoice exists, +and no time record was kept for the procurement and drill work. + +`internal_labor_hours` being null deserves a note: real operator hours were +spent this period on procurement, credential custody, and two restore drills. +Recording zero would have been false, and estimating would have been invention. +It is null with the reason attached, and the next period starts a time record. + +## Why August produces no variance + +`make variance ACTUAL=data/actuals/2026-08.json` returns `no-forecast` for the +period. The decision forecast begins at 2026-09, because the resource was not +expected to exist in August. This is the correct answer, not a defect: August +is a commissioning baseline and **2026-09 is the first comparable month**. + +The threshold run reflects the same thing without pretending. Of nine +thresholds: two `within`, one `not_applicable`, six `unmeasured`, zero +breaches. + +- `within` — `stale-backup` (100%) and `restore-rto-regression` (1.08 min against + a 3.24 min limit). These are the two things that were actually proven. +- `not_applicable` — `unused-commitment`. Scaleway Multi-AZ is usage-based with + no commitment, so there is nothing to under-use. Distinguished from passing. +- `unmeasured` — everything requiring an invoice, a request metric, a WAL split, + or a forecast row. + +`unmeasured` is deliberately not `within`. A threshold that silently passes on +absent evidence is worse than no threshold: it reports safety it never checked. + +## Thresholds set + +Budget variance at 10% on infrastructure and 1 hour on labour; abnormal growth +at 20% on stored bytes and WAL; any egress not explained by a recorded drill; +backup success below 100% on any single run, not averaged across a month; WAL +archive gap above 15 minutes, three times the 5-minute `archive_timeout` beyond +which the RPO requirement fails; restore RTO above 3.24 minutes, three times the +measured baseline. + +Two are declared rather than active. `wal_archive_gap_minutes` needs an archive +health metric from rapp-postgres. No euro budget ceiling exists: Scaleway has no +hard spend stop, the founder's email alert is the accepted backstop, and an +authoritative ceiling belongs to fin-hub. + +## Feed to fin-hub + +`financial_exchange.py usage` emits `technical_usage` records under the +contract's resource-control-to-fin-hub direction: resource, service, workload, +attribution key, period bounds, usage proxies, service evidence, and valued +internal labour — with `measurement_gaps` carried through so fin-hub sees why a +field is empty. + +Infrastructure cost is absent by design. fin-hub owns the booked fact; an +uninvoiced period has no amount to send, and a `null` is never normalized to +`0.00`. + +## The optimization case is now decided + +`opt:platform-audit-storage:2026-08` moved from `blocked_on_evidence` to +`approved`, approver "founder (human financial authority)", 2026-08-14. + +The schema gained `excluded` / `exclusion_reason` on an option, because the real +decision exposed a modelling gap: Host Europe never supplied written terms, and +the deciding authority chose to exclude it rather than delay recovery capability +indefinitely. Before this, an unevaluable alternative blocked its case forever, +which would have left the record claiming no decision had been made while the +bucket was in production. An excluded option no longer blocks, keeps its +recorded unknowns, and must state what would bring it back. + +Hetzner remains a fully costed `reject` at €29.14/month dearer. The rationale +records two accepted risks at decision time: no euro hard-stop at Scaleway, and +the independent second copy required by the correlated-failure rule still +outstanding as T06. + +## Boundary held + +No booked cost was originated. The resource record moved to `active` on +operational fact — production backup enabled, `ContinuousArchivingSuccess`, both +restores verified — not on the purchase. Implementation of anything the case +approves is delegated to `railiance-platform` and `rapp-postgres`. + +## Standing dependency + +Total cost stays unknown until Scaleway invoices and fin-hub books it under +`FIN-WP-0004`. The loop runs monthly regardless; 2026-09 is the first period +that will produce real variance. diff --git a/schemas/monthly-resource-observation.schema.json b/schemas/monthly-resource-observation.schema.json index 511f91d..797a774 100644 --- a/schemas/monthly-resource-observation.schema.json +++ b/schemas/monthly-resource-observation.schema.json @@ -4,40 +4,210 @@ "title": "Monthly resource forecast or technical usage observation", "type": "object", "additionalProperties": false, - "required": ["schema_version", "record_type", "workload", "cost_attribution_key", "provider_id", "created_at", "rows"], + "required": [ + "schema_version", + "record_type", + "workload", + "cost_attribution_key", + "provider_id", + "created_at", + "rows" + ], "properties": { - "schema_version": {"const": "0.1"}, - "record_type": {"enum": ["forecast", "usage_observation"]}, - "workload": {"type": "string"}, - "cost_attribution_key": {"type": "string"}, - "provider_id": {"type": "string"}, - "created_at": {"type": "string", "format": "date-time"}, - "scenario": {"type": ["string", "null"]}, - "forecast_ref": {"type": ["string", "null"]}, + "schema_version": { + "enum": [ + "0.1", + "0.2" + ], + "description": "0.2 permits null cost fields for periods the provider has not yet invoiced. Existing 0.1 records stay valid and are never rewritten." + }, + "record_type": { + "enum": [ + "forecast", + "usage_observation" + ] + }, + "workload": { + "type": "string" + }, + "cost_attribution_key": { + "type": "string" + }, + "provider_id": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "scenario": { + "type": [ + "string", + "null" + ] + }, + "forecast_ref": { + "type": [ + "string", + "null" + ] + }, "rows": { "type": "array", "items": { "type": "object", "additionalProperties": false, - "required": ["period", "database_gb", "stored_gb", "wal_gb", "restore_egress_gb", "write_requests", "read_requests", "infrastructure_eur", "internal_labor_hours", "internal_labor_eur", "total_eur"], + "required": [ + "period", + "database_gb", + "stored_gb", + "wal_gb", + "restore_egress_gb", + "write_requests", + "read_requests", + "infrastructure_eur", + "internal_labor_hours", + "internal_labor_eur", + "total_eur" + ], "properties": { - "period": {"type": "string", "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])$"}, - "database_gb": {"type": "number", "minimum": 0}, - "stored_gb": {"type": "number", "minimum": 0}, - "wal_gb": {"type": "number", "minimum": 0}, - "restore_egress_gb": {"type": "number", "minimum": 0}, - "write_requests": {"type": "integer", "minimum": 0}, - "read_requests": {"type": "integer", "minimum": 0}, - "infrastructure_eur": {"type": "number", "minimum": 0}, - "internal_labor_hours": {"type": "number", "minimum": 0}, - "internal_labor_eur": {"type": "number", "minimum": 0}, - "total_eur": {"type": "number", "minimum": 0}, - "backup_success_pct": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, - "restore_rto_minutes": {"type": ["number", "null"], "minimum": 0}, - "max_rpo_minutes": {"type": ["number", "null"], "minimum": 0}, - "evidence": {"type": "array", "items": {"type": "string"}} + "period": { + "type": "string", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])$" + }, + "database_gb": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "Null when the period's evidence does not separate or measure this proxy. Null is unknown, never zero." + }, + "stored_gb": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "Null when the period's evidence does not separate or measure this proxy. Null is unknown, never zero." + }, + "wal_gb": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "Null when the period's evidence does not separate or measure this proxy. Null is unknown, never zero." + }, + "restore_egress_gb": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "Null when the period's evidence does not separate or measure this proxy. Null is unknown, never zero." + }, + "write_requests": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "Null when no request metric is available for the period. Null is unknown, never zero." + }, + "read_requests": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "Null when no request metric is available for the period. Null is unknown, never zero." + }, + "infrastructure_eur": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "EUR, or null when no invoice or valued time record exists for the period. Null is unknown, never zero." + }, + "internal_labor_hours": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "Null when no time record exists for the period. Null is unknown, never zero." + }, + "internal_labor_eur": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "EUR, or null when no invoice or valued time record exists for the period. Null is unknown, never zero." + }, + "total_eur": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "description": "EUR, or null when no invoice or valued time record exists for the period. Null is unknown, never zero." + }, + "backup_success_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "restore_rto_minutes": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "max_rpo_minutes": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "evidence": { + "type": "array", + "items": { + "type": "string" + } + }, + "measurement_gaps": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "description": "Named reasons a proxy is null, each attributable to an owner or a missing source." + } } } + }, + "evidence": { + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true, + "description": "Non-secret references to the provider usage export or invoice, the database and CNPG measurement, the restore drill record, and any internal time record." + }, + "notes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Conditions a reader needs to interpret the row correctly, such as a partial first period." } } } diff --git a/schemas/optimization-case.schema.json b/schemas/optimization-case.schema.json index 3f566b8..fdcb69a 100644 --- a/schemas/optimization-case.schema.json +++ b/schemas/optimization-case.schema.json @@ -20,12 +20,20 @@ "evidence" ], "properties": { - "schema_version": { "const": "0.1" }, + "schema_version": { + "const": "0.1" + }, "record_scope": { "description": "operational cases assert real facts; illustrative cases exist only to exercise the mechanism.", - "enum": ["operational", "illustrative"] + "enum": [ + "operational", + "illustrative" + ] + }, + "case_id": { + "type": "string", + "pattern": "^opt:[a-z0-9][a-z0-9:._-]+$" }, - "case_id": { "type": "string", "pattern": "^opt:[a-z0-9][a-z0-9:._-]+$" }, "case_type": { "enum": [ "rightsizing", @@ -38,42 +46,94 @@ ] }, "trigger": { - "enum": ["cadence", "renewal", "variance", "incident", "request", "procurement"] + "enum": [ + "cadence", + "renewal", + "variance", + "incident", + "request", + "procurement" + ] + }, + "review_period": { + "type": "string", + "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])$" + }, + "created_at": { + "type": "string", + "format": "date-time" }, - "review_period": { "type": "string", "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])$" }, - "created_at": { "type": "string", "format": "date-time" }, "resource_ids": { "type": "array", "minItems": 1, "uniqueItems": true, - "items": { "type": "string", "pattern": "^resource:" } + "items": { + "type": "string", + "pattern": "^resource:" + } + }, + "baseline": { + "$ref": "#/$defs/option" }, - "baseline": { "$ref": "#/$defs/option" }, "alternatives": { "type": "array", "minItems": 1, - "items": { "$ref": "#/$defs/option" } + "items": { + "$ref": "#/$defs/option" + } }, "decision": { "type": "object", "additionalProperties": false, - "required": ["state", "recommended_option_id", "rationale", "approver", "approved_on", "delegated_to"], + "required": [ + "state", + "recommended_option_id", + "rationale", + "approver", + "approved_on", + "delegated_to" + ], "properties": { "state": { "description": "blocked_on_evidence until every decision field of the compared options is known; proposed once the case computes; then approved or rejected by the named authority.", - "enum": ["blocked_on_evidence", "proposed", "approved", "rejected", "superseded"] + "enum": [ + "blocked_on_evidence", + "proposed", + "approved", + "rejected", + "superseded" + ] + }, + "recommended_option_id": { + "type": [ + "string", + "null" + ] + }, + "rationale": { + "type": "string", + "minLength": 1 }, - "recommended_option_id": { "type": ["string", "null"] }, - "rationale": { "type": "string", "minLength": 1 }, "approver": { "description": "Named human or repository authority. Null while the case is not yet decided.", - "type": ["string", "null"] + "type": [ + "string", + "null" + ] + }, + "approved_on": { + "type": [ + "string", + "null" + ], + "format": "date" }, - "approved_on": { "type": ["string", "null"], "format": "date" }, "delegated_to": { "description": "Repositories that own any approved implementation. resource-control never implements.", "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "uniqueItems": true } } @@ -82,27 +142,49 @@ "description": "What is sent to fin-hub when the case is approved.", "type": "object", "additionalProperties": false, - "required": ["cost_attribution_key", "sent"], + "required": [ + "cost_attribution_key", + "sent" + ], "properties": { - "cost_attribution_key": { "type": ["string", "null"] }, - "sent": { "type": "boolean" }, - "reference": { "type": ["string", "null"] } + "cost_attribution_key": { + "type": [ + "string", + "null" + ] + }, + "sent": { + "type": "boolean" + }, + "reference": { + "type": [ + "string", + "null" + ] + } } }, "outcome": { "description": "Closes the loop: which control-cycle records will show whether the case was right.", "type": "object", "additionalProperties": false, - "required": ["feeds_forecast", "actual_refs"], + "required": [ + "feeds_forecast", + "actual_refs" + ], "properties": { "feeds_forecast": { "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "uniqueItems": true }, "actual_refs": { "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "uniqueItems": true } } @@ -110,39 +192,84 @@ "evidence": { "type": "array", "minItems": 1, - "items": { "$ref": "#/$defs/evidence_ref" } + "items": { + "$ref": "#/$defs/evidence_ref" + } } }, "$defs": { "measurement": { "type": "object", "additionalProperties": false, - "required": ["value", "unit"], + "required": [ + "value", + "unit" + ], "properties": { - "value": { "type": ["number", "null"] }, - "unit": { "type": "string", "minLength": 1 } + "value": { + "type": [ + "number", + "null" + ] + }, + "unit": { + "type": "string", + "minLength": 1 + } } }, "utilization": { "type": "object", "additionalProperties": false, - "required": ["provisioned", "used"], + "required": [ + "provisioned", + "used" + ], "properties": { - "provisioned": { "$ref": "#/$defs/measurement" }, - "used": { "$ref": "#/$defs/measurement" } + "provisioned": { + "$ref": "#/$defs/measurement" + }, + "used": { + "$ref": "#/$defs/measurement" + } } }, "evidence_ref": { "type": "object", "additionalProperties": false, - "required": ["kind", "ref", "authority"], + "required": [ + "kind", + "ref", + "authority" + ], "properties": { "kind": { - "enum": ["telemetry", "quote", "contract", "invoice", "forecast", "workplan", "document", "decision"] + "enum": [ + "telemetry", + "quote", + "contract", + "invoice", + "forecast", + "workplan", + "document", + "decision" + ] }, - "ref": { "type": "string", "minLength": 1 }, - "authority": { "type": "string", "minLength": 1 }, - "observed_at": { "type": ["string", "null"], "format": "date" } + "ref": { + "type": "string", + "minLength": 1 + }, + "authority": { + "type": "string", + "minLength": 1 + }, + "observed_at": { + "type": [ + "string", + "null" + ], + "format": "date" + } } }, "option": { @@ -164,41 +291,110 @@ "unknowns" ], "properties": { - "option_id": { "type": "string", "minLength": 1 }, - "label": { "type": "string", "minLength": 1 }, - "one_time_eur": { "type": ["number", "null"], "minimum": 0 }, - "recurring_infrastructure_eur_month": { "type": ["number", "null"], "minimum": 0 }, - "recurring_internal_labor_eur_month": { "type": ["number", "null"], "minimum": 0 }, - "recurring_external_labor_eur_month": { "type": ["number", "null"], "minimum": 0 }, + "option_id": { + "type": "string", + "minLength": 1 + }, + "label": { + "type": "string", + "minLength": 1 + }, + "one_time_eur": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "recurring_infrastructure_eur_month": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "recurring_internal_labor_eur_month": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "recurring_external_labor_eur_month": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, "utilization": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/utilization" } + "additionalProperties": { + "$ref": "#/$defs/utilization" + } }, "uncertainty": { "type": "object", "additionalProperties": false, - "required": ["level", "notes"], + "required": [ + "level", + "notes" + ], "properties": { - "level": { "enum": ["low", "medium", "high"] }, - "notes": { "type": "array", "items": { "type": "string" } } + "level": { + "enum": [ + "low", + "medium", + "high" + ] + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + } } }, "service_constraints": { "type": "object", - "additionalProperties": { "$ref": "#/$defs/measurement" } + "additionalProperties": { + "$ref": "#/$defs/measurement" + } }, "failure_domains": { "type": "array", "minItems": 1, - "items": { "type": "string" }, + "items": { + "type": "string" + }, "uniqueItems": true }, - "exit_path": { "type": ["string", "null"] }, + "exit_path": { + "type": [ + "string", + "null" + ] + }, "unknowns": { "description": "Named blocking evidence gaps, each attributable to an owning repository or authority.", "type": "array", - "items": { "type": "string" }, + "items": { + "type": "string" + }, "uniqueItems": true + }, + "excluded": { + "type": "boolean", + "default": false, + "description": "True when an alternative could not be evaluated and the deciding authority chose to exclude it rather than wait. An excluded option does not block the case, and its exclusion_reason becomes part of the decision record." + }, + "exclusion_reason": { + "type": [ + "string", + "null" + ], + "description": "Required when excluded is true: why the option was set aside, and what would bring it back." } } } diff --git a/tests/test_optimization.py b/tests/test_optimization.py index ab29751..c1c50eb 100644 --- a/tests/test_optimization.py +++ b/tests/test_optimization.py @@ -214,11 +214,13 @@ class RegisteredCasesTest(unittest.TestCase): self.assertIn("opt:platform-audit-storage:2026-08", types) self.assertIn("opt:reef-railiance-k3s:2026-08", types) - def test_storage_case_computes_hetzner_and_blocks_host_europe(self): + def test_storage_case_computes_hetzner_and_excludes_host_europe(self): report = evaluate(self.cases["platform-audit-storage-2026-08"]) verdicts = {c["option_id"]: c["verdict"] for c in report["comparisons"]} self.assertEqual("reject", verdicts["hetzner-object-storage"]) - self.assertEqual("blocked_on_evidence", verdicts["host-europe-cloud-storage"]) + # Host Europe never supplied written terms and was excluded at decision + # time rather than left blocking the case indefinitely. + self.assertEqual("excluded", verdicts["host-europe-cloud-storage"]) hetzner = next(c for c in report["comparisons"] if c["option_id"] == "hetzner-object-storage") self.assertEqual(29.14, hetzner["monthly_delta_eur"]) @@ -244,5 +246,69 @@ class RegisteredCasesTest(unittest.TestCase): self.assertEqual(before, record, name) +class ExcludedOptionTest(unittest.TestCase): + """An alternative the deciding authority set aside must not block forever.""" + + def test_excluded_option_does_not_block_the_case(self): + alternative = option("alt", one_time_eur=None, excluded=True, + exclusion_reason="no written terms within the decision window") + result = compare_option(option("baseline"), alternative) + self.assertEqual("excluded", result["verdict"]) + self.assertEqual([], result["blocking_evidence"]) + + def test_exclusion_reason_travels_with_the_verdict(self): + alternative = option("alt", excluded=True, exclusion_reason="supplier never replied") + self.assertEqual("supplier never replied", compare_option(option("baseline"), alternative)["exclusion_reason"]) + + def test_excluding_without_a_reason_is_rejected(self): + record = case(alternatives=[option("alt", one_time_eur=None, excluded=True)]) + with self.assertRaisesRegex(ValueError, "exclusion_reason"): + validate_case(record) + + def test_case_with_only_excluded_alternatives_can_be_approved(self): + record = case(alternatives=[option("alt", one_time_eur=None, excluded=True, + exclusion_reason="unevaluable")]) + record["decision"].update({"state": "approved", "recommended_option_id": "baseline", + "approver": "founder", "approved_on": "2026-08-14"}) + validate_case(record) + + def test_excluded_option_is_never_recommended(self): + record = case(alternatives=[option("alt", recurring_infrastructure_eur_month=0, excluded=True, + exclusion_reason="unevaluable")]) + record["decision"].update({"state": "approved", "recommended_option_id": "baseline", + "approver": "founder", "approved_on": "2026-08-14"}) + self.assertIsNone(evaluate(record)["best_option_id"]) + + +class ApprovedStorageCaseTest(unittest.TestCase): + """The backup decision was made and executed on 2026-08-14.""" + + def setUp(self): + path = CASE_DIR / "platform-audit-storage-2026-08.json" + self.case = json.loads(path.read_text()) + + def test_case_is_approved_by_a_named_authority(self): + decision = self.case["decision"] + self.assertEqual("approved", decision["state"]) + self.assertEqual("scaleway-standard-multi-az", decision["recommended_option_id"]) + self.assertEqual("2026-08-14", decision["approved_on"]) + self.assertTrue(decision["approver"]) + + def test_host_europe_is_excluded_with_a_reason_not_silently_dropped(self): + host_europe = next(a for a in self.case["alternatives"] if a["option_id"] == "host-europe-cloud-storage") + self.assertTrue(host_europe["excluded"]) + self.assertIn("RESOURCE-WP-0002-T02", host_europe["exclusion_reason"]) + self.assertTrue(host_europe["unknowns"], "the unknowns stay recorded after exclusion") + + def test_hetzner_stays_a_costed_rejection(self): + verdicts = {c["option_id"]: c["verdict"] for c in evaluate(self.case)["comparisons"]} + self.assertEqual("reject", verdicts["hetzner-object-storage"]) + self.assertEqual("excluded", verdicts["host-europe-cloud-storage"]) + + def test_outcome_points_at_the_real_actuals(self): + self.assertIn("data/actuals/2026-08.json", self.case["outcome"]["actual_refs"]) + self.assertTrue(self.case["financial_handoff"]["sent"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py index a4c0ae4..57ca423 100644 --- a/tests/test_portfolio.py +++ b/tests/test_portfolio.py @@ -113,7 +113,10 @@ class ReefViewTest(unittest.TestCase): [row["resource_id"] for row in view["resources"]], ) self.assertEqual(["rapp-postgres"], view["consumers_potential"]) - self.assertEqual([], view["consumers_actual"]) + # rapp-postgres became an actual consumer on 2026-08-14: continuous WAL + # archiving and a proven restore make it a realised consumer, not a + # potential one. + self.assertEqual(["rapp-postgres"], view["consumers_actual"]) if __name__ == "__main__": diff --git a/tests/test_portfolio_report.py b/tests/test_portfolio_report.py index b7d8d2d..f2fb2d3 100644 --- a/tests/test_portfolio_report.py +++ b/tests/test_portfolio_report.py @@ -146,9 +146,15 @@ class LivePortfolioTest(unittest.TestCase): def test_missing_contract_dates_are_surfaced_rather_than_read_as_no_commitments(self): self.assertTrue(self.report["renewals"]["undated"]) - def test_open_optimization_cases_are_listed_with_their_blockers(self): - self.assertEqual(2, len(self.report["optimization"]["cases"])) - self.assertEqual(2, len(self.report["optimization"]["undecided"])) + def test_optimization_cases_are_listed_with_their_state(self): + cases = self.report["optimization"]["cases"] + self.assertEqual(2, len(cases)) + states = {c["case_id"]: c["state"] for c in cases} + # The storage case was decided and executed on 2026-08-14; the cluster + # rightsizing case is still blocked on delegated evidence. + self.assertEqual("approved", states["opt:platform-audit-storage:2026-08"]) + self.assertEqual("blocked_on_evidence", states["opt:reef-railiance-k3s:2026-08"]) + self.assertEqual(["opt:reef-railiance-k3s:2026-08"], self.report["optimization"]["undecided"]) self.assertTrue(self.report["optimization"]["blocked_on_evidence"]) def test_next_actions_name_an_owning_repository_for_each_gap(self): diff --git a/tests/test_thresholds.py b/tests/test_thresholds.py new file mode 100644 index 0000000..3ace625 --- /dev/null +++ b/tests/test_thresholds.py @@ -0,0 +1,128 @@ +import json +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parents[1] / "tools")) +from thresholds import evaluate, evaluate_threshold + +ROOT = Path(__file__).parents[1] +CONFIG = json.loads((ROOT / "data" / "thresholds" / "platform-audit-storage.json").read_text()) +OBSERVATION = json.loads((ROOT / "data" / "actuals" / "2026-08.json").read_text()) +FORECAST = json.loads( + (ROOT / "data" / "forecasts" / "platform-audit-storage-scaleway-base-2026-08.json").read_text() +) + + +def threshold(**overrides): + result = { + "id": "t", "kind": "budget_variance", "metric": "infrastructure_eur", + "comparison": "absolute_percentage_error", "limit": 10, "action": "investigate", + } + result.update(overrides) + return result + + +def observed(**overrides): + result = {"period": "2026-09", "infrastructure_eur": 3.0, "backup_success_pct": 100, "measurement_gaps": []} + result.update(overrides) + return result + + +class ThresholdVerdictTest(unittest.TestCase): + def test_variance_within_limit_passes(self): + compared = {"infrastructure_eur": {"absolute_percentage_error": 4.0, "error": 0.11}} + self.assertEqual("within", evaluate_threshold(threshold(), observed(), compared)["verdict"]) + + def test_variance_above_limit_breaches(self): + compared = {"infrastructure_eur": {"absolute_percentage_error": 42.0, "error": 1.2}} + result = evaluate_threshold(threshold(), observed(), compared) + self.assertEqual("breach", result["verdict"]) + self.assertEqual(42.0, result["measured"]) + + def test_absolute_error_comparison_uses_magnitude_not_sign(self): + spec = threshold(metric="internal_labor_hours", comparison="absolute_error", limit=1) + compared = {"internal_labor_hours": {"error": -2.5, "absolute_percentage_error": 250.0}} + result = evaluate_threshold(spec, observed(internal_labor_hours=1.5), compared) + self.assertEqual("breach", result["verdict"]) + self.assertEqual(2.5, result["measured"]) + + def test_minimum_comparison_breaches_below_the_limit(self): + spec = threshold(id="backup", metric="backup_success_pct", comparison="minimum", limit=100) + self.assertEqual("breach", evaluate_threshold(spec, observed(backup_success_pct=99), None)["verdict"]) + self.assertEqual("within", evaluate_threshold(spec, observed(), None)["verdict"]) + + def test_maximum_comparison_breaches_above_the_limit(self): + spec = threshold(id="rto", metric="restore_rto_minutes", comparison="maximum", limit=3.24) + self.assertEqual("breach", evaluate_threshold(spec, observed(restore_rto_minutes=9.0), None)["verdict"]) + self.assertEqual("within", evaluate_threshold(spec, observed(restore_rto_minutes=1.08), None)["verdict"]) + + +class FailClosedTest(unittest.TestCase): + """An unmeasured value must never pass a threshold.""" + + def test_missing_value_is_unmeasured_not_within(self): + result = evaluate_threshold(threshold(), observed(infrastructure_eur=None), None) + self.assertEqual("unmeasured", result["verdict"]) + + def test_unmeasured_result_carries_the_named_gap(self): + row = observed( + infrastructure_eur=None, + measurement_gaps=["infrastructure_eur: no invoice yet (owner: fin-hub)"], + ) + result = evaluate_threshold(threshold(), row, None) + self.assertIn("owner: fin-hub", result["detail"]) + + def test_missing_forecast_row_is_unmeasured_not_within(self): + result = evaluate_threshold(threshold(), observed(), None) + self.assertEqual("unmeasured", result["verdict"]) + self.assertIn("no forecast row", result["detail"]) + + def test_unknown_variance_entry_is_unmeasured(self): + compared = {"infrastructure_eur": {"forecast": 2.89, "actual": None, "status": "unknown"}} + self.assertEqual("unmeasured", evaluate_threshold(threshold(), observed(), compared)["verdict"]) + + def test_zero_forecast_does_not_pass_on_undefined_percentage(self): + compared = {"infrastructure_eur": {"absolute_percentage_error": None, "error": 3.0}} + result = evaluate_threshold(threshold(), observed(), compared) + self.assertEqual("unmeasured", result["verdict"]) + self.assertIn("percentage error is undefined", result["detail"]) + + def test_not_applicable_threshold_is_distinct_from_passing(self): + spec = threshold(id="commitment", metric="commitment_utilization", comparison="minimum", + limit=None, status="not_applicable") + self.assertEqual("not_applicable", evaluate_threshold(spec, observed(), None)["verdict"]) + + +class AugustObservationTest(unittest.TestCase): + """The real first period: live for four hours, no invoice, no request metrics.""" + + @classmethod + def setUpClass(cls): + cls.report = evaluate(CONFIG, OBSERVATION, FORECAST, "2026-08") + + def test_no_breach_and_no_false_pass(self): + self.assertEqual([], self.report["breaches"]) + self.assertEqual(2, self.report["summary"]["within"]) + self.assertEqual(6, self.report["summary"]["unmeasured"]) + self.assertEqual(1, self.report["summary"]["not_applicable"]) + + def test_august_has_no_forecast_so_variance_thresholds_cannot_pass(self): + self.assertFalse(self.report["forecast_available"]) + self.assertIn("abnormal-growth-stored", self.report["unmeasured"]) + + def test_what_was_actually_proven_reads_as_within(self): + within = [r["id"] for r in self.report["results"] if r["verdict"] == "within"] + self.assertEqual(["stale-backup", "restore-rto-regression"], within) + + def test_commitment_threshold_is_not_applicable_without_a_commitment(self): + commitment = next(r for r in self.report["results"] if r["id"] == "unused-commitment") + self.assertEqual("not_applicable", commitment["verdict"]) + + def test_missing_period_is_an_error_not_an_empty_pass(self): + with self.assertRaises(ValueError): + evaluate(CONFIG, OBSERVATION, FORECAST, "2026-07") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/financial_exchange.py b/tools/financial_exchange.py index 0609346..23e31d0 100644 --- a/tools/financial_exchange.py +++ b/tools/financial_exchange.py @@ -76,6 +76,67 @@ def forecast_records(payload: dict, *, resource_id: str, source_ref: str) -> lis return records +def usage_records(payload: dict, *, resource_id: str, source_ref: str) -> list[dict]: + """Normalize a monthly technical observation for the fin-hub exchange. + + Only the resource-control-to-fin-hub direction: technical usage and valued + internal labour. Infrastructure cost is deliberately absent — fin-hub owns + the booked fact, and an uninvoiced period carries no amount to send. A null + stays null; it is never sent as 0.00. + """ + if payload.get("schema_version") not in {"0.1", "0.2"}: + raise ValueError("expected a resource-control observation v0.1 or v0.2") + if payload.get("record_type") != "usage_observation": + raise ValueError("expected record_type usage_observation") + if not resource_id.startswith("resource:"): + raise ValueError("resource_id must use the resource: namespace") + + records = [] + for row in payload["rows"]: + year, month = (int(part) for part in row["period"].split("-")) + period_start = date(year, month, 1) + period_end = date(year, month, monthrange(year, month)[1]) + labor = row["internal_labor_eur"] + records.append({ + "schema_version": "0.1", + "record_type": "technical_usage", + "record_id": ( + f"usage:{payload['provider_id']}:{payload['cost_attribution_key']}:{row['period']}" + ), + "resource_id": resource_id, + "service_id": payload["provider_id"], + "workload_id": payload["workload"], + "tenant_id": None, + "environment": "production", + "cost_attribution_key": payload["cost_attribution_key"], + "period_start": period_start.isoformat(), + "period_end": period_end.isoformat(), + "currency": "EUR", + "created_at": payload["created_at"], + "source_evidence": [source_ref, *payload.get("evidence", [])], + "usage": { + key: row[key] + for key in ( + "database_gb", "stored_gb", "wal_gb", "restore_egress_gb", + "write_requests", "read_requests", + ) + }, + "service_evidence": { + key: row.get(key) + for key in ("backup_success_pct", "restore_rto_minutes") + }, + "costs": { + # resource-control originates internal labour only. Booked + # infrastructure cost is fin-hub's to record, not ours to echo. + "internal_labor": None if labor is None else money_text(labor), + "internal_labor_hours": row["internal_labor_hours"], + }, + "measurement_gaps": row.get("measurement_gaps", []), + "notes": payload.get("notes", []), + }) + return records + + def validate_booked_cost(record: dict) -> dict: required = { "schema_version", "record_type", "financial_fact_id", "adjustment_kind", @@ -126,13 +187,19 @@ def reconcile(forecasts: list[dict], booked_costs: list[dict]) -> list[dict]: def main() -> int: - if len(sys.argv) < 3 or sys.argv[1] not in {"forecast", "reconcile"}: - print(f"usage: {sys.argv[0]} forecast FORECAST.json [RESOURCE_ID] | reconcile FORECAST_EVIDENCE.json BOOKED_COST.json", file=sys.stderr) + if len(sys.argv) < 3 or sys.argv[1] not in {"forecast", "usage", "reconcile"}: + print( + f"usage: {sys.argv[0]} forecast FORECAST.json [RESOURCE_ID]" + f" | usage OBSERVATION.json [RESOURCE_ID]" + f" | reconcile FORECAST_EVIDENCE.json BOOKED_COST.json", + file=sys.stderr, + ) return 2 - if sys.argv[1] == "forecast": + if sys.argv[1] in {"forecast", "usage"}: payload = json.loads(Path(sys.argv[2]).read_text()) resource_id = sys.argv[3] if len(sys.argv) > 3 else "resource:platform_audit_storage" - print(json.dumps(forecast_records(payload, resource_id=resource_id, source_ref=sys.argv[2]), indent=2)) + build = forecast_records if sys.argv[1] == "forecast" else usage_records + print(json.dumps(build(payload, resource_id=resource_id, source_ref=sys.argv[2]), indent=2)) return 0 forecasts = json.loads(Path(sys.argv[2]).read_text()) booked = json.loads(Path(sys.argv[3]).read_text()) diff --git a/tools/optimization.py b/tools/optimization.py index 7514103..bea460d 100644 --- a/tools/optimization.py +++ b/tools/optimization.py @@ -64,6 +64,18 @@ def utilization_ratios(option: dict) -> dict: def compare_option(baseline: dict, alternative: dict) -> dict: """Compare one alternative against the baseline on the decision fields.""" + if alternative.get("excluded"): + # The deciding authority set this option aside rather than wait for + # evidence that was not going to arrive. It no longer blocks the case, + # and the reason travels with the decision. + return { + "option_id": alternative["option_id"], + "label": alternative["label"], + "verdict": "excluded", + "exclusion_reason": alternative.get("exclusion_reason"), + "blocking_evidence": [], + } + blocking = sorted(set( [f"baseline.{field}" for field in missing_fields(baseline)] + [f"{alternative['option_id']}.{field}" for field in missing_fields(alternative)] @@ -150,6 +162,12 @@ def validate_case(case: dict) -> None: if len(set(option_ids)) != len(option_ids): raise ValueError("option identifiers must be unique within a case") + for alternative in case["alternatives"]: + if alternative.get("excluded") and not alternative.get("exclusion_reason"): + raise ValueError( + f"excluded option {alternative['option_id']} must record an exclusion_reason" + ) + report = evaluate(case) blocked = any(r["verdict"] == "blocked_on_evidence" for r in report["comparisons"]) diff --git a/tools/thresholds.py b/tools/thresholds.py new file mode 100644 index 0000000..f90cafb --- /dev/null +++ b/tools/thresholds.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Evaluate a monthly observation against a resource's declared thresholds. + +Fail-closed in both directions. An unknown measurement never passes a +threshold — a missing number is reported as `unmeasured`, because a threshold +that silently passes on absent evidence is worse than no threshold at all. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import variance + +# Verdicts, in the order a reviewer should read them. +BREACH = "breach" +UNMEASURED = "unmeasured" +NOT_APPLICABLE = "not_applicable" +WITHIN = "within" + + +def _row_for(payload: dict, period: str) -> dict | None: + for row in payload["rows"]: + if row["period"] == period: + return row + return None + + +def evaluate_threshold(threshold: dict, observed: dict, compared: dict | None) -> dict: + metric = threshold["metric"] + result = { + "id": threshold["id"], + "kind": threshold["kind"], + "metric": metric, + "limit": threshold["limit"], + "action": threshold["action"], + } + + if threshold.get("status") == "not_applicable": + return {**result, "verdict": NOT_APPLICABLE, "detail": threshold["action"]} + + value = observed.get(metric) + if value is None: + gaps = [gap for gap in observed.get("measurement_gaps", []) if gap.startswith(f"{metric}:")] + return { + **result, + "verdict": UNMEASURED, + "detail": gaps[0] if gaps else f"{metric} is not present in the observation", + } + + comparison = threshold["comparison"] + if comparison == "minimum": + breached = value < threshold["limit"] + return {**result, "verdict": BREACH if breached else WITHIN, "observed": value} + if comparison == "maximum": + breached = value > threshold["limit"] + return {**result, "verdict": BREACH if breached else WITHIN, "observed": value} + if comparison == "unplanned": + breached = value > threshold["limit"] + return {**result, "verdict": BREACH if breached else WITHIN, "observed": value} + + # The remaining comparisons need a forecast to compare against. + if compared is None or metric not in compared: + return { + **result, + "verdict": UNMEASURED, + "observed": value, + "detail": "no forecast row exists for this period, so no variance can be computed", + } + entry = compared[metric] + if entry.get("status") == "unknown": + return {**result, "verdict": UNMEASURED, "observed": value, "detail": "forecast or actual amount is unknown"} + if comparison == "absolute_percentage_error": + measured = entry["absolute_percentage_error"] + if measured is None: + return { + **result, + "verdict": UNMEASURED, + "observed": value, + "detail": "forecast is zero, so percentage error is undefined; review the absolute error instead", + } + else: + measured = abs(entry["error"]) + return { + **result, + "verdict": BREACH if measured > threshold["limit"] else WITHIN, + "observed": value, + "measured": measured, + } + + +def evaluate(config: dict, observation: dict, forecast: dict | None, period: str) -> dict: + observed = _row_for(observation, period) + if observed is None: + raise ValueError(f"observation has no row for {period}") + + compared = None + if forecast is not None: + report = variance.compare(forecast, observation) + for row in report["rows"]: + if row["period"] == period and row["status"] == "compared": + compared = row["metrics"] + + results = [evaluate_threshold(t, observed, compared) for t in config["thresholds"]] + counts: dict[str, int] = {} + for result in results: + counts[result["verdict"]] = counts.get(result["verdict"], 0) + 1 + return { + "resource_id": config["resource_id"], + "period": period, + "forecast_available": compared is not None, + "results": results, + "summary": dict(sorted(counts.items())), + "breaches": [r["id"] for r in results if r["verdict"] == BREACH], + "unmeasured": [r["id"] for r in results if r["verdict"] == UNMEASURED], + "known_gaps": config.get("known_gaps", []), + } + + +def main() -> int: + if len(sys.argv) < 4: + print(f"usage: {sys.argv[0]} THRESHOLDS.json OBSERVATION.json PERIOD [FORECAST.json]", file=sys.stderr) + return 2 + config = json.loads(Path(sys.argv[1]).read_text()) + observation = json.loads(Path(sys.argv[2]).read_text()) + period = sys.argv[3] + forecast = json.loads(Path(sys.argv[4]).read_text()) if len(sys.argv) > 4 else None + report = evaluate(config, observation, forecast, period) + print(json.dumps(report, indent=2)) + # A breach is a non-zero exit so the monthly cadence can gate on it. + return 1 if report["breaches"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/variance.py b/tools/variance.py index 809f56a..197e45d 100644 --- a/tools/variance.py +++ b/tools/variance.py @@ -26,6 +26,16 @@ def compare(forecast: dict, actual: dict) -> dict: for metric in METRICS: planned = expected[period][metric] measured = observed[metric] + # An uninvoiced period has no cost to compare against. Treating a + # missing amount as zero would report a 100% favourable variance. + if planned is None or measured is None: + metrics[metric] = { + "forecast": planned, + "actual": measured, + "status": "unknown", + "category": "data_quality", + } + continue error = measured - planned metrics[metric] = { "forecast": planned, diff --git a/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md b/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md index e3cb79b..5330655 100644 --- a/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md +++ b/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md @@ -323,7 +323,7 @@ Evidence `docs/evidence/RESOURCE-WP-0002-T06-logical-offsite-2026-08-14.md`. ```task id: RESOURCE-WP-0002-T07 -status: todo +status: done priority: medium state_hub_task_id: "e6c848ce-ea4b-4ac6-99da-5890de242b50" ``` @@ -340,3 +340,23 @@ contract timing, and credential revocation—not only a cheaper unit price. Done when fin-hub receives normalized evidence and resource-control can identify rightsizing or provider-switch opportunities from observed utilization and total cost. + +Done 2026-08-14: the loop runs on real evidence. First observation +`data/actuals/2026-08.json` records database 0.6365 GB, stored 0.0066 GB over +8 objects, backup success 1/1, restore RTO 1.08 min, and five null proxies each +with a named owner. Thresholds are declared in +`data/thresholds/platform-audit-storage.json` and evaluated fail-closed by +`make thresholds PERIOD=2026-08`: two within, one not applicable, six +unmeasured, zero breaches — `unmeasured` never counts as passing. +`make exchange-usage` emits `technical_usage` records to fin-hub with +measurement gaps carried through and no infrastructure amount, since fin-hub +owns the booked fact. The A/B/C case +`opt:platform-audit-storage:2026-08` is now `approved` by the founder with +Hetzner a costed reject and Host Europe excluded for want of written terms. +The resource moved `ordered` → `active`, commissioned 2026-08-14. Evidence: +`docs/evidence/RESOURCE-WP-0002-T07-control-loop-2026-08-14.md`. + +August produces no variance and should not: the decision forecast starts at +2026-09, so August is a commissioning baseline and 2026-09 is the first +comparable month. Total cost stays unknown until Scaleway invoices and fin-hub +books it under `FIN-WP-0004`.