From 34a014a896ff42d92859370ad52e4a7cab1ab1be Mon Sep 17 00:00:00 2001 From: tegwick Date: Sat, 15 Aug 2026 02:40:43 +0200 Subject: [PATCH] feat: inventory schema 0.3 and close WP-0006 T01 T02 T04 v0.2 records stay valid. v0.3 requires the five facets. Validators reject inline Scaleway endpoints and secret-looking strings. The backup record is the first 0.3 object. Reef views already met T02. --- data/resources/platform-audit-storage.json | 2 +- docs/operational-reference-convention.md | 2 +- schemas/resource-inventory.schema.json | 16 ++++- tests/test_portfolio.py | 31 ++++++++++ tools/portfolio.py | 62 ++++++++++++++++++- tools/validate.py | 11 +++- ...WP-0002-procure-postgres-backup-storage.md | 6 ++ ...E-WP-0006-resource-object-and-reef-refs.md | 22 +++++-- 8 files changed, 140 insertions(+), 12 deletions(-) diff --git a/data/resources/platform-audit-storage.json b/data/resources/platform-audit-storage.json index 21ec359..abaa3c3 100644 --- a/data/resources/platform-audit-storage.json +++ b/data/resources/platform-audit-storage.json @@ -1,5 +1,5 @@ { - "schema_version": "0.2", + "schema_version": "0.3", "record_scope": "inventory", "id": "resource:platform:audit-storage", "financial_entity_id": "entity:railiance", diff --git a/docs/operational-reference-convention.md b/docs/operational-reference-convention.md index 4ae1be1..c6f8017 100644 --- a/docs/operational-reference-convention.md +++ b/docs/operational-reference-convention.md @@ -45,7 +45,7 @@ reef:/# - Fragment is a dotted key into that file (`endpoint`, `bucket.prefix`). - The file is committed, non-secret, and the reef is authoritative for it. -Examples (illustrative until `reef-storage` exists): +Examples (live on `reef-storage` and `reef-railiance`): ```text reef:storage/substrate/object-stores/platform-audit-storage.yaml#endpoint diff --git a/schemas/resource-inventory.schema.json b/schemas/resource-inventory.schema.json index 33b16cb..c9a7f11 100644 --- a/schemas/resource-inventory.schema.json +++ b/schemas/resource-inventory.schema.json @@ -1,12 +1,24 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://coulomb.social/resource-control/resource-inventory.schema.json", - "title": "Railiance managed-infrastructure portfolio resource v0.2", + "title": "Railiance managed-infrastructure portfolio resource v0.2/v0.3", "type": "object", "additionalProperties": false, "required": ["schema_version", "record_scope", "id", "resource_class", "status", "management_model", "provider", "service", "location", "capacity", "ownership", "cost", "lifecycle", "relationships", "requirements", "evidence"], + "if": { + "properties": {"schema_version": {"const": "0.3"}} + }, + "then": { + "required": [ + "schema_version", "record_scope", "id", "resource_class", "status", + "management_model", "provider", "service", "location", "capacity", + "ownership", "cost", "lifecycle", "relationships", "requirements", + "evidence", "description", "decision", "operational_refs", + "credential_handles", "consumers" + ] + }, "properties": { - "schema_version": {"const": "0.2"}, + "schema_version": {"enum": ["0.2", "0.3"]}, "record_scope": {"enum": ["inventory", "example"]}, "id": {"type": "string", "pattern": "^resource:[a-z0-9][a-z0-9:_-]+$"}, "financial_entity_id": {"type": ["string", "null"], "pattern": "^entity:[a-z0-9]+$"}, diff --git a/tests/test_portfolio.py b/tests/test_portfolio.py index 57ca423..877033a 100644 --- a/tests/test_portfolio.py +++ b/tests/test_portfolio.py @@ -47,6 +47,37 @@ class PortfolioTest(unittest.TestCase): record["decision"]["approved_on"] = "2026-08-14" validate_record(record) + def test_v03_requires_five_facets(self): + record = deepcopy(next( + r for _, r in self.records() if r["id"] == "resource:platform:audit-storage" + )) + self.assertEqual(record["schema_version"], "0.3") + del record["consumers"] + with self.assertRaisesRegex(ValueError, "five facets"): + validate_record(record) + + def test_inline_endpoint_is_rejected(self): + record = deepcopy(next( + r for _, r in self.records() if r["id"] == "resource:platform:audit-storage" + )) + record["description"] += " https://s3.nl-ams.scw.cloud" + with self.assertRaisesRegex(ValueError, "inline provider endpoint"): + validate_record(record) + + def test_inline_secret_is_rejected(self): + record = deepcopy(next( + r for _, r in self.records() if r["id"] == "resource:platform:audit-storage" + )) + record["description"] += " AGE-SECRET-KEY-1TEST" + with self.assertRaisesRegex(ValueError, "inline secret"): + validate_record(record) + + def test_v02_records_remain_valid(self): + v02 = [r for _, r in self.records() if r.get("schema_version") == "0.2"] + self.assertGreaterEqual(len(v02), 1) + for record in v02: + validate_record(record) + def test_unknown_commission_date_is_preserved(self): record = deepcopy(next(r for _, r in self.records() if r["status"] == "active")) record["lifecycle"]["commissioned_on"] = None diff --git a/tools/portfolio.py b/tools/portfolio.py index e9e39bf..cd5174e 100644 --- a/tools/portfolio.py +++ b/tools/portfolio.py @@ -12,6 +12,24 @@ from pathlib import Path from entities import association_ok RESOURCE_ID = re.compile(r"^resource:[a-z0-9][a-z0-9:_-]+$") +INLINE_ENDPOINT = re.compile( + r"(https?://\S*s3\S*|https?://\S+\.scw\.cloud|\bs3://)", + re.IGNORECASE, +) +INLINE_SECRET = re.compile( + r"(BEGIN [A-Z ]*PRIVATE KEY|AGE-SECRET-KEY-|AKIA[0-9A-Z]{16}|sk_live_|SCW[A-Z0-9]{17})" +) +V03_FACETS = ( + "description", "decision", "operational_refs", "credential_handles", "consumers", +) +# Evidence and decision refs may cite provider HTTPS docs. Those are not +# operating attributes and must not be treated as inline endpoints. +SKIP_INLINE_PATHS = { + ("evidence", "ref"), + ("decision", "ref"), + ("requirements", "ref"), + ("cost", "price_evidence"), +} STATUSES = { "proposed", "ordered", "commissioning", "active", "suspended", "retiring", "retired", "rejected", @@ -39,9 +57,27 @@ def validate_transition(current: str, target: str) -> None: raise ValueError(f"invalid lifecycle transition {current} -> {target}") +def _walk_strings(node, path=()): + if isinstance(node, dict): + for key, value in node.items(): + yield from _walk_strings(value, path + (key,)) + elif isinstance(node, list): + for value in node: + yield from _walk_strings(value, path) + elif isinstance(node, str): + yield path, node + + +def _skip_inline(path: tuple) -> bool: + if path and path[0] in {"operational_refs", "credential_handles"}: + return True + return len(path) >= 2 and (path[0], path[-1]) in SKIP_INLINE_PATHS + + def validate_record(record: dict) -> None: - if record.get("schema_version") != "0.2": - raise ValueError("portfolio records must use schema_version 0.2") + version = record.get("schema_version") + if version not in {"0.2", "0.3"}: + raise ValueError("portfolio records must use schema_version 0.2 or 0.3") if not RESOURCE_ID.fullmatch(record.get("id", "")): raise ValueError("invalid resource id") if record.get("status") not in STATUSES: @@ -51,6 +87,14 @@ def validate_record(record: dict) -> None: association_ok(record) + if version == "0.3": + missing = [name for name in V03_FACETS if name not in record] + if missing: + raise ValueError("schema 0.3 requires five facets: " + ",".join(missing)) + consumers = record.get("consumers") or {} + if "potential" not in consumers or "actual" not in consumers: + raise ValueError("schema 0.3 consumers must list potential and actual") + decision = record.get("decision") if record.get("status") in {"ordered", "commissioning"} and ( not decision or decision.get("status") != "approved" @@ -63,6 +107,20 @@ def validate_record(record: dict) -> None: if not str(ref).startswith("secret:"): raise ValueError(f"credential_handles must be secret: references: {ref}") + for path, value in _walk_strings(record): + if _skip_inline(path): + continue + if INLINE_ENDPOINT.search(value): + raise ValueError( + "inline provider endpoint must be a reef: reference, not " + + ".".join(str(part) for part in path) + ) + if INLINE_SECRET.search(value): + raise ValueError( + "inline secret material is forbidden; use a secret: handle at " + + ".".join(str(part) for part in path) + ) + allocation = record["ownership"]["allocation"] if allocation["mode"] == "unattributed": if allocation["cost_attribution_key"] is not None: diff --git a/tools/validate.py b/tools/validate.py index 2c9c937..f5e2df0 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -41,11 +41,16 @@ def main() -> int: assert forecast["record_type"] == "forecast" assert len({row["period"] for row in forecast["rows"]}) == len(forecast["rows"]) assert all(row["total_eur"] == round(row["infrastructure_eur"] + row["internal_labor_eur"], 2) for row in forecast["rows"]) - assert schema["properties"]["schema_version"]["const"] == "0.2" + assert set(schema["properties"]["schema_version"]["enum"]) == {"0.2", "0.3"} required = set(schema["required"]) + v03 = {"description", "decision", "operational_refs", "credential_handles", "consumers"} for resource in resources: - assert not required - resource.keys(), f"missing fields: {required - resource.keys()}" + need = set(required) + if resource.get("schema_version") == "0.3": + need |= v03 + assert not need - resource.keys(), f"missing fields: {need - resource.keys()}" validate_record(resource) + assert any(resource.get("schema_version") == "0.3" for resource in resources) operational_records = [load(str(path)) for path in Path("data/control-cycle").glob("*.json")] control_records += operational_records record_ids = {record["record_id"] for record in control_records} @@ -126,6 +131,8 @@ def main() -> int: if view["reef_id"] == "reef-railiance": assert view["role"] == "compute_substrate" assert any("reef-storage" in note for note in view.get("notes") or []) + if view["reef_id"] == "reef-storage": + assert view["role"] == "storage_substrate" print("resource-control declarations: valid") return 0 diff --git a/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md b/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md index a35f365..32c32ad 100644 --- a/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md +++ b/workplans/RESOURCE-WP-0002-procure-postgres-backup-storage.md @@ -32,6 +32,12 @@ The candidate provider baseline is: This workplan chooses on evidence. It does not presume that keeping compute and backup at one provider is cheaper or safer. +Inventory output follows `docs/operational-reference-convention.md` +(`RESOURCE-WP-0006`). Operating attributes are `reef:storage/…` references. +Credentials are `secret:railiance-platform/backup`. Do not write a Scaleway +endpoint or key into `data/resources/`. `rapp-postgres` consumes the +reviewed destination; it does not procure it. + ## Context and current evidence `rapp-postgres` has deployed `platform-pg` on reef-railiance at Host Europe. diff --git a/workplans/RESOURCE-WP-0006-resource-object-and-reef-refs.md b/workplans/RESOURCE-WP-0006-resource-object-and-reef-refs.md index acb86a1..82df55f 100644 --- a/workplans/RESOURCE-WP-0006-resource-object-and-reef-refs.md +++ b/workplans/RESOURCE-WP-0006-resource-object-and-reef-refs.md @@ -8,7 +8,7 @@ status: active owner: grok topic_slug: railiance created: "2026-08-14" -updated: "2026-08-14" +updated: "2026-08-15" related: - RESOURCE-WP-0002 - RESOURCE-WP-0003 @@ -51,7 +51,7 @@ object storage and own the non-secret attributes this repo will cite. ```task id: RESOURCE-WP-0006-T01 -status: todo +status: done priority: high state_hub_task_id: "55d3ebc2-51b7-49d1-8d0d-57ccc6379ae7" ``` @@ -67,11 +67,16 @@ an inline secret or endpoint that should be a `reef:` / `secret:` ref, and accept the proposed backup record with an explicit attribute-ref gap until `reef-storage` exists. +Done 2026-08-15: schema accepts `0.2` and `0.3`. v0.3 requires the five +facets. `portfolio.validate_record` rejects inline `s3://` / Scaleway +endpoints and secret-looking strings. The live backup record is `0.3`. +Other inventory stays `0.2` until T03. + ## T02 — Reef views and the reef-railiance projection ```task id: RESOURCE-WP-0006-T02 -status: progress +status: done priority: high state_hub_task_id: "f2b38140-3d73-45a2-a2ff-fea12733e7b0" ``` @@ -87,6 +92,11 @@ is out of `reef-railiance`. Started 2026-08-14: convention published; first reef-railiance view committed. +Done 2026-08-15: `data/reefs/reef-railiance.json` and `reef-storage.json` +validate. Every cited `resource_id` exists in inventory. railiance view +states S3 backup is out of that reef. `reef-storage` view exists now that +the repo does. + ## T03 — Migrate live inventory to the five facets ```task @@ -110,7 +120,7 @@ only consumer list. ```task id: RESOURCE-WP-0006-T04 -status: todo +status: done priority: medium state_hub_task_id: "a34fa806-ee34-4d3b-a823-2797fe260bbc" ``` @@ -122,3 +132,7 @@ remains a consumer, not the procurer. Done when WP-0002 cites this convention and will not accept an inventory update that inlines Scaleway endpoint or keys. + +Done 2026-08-15: WP-0002 goal cites the convention. The live inventory +record is v0.3 with reef/secret refs; validators reject an inline +Scaleway endpoint on that record.