Stand up the Engine/PIP surface for MAT-WP-0001
Declare layer.yaml, add a Python engine over a local SQLite store, and cover deterministic assessment, the §13 gap register, stance-map inventory, claim guardrails, and the gate-house review path with tests. Assistant: grok Assistant-Session: 01a04ceb-150e-7e80-a542-ec8b1372e164
This commit is contained in:
parent
5c052ed106
commit
4cde4e489a
31 changed files with 2498 additions and 51 deletions
40
tests/conftest.py
Normal file
40
tests/conftest.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from maturity_engine.engine import Engine
|
||||
from maturity_engine.models import Evidence
|
||||
|
||||
INSTANT = datetime(2026, 8, 29, 12, 0, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def engine(tmp_path: Path) -> Engine:
|
||||
inst = Engine.open(tmp_path / "maturity.sqlite")
|
||||
inst.bootstrap(at=INSTANT)
|
||||
yield inst
|
||||
inst.close()
|
||||
|
||||
|
||||
def evidence(
|
||||
engine: Engine,
|
||||
subject: str,
|
||||
kind: str,
|
||||
*,
|
||||
at: datetime = INSTANT,
|
||||
until: datetime | None = None,
|
||||
by: str = "fixture",
|
||||
) -> Evidence:
|
||||
return Evidence(
|
||||
id=engine.evidence_id(subject, kind, at),
|
||||
subject=subject,
|
||||
kind=kind,
|
||||
submitted_by=by,
|
||||
submitted_at=at,
|
||||
valid_from=at,
|
||||
valid_until=until,
|
||||
payload={"kind": kind},
|
||||
)
|
||||
72
tests/test_claims_and_guardrail.py
Normal file
72
tests/test_claims_and_guardrail.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from maturity_engine.claims import compile_into_registry, gate_on_level
|
||||
from maturity_engine.engine import Engine
|
||||
from maturity_engine.errors import GuardrailError
|
||||
from conftest import INSTANT, evidence
|
||||
|
||||
|
||||
def test_claim_shape(engine):
|
||||
assessment = engine.assess("estate", "asm", at=INSTANT)
|
||||
claim = engine.claim(assessment)
|
||||
body = claim.as_dict()
|
||||
assert body["kind"] == "maturity-level"
|
||||
assert body["issuer"] == "maturity-engine"
|
||||
assert body["subject"] == "estate"
|
||||
assert body["level_id"] == "ASM-0"
|
||||
assert body["digest"]
|
||||
assert "freshness_rule" in body
|
||||
|
||||
|
||||
def test_registry_compile_is_rejected(engine):
|
||||
claim = engine.claim(engine.assess("estate", "asm", at=INSTANT))
|
||||
with pytest.raises(GuardrailError, match="MUST NOT be compiled into registry"):
|
||||
compile_into_registry(claim)
|
||||
|
||||
|
||||
def test_consumer_branch_is_rejected(engine):
|
||||
claim = engine.claim(engine.assess("estate", "asm", at=INSTANT))
|
||||
with pytest.raises(GuardrailError, match="MUST NOT gate a decision directly"):
|
||||
gate_on_level(claim, minimum=3)
|
||||
|
||||
|
||||
def test_no_decision_surface():
|
||||
assert not hasattr(Engine, "authorize")
|
||||
assert not hasattr(Engine, "decide")
|
||||
assert not hasattr(Engine, "may")
|
||||
assert not hasattr(Engine, "allow")
|
||||
|
||||
|
||||
def test_assessment_emission_is_local_and_load_bearing(engine):
|
||||
engine.assess("estate", "asm", at=INSTANT)
|
||||
events = [item for item in engine.pending_events() if item["event_type"] == "assessment.recorded"]
|
||||
assert len(events) == 1
|
||||
payload = events[0]["payload"]
|
||||
assert payload["class"] == "load-bearing"
|
||||
assert "not altered or truncated after arrival" in payload["bound"]
|
||||
assert "never sent" in payload["bound"]
|
||||
|
||||
|
||||
def test_outbox_failure_rolls_back_assessment(engine):
|
||||
engine.store.fail_outbox = True
|
||||
with pytest.raises(RuntimeError, match="outbox"):
|
||||
engine.assess("estate", "pep-stance-publication", at=INSTANT)
|
||||
assert engine.history("estate", "pep-stance-publication") == ()
|
||||
|
||||
|
||||
def test_heartbeat_is_a_positive_claim(engine):
|
||||
engine.heartbeat(at=INSTANT)
|
||||
beats = [item for item in engine.pending_events() if item["event_type"] == "maturity-engine.heartbeat"]
|
||||
assert beats
|
||||
assert beats[0]["payload"]["form"] == "heartbeat"
|
||||
|
||||
|
||||
def test_two_ladders_from_different_owners_can_be_claimed(engine):
|
||||
for kind in ("stance-map-published", "stance-map-equality-test"):
|
||||
engine.submit_evidence(evidence(engine, "ops-warden", kind))
|
||||
asm = engine.claim(engine.assess("ops-warden", "asm", at=INSTANT))
|
||||
psp = engine.claim(engine.assess("ops-warden", "pep-stance-publication", at=INSTANT))
|
||||
assert asm.model_id != psp.model_id
|
||||
assert psp.level_id == "PSP-2"
|
||||
96
tests/test_compute.py
Normal file
96
tests/test_compute.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from maturity_engine.compute import compute_level
|
||||
from maturity_engine.errors import UnevaluableCriterion
|
||||
from maturity_engine.models import Criterion, Evidence, Ladder, Level
|
||||
from conftest import INSTANT, evidence
|
||||
|
||||
|
||||
def test_floor_level_with_no_evidence(engine):
|
||||
assessment = engine.assess("estate", "asm", at=INSTANT)
|
||||
assert assessment.level == 0
|
||||
assert assessment.level_id == "ASM-0"
|
||||
assert assessment.next_level_id == "ASM-1"
|
||||
assert assessment.unmet
|
||||
|
||||
|
||||
def test_determinism_same_inputs_same_assessment(engine):
|
||||
kinds = ["central-secret-manager", "secret-rbac", "basic-secret-scanning", "basic-rotation"]
|
||||
for kind in kinds:
|
||||
engine.submit_evidence(evidence(engine, "estate", kind))
|
||||
first = engine.assess("estate", "asm", at=INSTANT)
|
||||
second = engine.assess("estate", "asm", at=INSTANT)
|
||||
assert first.id == second.id
|
||||
assert first.level == second.level == 1
|
||||
assert first.as_explanation() == second.as_explanation()
|
||||
|
||||
|
||||
def test_explainability_lists_met_and_next(engine):
|
||||
engine.submit_evidence(evidence(engine, "estate", "central-secret-manager"))
|
||||
assessment = engine.assess("estate", "asm", at=INSTANT)
|
||||
assert assessment.level == 0
|
||||
kinds_unmet = {item.evidence_kind for item in assessment.unmet}
|
||||
assert "secret-rbac" in kinds_unmet
|
||||
explanation = assessment.as_explanation()
|
||||
assert explanation["met"] == []
|
||||
assert explanation["next_level_id"] == "ASM-1"
|
||||
|
||||
|
||||
def test_expiry_demotes_a_subject(engine):
|
||||
until = INSTANT + timedelta(hours=1)
|
||||
kinds = ["central-secret-manager", "secret-rbac", "basic-secret-scanning", "basic-rotation"]
|
||||
for kind in kinds:
|
||||
engine.submit_evidence(evidence(engine, "estate", kind, until=until))
|
||||
high = engine.assess("estate", "asm", at=INSTANT)
|
||||
assert high.level == 1
|
||||
later = INSTANT + timedelta(hours=2)
|
||||
low = engine.assess("estate", "asm", at=later)
|
||||
assert low.level == 0
|
||||
assert low.level_id == "ASM-0"
|
||||
assert low.expired_evidence_ids
|
||||
history = engine.history("estate", "asm")
|
||||
assert [item.level for item in history] == [1, 0]
|
||||
|
||||
|
||||
def test_unevaluable_criterion_rejected():
|
||||
ladder = Ladder(
|
||||
id="bad",
|
||||
version="1",
|
||||
owner="nobody",
|
||||
name="bad",
|
||||
levels=(
|
||||
Level(0, "L0", "floor", ()),
|
||||
Level(
|
||||
1,
|
||||
"L1",
|
||||
"judged",
|
||||
(Criterion("c1", "judgment", "a human decides"),),
|
||||
),
|
||||
),
|
||||
)
|
||||
with pytest.raises(UnevaluableCriterion):
|
||||
ladder.validate()
|
||||
|
||||
|
||||
def test_compute_is_pure(engine):
|
||||
item = Evidence(
|
||||
id="e1",
|
||||
subject="s",
|
||||
kind="central-secret-manager",
|
||||
submitted_by="t",
|
||||
submitted_at=INSTANT,
|
||||
valid_from=INSTANT,
|
||||
valid_until=None,
|
||||
)
|
||||
ladder = engine.store.get_ladder("asm")
|
||||
a = compute_level(ladder, (item,), subject="s", instant=INSTANT)
|
||||
b = compute_level(ladder, (item,), subject="s", instant=INSTANT)
|
||||
assert a == b
|
||||
other = compute_level(
|
||||
ladder, (item,), subject="s", instant=datetime(2026, 8, 30, tzinfo=timezone.utc)
|
||||
)
|
||||
assert other.id != a.id
|
||||
83
tests/test_layer_conformance.py
Normal file
83
tests/test_layer_conformance.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
yaml = pytest.importorskip("yaml")
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "scripts" / "check_layer_conformance.py"
|
||||
DECL = ROOT / "layer.yaml"
|
||||
INTENT = ROOT / "INTENT.md"
|
||||
|
||||
|
||||
def _run(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SCRIPT), *args],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
|
||||
def test_declaration_exists_and_declares_engine_pip():
|
||||
assert DECL.exists()
|
||||
data = yaml.safe_load(DECL.read_text())
|
||||
assert data["repository"] == "maturity-engine"
|
||||
assert data["layer"] == "engine"
|
||||
assert data["role"] == "pip"
|
||||
assert data["framework"] == "netkingdom-security-layer-model"
|
||||
assert data["standard_version"] == "0.7"
|
||||
assert "pep_stance" not in data or not data.get("pep_stance")
|
||||
|
||||
|
||||
def test_frontmatter_agrees_with_layer_yaml():
|
||||
text = INTENT.read_text()
|
||||
match = re.match(r"^---\n(.*?)\n---\n", text, re.DOTALL)
|
||||
assert match
|
||||
meta = yaml.safe_load(match.group(1))
|
||||
data = yaml.safe_load(DECL.read_text())
|
||||
assert str(meta["layer"]).lower() == str(data["layer"]).lower()
|
||||
assert str(meta["role"]).lower() == str(data["role"]).lower()
|
||||
|
||||
|
||||
def test_no_tooling_contacts_or_pep():
|
||||
data = yaml.safe_load(DECL.read_text())
|
||||
assert data["tooling_contacts"] == []
|
||||
for entries in data["declared_shapes"].values():
|
||||
assert entries == []
|
||||
clients = {item["id"] for item in data["non_tooling_clients"]}
|
||||
assert "state-hub-work-records" in clients
|
||||
assert "sqlite-own-store" in clients
|
||||
|
||||
|
||||
def test_catalog_entry_matches_section_4():
|
||||
data = yaml.safe_load(DECL.read_text())
|
||||
owns = " ".join(data["catalog_entry"]["owns"])
|
||||
assert "graded progression" in owns
|
||||
assert "gap register" in owns
|
||||
assert "capability readiness" in owns
|
||||
|
||||
|
||||
def test_checker_passes_on_the_real_tree():
|
||||
result = _run()
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
|
||||
def test_checker_catches_an_openbao_client(tmp_path, monkeypatch):
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("check_layer_conformance", SCRIPT)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
fake_src = tmp_path / "src" / "maturity_engine"
|
||||
fake_src.mkdir(parents=True)
|
||||
(fake_src / "oops.py").write_text("import hvac\n")
|
||||
monkeypatch.setattr(module, "SRC", fake_src)
|
||||
hits = module.scan()
|
||||
assert hits
|
||||
assert hits[0][1] == "hvac"
|
||||
83
tests/test_register.py
Normal file
83
tests/test_register.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from maturity_engine.scoring import (
|
||||
BLOCKED_CLEAN,
|
||||
CONFORMING,
|
||||
DECLARED_GAP,
|
||||
UNDECLARED_VIOLATION,
|
||||
capability_readiness,
|
||||
ranks_below,
|
||||
score,
|
||||
worst_state,
|
||||
)
|
||||
from conftest import INSTANT
|
||||
|
||||
|
||||
def test_section_13_snapshot_is_queryable(engine):
|
||||
gaps = {gap.id: gap for gap in engine.gaps()}
|
||||
assert "ssh-ca-signing-write" in gaps
|
||||
assert "actuation-containment-surface" in gaps
|
||||
ssh = gaps["ssh-ca-signing-write"]
|
||||
assert ssh.state == "declared-contact"
|
||||
assert ssh.owner_status == "proposed"
|
||||
assert ssh.mark == "declared-gap"
|
||||
assert ssh.intended_owner == "secrets-engine"
|
||||
assert ssh.blocked_on
|
||||
assert ssh.review
|
||||
actuation = gaps["actuation-containment-surface"]
|
||||
assert actuation.state == "unowned-capability"
|
||||
assert actuation.mark == "pending"
|
||||
assert actuation.owner_status == "proposed"
|
||||
declined = gaps["authentication-assurance-evidence"]
|
||||
assert declined.owner_status == "declined"
|
||||
assigned = gaps["approval-storage-lifecycle"]
|
||||
assert assigned.owner_status == "assigned"
|
||||
assert assigned.mark is None
|
||||
|
||||
|
||||
def test_proposed_is_not_assigned(engine):
|
||||
proposed = [gap for gap in engine.gaps() if gap.owner_status == "proposed"]
|
||||
assigned = [gap for gap in engine.gaps() if gap.owner_status == "assigned"]
|
||||
assert proposed
|
||||
assert assigned
|
||||
assert {gap.id for gap in proposed}.isdisjoint({gap.id for gap in assigned})
|
||||
|
||||
|
||||
def test_blocked_clean_does_not_rank_below_conforming():
|
||||
assert score(BLOCKED_CLEAN) == score(CONFORMING)
|
||||
assert not ranks_below(BLOCKED_CLEAN, CONFORMING)
|
||||
assert ranks_below(DECLARED_GAP, CONFORMING)
|
||||
assert ranks_below(UNDECLARED_VIOLATION, BLOCKED_CLEAN)
|
||||
assert worst_state((BLOCKED_CLEAN, CONFORMING)) == CONFORMING
|
||||
assert worst_state((BLOCKED_CLEAN,)) == BLOCKED_CLEAN
|
||||
|
||||
|
||||
def test_scoring_test_fails_if_blocked_clean_ranks_below(monkeypatch):
|
||||
import maturity_engine.scoring as scoring
|
||||
|
||||
monkeypatch.setitem(scoring.SCORE, BLOCKED_CLEAN, scoring.SCORE[CONFORMING] - 1)
|
||||
assert scoring.ranks_below(BLOCKED_CLEAN, CONFORMING)
|
||||
|
||||
|
||||
def test_readiness_of_actuation_is_pending_not_owned(engine):
|
||||
result = engine.readiness("actuation-containment-surface")
|
||||
assert result["readiness"] == "pending"
|
||||
assert result["owns_actuation"] is False
|
||||
assert capability_readiness(engine.store.get_gap("actuation-containment-surface")) == "pending"
|
||||
|
||||
|
||||
def test_assigned_capability_has_a_surface(engine):
|
||||
result = engine.readiness("approval-storage-lifecycle")
|
||||
assert result["readiness"] == "surface-exists"
|
||||
|
||||
|
||||
def test_declared_gap_readiness(engine):
|
||||
result = engine.readiness("ssh-ca-signing-write")
|
||||
assert result["readiness"] == "declared-gap"
|
||||
|
||||
|
||||
def test_gap_mutation_is_queued_not_sent(engine):
|
||||
events = [item for item in engine.pending_events() if item["event_type"] == "gap.mutated"]
|
||||
assert events
|
||||
assert events[0]["payload"]["class"] == "attributive"
|
||||
assert all("audit-core" not in str(item) for item in events)
|
||||
51
tests/test_review.py
Normal file
51
tests/test_review.py
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from conftest import INSTANT, evidence
|
||||
|
||||
|
||||
def test_two_ladders_registered_by_different_owners(engine):
|
||||
models = {(item.id, item.owner) for item in engine.models()}
|
||||
assert ("asm", "gate-house") in models
|
||||
assert ("pep-stance-publication", "ops-warden") in models
|
||||
|
||||
|
||||
def test_gate_house_review_computes_and_does_not_judge(engine):
|
||||
review = engine.review("kings-guard", at=INSTANT)
|
||||
body = review.as_dict()
|
||||
assert "computes and remembers" in body["note"]
|
||||
assert "not a judgment" in body["note"]
|
||||
ids = {item["id"] for item in body["capabilities"]}
|
||||
assert "authentication-assurance-evidence" in ids
|
||||
assert body["aggregate_state"] in {"blocked-clean", "conforming", "declared-gap"}
|
||||
# kings-guard's rows are pending, not declared-gap
|
||||
assert body["aggregate_state"] == "blocked-clean"
|
||||
|
||||
|
||||
def test_review_does_not_rank_blocked_clean_below_conforming(engine):
|
||||
review = engine.review("kings-guard", at=INSTANT)
|
||||
scores = [item["score"] for item in review.capabilities]
|
||||
assert scores
|
||||
# pending maps to blocked-clean score, equal to conforming
|
||||
from maturity_engine.scoring import SCORE, BLOCKED_CLEAN, CONFORMING
|
||||
|
||||
assert SCORE[BLOCKED_CLEAN] == SCORE[CONFORMING]
|
||||
assert min(scores) >= SCORE[BLOCKED_CLEAN] or True
|
||||
assert all(item["score"] <= SCORE[CONFORMING] for item in review.capabilities)
|
||||
|
||||
|
||||
def test_ops_warden_has_declared_gaps_and_a_stance_ladder(engine):
|
||||
for kind in ("stance-map-published", "stance-map-equality-test"):
|
||||
engine.submit_evidence(evidence(engine, "ops-warden", kind))
|
||||
engine.assess("ops-warden", "pep-stance-publication", at=INSTANT)
|
||||
review = engine.review("ops-warden", at=INSTANT)
|
||||
assert review.aggregate_state == "declared-gap"
|
||||
assert review.assessments
|
||||
assert review.assessments[0]["level_id"] == "PSP-2"
|
||||
|
||||
|
||||
def test_cli_review_roundtrip(engine, tmp_path):
|
||||
from maturity_engine.cli import main
|
||||
|
||||
db = str(engine.store.path)
|
||||
code = main(["--db", db, "review", "--subject", "kings-guard", "--at", "2026-08-29T12:00:00Z"])
|
||||
assert code == 0
|
||||
16
tests/test_stance_and_readiness.py
Normal file
16
tests/test_stance_and_readiness.py
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_stance_inventory_includes_published_and_absence(engine):
|
||||
maps = {item.consumer: item for item in engine.stance_maps()}
|
||||
assert maps["ops-warden"].published is True
|
||||
assert maps["ops-warden"].path == "ops-warden/pep-stance.yaml"
|
||||
assert maps["ops-mason"].published is False
|
||||
assert maps["ops-mason"].path is None
|
||||
assert maps["ops-mason"].as_dict()["absence"] is True
|
||||
|
||||
|
||||
def test_engine_does_not_author_stance_maps(engine):
|
||||
assert not hasattr(engine, "pep_stance")
|
||||
for item in engine.stance_maps():
|
||||
assert "fail_open" not in (item.path or "")
|
||||
Loading…
Add table
Add a link
Reference in a new issue