304 lines
11 KiB
Python
304 lines
11 KiB
Python
|
"""
|
||
|
|
Unit tests for quality gate models and individual gate implementations.
|
||
|
|
|
||
|
|
Tests QualityGate models, SchemaValidationGate, and PatternValidationGate.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from markitect.prompts.quality.models import (
|
||
|
|
GateType,
|
||
|
|
ValidationStatus,
|
||
|
|
ValidationDiagnostic,
|
||
|
|
ValidationResult,
|
||
|
|
QualityPolicy,
|
||
|
|
HaltDecision,
|
||
|
|
HaltingRecord,
|
||
|
|
)
|
||
|
|
from markitect.prompts.quality.gates.schema_gate import SchemaValidationGate
|
||
|
|
from markitect.prompts.quality.gates.pattern_gate import PatternValidationGate
|
||
|
|
|
||
|
|
|
||
|
|
class TestValidationDiagnostic:
|
||
|
|
"""Tests for ValidationDiagnostic."""
|
||
|
|
|
||
|
|
def test_create_diagnostic(self):
|
||
|
|
"""Test creating a diagnostic."""
|
||
|
|
diag = ValidationDiagnostic(
|
||
|
|
code="TEST_ERROR",
|
||
|
|
message="Something went wrong",
|
||
|
|
severity="error",
|
||
|
|
)
|
||
|
|
assert diag.code == "TEST_ERROR"
|
||
|
|
assert diag.severity == "error"
|
||
|
|
|
||
|
|
def test_to_dict(self):
|
||
|
|
"""Test diagnostic serialization."""
|
||
|
|
diag = ValidationDiagnostic(code="E1", message="msg", severity="warning")
|
||
|
|
d = diag.to_dict()
|
||
|
|
assert d["code"] == "E1"
|
||
|
|
assert d["severity"] == "warning"
|
||
|
|
|
||
|
|
def test_from_dict(self):
|
||
|
|
"""Test diagnostic deserialization."""
|
||
|
|
data = {"code": "E1", "message": "msg", "severity": "info"}
|
||
|
|
diag = ValidationDiagnostic.from_dict(data)
|
||
|
|
assert diag.code == "E1"
|
||
|
|
assert diag.severity == "info"
|
||
|
|
|
||
|
|
|
||
|
|
class TestValidationResult:
|
||
|
|
"""Tests for ValidationResult."""
|
||
|
|
|
||
|
|
def test_create_result(self):
|
||
|
|
"""Test creating a validation result."""
|
||
|
|
result = ValidationResult.create(
|
||
|
|
gate_id="gate-1",
|
||
|
|
gate_type=GateType.SCHEMA,
|
||
|
|
artifact_id="art-1",
|
||
|
|
status=ValidationStatus.PASS,
|
||
|
|
score=1.0,
|
||
|
|
)
|
||
|
|
assert result.gate_id == "gate-1"
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
assert result.score == 1.0
|
||
|
|
|
||
|
|
def test_result_unique_ids(self):
|
||
|
|
"""Test that each result gets a unique ID."""
|
||
|
|
r1 = ValidationResult.create(
|
||
|
|
gate_id="g", gate_type=GateType.PATTERN,
|
||
|
|
artifact_id="a", status=ValidationStatus.PASS,
|
||
|
|
)
|
||
|
|
r2 = ValidationResult.create(
|
||
|
|
gate_id="g", gate_type=GateType.PATTERN,
|
||
|
|
artifact_id="a", status=ValidationStatus.PASS,
|
||
|
|
)
|
||
|
|
assert r1.id != r2.id
|
||
|
|
|
||
|
|
def test_to_dict_from_dict(self):
|
||
|
|
"""Test round-trip serialization."""
|
||
|
|
result = ValidationResult.create(
|
||
|
|
gate_id="g1",
|
||
|
|
gate_type=GateType.SCHEMA,
|
||
|
|
artifact_id="a1",
|
||
|
|
status=ValidationStatus.FAIL,
|
||
|
|
score=0.5,
|
||
|
|
diagnostics=[
|
||
|
|
ValidationDiagnostic(code="E1", message="err", severity="error"),
|
||
|
|
],
|
||
|
|
)
|
||
|
|
d = result.to_dict()
|
||
|
|
restored = ValidationResult.from_dict(d)
|
||
|
|
assert restored.id == result.id
|
||
|
|
assert restored.status == ValidationStatus.FAIL
|
||
|
|
assert restored.score == 0.5
|
||
|
|
assert len(restored.diagnostics) == 1
|
||
|
|
|
||
|
|
|
||
|
|
class TestQualityPolicy:
|
||
|
|
"""Tests for QualityPolicy."""
|
||
|
|
|
||
|
|
def test_default_policy(self):
|
||
|
|
"""Test default policy values."""
|
||
|
|
policy = QualityPolicy()
|
||
|
|
assert policy.max_iterations == 3
|
||
|
|
assert policy.min_improvement == 0.05
|
||
|
|
assert policy.fail_on_gate_failure is True
|
||
|
|
assert policy.resource_budget == 10
|
||
|
|
|
||
|
|
def test_to_dict_from_dict(self):
|
||
|
|
"""Test round-trip serialization."""
|
||
|
|
policy = QualityPolicy(
|
||
|
|
max_iterations=5,
|
||
|
|
min_improvement=0.1,
|
||
|
|
required_gate_ids=["g1", "g2"],
|
||
|
|
)
|
||
|
|
d = policy.to_dict()
|
||
|
|
restored = QualityPolicy.from_dict(d)
|
||
|
|
assert restored.max_iterations == 5
|
||
|
|
assert restored.min_improvement == 0.1
|
||
|
|
assert restored.required_gate_ids == ["g1", "g2"]
|
||
|
|
|
||
|
|
|
||
|
|
class TestSchemaValidationGate:
|
||
|
|
"""Tests for SchemaValidationGate."""
|
||
|
|
|
||
|
|
def test_valid_json_passes(self):
|
||
|
|
"""Test valid JSON against schema passes."""
|
||
|
|
schema = {
|
||
|
|
"type": "object",
|
||
|
|
"required": ["name", "version"],
|
||
|
|
"properties": {
|
||
|
|
"name": {"type": "string"},
|
||
|
|
"version": {"type": "integer"},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
gate = SchemaValidationGate(schema=schema, name="test-schema")
|
||
|
|
content = json.dumps({"name": "test", "version": 1})
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
assert result.score == 1.0
|
||
|
|
assert len(result.diagnostics) == 0
|
||
|
|
|
||
|
|
def test_missing_required_field_fails(self):
|
||
|
|
"""Test missing required field fails validation."""
|
||
|
|
schema = {
|
||
|
|
"type": "object",
|
||
|
|
"required": ["name", "version"],
|
||
|
|
"properties": {
|
||
|
|
"name": {"type": "string"},
|
||
|
|
"version": {"type": "integer"},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
gate = SchemaValidationGate(schema=schema)
|
||
|
|
content = json.dumps({"name": "test"})
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert any(d.code == "SCHEMA_VIOLATION" for d in result.diagnostics)
|
||
|
|
|
||
|
|
def test_wrong_type_fails(self):
|
||
|
|
"""Test wrong type fails validation."""
|
||
|
|
schema = {
|
||
|
|
"type": "object",
|
||
|
|
"properties": {
|
||
|
|
"count": {"type": "integer"},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
gate = SchemaValidationGate(schema=schema)
|
||
|
|
content = json.dumps({"count": "not-a-number"})
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
|
||
|
|
def test_invalid_json_fails(self):
|
||
|
|
"""Test invalid JSON content fails."""
|
||
|
|
schema = {"type": "object"}
|
||
|
|
gate = SchemaValidationGate(schema=schema)
|
||
|
|
|
||
|
|
result = gate.validate("not json {{{", "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert result.score == 0.0
|
||
|
|
assert any(d.code == "INVALID_JSON" for d in result.diagnostics)
|
||
|
|
|
||
|
|
def test_gate_has_correct_type(self):
|
||
|
|
"""Test gate type is SCHEMA."""
|
||
|
|
gate = SchemaValidationGate(schema={"type": "object"})
|
||
|
|
assert gate.gate_type == GateType.SCHEMA
|
||
|
|
|
||
|
|
def test_empty_schema_passes_any_object(self):
|
||
|
|
"""Test empty schema passes any valid JSON."""
|
||
|
|
gate = SchemaValidationGate(schema={})
|
||
|
|
result = gate.validate(json.dumps({"any": "thing"}), "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
|
||
|
|
def test_score_reflects_error_count(self):
|
||
|
|
"""Test that score decreases with more errors."""
|
||
|
|
schema = {
|
||
|
|
"type": "object",
|
||
|
|
"required": ["a", "b", "c"],
|
||
|
|
"properties": {
|
||
|
|
"a": {"type": "string"},
|
||
|
|
"b": {"type": "string"},
|
||
|
|
"c": {"type": "string"},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
gate = SchemaValidationGate(schema=schema)
|
||
|
|
|
||
|
|
# Missing all 3 required fields
|
||
|
|
result = gate.validate(json.dumps({}), "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert result.score < 1.0
|
||
|
|
|
||
|
|
|
||
|
|
class TestPatternValidationGate:
|
||
|
|
"""Tests for PatternValidationGate."""
|
||
|
|
|
||
|
|
def test_required_pattern_present(self):
|
||
|
|
"""Test content with required patterns passes."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
required_patterns=[r"## Endpoints", r"### Authentication"],
|
||
|
|
)
|
||
|
|
content = "# API\n## Endpoints\n### Authentication\nDetails here."
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
assert result.score == 1.0
|
||
|
|
|
||
|
|
def test_required_pattern_missing(self):
|
||
|
|
"""Test missing required pattern fails."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
required_patterns=[r"## Endpoints", r"### Authentication"],
|
||
|
|
)
|
||
|
|
content = "# API\n## Endpoints\nNo auth section."
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert any(d.code == "MISSING_PATTERN" for d in result.diagnostics)
|
||
|
|
|
||
|
|
def test_forbidden_pattern_absent(self):
|
||
|
|
"""Test content without forbidden patterns passes."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
forbidden_patterns=[r"TODO", r"FIXME"],
|
||
|
|
)
|
||
|
|
content = "This is a clean document."
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
|
||
|
|
def test_forbidden_pattern_present(self):
|
||
|
|
"""Test content with forbidden pattern fails."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
forbidden_patterns=[r"TODO", r"FIXME"],
|
||
|
|
)
|
||
|
|
content = "This needs work. TODO: fix this."
|
||
|
|
|
||
|
|
result = gate.validate(content, "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert any(d.code == "FORBIDDEN_PATTERN" for d in result.diagnostics)
|
||
|
|
|
||
|
|
def test_combined_required_and_forbidden(self):
|
||
|
|
"""Test both required and forbidden patterns together."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
required_patterns=[r"## Summary"],
|
||
|
|
forbidden_patterns=[r"FIXME"],
|
||
|
|
)
|
||
|
|
# Has required, no forbidden
|
||
|
|
result1 = gate.validate("## Summary\nAll good.", "art-1")
|
||
|
|
assert result1.status == ValidationStatus.PASS
|
||
|
|
|
||
|
|
# Has required AND forbidden
|
||
|
|
result2 = gate.validate("## Summary\nFIXME: broken.", "art-1")
|
||
|
|
assert result2.status == ValidationStatus.FAIL
|
||
|
|
|
||
|
|
def test_no_patterns_passes(self):
|
||
|
|
"""Test gate with no patterns always passes."""
|
||
|
|
gate = PatternValidationGate()
|
||
|
|
result = gate.validate("anything", "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|
||
|
|
|
||
|
|
def test_gate_has_correct_type(self):
|
||
|
|
"""Test gate type is PATTERN."""
|
||
|
|
gate = PatternValidationGate()
|
||
|
|
assert gate.gate_type == GateType.PATTERN
|
||
|
|
|
||
|
|
def test_score_proportional_to_failures(self):
|
||
|
|
"""Test score is proportional to number of checks passed."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
required_patterns=[r"A", r"B", r"C", r"D"],
|
||
|
|
)
|
||
|
|
# Only A is present → 3 out of 4 fail
|
||
|
|
result = gate.validate("A", "art-1")
|
||
|
|
assert result.status == ValidationStatus.FAIL
|
||
|
|
assert 0.0 < result.score < 1.0
|
||
|
|
|
||
|
|
def test_regex_pattern_matching(self):
|
||
|
|
"""Test regex patterns work correctly."""
|
||
|
|
gate = PatternValidationGate(
|
||
|
|
required_patterns=[r"\d{3}-\d{4}"],
|
||
|
|
)
|
||
|
|
result = gate.validate("Call 555-1234 for info", "art-1")
|
||
|
|
assert result.status == ValidationStatus.PASS
|