markitect-main/tests/integration/prompts/test_quality_validation.py

"""
Integration tests for full quality validation workflow.

Tests applying quality gates to artifacts with real DB persistence,
manifest integration, and multi-gate validation.
"""

import json
import pytest
import tempfile
from pathlib import Path

from markitect.prompts.models import Artifact, ArtifactType
from markitect.prompts.repositories.sqlite import SQLiteArtifactRepository
from markitect.prompts.quality.models import (
    GateType,
    ValidationStatus,
)
from markitect.prompts.quality.gates.schema_gate import SchemaValidationGate
from markitect.prompts.quality.gates.pattern_gate import PatternValidationGate
from markitect.prompts.quality.validator import QualityValidator


@pytest.fixture
def temp_db():
    """Create temporary database for testing."""
    with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f:
        db_path = f.name
    yield db_path
    Path(db_path).unlink(missing_ok=True)


@pytest.fixture
def artifact_repo(temp_db):
    """Create artifact repository."""
    return SQLiteArtifactRepository(temp_db)


def _create_artifact(repo, name, content, art_type=ArtifactType.GENERATED):
    """Helper to create and persist an artifact."""
    artifact = Artifact.create(
        space_id="space-1",
        name=name,
        content=content,
        artifact_type=art_type,
    )
    return repo.create(artifact)


class TestSchemaValidationWorkflow:
    """Full schema validation workflow with real DB."""

    def test_validate_json_artifact_passes(self, temp_db, artifact_repo):
        """Test validating a valid JSON artifact."""
        content = json.dumps({
            "name": "API Spec",
            "version": 1,
            "endpoints": ["/users", "/auth"],
        })
        artifact = _create_artifact(artifact_repo, "api-spec", content)

        schema = {
            "type": "object",
            "required": ["name", "version", "endpoints"],
            "properties": {
                "name": {"type": "string"},
                "version": {"type": "integer"},
                "endpoints": {"type": "array", "items": {"type": "string"}},
            },
        }
        gate = SchemaValidationGate(schema=schema, gate_id="schema-api")
        validator = QualityValidator(gates=[gate], db_path=temp_db)

        results = validator.validate_artifact(
            content, artifact.id, run_id="run-1"
        )

        assert len(results) == 1
        assert results[0].status == ValidationStatus.PASS

        # Verify persisted
        persisted = validator.get_results_for_run("run-1")
        assert len(persisted) == 1
        assert persisted[0]["status"] == "pass"

    def test_validate_json_artifact_fails(self, temp_db, artifact_repo):
        """Test validating an invalid JSON artifact."""
        content = json.dumps({"name": "Incomplete"})
        artifact = _create_artifact(artifact_repo, "bad-spec", content)

        schema = {
            "type": "object",
            "required": ["name", "version"],
        }
        gate = SchemaValidationGate(schema=schema, gate_id="schema-strict")
        validator = QualityValidator(gates=[gate], db_path=temp_db)

        results = validator.validate_artifact(
            content, artifact.id, run_id="run-2"
        )

        assert results[0].status == ValidationStatus.FAIL
        assert len(results[0].diagnostics) > 0

        persisted = validator.get_results_for_run("run-2")
        assert persisted[0]["status"] == "fail"


class TestPatternValidationWorkflow:
    """Full pattern validation workflow with real DB."""

    def test_validate_markdown_artifact(self, temp_db, artifact_repo):
        """Test validating a markdown artifact against patterns."""
        content = "# API Documentation\n## Endpoints\n### Authentication\nOAuth2 flow."
        artifact = _create_artifact(artifact_repo, "api-docs", content)

        gate = PatternValidationGate(
            required_patterns=[r"## Endpoints", r"### Authentication"],
            forbidden_patterns=[r"TODO", r"FIXME"],
            gate_id="pattern-api",
        )
        validator = QualityValidator(gates=[gate], db_path=temp_db)

        results = validator.validate_artifact(
            content, artifact.id, run_id="run-3"
        )

        assert results[0].status == ValidationStatus.PASS

    def test_forbidden_pattern_detected(self, temp_db, artifact_repo):
        """Test that forbidden patterns are caught."""
        content = "# Draft\n## Endpoints\nTODO: Add authentication."
        artifact = _create_artifact(artifact_repo, "draft-docs", content)

        gate = PatternValidationGate(
            required_patterns=[r"## Endpoints"],
            forbidden_patterns=[r"TODO"],
            gate_id="pattern-clean",
        )
        validator = QualityValidator(gates=[gate], db_path=temp_db)

        results = validator.validate_artifact(
            content, artifact.id, run_id="run-4"
        )

        assert results[0].status == ValidationStatus.FAIL


class TestMultiGateWorkflow:
    """Tests applying multiple gates in a single validation."""

    def test_multi_gate_validation(self, temp_db, artifact_repo):
        """Test applying schema + pattern gates to an artifact."""
        content = json.dumps({
            "title": "Design Doc",
            "sections": ["## Overview", "## Details"],
        })
        artifact = _create_artifact(artifact_repo, "design-doc", content)

        schema_gate = SchemaValidationGate(
            schema={
                "type": "object",
                "required": ["title", "sections"],
            },
            gate_id="schema-doc",
        )
        pattern_gate = PatternValidationGate(
            forbidden_patterns=[r"FIXME"],
            gate_id="pattern-clean",
        )
        validator = QualityValidator(
            gates=[schema_gate, pattern_gate],
            db_path=temp_db,
        )

        results = validator.validate_artifact(
            content, artifact.id, run_id="run-5"
        )

        assert len(results) == 2
        assert all(r.status == ValidationStatus.PASS for r in results)

        # Check manifest dict
        manifest = validator.results_to_manifest_dict(results)
        assert manifest["all_passed"] is True
        assert manifest["aggregate_score"] == 1.0

        # Verify all persisted
        persisted = validator.get_results_for_run("run-5")
        assert len(persisted) == 2

    def test_retrieve_by_artifact(self, temp_db, artifact_repo):
        """Test retrieving results by artifact across multiple runs."""
        content = json.dumps({"name": "test"})
        artifact = _create_artifact(artifact_repo, "test-art", content)

        gate = SchemaValidationGate(
            schema={"type": "object", "required": ["name"]},
            gate_id="schema-1",
        )
        validator = QualityValidator(gates=[gate], db_path=temp_db)

        # Validate across two runs
        validator.validate_artifact(content, artifact.id, run_id="run-a")
        validator.validate_artifact(content, artifact.id, run_id="run-b")

        results = validator.get_results_for_artifact(artifact.id)
        assert len(results) == 2