canned-prompts/reference/tests/test_canned_prompts.py

from pathlib import Path

import pytest

import canned_prompts as cp


@pytest.fixture()
def package(tmp_path: Path) -> Path:
    pkg = tmp_path / "pkg"
    pkg.mkdir()
    (pkg / "prompt.yaml").write_text(
        """\
format: canned-prompt/v0.1
id: demo/hello
name: Hello
version: 1.0.0
summary: Say hello.
template: prompt.md
inputs:
  - name: person
    required: true
parameters:
  tone:
    type: enum
    values: [warm, formal]
    default: warm
""",
        encoding="utf-8",
    )
    (pkg / "prompt.md").write_text(
        "Say hello to {{ person }} in a {{ tone }} tone.\n", encoding="utf-8"
    )
    return pkg


def test_validate_and_render(package: Path) -> None:
    manifest = cp.validate_package(package)
    values = cp.resolve_values(manifest, {"person": "Ada"})
    rendered = cp.render_template((package / "prompt.md").read_text(), values)
    assert rendered == "Say hello to Ada in a warm tone.\n"


def test_missing_required_input_fails(package: Path) -> None:
    manifest = cp.validate_package(package)
    with pytest.raises(cp.CannedPromptError, match="missing required input"):
        cp.resolve_values(manifest, {})


def test_undeclared_placeholder_fails(package: Path) -> None:
    (package / "prompt.md").write_text("{{ missing }}\n", encoding="utf-8")
    with pytest.raises(cp.CannedPromptError, match="undeclared placeholders"):
        cp.validate_package(package)


def write_pkg(pkg: Path, manifest: str, template: str = "{{ greeting }}\n") -> Path:
    pkg.mkdir(exist_ok=True)
    (pkg / "prompt.yaml").write_text(manifest, encoding="utf-8")
    (pkg / "prompt.md").write_text(template, encoding="utf-8")
    return pkg


BASE = """\
format: canned-prompt/v0.1
id: demo/defaults
name: Defaults
version: 1.0.0
summary: Exercise input defaults.
template: prompt.md
"""


def test_static_default_fills_optional_input(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: false
    default: "hello there"
""",
    )
    manifest = cp.validate_package(pkg)
    resolution = cp.resolve_inputs(manifest, {})
    assert resolution.values["greeting"] == "hello there"
    assert resolution.origins["greeting"] == "default"
    assert cp.render_template("{{ greeting }}", resolution.values) == "hello there"


def test_supplied_value_overrides_static_default(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: false
    default: "hello there"
""",
    )
    manifest = cp.validate_package(pkg)
    resolution = cp.resolve_inputs(manifest, {"greeting": "hi"})
    assert resolution.values["greeting"] == "hi"
    assert resolution.origins["greeting"] == "supplied"


def test_derived_default_uses_static_fallback(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
dependencies:
  prompts:
    - id: context/greeting
      version: 1.0.0
      requirement: generate
inputs:
  - name: greeting
    required: false
    default:
      derive: context/greeting
      value: "(none)"
""",
    )
    manifest = cp.validate_package(pkg)
    resolution = cp.resolve_inputs(manifest, {})
    assert resolution.values["greeting"] == "(none)"
    assert resolution.origins["greeting"] == "fallback (not derived)"
    assert resolution.underivable == []


def test_derived_default_without_fallback_is_underivable(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
dependencies:
  prompts:
    - id: context/greeting
      version: 1.0.0
      requirement: generate
inputs:
  - name: greeting
    required: false
    default:
      derive: context/greeting
""",
    )
    manifest = cp.validate_package(pkg)
    resolution = cp.resolve_inputs(manifest, {})
    assert resolution.underivable == ["greeting"]
    assert "greeting" not in resolution.values
    with pytest.raises(cp.CannedPromptError, match="unresolved placeholder"):
        cp.render_template("{{ greeting }}", resolution.values)


def test_inline_derive_is_valid(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: false
    default:
      derive:
        prompt: Produce a greeting suited to the audience.
      value: "(none)"
""",
    )
    manifest = cp.validate_package(pkg)
    assert cp.resolve_inputs(manifest, {}).values["greeting"] == "(none)"


def test_default_with_required_true_fails(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: true
    default: "hello"
""",
    )
    with pytest.raises(cp.CannedPromptError, match="required: true"):
        cp.validate_package(pkg)


def test_derive_reference_must_be_declared(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: false
    default:
      derive: context/greeting
""",
    )
    with pytest.raises(cp.CannedPromptError, match="not declared in dependencies"):
        cp.validate_package(pkg)


def test_inline_derive_cannot_also_reference(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
inputs:
  - name: greeting
    required: false
    default:
      derive:
        prompt: Produce a greeting.
        id: context/greeting
""",
    )
    with pytest.raises(cp.CannedPromptError, match="cannot also reference"):
        cp.validate_package(pkg)


MINIMAL = """\
format: canned-prompt/v0.1
id: practice/thing
name: Thing
version: 1.0.0
summary: A thing.
template: prompt.md
inputs:
  - name: greeting
    required: false
    default: hi
"""


def test_parse_reference() -> None:
    assert cp.parse_reference("practice/thing") == (None, "practice/thing")
    assert cp.parse_reference("house:practice/thing") == ("house", "practice/thing")
    with pytest.raises(cp.CannedPromptError, match="malformed reference"):
        cp.parse_reference("house:")


def test_registry_name_falls_back_to_basename(tmp_path: Path) -> None:
    registry = tmp_path / "upstream"
    registry.mkdir()
    assert cp.registry_name(registry) == "upstream"


def test_registry_name_from_manifest(tmp_path: Path) -> None:
    registry = tmp_path / "some-dir"
    registry.mkdir()
    (registry / "registry.yaml").write_text(
        "format: canned-prompt-registry/v0.1\nname: house\n", encoding="utf-8"
    )
    assert cp.registry_name(registry) == "house"


def test_registry_manifest_rejects_bad_format(tmp_path: Path) -> None:
    registry = tmp_path / "r"
    registry.mkdir()
    (registry / "registry.yaml").write_text(
        "format: something-else\nname: house\n", encoding="utf-8"
    )
    with pytest.raises(cp.CannedPromptError, match="unsupported registry format"):
        cp.read_registry_manifest(registry)


def test_registry_manifest_rejects_bad_policy(tmp_path: Path) -> None:
    registry = tmp_path / "r"
    registry.mkdir()
    (registry / "registry.yaml").write_text(
        "format: canned-prompt-registry/v0.1\n"
        "name: house\n"
        "namespaces:\n  practice:\n    policy: maybe\n",
        encoding="utf-8",
    )
    with pytest.raises(cp.CannedPromptError, match="policy must be"):
        cp.read_registry_manifest(registry)


def test_namespace_policy(tmp_path: Path) -> None:
    registry = tmp_path / "r"
    registry.mkdir()
    (registry / "registry.yaml").write_text(
        "format: canned-prompt-registry/v0.1\n"
        "name: house\n"
        "namespaces:\n  practice:\n    owner: Ada\n    policy: closed\n",
        encoding="utf-8",
    )
    assert cp.namespace_policy(registry, "practice/thing")[0] == "closed"
    assert cp.namespace_policy(registry, "scratch/thing")[0] == "open"


def install_into(catalog: Path, registry: str) -> None:
    """Place a package in the catalog under a given registry name."""
    dst = cp.catalog_package_path(catalog, registry, "practice/thing", "1.0.0")
    dst.mkdir(parents=True)
    (dst / "prompt.yaml").write_text(MINIMAL, encoding="utf-8")
    (dst / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8")


def test_same_id_from_two_registries_coexists(tmp_path: Path) -> None:
    catalog = tmp_path / "catalog"
    install_into(catalog, "house")
    install_into(catalog, "upstream")
    assert cp.catalog_registries(catalog) == ["house", "upstream"]

    package_dir, registry = cp.resolve_installed(catalog, "house:practice/thing", None)
    assert registry == "house"
    assert package_dir.is_dir()


def test_bare_id_in_two_registries_is_ambiguous(tmp_path: Path) -> None:
    catalog = tmp_path / "catalog"
    install_into(catalog, "house")
    install_into(catalog, "upstream")
    with pytest.raises(cp.CannedPromptError, match="more than one registry"):
        cp.resolve_installed(catalog, "practice/thing", None)


def test_bare_id_in_one_registry_resolves(tmp_path: Path) -> None:
    catalog = tmp_path / "catalog"
    install_into(catalog, "house")
    _, registry = cp.resolve_installed(catalog, "practice/thing", None)
    assert registry == "house"


def test_legacy_catalog_layout_is_reported(tmp_path: Path) -> None:
    catalog = tmp_path / "catalog"
    legacy = catalog / "practice" / "thing" / "1.0.0"
    legacy.mkdir(parents=True)
    (legacy / "prompt.yaml").write_text(MINIMAL, encoding="utf-8")
    (legacy / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8")
    with pytest.raises(cp.CannedPromptError, match="pre-registry-scoped"):
        cp.resolve_installed(catalog, "practice/thing", None)


# --- version selectors (§ 10.1) ---

AVAILABLE = ["2.1.0", "2.0.0", "1.5.0", "1.0.0"]


@pytest.mark.parametrize(
    "available,selector,expected",
    [
        (AVAILABLE, None, "2.1.0"),
        (AVAILABLE, "any", "2.1.0"),
        (AVAILABLE, "newest", "2.1.0"),
        (AVAILABLE, "1.5.0", "1.5.0"),
        (AVAILABLE, ">=2.0.0", "2.1.0"),
        (AVAILABLE, "9.9.9", None),
        (AVAILABLE, ">=9.9.9", None),
        (["1.5.0", "1.0.0"], ">=1.2.0", "1.5.0"),
        ([], "newest", None),
    ],
)
def test_select_version(available, selector, expected) -> None:
    assert cp.select_version(available, selector) == expected


@pytest.mark.parametrize("bad", ["^1.0.0", "~1.2", "1.x", ">=nope", "", "latest"])
def test_range_syntax_is_rejected(bad) -> None:
    with pytest.raises(cp.CannedPromptError):
        cp.validate_version_selector(bad, "dep")


# --- composition (§ 10.2) ---

FRAGMENT = """\
format: canned-prompt/v0.1
id: style/house
name: House Style
version: 1.0.0
summary: Shared style block.
type: fragment
template: prompt.md
parameters:
  tone:
    type: enum
    values: [neutral, blunt]
    default: neutral
"""

COMPOSER = """\
format: canned-prompt/v0.1
id: review/change
name: Change Review
version: 1.0.0
summary: Composes the shared style block.
template: prompt.md
dependencies:
  prompts:
    - id: style/house
      version: 1.0.0
inputs:
  - name: house_style
    required: false
    default:
      include: style/house
parameters:
  tone:
    type: enum
    values: [neutral, blunt]
    default: blunt
"""


def place(catalog: Path, registry: str, package_id: str, version: str,
          manifest: str, template: str) -> None:
    dst = cp.catalog_package_path(catalog, registry, package_id, version)
    dst.mkdir(parents=True)
    (dst / "prompt.yaml").write_text(manifest, encoding="utf-8")
    (dst / "prompt.md").write_text(template, encoding="utf-8")


@pytest.fixture()
def composed(tmp_path: Path) -> Path:
    catalog = tmp_path / "catalog"
    place(catalog, "local", "style/house", "1.0.0", FRAGMENT, "Tone is {{ tone }}.")
    place(catalog, "local", "review/change", "1.0.0", COMPOSER, "S: {{ house_style }}")
    return catalog


def test_include_inlines_rendered_template(composed: Path) -> None:
    package_dir, _ = cp.resolve_installed(composed, "review/change", None)
    manifest = cp.validate_package(package_dir)
    resolution = cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(composed))
    assert resolution.values["house_style"] == "Tone is blunt."
    assert resolution.origins["house_style"] == "included from style/house"


def test_include_inherits_outer_parameters(composed: Path) -> None:
    """The fragment's own default is neutral; the including package says blunt."""
    package_dir, _ = cp.resolve_installed(composed, "review/change", None)
    manifest = cp.validate_package(package_dir)
    resolution = cp.resolve_inputs(
        manifest, {"tone": "neutral"}, composer=cp.CatalogComposer(composed)
    )
    assert resolution.values["house_style"] == "Tone is neutral."


def test_include_without_composer_falls_back_to_unresolved(composed: Path) -> None:
    package_dir, _ = cp.resolve_installed(composed, "review/change", None)
    manifest = cp.validate_package(package_dir)
    resolution = cp.resolve_inputs(manifest, {})
    assert resolution.underivable == ["house_style"]


def test_inclusion_cycle_is_detected(tmp_path: Path) -> None:
    catalog = tmp_path / "catalog"
    for this, other in (("cyc/a", "cyc/b"), ("cyc/b", "cyc/a")):
        manifest = (
            "format: canned-prompt/v0.1\n"
            f"id: {this}\nname: X\nversion: 1.0.0\nsummary: s\ntemplate: prompt.md\n"
            f"dependencies:\n  prompts:\n    - id: {other}\n      version: 1.0.0\n"
            f"inputs:\n  - name: other\n    required: false\n"
            f"    default:\n      include: {other}\n"
        )
        place(catalog, "local", this, "1.0.0", manifest, "{{ other }}")
    package_dir, _ = cp.resolve_installed(catalog, "cyc/a", None)
    manifest = cp.validate_package(package_dir)
    with pytest.raises(cp.CannedPromptError, match="inclusion cycle"):
        cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(catalog))


def test_include_and_derive_together_is_rejected(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
dependencies:
  prompts:
    - id: style/house
      version: 1.0.0
inputs:
  - name: greeting
    required: false
    default:
      include: style/house
      derive: style/house
""",
    )
    with pytest.raises(cp.CannedPromptError, match="at most one of"):
        cp.validate_package(pkg)


def test_composed_dependency_must_declare_a_version(tmp_path: Path) -> None:
    pkg = write_pkg(
        tmp_path / "p",
        BASE
        + """\
dependencies:
  prompts:
    - id: style/house
inputs:
  - name: greeting
    required: false
    default:
      include: style/house
""",
    )
    with pytest.raises(cp.CannedPromptError, match="must declare a version"):
        cp.validate_package(pkg)


# --- evals (§ 12) ---

EVAL_BASE = """\
format: canned-prompt/v0.1
id: demo/evaluated
name: Evaluated
version: 1.0.0
summary: Exercise eval files.
template: prompt.md
examples:
- examples/basic.yaml
evals:
- evals/quality.yaml
"""


def write_evaluated(pkg: Path, eval_body: str, template: str = "Sum to 100%. {{ topic }}\n") -> Path:
    pkg.mkdir(exist_ok=True)
    (pkg / "prompt.yaml").write_text(
        EVAL_BASE + "inputs:\n  - name: topic\n    required: false\n    default: cats\n",
        encoding="utf-8",
    )
    (pkg / "prompt.md").write_text(template, encoding="utf-8")
    (pkg / "examples").mkdir(exist_ok=True)
    (pkg / "examples" / "basic.yaml").write_text(
        "name: basic\nvalues:\n  topic: dogs\n", encoding="utf-8"
    )
    (pkg / "evals").mkdir(exist_ok=True)
    (pkg / "evals" / "quality.yaml").write_text(eval_body, encoding="utf-8")
    return pkg


RUBRIC = """\
schema: canned-prompts/eval-rubric/v0.1
name: quality
example: examples/basic.yaml
render:
  - contains: "Sum to 100%"
  - not_contains: "{{"
  - resolves_all: true
output:
  criteria:
    - Answers the question.
"""


def test_valid_eval_passes_validation(tmp_path: Path) -> None:
    pkg = write_evaluated(tmp_path / "p", RUBRIC)
    assert cp.validate_package(pkg)["id"] == "demo/evaluated"


def test_eval_must_declare_a_schema(tmp_path: Path) -> None:
    pkg = write_evaluated(tmp_path / "p", "name: quality\nrender: []\n")
    with pytest.raises(cp.CannedPromptError, match="must declare a schema"):
        cp.validate_package(pkg)


def test_unknown_eval_schema_is_ignored(tmp_path: Path) -> None:
    pkg = write_evaluated(tmp_path / "p", "schema: someone/else/v1\nwhatever: true\n")
    assert cp.validate_package(pkg)["id"] == "demo/evaluated"


def test_eval_asserting_nothing_is_rejected(tmp_path: Path) -> None:
    pkg = write_evaluated(
        tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: empty\n"
    )
    with pytest.raises(cp.CannedPromptError, match="asserts nothing"):
        cp.validate_package(pkg)


def test_unknown_render_check_is_rejected(tmp_path: Path) -> None:
    pkg = write_evaluated(
        tmp_path / "p",
        "schema: canned-prompts/eval-rubric/v0.1\nname: q\nrender:\n  - matches: 'x.*'\n",
    )
    with pytest.raises(cp.CannedPromptError, match="unknown render check"):
        cp.validate_package(pkg)


def test_eval_example_must_be_declared(tmp_path: Path) -> None:
    pkg = write_evaluated(
        tmp_path / "p",
        "schema: canned-prompts/eval-rubric/v0.1\nname: q\n"
        "example: examples/missing.yaml\nrender:\n  - contains: x\n",
    )
    with pytest.raises(cp.CannedPromptError, match="not declared in the manifest"):
        cp.validate_package(pkg)


def test_render_checks_evaluate(tmp_path: Path) -> None:
    resolution = cp.Resolution(values={"topic": "dogs"}, origins={"topic": "supplied"})
    checks = [{"contains": "dogs"}, {"contains": "cats"}, {"not_contains": "cats"}]
    outcomes = cp.run_render_checks("about dogs", resolution, checks)
    assert [ok for ok, _ in outcomes] == [True, False, True]


def test_resolves_all_reports_unresolved_names() -> None:
    resolution = cp.Resolution(
        values={"a": 1}, origins={"a": "supplied", "b": "unresolved (no default)"}
    )
    outcomes = cp.run_render_checks("text", resolution, [{"resolves_all": True}])
    assert outcomes[0][0] is False
    assert "b" in outcomes[0][1]


def test_typed_fixture_values_are_not_reparsed(tmp_path: Path) -> None:
    """A YAML fixture carries real types; only CLI strings need parsing."""
    manifest = {"parameters": {"flag": {"type": "boolean", "default": False}}}
    assert cp.resolve_inputs(manifest, {"flag": True}).values["flag"] is True