from pathlib import Path import pytest import canned_prompts as cp @pytest.fixture() def package(tmp_path: Path) -> Path: pkg = tmp_path / "pkg" pkg.mkdir() (pkg / "prompt.yaml").write_text( """\ format: canned-prompt/v0.1 id: demo/hello name: Hello version: 1.0.0 summary: Say hello. template: prompt.md inputs: - name: person required: true parameters: tone: type: enum values: [warm, formal] default: warm """, encoding="utf-8", ) (pkg / "prompt.md").write_text( "Say hello to {{ person }} in a {{ tone }} tone.\n", encoding="utf-8" ) return pkg def test_validate_and_render(package: Path) -> None: manifest = cp.validate_package(package) values = cp.resolve_values(manifest, {"person": "Ada"}) rendered = cp.render_template((package / "prompt.md").read_text(), values) assert rendered == "Say hello to Ada in a warm tone.\n" def test_missing_required_input_fails(package: Path) -> None: manifest = cp.validate_package(package) with pytest.raises(cp.CannedPromptError, match="missing required input"): cp.resolve_values(manifest, {}) def test_undeclared_placeholder_fails(package: Path) -> None: (package / "prompt.md").write_text("{{ missing }}\n", encoding="utf-8") with pytest.raises(cp.CannedPromptError, match="undeclared placeholders"): cp.validate_package(package) def write_pkg(pkg: Path, manifest: str, template: str = "{{ greeting }}\n") -> Path: pkg.mkdir(exist_ok=True) (pkg / "prompt.yaml").write_text(manifest, encoding="utf-8") (pkg / "prompt.md").write_text(template, encoding="utf-8") return pkg BASE = """\ format: canned-prompt/v0.1 id: demo/defaults name: Defaults version: 1.0.0 summary: Exercise input defaults. template: prompt.md """ def test_static_default_fills_optional_input(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: false default: "hello there" """, ) manifest = cp.validate_package(pkg) resolution = cp.resolve_inputs(manifest, {}) assert resolution.values["greeting"] == "hello there" assert resolution.origins["greeting"] == "default" assert cp.render_template("{{ greeting }}", resolution.values) == "hello there" def test_supplied_value_overrides_static_default(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: false default: "hello there" """, ) manifest = cp.validate_package(pkg) resolution = cp.resolve_inputs(manifest, {"greeting": "hi"}) assert resolution.values["greeting"] == "hi" assert resolution.origins["greeting"] == "supplied" def test_derived_default_uses_static_fallback(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ dependencies: prompts: - id: context/greeting version: 1.0.0 requirement: generate inputs: - name: greeting required: false default: derive: context/greeting value: "(none)" """, ) manifest = cp.validate_package(pkg) resolution = cp.resolve_inputs(manifest, {}) assert resolution.values["greeting"] == "(none)" assert resolution.origins["greeting"] == "fallback (not derived)" assert resolution.underivable == [] def test_derived_default_without_fallback_is_underivable(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ dependencies: prompts: - id: context/greeting version: 1.0.0 requirement: generate inputs: - name: greeting required: false default: derive: context/greeting """, ) manifest = cp.validate_package(pkg) resolution = cp.resolve_inputs(manifest, {}) assert resolution.underivable == ["greeting"] assert "greeting" not in resolution.values with pytest.raises(cp.CannedPromptError, match="unresolved placeholder"): cp.render_template("{{ greeting }}", resolution.values) def test_inline_derive_is_valid(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: false default: derive: prompt: Produce a greeting suited to the audience. value: "(none)" """, ) manifest = cp.validate_package(pkg) assert cp.resolve_inputs(manifest, {}).values["greeting"] == "(none)" def test_default_with_required_true_fails(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: true default: "hello" """, ) with pytest.raises(cp.CannedPromptError, match="required: true"): cp.validate_package(pkg) def test_derive_reference_must_be_declared(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: false default: derive: context/greeting """, ) with pytest.raises(cp.CannedPromptError, match="not declared in dependencies"): cp.validate_package(pkg) def test_inline_derive_cannot_also_reference(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ inputs: - name: greeting required: false default: derive: prompt: Produce a greeting. id: context/greeting """, ) with pytest.raises(cp.CannedPromptError, match="cannot also reference"): cp.validate_package(pkg) MINIMAL = """\ format: canned-prompt/v0.1 id: practice/thing name: Thing version: 1.0.0 summary: A thing. template: prompt.md inputs: - name: greeting required: false default: hi """ def test_parse_reference() -> None: assert cp.parse_reference("practice/thing") == (None, "practice/thing") assert cp.parse_reference("house:practice/thing") == ("house", "practice/thing") with pytest.raises(cp.CannedPromptError, match="malformed reference"): cp.parse_reference("house:") def test_registry_name_falls_back_to_basename(tmp_path: Path) -> None: registry = tmp_path / "upstream" registry.mkdir() assert cp.registry_name(registry) == "upstream" def test_registry_name_from_manifest(tmp_path: Path) -> None: registry = tmp_path / "some-dir" registry.mkdir() (registry / "registry.yaml").write_text( "format: canned-prompt-registry/v0.1\nname: house\n", encoding="utf-8" ) assert cp.registry_name(registry) == "house" def test_registry_manifest_rejects_bad_format(tmp_path: Path) -> None: registry = tmp_path / "r" registry.mkdir() (registry / "registry.yaml").write_text( "format: something-else\nname: house\n", encoding="utf-8" ) with pytest.raises(cp.CannedPromptError, match="unsupported registry format"): cp.read_registry_manifest(registry) def test_registry_manifest_rejects_bad_policy(tmp_path: Path) -> None: registry = tmp_path / "r" registry.mkdir() (registry / "registry.yaml").write_text( "format: canned-prompt-registry/v0.1\n" "name: house\n" "namespaces:\n practice:\n policy: maybe\n", encoding="utf-8", ) with pytest.raises(cp.CannedPromptError, match="policy must be"): cp.read_registry_manifest(registry) def test_namespace_policy(tmp_path: Path) -> None: registry = tmp_path / "r" registry.mkdir() (registry / "registry.yaml").write_text( "format: canned-prompt-registry/v0.1\n" "name: house\n" "namespaces:\n practice:\n owner: Ada\n policy: closed\n", encoding="utf-8", ) assert cp.namespace_policy(registry, "practice/thing")[0] == "closed" assert cp.namespace_policy(registry, "scratch/thing")[0] == "open" def install_into(catalog: Path, registry: str) -> None: """Place a package in the catalog under a given registry name.""" dst = cp.catalog_package_path(catalog, registry, "practice/thing", "1.0.0") dst.mkdir(parents=True) (dst / "prompt.yaml").write_text(MINIMAL, encoding="utf-8") (dst / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8") def test_same_id_from_two_registries_coexists(tmp_path: Path) -> None: catalog = tmp_path / "catalog" install_into(catalog, "house") install_into(catalog, "upstream") assert cp.catalog_registries(catalog) == ["house", "upstream"] package_dir, registry = cp.resolve_installed(catalog, "house:practice/thing", None) assert registry == "house" assert package_dir.is_dir() def test_bare_id_in_two_registries_is_ambiguous(tmp_path: Path) -> None: catalog = tmp_path / "catalog" install_into(catalog, "house") install_into(catalog, "upstream") with pytest.raises(cp.CannedPromptError, match="more than one registry"): cp.resolve_installed(catalog, "practice/thing", None) def test_bare_id_in_one_registry_resolves(tmp_path: Path) -> None: catalog = tmp_path / "catalog" install_into(catalog, "house") _, registry = cp.resolve_installed(catalog, "practice/thing", None) assert registry == "house" def test_legacy_catalog_layout_is_reported(tmp_path: Path) -> None: catalog = tmp_path / "catalog" legacy = catalog / "practice" / "thing" / "1.0.0" legacy.mkdir(parents=True) (legacy / "prompt.yaml").write_text(MINIMAL, encoding="utf-8") (legacy / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8") with pytest.raises(cp.CannedPromptError, match="pre-registry-scoped"): cp.resolve_installed(catalog, "practice/thing", None) # --- version selectors (§ 10.1) --- AVAILABLE = ["2.1.0", "2.0.0", "1.5.0", "1.0.0"] @pytest.mark.parametrize( "available,selector,expected", [ (AVAILABLE, None, "2.1.0"), (AVAILABLE, "any", "2.1.0"), (AVAILABLE, "newest", "2.1.0"), (AVAILABLE, "1.5.0", "1.5.0"), (AVAILABLE, ">=2.0.0", "2.1.0"), (AVAILABLE, "9.9.9", None), (AVAILABLE, ">=9.9.9", None), (["1.5.0", "1.0.0"], ">=1.2.0", "1.5.0"), ([], "newest", None), ], ) def test_select_version(available, selector, expected) -> None: assert cp.select_version(available, selector) == expected @pytest.mark.parametrize("bad", ["^1.0.0", "~1.2", "1.x", ">=nope", "", "latest"]) def test_range_syntax_is_rejected(bad) -> None: with pytest.raises(cp.CannedPromptError): cp.validate_version_selector(bad, "dep") # --- composition (§ 10.2) --- FRAGMENT = """\ format: canned-prompt/v0.1 id: style/house name: House Style version: 1.0.0 summary: Shared style block. type: fragment template: prompt.md parameters: tone: type: enum values: [neutral, blunt] default: neutral """ COMPOSER = """\ format: canned-prompt/v0.1 id: review/change name: Change Review version: 1.0.0 summary: Composes the shared style block. template: prompt.md dependencies: prompts: - id: style/house version: 1.0.0 inputs: - name: house_style required: false default: include: style/house parameters: tone: type: enum values: [neutral, blunt] default: blunt """ def place(catalog: Path, registry: str, package_id: str, version: str, manifest: str, template: str) -> None: dst = cp.catalog_package_path(catalog, registry, package_id, version) dst.mkdir(parents=True) (dst / "prompt.yaml").write_text(manifest, encoding="utf-8") (dst / "prompt.md").write_text(template, encoding="utf-8") @pytest.fixture() def composed(tmp_path: Path) -> Path: catalog = tmp_path / "catalog" place(catalog, "local", "style/house", "1.0.0", FRAGMENT, "Tone is {{ tone }}.") place(catalog, "local", "review/change", "1.0.0", COMPOSER, "S: {{ house_style }}") return catalog def test_include_inlines_rendered_template(composed: Path) -> None: package_dir, _ = cp.resolve_installed(composed, "review/change", None) manifest = cp.validate_package(package_dir) resolution = cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(composed)) assert resolution.values["house_style"] == "Tone is blunt." assert resolution.origins["house_style"] == "included from style/house" def test_include_inherits_outer_parameters(composed: Path) -> None: """The fragment's own default is neutral; the including package says blunt.""" package_dir, _ = cp.resolve_installed(composed, "review/change", None) manifest = cp.validate_package(package_dir) resolution = cp.resolve_inputs( manifest, {"tone": "neutral"}, composer=cp.CatalogComposer(composed) ) assert resolution.values["house_style"] == "Tone is neutral." def test_include_without_composer_falls_back_to_unresolved(composed: Path) -> None: package_dir, _ = cp.resolve_installed(composed, "review/change", None) manifest = cp.validate_package(package_dir) resolution = cp.resolve_inputs(manifest, {}) assert resolution.underivable == ["house_style"] def test_inclusion_cycle_is_detected(tmp_path: Path) -> None: catalog = tmp_path / "catalog" for this, other in (("cyc/a", "cyc/b"), ("cyc/b", "cyc/a")): manifest = ( "format: canned-prompt/v0.1\n" f"id: {this}\nname: X\nversion: 1.0.0\nsummary: s\ntemplate: prompt.md\n" f"dependencies:\n prompts:\n - id: {other}\n version: 1.0.0\n" f"inputs:\n - name: other\n required: false\n" f" default:\n include: {other}\n" ) place(catalog, "local", this, "1.0.0", manifest, "{{ other }}") package_dir, _ = cp.resolve_installed(catalog, "cyc/a", None) manifest = cp.validate_package(package_dir) with pytest.raises(cp.CannedPromptError, match="inclusion cycle"): cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(catalog)) def test_include_and_derive_together_is_rejected(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ dependencies: prompts: - id: style/house version: 1.0.0 inputs: - name: greeting required: false default: include: style/house derive: style/house """, ) with pytest.raises(cp.CannedPromptError, match="at most one of"): cp.validate_package(pkg) def test_composed_dependency_must_declare_a_version(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE + """\ dependencies: prompts: - id: style/house inputs: - name: greeting required: false default: include: style/house """, ) with pytest.raises(cp.CannedPromptError, match="must declare a version"): cp.validate_package(pkg) # --- evals (§ 12) --- EVAL_BASE = """\ format: canned-prompt/v0.1 id: demo/evaluated name: Evaluated version: 1.0.0 summary: Exercise eval files. template: prompt.md examples: - examples/basic.yaml evals: - evals/quality.yaml """ def write_evaluated(pkg: Path, eval_body: str, template: str = "Sum to 100%. {{ topic }}\n") -> Path: pkg.mkdir(exist_ok=True) (pkg / "prompt.yaml").write_text( EVAL_BASE + "inputs:\n - name: topic\n required: false\n default: cats\n", encoding="utf-8", ) (pkg / "prompt.md").write_text(template, encoding="utf-8") (pkg / "examples").mkdir(exist_ok=True) (pkg / "examples" / "basic.yaml").write_text( "name: basic\nvalues:\n topic: dogs\n", encoding="utf-8" ) (pkg / "evals").mkdir(exist_ok=True) (pkg / "evals" / "quality.yaml").write_text(eval_body, encoding="utf-8") return pkg RUBRIC = """\ schema: canned-prompts/eval-rubric/v0.1 name: quality example: examples/basic.yaml render: - contains: "Sum to 100%" - not_contains: "{{" - resolves_all: true output: criteria: - Answers the question. """ def test_valid_eval_passes_validation(tmp_path: Path) -> None: pkg = write_evaluated(tmp_path / "p", RUBRIC) assert cp.validate_package(pkg)["id"] == "demo/evaluated" def test_eval_must_declare_a_schema(tmp_path: Path) -> None: pkg = write_evaluated(tmp_path / "p", "name: quality\nrender: []\n") with pytest.raises(cp.CannedPromptError, match="must declare a schema"): cp.validate_package(pkg) def test_unknown_eval_schema_is_ignored(tmp_path: Path) -> None: pkg = write_evaluated(tmp_path / "p", "schema: someone/else/v1\nwhatever: true\n") assert cp.validate_package(pkg)["id"] == "demo/evaluated" def test_eval_asserting_nothing_is_rejected(tmp_path: Path) -> None: pkg = write_evaluated( tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: empty\n" ) with pytest.raises(cp.CannedPromptError, match="asserts nothing"): cp.validate_package(pkg) def test_unknown_render_check_is_rejected(tmp_path: Path) -> None: pkg = write_evaluated( tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: q\nrender:\n - matches: 'x.*'\n", ) with pytest.raises(cp.CannedPromptError, match="unknown render check"): cp.validate_package(pkg) def test_eval_example_must_be_declared(tmp_path: Path) -> None: pkg = write_evaluated( tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: q\n" "example: examples/missing.yaml\nrender:\n - contains: x\n", ) with pytest.raises(cp.CannedPromptError, match="not declared in the manifest"): cp.validate_package(pkg) def test_render_checks_evaluate(tmp_path: Path) -> None: resolution = cp.Resolution(values={"topic": "dogs"}, origins={"topic": "supplied"}) checks = [{"contains": "dogs"}, {"contains": "cats"}, {"not_contains": "cats"}] outcomes = cp.run_render_checks("about dogs", resolution, checks) assert [ok for ok, _ in outcomes] == [True, False, True] def test_resolves_all_reports_unresolved_names() -> None: resolution = cp.Resolution( values={"a": 1}, origins={"a": "supplied", "b": "unresolved (no default)"} ) outcomes = cp.run_render_checks("text", resolution, [{"resolves_all": True}]) assert outcomes[0][0] is False assert "b" in outcomes[0][1] def test_typed_fixture_values_are_not_reparsed(tmp_path: Path) -> None: """A YAML fixture carries real types; only CLI strings need parsing.""" manifest = {"parameters": {"flag": {"type": "boolean", "default": False}}} assert cp.resolve_inputs(manifest, {"flag": True}).values["flag"] is True # --- context and capability dependencies (§ 10.1, § 10.2) --- def ctx_pkg(tmp_path: Path, body: str) -> Path: return write_pkg(tmp_path / "p", BASE + body, "hello\n") def test_context_dependency_requires_name_and_description(tmp_path: Path) -> None: pkg = ctx_pkg( tmp_path, "dependencies:\n context:\n - name: repository-tree\n" " description: A listing of the repository.\n", ) assert cp.validate_package(pkg)["id"] == "demo/defaults" def test_context_dependency_without_description_is_rejected(tmp_path: Path) -> None: pkg = ctx_pkg(tmp_path, "dependencies:\n context:\n - name: repository-tree\n") with pytest.raises(cp.CannedPromptError, match="must declare a description"): cp.validate_package(pkg) def test_context_dependency_using_id_is_rejected(tmp_path: Path) -> None: """Context entries use `name`; `id` would imply resolvable package identity.""" pkg = ctx_pkg( tmp_path, "dependencies:\n context:\n - id: policy/security\n" " description: A policy.\n", ) with pytest.raises(cp.CannedPromptError, match="not 'id'"): cp.validate_package(pkg) def test_context_requirement_generate_is_rejected(tmp_path: Path) -> None: pkg = ctx_pkg( tmp_path, "dependencies:\n context:\n - name: repository-tree\n" " description: A listing.\n requirement: generate\n", ) with pytest.raises(cp.CannedPromptError, match="required' or 'optional"): cp.validate_package(pkg) @pytest.mark.parametrize("name", ["web-search", "code-analysis", "vision", "a1-b2"]) def test_valid_capability_names(name) -> None: assert cp.validate_capabilities([name], "dependencies.capabilities") == [name] @pytest.mark.parametrize("name", ["Web-Search", "web_search", "-lead", "trail-", ""]) def test_invalid_capability_names(name) -> None: with pytest.raises(cp.CannedPromptError, match="kebab-case"): cp.validate_capabilities([name], "dependencies.capabilities") def test_capability_may_be_both_required_and_observed(tmp_path: Path) -> None: """§ 9.1: required to run at all, and separately observed to work well.""" pkg = ctx_pkg( tmp_path, "dependencies:\n capabilities:\n - web-search\n" "compatibility:\n capabilities:\n - web-search\n - long-context\n", ) manifest = cp.validate_package(pkg) assert manifest["dependencies"]["capabilities"] == ["web-search"] assert "long-context" in manifest["compatibility"]["capabilities"] # --- semver precedence and prereleases (§ 17.1) --- def test_release_outranks_its_prerelease() -> None: assert cp.parse_semver("1.0.0") > cp.parse_semver("1.0.0-rc1") def test_prerelease_ordering_follows_semver() -> None: ordered = sorted( ["1.0.0", "1.0.0-rc.2", "1.0.0-rc.10", "1.0.0-alpha", "0.9.0"], key=cp.parse_semver, reverse=True, ) assert ordered == ["1.0.0", "1.0.0-rc.10", "1.0.0-rc.2", "1.0.0-alpha", "0.9.0"] def test_build_metadata_is_ignored_for_precedence() -> None: assert cp.parse_semver("1.0.0+build.1") == cp.parse_semver("1.0.0+build.2") @pytest.mark.parametrize("selector", ["newest", "any", ">=1.0.0", None]) def test_prerelease_is_not_selected_implicitly(selector) -> None: available = ["1.1.0-rc1", "1.0.0"] assert cp.select_version(available, selector) == "1.0.0" def test_exact_pin_selects_a_prerelease() -> None: assert cp.select_version(["1.1.0-rc1", "1.0.0"], "1.1.0-rc1") == "1.1.0-rc1" def test_only_prereleases_available_selects_nothing() -> None: assert cp.select_version(["1.0.0-rc1"], "newest") is None def test_prerelease_only_package_reports_why(tmp_path: Path) -> None: catalog = tmp_path / "catalog" place(catalog, "local", "practice/thing", "1.0.0-rc1", MINIMAL.replace( "version: 1.0.0", "version: 1.0.0-rc1"), "{{ greeting }}\n") with pytest.raises(cp.CannedPromptError, match="only prerelease versions"): cp.resolve_installed(catalog, "practice/thing", None) # --- strict packaging (§ 2) --- def test_only_reserved_and_referenced_paths_are_packaged(tmp_path: Path) -> None: src = write_evaluated(tmp_path / "src", RUBRIC) (src / "LICENSE").write_text("MIT\n", encoding="utf-8") (src / "notes.txt").write_text("scratch\n", encoding="utf-8") (src / ".git").mkdir() (src / ".git" / "config").write_text("junk\n", encoding="utf-8") manifest = cp.validate_package(src) dst = tmp_path / "out" skipped = cp.copy_package(src, dst, manifest, "package") packaged = sorted(p.relative_to(dst).as_posix() for p in dst.rglob("*") if p.is_file()) assert packaged == [ "LICENSE", "evals/quality.yaml", "examples/basic.yaml", "prompt.md", "prompt.yaml", ] assert skipped == [".git/", "notes.txt"] def test_packaged_copy_is_still_valid(tmp_path: Path) -> None: src = write_evaluated(tmp_path / "src", RUBRIC) manifest = cp.validate_package(src) dst = tmp_path / "out" cp.copy_package(src, dst, manifest, "package") assert cp.validate_package(dst)["id"] == "demo/evaluated" def test_copy_refuses_to_overwrite(tmp_path: Path) -> None: src = write_evaluated(tmp_path / "src", RUBRIC) manifest = cp.validate_package(src) dst = tmp_path / "out" cp.copy_package(src, dst, manifest, "package") with pytest.raises(cp.CannedPromptError, match="already exists"): cp.copy_package(src, dst, manifest, "package") # --- format revisions (§ 3.2) --- @pytest.mark.parametrize("declared", ["canned-prompt/v0.2", "canned-prompt/v0.1"]) def test_both_format_revisions_are_accepted(tmp_path: Path, declared) -> None: """v0.2 is additive, so a v0.1 package means exactly what it always meant.""" pkg = write_pkg( tmp_path / "p", BASE.replace("canned-prompt/v0.1", declared) + "inputs:\n - name: greeting\n required: false\n default: hi\n", ) assert cp.validate_package(pkg)["format"] == declared def test_unknown_format_is_rejected(tmp_path: Path) -> None: pkg = write_pkg( tmp_path / "p", BASE.replace("canned-prompt/v0.1", "canned-prompt/v9.9") + "inputs:\n - name: greeting\n required: false\n default: hi\n", ) with pytest.raises(cp.CannedPromptError, match="unsupported format"): cp.validate_package(pkg)