tegwick
b3280df742
CANP-WP-0004. The operator asked canned-prompts to build a database of versioned prompts recording where each came from and when — the first step toward a platform for collaborative prompting. The format had nowhere to put that. `provenance` records who wrote a prompt and where the idea came from; nothing recorded how a copy arrived in a particular store. Section 20.3 now specifies an `index.yaml` as store metadata rather than package data: how a copy arrived differs for every consumer, and recording an arrival must never rewrite the package that arrived. `add`, `install` and `publish` record registry, id, version, name, source, method, first-inclusion date, and the package's declared author, source and licence — the last three copied so a listing is readable without opening every package. A new `index` verb lists it. `included_at` is never overwritten; a re-run updates `last_seen_at`, because when a package first entered a collection is a fact about history rather than about the last command run. Tests 84 -> 90. Also records two findings from actually using the format: CANP-WP-0004-T03 — inclusion has no deduplication, so a diamond dependency renders shared content once per path. Found by composing a real collection. Not fixed here: deduplicating means choosing which occurrence survives and deciding what happens when two paths resolve different versions, which is resolver behaviour that section 10.4 deliberately avoids. Handed to CANP-WP-0005 with a leaning: document it, warn at validation time, do not deduplicate. The add_ons workaround in practice/pqrst-estimate is evidence about section 23's deferred "richer template syntax" — an optional appendix has to be an input with an empty default, because CPF has no conditionals. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM Assistant: claude-code Assistant-Model: opus Assistant-Process: 388925@bnt-lap001 Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502
869 lines
28 KiB
Python
869 lines
28 KiB
Python
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
import canned_prompts as cp
|
|
|
|
|
|
@pytest.fixture()
|
|
def package(tmp_path: Path) -> Path:
|
|
pkg = tmp_path / "pkg"
|
|
pkg.mkdir()
|
|
(pkg / "prompt.yaml").write_text(
|
|
"""\
|
|
format: canned-prompt/v0.1
|
|
id: demo/hello
|
|
name: Hello
|
|
version: 1.0.0
|
|
summary: Say hello.
|
|
template: prompt.md
|
|
inputs:
|
|
- name: person
|
|
required: true
|
|
parameters:
|
|
tone:
|
|
type: enum
|
|
values: [warm, formal]
|
|
default: warm
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
(pkg / "prompt.md").write_text(
|
|
"Say hello to {{ person }} in a {{ tone }} tone.\n", encoding="utf-8"
|
|
)
|
|
return pkg
|
|
|
|
|
|
def test_validate_and_render(package: Path) -> None:
|
|
manifest = cp.validate_package(package)
|
|
values = cp.resolve_values(manifest, {"person": "Ada"})
|
|
rendered = cp.render_template((package / "prompt.md").read_text(), values)
|
|
assert rendered == "Say hello to Ada in a warm tone.\n"
|
|
|
|
|
|
def test_missing_required_input_fails(package: Path) -> None:
|
|
manifest = cp.validate_package(package)
|
|
with pytest.raises(cp.CannedPromptError, match="missing required input"):
|
|
cp.resolve_values(manifest, {})
|
|
|
|
|
|
def test_undeclared_placeholder_fails(package: Path) -> None:
|
|
(package / "prompt.md").write_text("{{ missing }}\n", encoding="utf-8")
|
|
with pytest.raises(cp.CannedPromptError, match="undeclared placeholders"):
|
|
cp.validate_package(package)
|
|
|
|
|
|
def write_pkg(pkg: Path, manifest: str, template: str = "{{ greeting }}\n") -> Path:
|
|
pkg.mkdir(exist_ok=True)
|
|
(pkg / "prompt.yaml").write_text(manifest, encoding="utf-8")
|
|
(pkg / "prompt.md").write_text(template, encoding="utf-8")
|
|
return pkg
|
|
|
|
|
|
BASE = """\
|
|
format: canned-prompt/v0.1
|
|
id: demo/defaults
|
|
name: Defaults
|
|
version: 1.0.0
|
|
summary: Exercise input defaults.
|
|
template: prompt.md
|
|
"""
|
|
|
|
|
|
def test_static_default_fills_optional_input(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default: "hello there"
|
|
""",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
resolution = cp.resolve_inputs(manifest, {})
|
|
assert resolution.values["greeting"] == "hello there"
|
|
assert resolution.origins["greeting"] == "default"
|
|
assert cp.render_template("{{ greeting }}", resolution.values) == "hello there"
|
|
|
|
|
|
def test_supplied_value_overrides_static_default(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default: "hello there"
|
|
""",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
resolution = cp.resolve_inputs(manifest, {"greeting": "hi"})
|
|
assert resolution.values["greeting"] == "hi"
|
|
assert resolution.origins["greeting"] == "supplied"
|
|
|
|
|
|
def test_derived_default_uses_static_fallback(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
dependencies:
|
|
prompts:
|
|
- id: context/greeting
|
|
version: 1.0.0
|
|
requirement: generate
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
derive: context/greeting
|
|
value: "(none)"
|
|
""",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
resolution = cp.resolve_inputs(manifest, {})
|
|
assert resolution.values["greeting"] == "(none)"
|
|
assert resolution.origins["greeting"] == "fallback (not derived)"
|
|
assert resolution.underivable == []
|
|
|
|
|
|
def test_derived_default_without_fallback_is_underivable(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
dependencies:
|
|
prompts:
|
|
- id: context/greeting
|
|
version: 1.0.0
|
|
requirement: generate
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
derive: context/greeting
|
|
""",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
resolution = cp.resolve_inputs(manifest, {})
|
|
assert resolution.underivable == ["greeting"]
|
|
assert "greeting" not in resolution.values
|
|
with pytest.raises(cp.CannedPromptError, match="unresolved placeholder"):
|
|
cp.render_template("{{ greeting }}", resolution.values)
|
|
|
|
|
|
def test_inline_derive_is_valid(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
derive:
|
|
prompt: Produce a greeting suited to the audience.
|
|
value: "(none)"
|
|
""",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
assert cp.resolve_inputs(manifest, {}).values["greeting"] == "(none)"
|
|
|
|
|
|
def test_default_with_required_true_fails(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: true
|
|
default: "hello"
|
|
""",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="required: true"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_derive_reference_must_be_declared(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
derive: context/greeting
|
|
""",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="not declared in dependencies"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_inline_derive_cannot_also_reference(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
derive:
|
|
prompt: Produce a greeting.
|
|
id: context/greeting
|
|
""",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="cannot also reference"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
MINIMAL = """\
|
|
format: canned-prompt/v0.1
|
|
id: practice/thing
|
|
name: Thing
|
|
version: 1.0.0
|
|
summary: A thing.
|
|
template: prompt.md
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default: hi
|
|
"""
|
|
|
|
|
|
def test_parse_reference() -> None:
|
|
assert cp.parse_reference("practice/thing") == (None, "practice/thing")
|
|
assert cp.parse_reference("house:practice/thing") == ("house", "practice/thing")
|
|
with pytest.raises(cp.CannedPromptError, match="malformed reference"):
|
|
cp.parse_reference("house:")
|
|
|
|
|
|
def test_registry_name_falls_back_to_basename(tmp_path: Path) -> None:
|
|
registry = tmp_path / "upstream"
|
|
registry.mkdir()
|
|
assert cp.registry_name(registry) == "upstream"
|
|
|
|
|
|
def test_registry_name_from_manifest(tmp_path: Path) -> None:
|
|
registry = tmp_path / "some-dir"
|
|
registry.mkdir()
|
|
(registry / "registry.yaml").write_text(
|
|
"format: canned-prompt-registry/v0.1\nname: house\n", encoding="utf-8"
|
|
)
|
|
assert cp.registry_name(registry) == "house"
|
|
|
|
|
|
def test_registry_manifest_rejects_bad_format(tmp_path: Path) -> None:
|
|
registry = tmp_path / "r"
|
|
registry.mkdir()
|
|
(registry / "registry.yaml").write_text(
|
|
"format: something-else\nname: house\n", encoding="utf-8"
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="unsupported registry format"):
|
|
cp.read_registry_manifest(registry)
|
|
|
|
|
|
def test_registry_manifest_rejects_bad_policy(tmp_path: Path) -> None:
|
|
registry = tmp_path / "r"
|
|
registry.mkdir()
|
|
(registry / "registry.yaml").write_text(
|
|
"format: canned-prompt-registry/v0.1\n"
|
|
"name: house\n"
|
|
"namespaces:\n practice:\n policy: maybe\n",
|
|
encoding="utf-8",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="policy must be"):
|
|
cp.read_registry_manifest(registry)
|
|
|
|
|
|
def test_namespace_policy(tmp_path: Path) -> None:
|
|
registry = tmp_path / "r"
|
|
registry.mkdir()
|
|
(registry / "registry.yaml").write_text(
|
|
"format: canned-prompt-registry/v0.1\n"
|
|
"name: house\n"
|
|
"namespaces:\n practice:\n owner: Ada\n policy: closed\n",
|
|
encoding="utf-8",
|
|
)
|
|
assert cp.namespace_policy(registry, "practice/thing")[0] == "closed"
|
|
assert cp.namespace_policy(registry, "scratch/thing")[0] == "open"
|
|
|
|
|
|
def install_into(catalog: Path, registry: str) -> None:
|
|
"""Place a package in the catalog under a given registry name."""
|
|
dst = cp.catalog_package_path(catalog, registry, "practice/thing", "1.0.0")
|
|
dst.mkdir(parents=True)
|
|
(dst / "prompt.yaml").write_text(MINIMAL, encoding="utf-8")
|
|
(dst / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8")
|
|
|
|
|
|
def test_same_id_from_two_registries_coexists(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
install_into(catalog, "house")
|
|
install_into(catalog, "upstream")
|
|
assert cp.catalog_registries(catalog) == ["house", "upstream"]
|
|
|
|
package_dir, registry = cp.resolve_installed(catalog, "house:practice/thing", None)
|
|
assert registry == "house"
|
|
assert package_dir.is_dir()
|
|
|
|
|
|
def test_bare_id_in_two_registries_is_ambiguous(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
install_into(catalog, "house")
|
|
install_into(catalog, "upstream")
|
|
with pytest.raises(cp.CannedPromptError, match="more than one registry"):
|
|
cp.resolve_installed(catalog, "practice/thing", None)
|
|
|
|
|
|
def test_bare_id_in_one_registry_resolves(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
install_into(catalog, "house")
|
|
_, registry = cp.resolve_installed(catalog, "practice/thing", None)
|
|
assert registry == "house"
|
|
|
|
|
|
def test_legacy_catalog_layout_is_reported(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
legacy = catalog / "practice" / "thing" / "1.0.0"
|
|
legacy.mkdir(parents=True)
|
|
(legacy / "prompt.yaml").write_text(MINIMAL, encoding="utf-8")
|
|
(legacy / "prompt.md").write_text("{{ greeting }}\n", encoding="utf-8")
|
|
with pytest.raises(cp.CannedPromptError, match="pre-registry-scoped"):
|
|
cp.resolve_installed(catalog, "practice/thing", None)
|
|
|
|
|
|
# --- version selectors (§ 10.1) ---
|
|
|
|
AVAILABLE = ["2.1.0", "2.0.0", "1.5.0", "1.0.0"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"available,selector,expected",
|
|
[
|
|
(AVAILABLE, None, "2.1.0"),
|
|
(AVAILABLE, "any", "2.1.0"),
|
|
(AVAILABLE, "newest", "2.1.0"),
|
|
(AVAILABLE, "1.5.0", "1.5.0"),
|
|
(AVAILABLE, ">=2.0.0", "2.1.0"),
|
|
(AVAILABLE, "9.9.9", None),
|
|
(AVAILABLE, ">=9.9.9", None),
|
|
(["1.5.0", "1.0.0"], ">=1.2.0", "1.5.0"),
|
|
([], "newest", None),
|
|
],
|
|
)
|
|
def test_select_version(available, selector, expected) -> None:
|
|
assert cp.select_version(available, selector) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("bad", ["^1.0.0", "~1.2", "1.x", ">=nope", "", "latest"])
|
|
def test_range_syntax_is_rejected(bad) -> None:
|
|
with pytest.raises(cp.CannedPromptError):
|
|
cp.validate_version_selector(bad, "dep")
|
|
|
|
|
|
# --- composition (§ 10.2) ---
|
|
|
|
FRAGMENT = """\
|
|
format: canned-prompt/v0.1
|
|
id: style/house
|
|
name: House Style
|
|
version: 1.0.0
|
|
summary: Shared style block.
|
|
type: fragment
|
|
template: prompt.md
|
|
parameters:
|
|
tone:
|
|
type: enum
|
|
values: [neutral, blunt]
|
|
default: neutral
|
|
"""
|
|
|
|
COMPOSER = """\
|
|
format: canned-prompt/v0.1
|
|
id: review/change
|
|
name: Change Review
|
|
version: 1.0.0
|
|
summary: Composes the shared style block.
|
|
template: prompt.md
|
|
dependencies:
|
|
prompts:
|
|
- id: style/house
|
|
version: 1.0.0
|
|
inputs:
|
|
- name: house_style
|
|
required: false
|
|
default:
|
|
include: style/house
|
|
parameters:
|
|
tone:
|
|
type: enum
|
|
values: [neutral, blunt]
|
|
default: blunt
|
|
"""
|
|
|
|
|
|
def place(catalog: Path, registry: str, package_id: str, version: str,
|
|
manifest: str, template: str) -> None:
|
|
dst = cp.catalog_package_path(catalog, registry, package_id, version)
|
|
dst.mkdir(parents=True)
|
|
(dst / "prompt.yaml").write_text(manifest, encoding="utf-8")
|
|
(dst / "prompt.md").write_text(template, encoding="utf-8")
|
|
|
|
|
|
@pytest.fixture()
|
|
def composed(tmp_path: Path) -> Path:
|
|
catalog = tmp_path / "catalog"
|
|
place(catalog, "local", "style/house", "1.0.0", FRAGMENT, "Tone is {{ tone }}.")
|
|
place(catalog, "local", "review/change", "1.0.0", COMPOSER, "S: {{ house_style }}")
|
|
return catalog
|
|
|
|
|
|
def test_include_inlines_rendered_template(composed: Path) -> None:
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
resolution = cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(composed))
|
|
assert resolution.values["house_style"] == "Tone is blunt."
|
|
assert resolution.origins["house_style"] == "included from style/house"
|
|
|
|
|
|
def test_include_inherits_outer_parameters(composed: Path) -> None:
|
|
"""The fragment's own default is neutral; the including package says blunt."""
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
resolution = cp.resolve_inputs(
|
|
manifest, {"tone": "neutral"}, composer=cp.CatalogComposer(composed)
|
|
)
|
|
assert resolution.values["house_style"] == "Tone is neutral."
|
|
|
|
|
|
def test_include_without_composer_falls_back_to_unresolved(composed: Path) -> None:
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
resolution = cp.resolve_inputs(manifest, {})
|
|
assert resolution.underivable == ["house_style"]
|
|
|
|
|
|
def test_inclusion_cycle_is_detected(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
for this, other in (("cyc/a", "cyc/b"), ("cyc/b", "cyc/a")):
|
|
manifest = (
|
|
"format: canned-prompt/v0.1\n"
|
|
f"id: {this}\nname: X\nversion: 1.0.0\nsummary: s\ntemplate: prompt.md\n"
|
|
f"dependencies:\n prompts:\n - id: {other}\n version: 1.0.0\n"
|
|
f"inputs:\n - name: other\n required: false\n"
|
|
f" default:\n include: {other}\n"
|
|
)
|
|
place(catalog, "local", this, "1.0.0", manifest, "{{ other }}")
|
|
package_dir, _ = cp.resolve_installed(catalog, "cyc/a", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
with pytest.raises(cp.CannedPromptError, match="inclusion cycle"):
|
|
cp.resolve_inputs(manifest, {}, composer=cp.CatalogComposer(catalog))
|
|
|
|
|
|
def test_include_and_derive_together_is_rejected(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
dependencies:
|
|
prompts:
|
|
- id: style/house
|
|
version: 1.0.0
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
include: style/house
|
|
derive: style/house
|
|
""",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="at most one of"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_composed_dependency_must_declare_a_version(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE
|
|
+ """\
|
|
dependencies:
|
|
prompts:
|
|
- id: style/house
|
|
inputs:
|
|
- name: greeting
|
|
required: false
|
|
default:
|
|
include: style/house
|
|
""",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="must declare a version"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
# --- evals (§ 12) ---
|
|
|
|
EVAL_BASE = """\
|
|
format: canned-prompt/v0.1
|
|
id: demo/evaluated
|
|
name: Evaluated
|
|
version: 1.0.0
|
|
summary: Exercise eval files.
|
|
template: prompt.md
|
|
examples:
|
|
- examples/basic.yaml
|
|
evals:
|
|
- evals/quality.yaml
|
|
"""
|
|
|
|
|
|
def write_evaluated(pkg: Path, eval_body: str, template: str = "Sum to 100%. {{ topic }}\n") -> Path:
|
|
pkg.mkdir(exist_ok=True)
|
|
(pkg / "prompt.yaml").write_text(
|
|
EVAL_BASE + "inputs:\n - name: topic\n required: false\n default: cats\n",
|
|
encoding="utf-8",
|
|
)
|
|
(pkg / "prompt.md").write_text(template, encoding="utf-8")
|
|
(pkg / "examples").mkdir(exist_ok=True)
|
|
(pkg / "examples" / "basic.yaml").write_text(
|
|
"name: basic\nvalues:\n topic: dogs\n", encoding="utf-8"
|
|
)
|
|
(pkg / "evals").mkdir(exist_ok=True)
|
|
(pkg / "evals" / "quality.yaml").write_text(eval_body, encoding="utf-8")
|
|
return pkg
|
|
|
|
|
|
RUBRIC = """\
|
|
schema: canned-prompts/eval-rubric/v0.1
|
|
name: quality
|
|
example: examples/basic.yaml
|
|
render:
|
|
- contains: "Sum to 100%"
|
|
- not_contains: "{{"
|
|
- resolves_all: true
|
|
output:
|
|
criteria:
|
|
- Answers the question.
|
|
"""
|
|
|
|
|
|
def test_valid_eval_passes_validation(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(tmp_path / "p", RUBRIC)
|
|
assert cp.validate_package(pkg)["id"] == "demo/evaluated"
|
|
|
|
|
|
def test_eval_must_declare_a_schema(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(tmp_path / "p", "name: quality\nrender: []\n")
|
|
with pytest.raises(cp.CannedPromptError, match="must declare a schema"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_unknown_eval_schema_is_ignored(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(tmp_path / "p", "schema: someone/else/v1\nwhatever: true\n")
|
|
assert cp.validate_package(pkg)["id"] == "demo/evaluated"
|
|
|
|
|
|
def test_eval_asserting_nothing_is_rejected(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(
|
|
tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: empty\n"
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="asserts nothing"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_unknown_render_check_is_rejected(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(
|
|
tmp_path / "p",
|
|
"schema: canned-prompts/eval-rubric/v0.1\nname: q\nrender:\n - matches: 'x.*'\n",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="unknown render check"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_eval_example_must_be_declared(tmp_path: Path) -> None:
|
|
pkg = write_evaluated(
|
|
tmp_path / "p",
|
|
"schema: canned-prompts/eval-rubric/v0.1\nname: q\n"
|
|
"example: examples/missing.yaml\nrender:\n - contains: x\n",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="not declared in the manifest"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_render_checks_evaluate(tmp_path: Path) -> None:
|
|
resolution = cp.Resolution(values={"topic": "dogs"}, origins={"topic": "supplied"})
|
|
checks = [{"contains": "dogs"}, {"contains": "cats"}, {"not_contains": "cats"}]
|
|
outcomes = cp.run_render_checks("about dogs", resolution, checks)
|
|
assert [ok for ok, _ in outcomes] == [True, False, True]
|
|
|
|
|
|
def test_resolves_all_reports_unresolved_names() -> None:
|
|
resolution = cp.Resolution(
|
|
values={"a": 1}, origins={"a": "supplied", "b": "unresolved (no default)"}
|
|
)
|
|
outcomes = cp.run_render_checks("text", resolution, [{"resolves_all": True}])
|
|
assert outcomes[0][0] is False
|
|
assert "b" in outcomes[0][1]
|
|
|
|
|
|
def test_typed_fixture_values_are_not_reparsed(tmp_path: Path) -> None:
|
|
"""A YAML fixture carries real types; only CLI strings need parsing."""
|
|
manifest = {"parameters": {"flag": {"type": "boolean", "default": False}}}
|
|
assert cp.resolve_inputs(manifest, {"flag": True}).values["flag"] is True
|
|
|
|
|
|
# --- context and capability dependencies (§ 10.1, § 10.2) ---
|
|
|
|
def ctx_pkg(tmp_path: Path, body: str) -> Path:
|
|
return write_pkg(tmp_path / "p", BASE + body, "hello\n")
|
|
|
|
|
|
def test_context_dependency_requires_name_and_description(tmp_path: Path) -> None:
|
|
pkg = ctx_pkg(
|
|
tmp_path,
|
|
"dependencies:\n context:\n - name: repository-tree\n"
|
|
" description: A listing of the repository.\n",
|
|
)
|
|
assert cp.validate_package(pkg)["id"] == "demo/defaults"
|
|
|
|
|
|
def test_context_dependency_without_description_is_rejected(tmp_path: Path) -> None:
|
|
pkg = ctx_pkg(tmp_path, "dependencies:\n context:\n - name: repository-tree\n")
|
|
with pytest.raises(cp.CannedPromptError, match="must declare a description"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_context_dependency_using_id_is_rejected(tmp_path: Path) -> None:
|
|
"""Context entries use `name`; `id` would imply resolvable package identity."""
|
|
pkg = ctx_pkg(
|
|
tmp_path,
|
|
"dependencies:\n context:\n - id: policy/security\n"
|
|
" description: A policy.\n",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="not 'id'"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
def test_context_requirement_generate_is_rejected(tmp_path: Path) -> None:
|
|
pkg = ctx_pkg(
|
|
tmp_path,
|
|
"dependencies:\n context:\n - name: repository-tree\n"
|
|
" description: A listing.\n requirement: generate\n",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="required' or 'optional"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["web-search", "code-analysis", "vision", "a1-b2"])
|
|
def test_valid_capability_names(name) -> None:
|
|
assert cp.validate_capabilities([name], "dependencies.capabilities") == [name]
|
|
|
|
|
|
@pytest.mark.parametrize("name", ["Web-Search", "web_search", "-lead", "trail-", ""])
|
|
def test_invalid_capability_names(name) -> None:
|
|
with pytest.raises(cp.CannedPromptError, match="kebab-case"):
|
|
cp.validate_capabilities([name], "dependencies.capabilities")
|
|
|
|
|
|
def test_capability_may_be_both_required_and_observed(tmp_path: Path) -> None:
|
|
"""§ 9.1: required to run at all, and separately observed to work well."""
|
|
pkg = ctx_pkg(
|
|
tmp_path,
|
|
"dependencies:\n capabilities:\n - web-search\n"
|
|
"compatibility:\n capabilities:\n - web-search\n - long-context\n",
|
|
)
|
|
manifest = cp.validate_package(pkg)
|
|
assert manifest["dependencies"]["capabilities"] == ["web-search"]
|
|
assert "long-context" in manifest["compatibility"]["capabilities"]
|
|
|
|
|
|
# --- semver precedence and prereleases (§ 17.1) ---
|
|
|
|
def test_release_outranks_its_prerelease() -> None:
|
|
assert cp.parse_semver("1.0.0") > cp.parse_semver("1.0.0-rc1")
|
|
|
|
|
|
def test_prerelease_ordering_follows_semver() -> None:
|
|
ordered = sorted(
|
|
["1.0.0", "1.0.0-rc.2", "1.0.0-rc.10", "1.0.0-alpha", "0.9.0"],
|
|
key=cp.parse_semver,
|
|
reverse=True,
|
|
)
|
|
assert ordered == ["1.0.0", "1.0.0-rc.10", "1.0.0-rc.2", "1.0.0-alpha", "0.9.0"]
|
|
|
|
|
|
def test_build_metadata_is_ignored_for_precedence() -> None:
|
|
assert cp.parse_semver("1.0.0+build.1") == cp.parse_semver("1.0.0+build.2")
|
|
|
|
|
|
@pytest.mark.parametrize("selector", ["newest", "any", ">=1.0.0", None])
|
|
def test_prerelease_is_not_selected_implicitly(selector) -> None:
|
|
available = ["1.1.0-rc1", "1.0.0"]
|
|
assert cp.select_version(available, selector) == "1.0.0"
|
|
|
|
|
|
def test_exact_pin_selects_a_prerelease() -> None:
|
|
assert cp.select_version(["1.1.0-rc1", "1.0.0"], "1.1.0-rc1") == "1.1.0-rc1"
|
|
|
|
|
|
def test_only_prereleases_available_selects_nothing() -> None:
|
|
assert cp.select_version(["1.0.0-rc1"], "newest") is None
|
|
|
|
|
|
def test_prerelease_only_package_reports_why(tmp_path: Path) -> None:
|
|
catalog = tmp_path / "catalog"
|
|
place(catalog, "local", "practice/thing", "1.0.0-rc1", MINIMAL.replace(
|
|
"version: 1.0.0", "version: 1.0.0-rc1"), "{{ greeting }}\n")
|
|
with pytest.raises(cp.CannedPromptError, match="only prerelease versions"):
|
|
cp.resolve_installed(catalog, "practice/thing", None)
|
|
|
|
|
|
# --- strict packaging (§ 2) ---
|
|
|
|
def test_only_reserved_and_referenced_paths_are_packaged(tmp_path: Path) -> None:
|
|
src = write_evaluated(tmp_path / "src", RUBRIC)
|
|
(src / "LICENSE").write_text("MIT\n", encoding="utf-8")
|
|
(src / "notes.txt").write_text("scratch\n", encoding="utf-8")
|
|
(src / ".git").mkdir()
|
|
(src / ".git" / "config").write_text("junk\n", encoding="utf-8")
|
|
|
|
manifest = cp.validate_package(src)
|
|
dst = tmp_path / "out"
|
|
skipped = cp.copy_package(src, dst, manifest, "package")
|
|
|
|
packaged = sorted(p.relative_to(dst).as_posix() for p in dst.rglob("*") if p.is_file())
|
|
assert packaged == [
|
|
"LICENSE",
|
|
"evals/quality.yaml",
|
|
"examples/basic.yaml",
|
|
"prompt.md",
|
|
"prompt.yaml",
|
|
]
|
|
assert skipped == [".git/", "notes.txt"]
|
|
|
|
|
|
def test_packaged_copy_is_still_valid(tmp_path: Path) -> None:
|
|
src = write_evaluated(tmp_path / "src", RUBRIC)
|
|
manifest = cp.validate_package(src)
|
|
dst = tmp_path / "out"
|
|
cp.copy_package(src, dst, manifest, "package")
|
|
assert cp.validate_package(dst)["id"] == "demo/evaluated"
|
|
|
|
|
|
def test_copy_refuses_to_overwrite(tmp_path: Path) -> None:
|
|
src = write_evaluated(tmp_path / "src", RUBRIC)
|
|
manifest = cp.validate_package(src)
|
|
dst = tmp_path / "out"
|
|
cp.copy_package(src, dst, manifest, "package")
|
|
with pytest.raises(cp.CannedPromptError, match="already exists"):
|
|
cp.copy_package(src, dst, manifest, "package")
|
|
|
|
|
|
# --- format revisions (§ 3.2) ---
|
|
|
|
@pytest.mark.parametrize("declared", ["canned-prompt/v0.2", "canned-prompt/v0.1"])
|
|
def test_both_format_revisions_are_accepted(tmp_path: Path, declared) -> None:
|
|
"""v0.2 is additive, so a v0.1 package means exactly what it always meant."""
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE.replace("canned-prompt/v0.1", declared)
|
|
+ "inputs:\n - name: greeting\n required: false\n default: hi\n",
|
|
)
|
|
assert cp.validate_package(pkg)["format"] == declared
|
|
|
|
|
|
def test_unknown_format_is_rejected(tmp_path: Path) -> None:
|
|
pkg = write_pkg(
|
|
tmp_path / "p",
|
|
BASE.replace("canned-prompt/v0.1", "canned-prompt/v9.9")
|
|
+ "inputs:\n - name: greeting\n required: false\n default: hi\n",
|
|
)
|
|
with pytest.raises(cp.CannedPromptError, match="unsupported format"):
|
|
cp.validate_package(pkg)
|
|
|
|
|
|
# --- unsatisfied dependencies are reported, not fetched (§ 10) ---
|
|
|
|
def test_missing_dependency_is_reported(composed: Path) -> None:
|
|
"""Installing a composing package without its fragment must not look fine."""
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
empty = composed.parent / "empty-catalog"
|
|
assert cp.missing_dependencies(empty, manifest) == ["style/house@1.0.0"]
|
|
|
|
|
|
def test_present_dependency_is_not_reported(composed: Path) -> None:
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
assert cp.missing_dependencies(composed, manifest) == []
|
|
|
|
|
|
def test_dependency_present_but_wrong_version_is_reported(composed: Path) -> None:
|
|
package_dir, _ = cp.resolve_installed(composed, "review/change", None)
|
|
manifest = cp.validate_package(package_dir)
|
|
manifest["dependencies"]["prompts"][0]["version"] = "9.9.9"
|
|
assert cp.missing_dependencies(composed, manifest) == ["style/house@9.9.9"]
|
|
|
|
|
|
# --- catalog index (§ 20.3) ---
|
|
|
|
def indexed(tmp_path: Path) -> tuple[Path, dict]:
|
|
store = tmp_path / "catalog"
|
|
manifest = {"id": "helix/thing", "version": "1.0.0", "name": "Thing",
|
|
"license": "MIT", "provenance": {"author": "Ada", "source": "~/somewhere"}}
|
|
return store, manifest
|
|
|
|
|
|
def test_index_starts_empty(tmp_path: Path) -> None:
|
|
store, _ = indexed(tmp_path)
|
|
assert cp.read_index(store)["entries"] == []
|
|
|
|
|
|
def test_record_inclusion_captures_source_and_date(tmp_path: Path) -> None:
|
|
store, manifest = indexed(tmp_path)
|
|
entry = cp.record_inclusion(store, "local", manifest, "/src/thing", "add")
|
|
assert entry["id"] == "helix/thing"
|
|
assert entry["source"] == "/src/thing"
|
|
assert entry["method"] == "add"
|
|
assert entry["declared_author"] == "Ada"
|
|
assert entry["declared_source"] == "~/somewhere"
|
|
assert entry["license"] == "MIT"
|
|
assert entry["included_at"].endswith("Z")
|
|
assert cp.read_index(store)["entries"] == [entry]
|
|
|
|
|
|
def test_reincluding_keeps_the_original_date(tmp_path: Path) -> None:
|
|
"""When a package first entered is a fact about history, not about reruns."""
|
|
store, manifest = indexed(tmp_path)
|
|
first = cp.record_inclusion(store, "local", manifest, "/src/thing", "add")
|
|
again = cp.record_inclusion(store, "local", manifest, "/elsewhere", "install")
|
|
assert again["included_at"] == first["included_at"]
|
|
assert "last_seen_at" in again
|
|
assert len(cp.read_index(store)["entries"]) == 1
|
|
|
|
|
|
def test_distinct_versions_are_separate_entries(tmp_path: Path) -> None:
|
|
store, manifest = indexed(tmp_path)
|
|
cp.record_inclusion(store, "local", manifest, "/src", "add")
|
|
cp.record_inclusion(store, "local", {**manifest, "version": "1.1.0"}, "/src", "add")
|
|
assert len(cp.read_index(store)["entries"]) == 2
|
|
|
|
|
|
def test_same_id_from_two_registries_is_two_entries(tmp_path: Path) -> None:
|
|
store, manifest = indexed(tmp_path)
|
|
cp.record_inclusion(store, "local", manifest, "/a", "add")
|
|
cp.record_inclusion(store, "house", manifest, "/b", "install")
|
|
assert len(cp.read_index(store)["entries"]) == 2
|
|
|
|
|
|
def test_bad_index_format_is_rejected(tmp_path: Path) -> None:
|
|
store, _ = indexed(tmp_path)
|
|
store.mkdir(parents=True)
|
|
(store / "index.yaml").write_text("format: something/else\nentries: []\n", encoding="utf-8")
|
|
with pytest.raises(cp.CannedPromptError, match="unsupported index format"):
|
|
cp.read_index(store)
|