CANP-WP-0002 T04: eval schema, render checks and output criteria
`evals/` was a reserved path holding unvalidated blobs: section 12 named the directory and gave an illustrative snippet, but nothing was specified, so no tool could act on an eval file. Every eval file must now declare a `schema`, and CPF defines exactly one — `canned-prompts/eval-rubric/v0.1`. Unrecognized schemas stay legal and are skipped rather than rejected, so the format gains something actionable without becoming an evaluation language, which remains a non-goal. The schema splits along the same seam as T01 and T03. Render checks (`contains`, `not_contains`, `resolves_all`) assert properties of the rendered prompt text, need no model, and are therefore run by the reference CLI. Output criteria describe a good result and are declared but not run, because judging them requires a model. That division is now the format's consistent answer to "deterministic locally, or not". An eval references a fixture already declared in the manifest's `examples` rather than carrying its own copy, so an example that is also an eval fixture stays honest — both break together. An eval declares assessment and must not record outcomes. Results are run evidence and live outside the immutable package, per INTENT.md and section 17. Spec: 12 rewritten with 12.1, 18 (rules 17-18), 21 (`eval` verb). Reference CLI: read_eval, validate_eval, load_example_values, run_render_checks, cmd_eval; a failed render check exits non-zero. Tests 42 -> 51. examples/pqrst-estimate/evals/quality.yaml is a real eval with four render checks and four output criteria, and it passes. Fixes a latent bug reaching a fixture exposed: coerce_value assumed every value was a command-line string, so a YAML fixture carrying a real type (include_rationale: true) crashed on .lower(). Typed values are now validated but not re-parsed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM Assistant: claude-code Assistant-Model: opus Assistant-Process: 388925@bnt-lap001 Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502
This commit is contained in:
parent
cac6bc135f
commit
c5f1640454
8 changed files with 444 additions and 18 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
This is intentionally a **small reference implementation**, not the intended final architecture.
|
||||
|
||||
It demonstrates seven verbs:
|
||||
It demonstrates eight verbs:
|
||||
|
||||
```text
|
||||
add PATH
|
||||
|
|
@ -10,6 +10,7 @@ search QUERY
|
|||
show ID
|
||||
resolve ID --set key=value
|
||||
render ID --set key=value
|
||||
eval ID
|
||||
install ID [--version VERSION]
|
||||
publish PATH
|
||||
```
|
||||
|
|
@ -71,6 +72,10 @@ rather than resolved by guessing.
|
|||
print qualified `<registry>:<id>` references.
|
||||
- `include` defaults are satisfied (inclusion is deterministic); `derive`
|
||||
defaults are reported, not run. Inclusion cycles are detected and named.
|
||||
- `eval` runs the deterministic render checks of any eval declaring the
|
||||
`canned-prompts/eval-rubric/v0.1` schema, and reports output criteria as
|
||||
declared but not run. Unrecognized schemas are skipped, not rejected. A
|
||||
failed render check exits non-zero.
|
||||
- An optional `registry.yaml` names a registry and records namespace claims.
|
||||
`publish` warns when a namespace is declared `closed` — it cannot
|
||||
authenticate a publisher, and says so rather than implying it checked.
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ FORMAT = "canned-prompt/v0.1"
|
|||
REGISTRY_FORMAT = "canned-prompt-registry/v0.1"
|
||||
LOCAL_REGISTRY = "local"
|
||||
REGISTRY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
EVAL_RUBRIC_SCHEMA = "canned-prompts/eval-rubric/v0.1"
|
||||
RENDER_CHECKS = ("contains", "not_contains", "resolves_all")
|
||||
PLACEHOLDER_RE = re.compile(r"{{\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*}}")
|
||||
SEMVER_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:[-+].*)?$")
|
||||
REQUIRED_FIELDS = ("format", "id", "name", "version", "summary", "template")
|
||||
|
|
@ -135,6 +137,64 @@ def prompt_dependencies(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|||
return declared
|
||||
|
||||
|
||||
def read_eval(package_dir: Path, relative: str) -> dict[str, Any]:
|
||||
path = safe_relative_file(package_dir, relative, "evals")
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as exc:
|
||||
raise CannedPromptError(f"invalid YAML in {relative}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise CannedPromptError(f"{relative}: an eval file must contain a mapping")
|
||||
if not isinstance(data.get("schema"), str) or not data["schema"].strip():
|
||||
raise CannedPromptError(f"{relative}: an eval file must declare a schema")
|
||||
return data
|
||||
|
||||
|
||||
def validate_eval(relative: str, spec: dict[str, Any], declared_examples: list[str]) -> None:
|
||||
"""Validation rules 17-18 of § 18. Unknown schemas are ignored, not rejected."""
|
||||
if spec["schema"] != EVAL_RUBRIC_SCHEMA:
|
||||
return
|
||||
if not isinstance(spec.get("name"), str) or not spec["name"].strip():
|
||||
raise CannedPromptError(f"{relative}: eval must declare a name")
|
||||
|
||||
render = spec.get("render") or []
|
||||
output = spec.get("output") or {}
|
||||
if not render and not output:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: eval asserts nothing; declare render checks or output criteria"
|
||||
)
|
||||
|
||||
if not isinstance(render, list):
|
||||
raise CannedPromptError(f"{relative}: render must be a list of checks")
|
||||
for check in render:
|
||||
if not isinstance(check, dict) or len(check) != 1:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: each render check must be a single-key mapping"
|
||||
)
|
||||
(kind,) = check
|
||||
if kind not in RENDER_CHECKS:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: unknown render check {kind!r}; expected one of "
|
||||
+ ", ".join(RENDER_CHECKS)
|
||||
)
|
||||
|
||||
if output:
|
||||
if not isinstance(output, dict):
|
||||
raise CannedPromptError(f"{relative}: output must be a mapping")
|
||||
criteria = output.get("criteria") or []
|
||||
if not isinstance(criteria, list) or not all(isinstance(c, str) for c in criteria):
|
||||
raise CannedPromptError(f"{relative}: output.criteria must be a list of strings")
|
||||
|
||||
example = spec.get("example")
|
||||
if example is not None:
|
||||
if not isinstance(example, str):
|
||||
raise CannedPromptError(f"{relative}: example must be a path string")
|
||||
if example not in declared_examples:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: example {example!r} is not declared in the manifest's examples"
|
||||
)
|
||||
|
||||
|
||||
def check_composition_reference(
|
||||
name: str, reference: str, field: str, declared: dict[str, dict[str, Any]]
|
||||
) -> None:
|
||||
|
|
@ -243,6 +303,10 @@ def validate_package(package_dir: Path) -> dict[str, Any]:
|
|||
for item in manifest.get("inputs") or []:
|
||||
validate_input_default(item, declared)
|
||||
|
||||
declared_examples = [str(path) for path in (manifest.get("examples") or [])]
|
||||
for relative in manifest.get("evals") or []:
|
||||
validate_eval(relative, read_eval(package_dir, relative), declared_examples)
|
||||
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
placeholders = set(PLACEHOLDER_RE.findall(template))
|
||||
undeclared = sorted(placeholders - names)
|
||||
|
|
@ -572,8 +636,16 @@ def cmd_show(args: argparse.Namespace) -> None:
|
|||
print(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True).rstrip())
|
||||
|
||||
|
||||
def coerce_value(raw: str, spec: dict[str, Any]) -> Any:
|
||||
def coerce_value(raw: Any, spec: dict[str, Any]) -> Any:
|
||||
kind = spec.get("type", "string")
|
||||
if not isinstance(raw, str):
|
||||
# Already typed — from a YAML fixture rather than the command line.
|
||||
# Validate membership, but do not try to parse it.
|
||||
if kind == "enum" and raw not in (spec.get("values") or []):
|
||||
raise CannedPromptError(
|
||||
f"invalid enum value {raw!r}; expected one of {spec.get('values') or []}"
|
||||
)
|
||||
return raw
|
||||
if kind == "boolean":
|
||||
lowered = raw.lower()
|
||||
if lowered in {"true", "1", "yes", "on"}:
|
||||
|
|
@ -875,6 +947,91 @@ def cmd_resolve(args: argparse.Namespace) -> None:
|
|||
)
|
||||
|
||||
|
||||
def load_example_values(package_dir: Path, relative: str) -> dict[str, Any]:
|
||||
path = safe_relative_file(package_dir, relative, "examples")
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as exc:
|
||||
raise CannedPromptError(f"invalid YAML in {relative}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise CannedPromptError(f"{relative}: an example must contain a mapping")
|
||||
values = data.get("values") or {}
|
||||
if not isinstance(values, dict):
|
||||
raise CannedPromptError(f"{relative}: example values must be a mapping")
|
||||
return values
|
||||
|
||||
|
||||
def run_render_checks(
|
||||
rendered: str, resolution: Resolution, checks: list[dict[str, Any]]
|
||||
) -> list[tuple[bool, str]]:
|
||||
"""Evaluate § 12.1 render checks. Deterministic: no model is involved."""
|
||||
results: list[tuple[bool, str]] = []
|
||||
for check in checks:
|
||||
(kind,) = check
|
||||
expected = check[kind]
|
||||
if kind == "contains":
|
||||
results.append((str(expected) in rendered, f'contains "{expected}"'))
|
||||
elif kind == "not_contains":
|
||||
results.append((str(expected) not in rendered, f'not_contains "{expected}"'))
|
||||
elif kind == "resolves_all":
|
||||
unresolved = sorted(set(resolution.origins) - set(resolution.values))
|
||||
ok = (not unresolved) if expected else bool(unresolved)
|
||||
detail = f" ({', '.join(unresolved)})" if unresolved else ""
|
||||
results.append((ok, f"resolves_all{detail}"))
|
||||
return results
|
||||
|
||||
|
||||
def cmd_eval(args: argparse.Namespace) -> None:
|
||||
catalog = Path(args.catalog).expanduser()
|
||||
package_dir, registry = resolve_installed(catalog, args.id, args.version)
|
||||
manifest = validate_package(package_dir)
|
||||
template_path = safe_relative_file(package_dir, manifest["template"], "template")
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
|
||||
relatives = manifest.get("evals") or []
|
||||
if not relatives:
|
||||
print(f"{registry}:{manifest['id']}@{manifest['version']} no evals declared")
|
||||
return
|
||||
|
||||
print(f"{registry}:{manifest['id']}@{manifest['version']}")
|
||||
failures = 0
|
||||
for relative in relatives:
|
||||
spec = read_eval(package_dir, relative)
|
||||
if spec["schema"] != EVAL_RUBRIC_SCHEMA:
|
||||
print(f" {relative} schema {spec['schema']} not recognized — skipped")
|
||||
continue
|
||||
|
||||
print(f" {relative} ({spec['name']})")
|
||||
checks = spec.get("render") or []
|
||||
if checks:
|
||||
values = (
|
||||
load_example_values(package_dir, spec["example"])
|
||||
if spec.get("example")
|
||||
else {}
|
||||
)
|
||||
resolution = resolve_inputs(
|
||||
manifest, values, composer=CatalogComposer(catalog)
|
||||
)
|
||||
missing = sorted(set(PLACEHOLDER_RE.findall(template)) - set(resolution.values))
|
||||
rendered = "" if missing else render_template(template, resolution.values)
|
||||
if missing:
|
||||
print(f" render FAIL cannot render: unresolved {', '.join(missing)}")
|
||||
failures += 1
|
||||
for ok, label in run_render_checks(rendered, resolution, checks):
|
||||
print(f" render {'PASS' if ok else 'FAIL'} {label}")
|
||||
failures += 0 if ok else 1
|
||||
|
||||
criteria = (spec.get("output") or {}).get("criteria") or []
|
||||
if criteria:
|
||||
print(
|
||||
f" output -- {len(criteria)} criteria declared "
|
||||
"(not run: judging output needs a model)"
|
||||
)
|
||||
|
||||
if failures:
|
||||
raise CannedPromptError(f"{failures} render check(s) failed")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="canned-prompts", description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
|
@ -911,6 +1068,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
render.add_argument("--set", dest="set_values", action="append", default=[], metavar="NAME=VALUE")
|
||||
render.set_defaults(func=cmd_render)
|
||||
|
||||
evaluate = sub.add_parser("eval", help="run an installed package's render checks")
|
||||
evaluate.add_argument("id")
|
||||
evaluate.add_argument("--version")
|
||||
evaluate.add_argument("--catalog", default=str(default_catalog()))
|
||||
evaluate.set_defaults(func=cmd_eval)
|
||||
|
||||
resolve = sub.add_parser("resolve", help="report how each input and parameter resolves")
|
||||
resolve.add_argument("id")
|
||||
resolve.add_argument("--version")
|
||||
|
|
|
|||
|
|
@ -504,3 +504,114 @@ inputs:
|
|||
)
|
||||
with pytest.raises(cp.CannedPromptError, match="must declare a version"):
|
||||
cp.validate_package(pkg)
|
||||
|
||||
|
||||
# --- evals (§ 12) ---
|
||||
|
||||
EVAL_BASE = """\
|
||||
format: canned-prompt/v0.1
|
||||
id: demo/evaluated
|
||||
name: Evaluated
|
||||
version: 1.0.0
|
||||
summary: Exercise eval files.
|
||||
template: prompt.md
|
||||
examples:
|
||||
- examples/basic.yaml
|
||||
evals:
|
||||
- evals/quality.yaml
|
||||
"""
|
||||
|
||||
|
||||
def write_evaluated(pkg: Path, eval_body: str, template: str = "Sum to 100%. {{ topic }}\n") -> Path:
|
||||
pkg.mkdir(exist_ok=True)
|
||||
(pkg / "prompt.yaml").write_text(
|
||||
EVAL_BASE + "inputs:\n - name: topic\n required: false\n default: cats\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(pkg / "prompt.md").write_text(template, encoding="utf-8")
|
||||
(pkg / "examples").mkdir(exist_ok=True)
|
||||
(pkg / "examples" / "basic.yaml").write_text(
|
||||
"name: basic\nvalues:\n topic: dogs\n", encoding="utf-8"
|
||||
)
|
||||
(pkg / "evals").mkdir(exist_ok=True)
|
||||
(pkg / "evals" / "quality.yaml").write_text(eval_body, encoding="utf-8")
|
||||
return pkg
|
||||
|
||||
|
||||
RUBRIC = """\
|
||||
schema: canned-prompts/eval-rubric/v0.1
|
||||
name: quality
|
||||
example: examples/basic.yaml
|
||||
render:
|
||||
- contains: "Sum to 100%"
|
||||
- not_contains: "{{"
|
||||
- resolves_all: true
|
||||
output:
|
||||
criteria:
|
||||
- Answers the question.
|
||||
"""
|
||||
|
||||
|
||||
def test_valid_eval_passes_validation(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(tmp_path / "p", RUBRIC)
|
||||
assert cp.validate_package(pkg)["id"] == "demo/evaluated"
|
||||
|
||||
|
||||
def test_eval_must_declare_a_schema(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(tmp_path / "p", "name: quality\nrender: []\n")
|
||||
with pytest.raises(cp.CannedPromptError, match="must declare a schema"):
|
||||
cp.validate_package(pkg)
|
||||
|
||||
|
||||
def test_unknown_eval_schema_is_ignored(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(tmp_path / "p", "schema: someone/else/v1\nwhatever: true\n")
|
||||
assert cp.validate_package(pkg)["id"] == "demo/evaluated"
|
||||
|
||||
|
||||
def test_eval_asserting_nothing_is_rejected(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(
|
||||
tmp_path / "p", "schema: canned-prompts/eval-rubric/v0.1\nname: empty\n"
|
||||
)
|
||||
with pytest.raises(cp.CannedPromptError, match="asserts nothing"):
|
||||
cp.validate_package(pkg)
|
||||
|
||||
|
||||
def test_unknown_render_check_is_rejected(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(
|
||||
tmp_path / "p",
|
||||
"schema: canned-prompts/eval-rubric/v0.1\nname: q\nrender:\n - matches: 'x.*'\n",
|
||||
)
|
||||
with pytest.raises(cp.CannedPromptError, match="unknown render check"):
|
||||
cp.validate_package(pkg)
|
||||
|
||||
|
||||
def test_eval_example_must_be_declared(tmp_path: Path) -> None:
|
||||
pkg = write_evaluated(
|
||||
tmp_path / "p",
|
||||
"schema: canned-prompts/eval-rubric/v0.1\nname: q\n"
|
||||
"example: examples/missing.yaml\nrender:\n - contains: x\n",
|
||||
)
|
||||
with pytest.raises(cp.CannedPromptError, match="not declared in the manifest"):
|
||||
cp.validate_package(pkg)
|
||||
|
||||
|
||||
def test_render_checks_evaluate(tmp_path: Path) -> None:
|
||||
resolution = cp.Resolution(values={"topic": "dogs"}, origins={"topic": "supplied"})
|
||||
checks = [{"contains": "dogs"}, {"contains": "cats"}, {"not_contains": "cats"}]
|
||||
outcomes = cp.run_render_checks("about dogs", resolution, checks)
|
||||
assert [ok for ok, _ in outcomes] == [True, False, True]
|
||||
|
||||
|
||||
def test_resolves_all_reports_unresolved_names() -> None:
|
||||
resolution = cp.Resolution(
|
||||
values={"a": 1}, origins={"a": "supplied", "b": "unresolved (no default)"}
|
||||
)
|
||||
outcomes = cp.run_render_checks("text", resolution, [{"resolves_all": True}])
|
||||
assert outcomes[0][0] is False
|
||||
assert "b" in outcomes[0][1]
|
||||
|
||||
|
||||
def test_typed_fixture_values_are_not_reparsed(tmp_path: Path) -> None:
|
||||
"""A YAML fixture carries real types; only CLI strings need parsing."""
|
||||
manifest = {"parameters": {"flag": {"type": "boolean", "default": False}}}
|
||||
assert cp.resolve_inputs(manifest, {"flag": True}).values["flag"] is True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue