CANP-WP-0002 T04: eval schema, render checks and output criteria
`evals/` was a reserved path holding unvalidated blobs: section 12 named the directory and gave an illustrative snippet, but nothing was specified, so no tool could act on an eval file. Every eval file must now declare a `schema`, and CPF defines exactly one — `canned-prompts/eval-rubric/v0.1`. Unrecognized schemas stay legal and are skipped rather than rejected, so the format gains something actionable without becoming an evaluation language, which remains a non-goal. The schema splits along the same seam as T01 and T03. Render checks (`contains`, `not_contains`, `resolves_all`) assert properties of the rendered prompt text, need no model, and are therefore run by the reference CLI. Output criteria describe a good result and are declared but not run, because judging them requires a model. That division is now the format's consistent answer to "deterministic locally, or not". An eval references a fixture already declared in the manifest's `examples` rather than carrying its own copy, so an example that is also an eval fixture stays honest — both break together. An eval declares assessment and must not record outcomes. Results are run evidence and live outside the immutable package, per INTENT.md and section 17. Spec: 12 rewritten with 12.1, 18 (rules 17-18), 21 (`eval` verb). Reference CLI: read_eval, validate_eval, load_example_values, run_render_checks, cmd_eval; a failed render check exits non-zero. Tests 42 -> 51. examples/pqrst-estimate/evals/quality.yaml is a real eval with four render checks and four output criteria, and it passes. Fixes a latent bug reaching a fixture exposed: coerce_value assumed every value was a command-line string, so a YAML fixture carrying a real type (include_rationale: true) crashed on .lower(). Typed values are now validated but not re-parsed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM Assistant: claude-code Assistant-Model: opus Assistant-Process: 388925@bnt-lap001 Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502
This commit is contained in:
parent
cac6bc135f
commit
c5f1640454
8 changed files with 444 additions and 18 deletions
|
|
@ -23,6 +23,8 @@ FORMAT = "canned-prompt/v0.1"
|
|||
REGISTRY_FORMAT = "canned-prompt-registry/v0.1"
|
||||
LOCAL_REGISTRY = "local"
|
||||
REGISTRY_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
EVAL_RUBRIC_SCHEMA = "canned-prompts/eval-rubric/v0.1"
|
||||
RENDER_CHECKS = ("contains", "not_contains", "resolves_all")
|
||||
PLACEHOLDER_RE = re.compile(r"{{\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*}}")
|
||||
SEMVER_RE = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:[-+].*)?$")
|
||||
REQUIRED_FIELDS = ("format", "id", "name", "version", "summary", "template")
|
||||
|
|
@ -135,6 +137,64 @@ def prompt_dependencies(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|||
return declared
|
||||
|
||||
|
||||
def read_eval(package_dir: Path, relative: str) -> dict[str, Any]:
|
||||
path = safe_relative_file(package_dir, relative, "evals")
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as exc:
|
||||
raise CannedPromptError(f"invalid YAML in {relative}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise CannedPromptError(f"{relative}: an eval file must contain a mapping")
|
||||
if not isinstance(data.get("schema"), str) or not data["schema"].strip():
|
||||
raise CannedPromptError(f"{relative}: an eval file must declare a schema")
|
||||
return data
|
||||
|
||||
|
||||
def validate_eval(relative: str, spec: dict[str, Any], declared_examples: list[str]) -> None:
|
||||
"""Validation rules 17-18 of § 18. Unknown schemas are ignored, not rejected."""
|
||||
if spec["schema"] != EVAL_RUBRIC_SCHEMA:
|
||||
return
|
||||
if not isinstance(spec.get("name"), str) or not spec["name"].strip():
|
||||
raise CannedPromptError(f"{relative}: eval must declare a name")
|
||||
|
||||
render = spec.get("render") or []
|
||||
output = spec.get("output") or {}
|
||||
if not render and not output:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: eval asserts nothing; declare render checks or output criteria"
|
||||
)
|
||||
|
||||
if not isinstance(render, list):
|
||||
raise CannedPromptError(f"{relative}: render must be a list of checks")
|
||||
for check in render:
|
||||
if not isinstance(check, dict) or len(check) != 1:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: each render check must be a single-key mapping"
|
||||
)
|
||||
(kind,) = check
|
||||
if kind not in RENDER_CHECKS:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: unknown render check {kind!r}; expected one of "
|
||||
+ ", ".join(RENDER_CHECKS)
|
||||
)
|
||||
|
||||
if output:
|
||||
if not isinstance(output, dict):
|
||||
raise CannedPromptError(f"{relative}: output must be a mapping")
|
||||
criteria = output.get("criteria") or []
|
||||
if not isinstance(criteria, list) or not all(isinstance(c, str) for c in criteria):
|
||||
raise CannedPromptError(f"{relative}: output.criteria must be a list of strings")
|
||||
|
||||
example = spec.get("example")
|
||||
if example is not None:
|
||||
if not isinstance(example, str):
|
||||
raise CannedPromptError(f"{relative}: example must be a path string")
|
||||
if example not in declared_examples:
|
||||
raise CannedPromptError(
|
||||
f"{relative}: example {example!r} is not declared in the manifest's examples"
|
||||
)
|
||||
|
||||
|
||||
def check_composition_reference(
|
||||
name: str, reference: str, field: str, declared: dict[str, dict[str, Any]]
|
||||
) -> None:
|
||||
|
|
@ -243,6 +303,10 @@ def validate_package(package_dir: Path) -> dict[str, Any]:
|
|||
for item in manifest.get("inputs") or []:
|
||||
validate_input_default(item, declared)
|
||||
|
||||
declared_examples = [str(path) for path in (manifest.get("examples") or [])]
|
||||
for relative in manifest.get("evals") or []:
|
||||
validate_eval(relative, read_eval(package_dir, relative), declared_examples)
|
||||
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
placeholders = set(PLACEHOLDER_RE.findall(template))
|
||||
undeclared = sorted(placeholders - names)
|
||||
|
|
@ -572,8 +636,16 @@ def cmd_show(args: argparse.Namespace) -> None:
|
|||
print(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True).rstrip())
|
||||
|
||||
|
||||
def coerce_value(raw: str, spec: dict[str, Any]) -> Any:
|
||||
def coerce_value(raw: Any, spec: dict[str, Any]) -> Any:
|
||||
kind = spec.get("type", "string")
|
||||
if not isinstance(raw, str):
|
||||
# Already typed — from a YAML fixture rather than the command line.
|
||||
# Validate membership, but do not try to parse it.
|
||||
if kind == "enum" and raw not in (spec.get("values") or []):
|
||||
raise CannedPromptError(
|
||||
f"invalid enum value {raw!r}; expected one of {spec.get('values') or []}"
|
||||
)
|
||||
return raw
|
||||
if kind == "boolean":
|
||||
lowered = raw.lower()
|
||||
if lowered in {"true", "1", "yes", "on"}:
|
||||
|
|
@ -875,6 +947,91 @@ def cmd_resolve(args: argparse.Namespace) -> None:
|
|||
)
|
||||
|
||||
|
||||
def load_example_values(package_dir: Path, relative: str) -> dict[str, Any]:
|
||||
path = safe_relative_file(package_dir, relative, "examples")
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except yaml.YAMLError as exc:
|
||||
raise CannedPromptError(f"invalid YAML in {relative}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise CannedPromptError(f"{relative}: an example must contain a mapping")
|
||||
values = data.get("values") or {}
|
||||
if not isinstance(values, dict):
|
||||
raise CannedPromptError(f"{relative}: example values must be a mapping")
|
||||
return values
|
||||
|
||||
|
||||
def run_render_checks(
|
||||
rendered: str, resolution: Resolution, checks: list[dict[str, Any]]
|
||||
) -> list[tuple[bool, str]]:
|
||||
"""Evaluate § 12.1 render checks. Deterministic: no model is involved."""
|
||||
results: list[tuple[bool, str]] = []
|
||||
for check in checks:
|
||||
(kind,) = check
|
||||
expected = check[kind]
|
||||
if kind == "contains":
|
||||
results.append((str(expected) in rendered, f'contains "{expected}"'))
|
||||
elif kind == "not_contains":
|
||||
results.append((str(expected) not in rendered, f'not_contains "{expected}"'))
|
||||
elif kind == "resolves_all":
|
||||
unresolved = sorted(set(resolution.origins) - set(resolution.values))
|
||||
ok = (not unresolved) if expected else bool(unresolved)
|
||||
detail = f" ({', '.join(unresolved)})" if unresolved else ""
|
||||
results.append((ok, f"resolves_all{detail}"))
|
||||
return results
|
||||
|
||||
|
||||
def cmd_eval(args: argparse.Namespace) -> None:
|
||||
catalog = Path(args.catalog).expanduser()
|
||||
package_dir, registry = resolve_installed(catalog, args.id, args.version)
|
||||
manifest = validate_package(package_dir)
|
||||
template_path = safe_relative_file(package_dir, manifest["template"], "template")
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
|
||||
relatives = manifest.get("evals") or []
|
||||
if not relatives:
|
||||
print(f"{registry}:{manifest['id']}@{manifest['version']} no evals declared")
|
||||
return
|
||||
|
||||
print(f"{registry}:{manifest['id']}@{manifest['version']}")
|
||||
failures = 0
|
||||
for relative in relatives:
|
||||
spec = read_eval(package_dir, relative)
|
||||
if spec["schema"] != EVAL_RUBRIC_SCHEMA:
|
||||
print(f" {relative} schema {spec['schema']} not recognized — skipped")
|
||||
continue
|
||||
|
||||
print(f" {relative} ({spec['name']})")
|
||||
checks = spec.get("render") or []
|
||||
if checks:
|
||||
values = (
|
||||
load_example_values(package_dir, spec["example"])
|
||||
if spec.get("example")
|
||||
else {}
|
||||
)
|
||||
resolution = resolve_inputs(
|
||||
manifest, values, composer=CatalogComposer(catalog)
|
||||
)
|
||||
missing = sorted(set(PLACEHOLDER_RE.findall(template)) - set(resolution.values))
|
||||
rendered = "" if missing else render_template(template, resolution.values)
|
||||
if missing:
|
||||
print(f" render FAIL cannot render: unresolved {', '.join(missing)}")
|
||||
failures += 1
|
||||
for ok, label in run_render_checks(rendered, resolution, checks):
|
||||
print(f" render {'PASS' if ok else 'FAIL'} {label}")
|
||||
failures += 0 if ok else 1
|
||||
|
||||
criteria = (spec.get("output") or {}).get("criteria") or []
|
||||
if criteria:
|
||||
print(
|
||||
f" output -- {len(criteria)} criteria declared "
|
||||
"(not run: judging output needs a model)"
|
||||
)
|
||||
|
||||
if failures:
|
||||
raise CannedPromptError(f"{failures} render check(s) failed")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="canned-prompts", description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
|
@ -911,6 +1068,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|||
render.add_argument("--set", dest="set_values", action="append", default=[], metavar="NAME=VALUE")
|
||||
render.set_defaults(func=cmd_render)
|
||||
|
||||
evaluate = sub.add_parser("eval", help="run an installed package's render checks")
|
||||
evaluate.add_argument("id")
|
||||
evaluate.add_argument("--version")
|
||||
evaluate.add_argument("--catalog", default=str(default_catalog()))
|
||||
evaluate.set_defaults(func=cmd_eval)
|
||||
|
||||
resolve = sub.add_parser("resolve", help="report how each input and parameter resolves")
|
||||
resolve.add_argument("id")
|
||||
resolve.add_argument("--version")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue