Add the catalog index, and package a real prompt collection

CANP-WP-0004. The operator asked canned-prompts to build a database of
versioned prompts recording where each came from and when — the first step
toward a platform for collaborative prompting.

The format had nowhere to put that. `provenance` records who wrote a prompt and
where the idea came from; nothing recorded how a copy arrived in a particular
store. Section 20.3 now specifies an `index.yaml` as store metadata rather than
package data: how a copy arrived differs for every consumer, and recording an
arrival must never rewrite the package that arrived.

`add`, `install` and `publish` record registry, id, version, name, source,
method, first-inclusion date, and the package's declared author, source and
licence — the last three copied so a listing is readable without opening every
package. A new `index` verb lists it. `included_at` is never overwritten; a
re-run updates `last_seen_at`, because when a package first entered a
collection is a fact about history rather than about the last command run.

Tests 84 -> 90.

Also records two findings from actually using the format:

CANP-WP-0004-T03 — inclusion has no deduplication, so a diamond dependency
renders shared content once per path. Found by composing a real collection.
Not fixed here: deduplicating means choosing which occurrence survives and
deciding what happens when two paths resolve different versions, which is
resolver behaviour that section 10.4 deliberately avoids. Handed to
CANP-WP-0005 with a leaning: document it, warn at validation time, do not
deduplicate.

The add_ons workaround in practice/pqrst-estimate is evidence about section
23's deferred "richer template syntax" — an optional appendix has to be an
input with an empty default, because CPF has no conditionals.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bjefh8NUiEiahN4JLwoSKM

Assistant: claude-code
Assistant-Model: opus
Assistant-Process: 388925@bnt-lap001
Assistant-Session: 3507023f-e0fd-4a1e-9d90-a0d4217d1502

reference/README.md

@@ -2,7 +2,7 @@
 
 This is intentionally a **small reference implementation**, not the intended final architecture.
 
-It demonstrates eight verbs:
+It demonstrates nine verbs:
 
 ```text
 add      PATH
@@ -11,6 +11,7 @@ show     ID
 resolve  ID --set key=value
 render   ID --set key=value
 eval     ID
+index    [ID]
 install  ID [--version VERSION]
 publish  PATH
 ```
@@ -87,6 +88,10 @@ rather than resolved by guessing.
 - `add` and `install` name declared prompt dependencies the catalog cannot
   satisfy. They do not fetch them — resolution is the consumer's job (§ 10) —
   but a package that looks installed and cannot render should say so.
+- `add`, `install` and `publish` record an entry in the store's `index.yaml`:
+  source, method, first-inclusion date, and the package's declared author,
+  source and licence. `index` lists it. Re-adding keeps the original
+  `included_at` and updates `last_seen_at`.
 - An optional `registry.yaml` names a registry and records namespace claims.
   `publish` warns when a namespace is declared `closed` — it cannot
   authenticate a publisher, and says so rather than implying it checked.

reference/canned_prompts.py

@@ -11,6 +11,7 @@ import argparse
 import json
 import os
 import re
+import datetime as dt
 import shutil
 import sys
 from dataclasses import dataclass, field
@@ -30,6 +31,8 @@ RENDER_CHECKS = ("contains", "not_contains", "resolves_all")
 CAPABILITY_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
 RESERVED_FILES = ("prompt.yaml", "README.md", "LICENSE")
 RESERVED_DIRS = ("examples", "evals", "assets")
+INDEX_FILE = "index.yaml"
+INDEX_FORMAT = "canned-prompt-index/v0.1"
 PLACEHOLDER_RE = re.compile(r"{{\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*}}")
 SEMVER_RE = re.compile(
     r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)"
@@ -678,6 +681,76 @@ def copy_package(src: Path, dst: Path, manifest: dict[str, Any], what: str) -> l
     return skipped
 
 
+def read_index(store: Path) -> dict[str, Any]:
+    """The catalog index: which packages entered this store, from where, when.
+
+    Registry-side metadata, deliberately outside the immutable package (§ 20).
+    A package records who wrote it; the index records how it got here.
+    """
+    path = store / INDEX_FILE
+    if not path.is_file():
+        return {"format": INDEX_FORMAT, "entries": []}
+    try:
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise CannedPromptError(f"invalid YAML in {path}: {exc}") from exc
+    if not isinstance(data, dict) or not isinstance(data.get("entries"), list):
+        raise CannedPromptError(f"{path}: index must be a mapping with an entries list")
+    if data.get("format") != INDEX_FORMAT:
+        raise CannedPromptError(f"unsupported index format: {data.get('format')!r}")
+    return data
+
+
+def index_entry_key(entry: dict[str, Any]) -> tuple[str, str, str]:
+    return (entry.get("registry", ""), entry.get("id", ""), entry.get("version", ""))
+
+
+def record_inclusion(
+    store: Path,
+    registry: str,
+    manifest: dict[str, Any],
+    source: str,
+    method: str,
+) -> dict[str, Any]:
+    """Record that a package version entered this store, and from where.
+
+    Re-recording the same registry/id/version keeps the original `included_at`:
+    the date a package first entered is a fact about history, not about the last
+    time someone re-ran a command.
+    """
+    index = read_index(store)
+    entry = {
+        "registry": registry,
+        "id": manifest["id"],
+        "version": manifest["version"],
+        "name": manifest.get("name"),
+        "source": source,
+        "method": method,
+        "included_at": dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    }
+    provenance = manifest.get("provenance") or {}
+    if isinstance(provenance, dict):
+        for field in ("author", "source"):
+            if provenance.get(field):
+                entry[f"declared_{field}"] = str(provenance[field])
+    if manifest.get("license"):
+        entry["license"] = manifest["license"]
+
+    existing = {index_entry_key(e): e for e in index["entries"]}
+    key = index_entry_key(entry)
+    if key in existing:
+        entry["included_at"] = existing[key].get("included_at", entry["included_at"])
+        entry["last_seen_at"] = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+    existing[key] = entry
+
+    index["entries"] = [existing[k] for k in sorted(existing)]
+    store.mkdir(parents=True, exist_ok=True)
+    (store / INDEX_FILE).write_text(
+        yaml.safe_dump(index, sort_keys=False, allow_unicode=True), encoding="utf-8"
+    )
+    return entry
+
+
 def missing_dependencies(catalog: Path, manifest: dict[str, Any]) -> list[str]:
     """Declared prompt dependencies that this catalog cannot satisfy.
 
@@ -737,6 +810,7 @@ def cmd_add(args: argparse.Namespace) -> None:
     registry = check_registry_name(args.as_registry)
     dst = catalog_package_path(catalog, registry, manifest["id"], manifest["version"])
     report_skipped(copy_package(src.resolve(), dst, manifest, "catalog package"))
+    record_inclusion(catalog, registry, manifest, str(src.resolve()), "add")
     report_missing_dependencies(catalog, manifest)
     print(f"added {registry}:{manifest['id']}@{manifest['version']} -> {dst}")
 
@@ -759,6 +833,7 @@ def cmd_publish(args: argparse.Namespace) -> None:
 
     dst = registry_package_path(registry, manifest["id"], manifest["version"])
     report_skipped(copy_package(src.resolve(), dst, manifest, "published package"))
+    record_inclusion(registry, name or "", manifest, str(src.resolve()), "publish")
     label = f"{name}:{manifest['id']}" if name else manifest["id"]
     print(f"published {label}@{manifest['version']} -> {dst}")
 
@@ -778,6 +853,7 @@ def cmd_install(args: argparse.Namespace) -> None:
     manifest = validate_package(src)
     dst = catalog_package_path(catalog, name, manifest["id"], manifest["version"])
     copy_package(src, dst, manifest, "catalog package")
+    record_inclusion(catalog, name, manifest, str(registry.resolve()), "install")
     report_missing_dependencies(catalog, manifest)
     print(f"installed {name}:{manifest['id']}@{manifest['version']} -> {dst}")
 
@@ -1238,6 +1314,28 @@ def cmd_eval(args: argparse.Namespace) -> None:
         raise CannedPromptError(f"{failures} render check(s) failed")
 
 
+def cmd_index(args: argparse.Namespace) -> None:
+    store = Path(args.catalog).expanduser()
+    entries = read_index(store)["entries"]
+    if args.id:
+        _, package_id = parse_reference(args.id)
+        entries = [e for e in entries if e.get("id") == package_id]
+    if args.json:
+        print(json.dumps(entries, indent=2, ensure_ascii=False))
+        return
+    if not entries:
+        print("no packages recorded in this catalog")
+        return
+    width = max(len(f"{e.get('registry')}:{e.get('id')}@{e.get('version')}") for e in entries)
+    for entry in entries:
+        label = f"{entry.get('registry')}:{entry.get('id')}@{entry.get('version')}"
+        print(f"{label:<{width}}  {entry.get('included_at')}  {entry.get('method')}")
+        print(f"{'':<{width}}  from {entry.get('source')}")
+        declared = entry.get("declared_source")
+        if declared:
+            print(f"{'':<{width}}  declares source {declared}")
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(prog="canned-prompts", description=__doc__)
     sub = parser.add_subparsers(dest="command", required=True)
@@ -1274,6 +1372,12 @@ def build_parser() -> argparse.ArgumentParser:
     render.add_argument("--set", dest="set_values", action="append", default=[], metavar="NAME=VALUE")
     render.set_defaults(func=cmd_render)
 
+    index = sub.add_parser("index", help="show what entered this catalog, from where and when")
+    index.add_argument("id", nargs="?")
+    index.add_argument("--catalog", default=str(default_catalog()))
+    index.add_argument("--json", action="store_true")
+    index.set_defaults(func=cmd_index)
+
     evaluate = sub.add_parser("eval", help="run an installed package's render checks")
     evaluate.add_argument("id")
     evaluate.add_argument("--version")

reference/tests/test_canned_prompts.py

@@ -808,3 +808,62 @@ def test_dependency_present_but_wrong_version_is_reported(composed: Path) -> Non
     manifest = cp.validate_package(package_dir)
     manifest["dependencies"]["prompts"][0]["version"] = "9.9.9"
     assert cp.missing_dependencies(composed, manifest) == ["style/house@9.9.9"]
+
+
+# --- catalog index (§ 20.3) ---
+
+def indexed(tmp_path: Path) -> tuple[Path, dict]:
+    store = tmp_path / "catalog"
+    manifest = {"id": "helix/thing", "version": "1.0.0", "name": "Thing",
+                "license": "MIT", "provenance": {"author": "Ada", "source": "~/somewhere"}}
+    return store, manifest
+
+
+def test_index_starts_empty(tmp_path: Path) -> None:
+    store, _ = indexed(tmp_path)
+    assert cp.read_index(store)["entries"] == []
+
+
+def test_record_inclusion_captures_source_and_date(tmp_path: Path) -> None:
+    store, manifest = indexed(tmp_path)
+    entry = cp.record_inclusion(store, "local", manifest, "/src/thing", "add")
+    assert entry["id"] == "helix/thing"
+    assert entry["source"] == "/src/thing"
+    assert entry["method"] == "add"
+    assert entry["declared_author"] == "Ada"
+    assert entry["declared_source"] == "~/somewhere"
+    assert entry["license"] == "MIT"
+    assert entry["included_at"].endswith("Z")
+    assert cp.read_index(store)["entries"] == [entry]
+
+
+def test_reincluding_keeps_the_original_date(tmp_path: Path) -> None:
+    """When a package first entered is a fact about history, not about reruns."""
+    store, manifest = indexed(tmp_path)
+    first = cp.record_inclusion(store, "local", manifest, "/src/thing", "add")
+    again = cp.record_inclusion(store, "local", manifest, "/elsewhere", "install")
+    assert again["included_at"] == first["included_at"]
+    assert "last_seen_at" in again
+    assert len(cp.read_index(store)["entries"]) == 1
+
+
+def test_distinct_versions_are_separate_entries(tmp_path: Path) -> None:
+    store, manifest = indexed(tmp_path)
+    cp.record_inclusion(store, "local", manifest, "/src", "add")
+    cp.record_inclusion(store, "local", {**manifest, "version": "1.1.0"}, "/src", "add")
+    assert len(cp.read_index(store)["entries"]) == 2
+
+
+def test_same_id_from_two_registries_is_two_entries(tmp_path: Path) -> None:
+    store, manifest = indexed(tmp_path)
+    cp.record_inclusion(store, "local", manifest, "/a", "add")
+    cp.record_inclusion(store, "house", manifest, "/b", "install")
+    assert len(cp.read_index(store)["entries"]) == 2
+
+
+def test_bad_index_format_is_rejected(tmp_path: Path) -> None:
+    store, _ = indexed(tmp_path)
+    store.mkdir(parents=True)
+    (store / "index.yaml").write_text("format: something/else\nentries: []\n", encoding="utf-8")
+    with pytest.raises(cp.CannedPromptError, match="unsupported index format"):
+        cp.read_index(store)