Automate policy source freshness and inventory

tools/build_site.py

@@ -149,8 +149,13 @@ def build(
     manifest_path = manifest_path.resolve()
     manifest = load_manifest(manifest_path)
     as_of = as_of or dt.date.today()
+    source_root = os.environ.get("POLICY_NEXUS_SOURCE_ROOT")
     repository_paths = {
-        name: (manifest_path.parent / config["path"]).resolve()
+        name: (
+            (Path(source_root) / name).resolve()
+            if source_root
+            else (manifest_path.parent / config["path"]).resolve()
+        )
         for name, config in manifest["repositories"].items()
     }
     output_parent = output.resolve().parent
@@ -255,6 +260,8 @@ def build(
                     "lifecycle": lifecycle,
                     "canonical_path": canonical.as_posix(),
                     "revision_path": revision_path.as_posix(),
+                    "source_repo": document["source_repo"],
+                    "source_path": document["source_path"],
                     "source_revision": source_revision,
                     "source_digest": source_digest,
                 }

tools/fetch_sources.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Fetch exact Forgejo source archives declared by the policy source inventory."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path, PurePosixPath
+import subprocess
+import tarfile
+import tempfile
+import urllib.request
+
+
+def _revision(remote: str, branch: str) -> str:
+    result = subprocess.run(
+        ["git", "ls-remote", remote, f"refs/heads/{branch}"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    revision = result.stdout.split(maxsplit=1)[0] if result.stdout.strip() else ""
+    if len(revision) != 40 or any(char not in "0123456789abcdef" for char in revision):
+        raise ValueError(f"{remote}: could not resolve a clean 40-hex {branch} revision")
+    return revision
+
+
+def _archive_url(remote: str, revision: str) -> str:
+    if not remote.startswith("https://") or not remote.endswith(".git"):
+        raise ValueError(f"archive source must be an HTTPS .git URL, got {remote!r}")
+    return f"{remote[:-4]}/archive/{revision[:7]}.tar.gz"
+
+
+def _extract(archive: Path, target: Path) -> None:
+    target.mkdir(parents=True, exist_ok=False)
+    with tarfile.open(archive, "r:gz") as bundle:
+        members = bundle.getmembers()
+        roots = {
+            PurePosixPath(member.name).parts[0]
+            for member in members
+            if PurePosixPath(member.name).parts
+        }
+        if len(roots) != 1:
+            raise ValueError(f"{archive}: expected exactly one archive root")
+        root = next(iter(roots))
+        for member in members:
+            path = PurePosixPath(member.name)
+            if not path.parts or path.parts[0] != root:
+                raise ValueError(f"{archive}: inconsistent archive root")
+            relative = PurePosixPath(*path.parts[1:])
+            if not relative.parts:
+                continue
+            if relative.is_absolute() or ".." in relative.parts:
+                raise ValueError(f"{archive}: unsafe member {member.name!r}")
+            destination = target.joinpath(*relative.parts)
+            if member.isdir():
+                destination.mkdir(parents=True, exist_ok=True)
+                continue
+            if not member.isfile():
+                # Links and special files are not part of the publishable source
+                # corpus. Skipping them avoids materializing archive links; if a
+                # selector ever names one, the inventory audit fails on absence.
+                continue
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            source = bundle.extractfile(member)
+            if source is None:
+                raise ValueError(f"{archive}: could not read {member.name!r}")
+            with destination.open("wb") as output:
+                while chunk := source.read(1024 * 1024):
+                    output.write(chunk)
+
+
+def fetch(config_path: Path, destination: Path, policy_revision: str) -> dict[str, object]:
+    if len(policy_revision) != 40 or any(
+        char not in "0123456789abcdef" for char in policy_revision
+    ):
+        raise ValueError("--policy-revision must be a clean 40-hex Git commit")
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    if config.get("schema_version") != 1:
+        raise ValueError("source inventory config schema_version must be 1")
+    destination.mkdir(parents=True, exist_ok=True)
+    if any(destination.iterdir()):
+        raise ValueError(f"destination must be empty: {destination}")
+
+    revisions: dict[str, str] = {}
+    repositories: dict[str, dict[str, str]] = {}
+    for name, repository in sorted(config["repositories"].items()):
+        if repository.get("local"):
+            revision = policy_revision
+            repositories[name] = {"revision": revision, "source": "workflow-checkout"}
+            revisions[name] = revision
+            continue
+        remote = repository["remote"]
+        branch = repository.get("branch", "main")
+        revision = _revision(remote, branch)
+        url = _archive_url(remote, revision)
+        with tempfile.NamedTemporaryFile(suffix=".tar.gz") as archive:
+            with urllib.request.urlopen(url, timeout=60) as response:
+                while chunk := response.read(1024 * 1024):
+                    archive.write(chunk)
+            archive.flush()
+            _extract(Path(archive.name), destination / name)
+        revisions[name] = revision
+        repositories[name] = {
+            "revision": revision,
+            "remote": remote,
+            "branch": branch,
+        }
+
+    canonical = json.dumps(revisions, sort_keys=True, separators=(",", ":")).encode()
+    lock: dict[str, object] = {
+        "schema_version": 1,
+        "source_set_digest": hashlib.sha256(canonical).hexdigest(),
+        "repositories": repositories,
+    }
+    (destination / "source-lock.json").write_text(
+        json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return lock
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=Path("source-inventory.config.json"))
+    parser.add_argument("--destination", type=Path, required=True)
+    parser.add_argument("--policy-revision", required=True)
+    args = parser.parse_args()
+    lock = fetch(args.config.resolve(), args.destination.resolve(), args.policy_revision)
+    print(
+        json.dumps(
+            {
+                "source_set_digest": lock["source_set_digest"],
+                "repositories": len(lock["repositories"]),
+            },
+            sort_keys=True,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/source_inventory.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""Audit the explicit canon/ADR corpus and emit deterministic source evidence."""
+
+from __future__ import annotations
+
+import argparse
+import fnmatch
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+from typing import Any
+
+
+SCHEMA_VERSION = "policy-nexus-source-inventory/v1"
+DISPOSITIONS = {"published", "metadata-pending", "excluded", "unsupported-format"}
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError(f"{path}: expected a JSON object")
+    return value
+
+
+def _tracked_files(repo: Path) -> list[str]:
+    if not repo.is_dir():
+        raise FileNotFoundError(f"source repository is unavailable: {repo}")
+    if (repo / ".git").exists():
+        result = subprocess.run(
+            ["git", "-C", str(repo), "ls-tree", "-r", "--name-only", "HEAD"],
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+        return [line for line in result.stdout.splitlines() if line]
+    return sorted(
+        path.relative_to(repo).as_posix()
+        for path in repo.rglob("*")
+        if path.is_file() and not path.is_symlink()
+    )
+
+
+def _revision(repo: Path) -> str:
+    if not (repo / ".git").exists():
+        raise ValueError(f"{repo}: source lock is required for an archive checkout")
+    result = subprocess.run(
+        ["git", "-C", str(repo), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    revision = result.stdout.strip()
+    if len(revision) != 40 or any(char not in "0123456789abcdef" for char in revision):
+        raise ValueError(f"{repo}: invalid Git revision {revision!r}")
+    return revision
+
+
+def _repo_paths(
+    config: dict[str, Any], *, policy_root: Path, source_root: Path
+) -> dict[str, Path]:
+    paths: dict[str, Path] = {}
+    for name, repository in config["repositories"].items():
+        paths[name] = policy_root if repository.get("local") else source_root / name
+    return paths
+
+
+def discover(
+    config: dict[str, Any], *, policy_root: Path, source_root: Path
+) -> tuple[list[dict[str, str]], dict[str, Path]]:
+    paths = _repo_paths(config, policy_root=policy_root, source_root=source_root)
+    sources: list[dict[str, str]] = []
+    for name, repository in sorted(config["repositories"].items()):
+        selectors = repository.get("selectors", [])
+        if not selectors:
+            raise ValueError(f"{name}: at least one source selector is required")
+        for path in _tracked_files(paths[name]):
+            if any(fnmatch.fnmatchcase(path, selector) for selector in selectors):
+                sources.append({"source_repo": name, "source_path": path})
+    return sources, paths
+
+
+def _published_sources(publication_path: Path) -> set[tuple[str, str]]:
+    publication = _read_json(publication_path)
+    return {
+        (document["source_repo"], document["source_path"])
+        for document in publication.get("documents", [])
+    }
+
+
+def _new_entry(source: dict[str, str], published: set[tuple[str, str]]) -> dict[str, str]:
+    key = (source["source_repo"], source["source_path"])
+    path = Path(source["source_path"])
+    if key in published:
+        disposition = "published"
+        reason = "Published through an explicit publication.json document entry."
+    elif path.name.lower() == "readme.md":
+        disposition = "excluded"
+        reason = "Directory index, not an architecture decision record."
+    elif path.suffix.lower() != ".md":
+        disposition = "unsupported-format"
+        reason = "Inventoried governing source is not Markdown and has no renderer yet."
+    else:
+        disposition = "metadata-pending"
+        reason = (
+            "In scope; awaits explicit publication addressing and owner/revision/review metadata."
+        )
+    return source | {"disposition": disposition, "reason": reason}
+
+
+def refresh(
+    config_path: Path,
+    inventory_path: Path,
+    publication_path: Path,
+    *,
+    policy_root: Path,
+    source_root: Path,
+) -> dict[str, Any]:
+    config = _read_json(config_path)
+    if config.get("schema_version") != 1:
+        raise ValueError("source inventory config schema_version must be 1")
+    discovered, _paths = discover(config, policy_root=policy_root, source_root=source_root)
+    published = _published_sources(publication_path)
+    existing: dict[tuple[str, str], dict[str, str]] = {}
+    if inventory_path.exists():
+        for source in _read_json(inventory_path).get("sources", []):
+            existing[(source["source_repo"], source["source_path"])] = source
+    sources = []
+    for source in discovered:
+        key = (source["source_repo"], source["source_path"])
+        sources.append(existing.get(key, _new_entry(source, published)))
+    inventory = {
+        "schema_version": 1,
+        "sources": sources,
+    }
+    inventory_path.write_text(
+        json.dumps(inventory, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return inventory
+
+
+def _load_revisions(
+    config: dict[str, Any], paths: dict[str, Path], lock_path: Path | None
+) -> tuple[dict[str, str], str]:
+    if lock_path:
+        lock = _read_json(lock_path)
+        if lock.get("schema_version") != 1:
+            raise ValueError("source lock schema_version must be 1")
+        repositories = lock.get("repositories", {})
+        revisions = {
+            name: repositories[name]["revision"] for name in config["repositories"]
+        }
+    else:
+        revisions = {name: _revision(paths[name]) for name in config["repositories"]}
+    for name, revision in revisions.items():
+        if len(revision) != 40 or any(char not in "0123456789abcdef" for char in revision):
+            raise ValueError(f"{name}: invalid locked revision {revision!r}")
+    canonical = json.dumps(revisions, sort_keys=True, separators=(",", ":")).encode()
+    digest = hashlib.sha256(canonical).hexdigest()
+    if lock_path:
+        locked_digest = lock.get("source_set_digest")
+        if locked_digest != digest:
+            raise ValueError(
+                f"source lock digest mismatch: recorded {locked_digest!r}, computed {digest}"
+            )
+    return revisions, digest
+
+
+def check(
+    config_path: Path,
+    inventory_path: Path,
+    publication_path: Path,
+    *,
+    policy_root: Path,
+    source_root: Path,
+    lock_path: Path | None = None,
+) -> dict[str, Any]:
+    config = _read_json(config_path)
+    inventory = _read_json(inventory_path)
+    if config.get("schema_version") != 1 or inventory.get("schema_version") != 1:
+        raise ValueError("source inventory config and inventory schema_version must be 1")
+    discovered, paths = discover(config, policy_root=policy_root, source_root=source_root)
+    discovered_keys = {
+        (source["source_repo"], source["source_path"]) for source in discovered
+    }
+    entries = inventory.get("sources", [])
+    inventory_keys: set[tuple[str, str]] = set()
+    for source in entries:
+        key = (source.get("source_repo", ""), source.get("source_path", ""))
+        if key in inventory_keys:
+            raise ValueError(f"duplicate source inventory entry: {key[0]}/{key[1]}")
+        inventory_keys.add(key)
+        if source.get("disposition") not in DISPOSITIONS:
+            raise ValueError(f"{key[0]}/{key[1]}: invalid disposition")
+        if not source.get("reason"):
+            raise ValueError(f"{key[0]}/{key[1]}: disposition reason is required")
+    missing = sorted(discovered_keys - inventory_keys)
+    stale = sorted(inventory_keys - discovered_keys)
+    if missing or stale:
+        details = []
+        if missing:
+            details.append("unreviewed sources: " + ", ".join(f"{r}/{p}" for r, p in missing))
+        if stale:
+            details.append("inventory entries no longer present: " + ", ".join(f"{r}/{p}" for r, p in stale))
+        raise ValueError("; ".join(details) + "; run source_inventory.py refresh and review the diff")
+
+    published = _published_sources(publication_path)
+    inventory_published = {
+        (source["source_repo"], source["source_path"])
+        for source in entries
+        if source["disposition"] == "published"
+    }
+    if published != inventory_published:
+        raise ValueError(
+            "published source inventory must exactly match publication.json: "
+            f"manifest_only={sorted(published - inventory_published)}, "
+            f"inventory_only={sorted(inventory_published - published)}"
+        )
+
+    revisions, source_set_digest = _load_revisions(config, paths, lock_path)
+    counts = {disposition: 0 for disposition in sorted(DISPOSITIONS)}
+    repository_counts = {name: 0 for name in config["repositories"]}
+    for source in entries:
+        counts[source["disposition"]] += 1
+        repository_counts[source["source_repo"]] += 1
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "source_set_digest": source_set_digest,
+        "dispositions": counts,
+        "repositories": [
+            {
+                "name": name,
+                "revision": revisions[name],
+                "source_count": repository_counts[name],
+            }
+            for name in sorted(config["repositories"])
+        ],
+        "excluded_scopes": config.get("excluded_scopes", []),
+        "sources": entries,
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=("check", "refresh"))
+    parser.add_argument("--config", type=Path, default=Path("source-inventory.config.json"))
+    parser.add_argument("--inventory", type=Path, default=Path("source-inventory.json"))
+    parser.add_argument("--publication", type=Path, default=Path("publication.json"))
+    parser.add_argument("--policy-root", type=Path)
+    parser.add_argument("--source-root", type=Path)
+    parser.add_argument("--lock", type=Path)
+    parser.add_argument("--report", type=Path)
+    args = parser.parse_args(argv)
+    config_path = args.config.resolve()
+    policy_root = (args.policy_root or config_path.parent).resolve()
+    source_root = (args.source_root or policy_root.parent).resolve()
+    try:
+        if args.command == "refresh":
+            inventory = refresh(
+                config_path,
+                args.inventory.resolve(),
+                args.publication.resolve(),
+                policy_root=policy_root,
+                source_root=source_root,
+            )
+            print(f"{args.inventory}: recorded {len(inventory['sources'])} source(s)")
+            return 0
+        report = check(
+            config_path,
+            args.inventory.resolve(),
+            args.publication.resolve(),
+            policy_root=policy_root,
+            source_root=source_root,
+            lock_path=args.lock.resolve() if args.lock else None,
+        )
+        if args.report:
+            args.report.parent.mkdir(parents=True, exist_ok=True)
+            args.report.write_text(
+                json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+            )
+        counts = report["dispositions"]
+        print(
+            f"source inventory ok: {len(report['sources'])} source(s), "
+            f"{counts['published']} published, {counts['metadata-pending']} metadata-pending, "
+            f"source-set {report['source_set_digest']}"
+        )
+        return 0
+    except (KeyError, OSError, ValueError, subprocess.CalledProcessError) as exc:
+        print(f"source inventory failed: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/verify_release.py

@@ -46,8 +46,11 @@ def verify(build: Path) -> dict[str, Any]:
 
     index = build / "index.html"
     manifest_path = build / "publication-manifest.json"
-    if not index.is_file() or not manifest_path.is_file():
-        raise ValueError("release requires index.html and publication-manifest.json")
+    inventory_path = build / "source-inventory.json"
+    if not index.is_file() or not manifest_path.is_file() or not inventory_path.is_file():
+        raise ValueError(
+            "release requires index.html, publication-manifest.json and source-inventory.json"
+        )
 
     manifest_bytes = manifest_path.read_bytes()
     manifest = json.loads(manifest_bytes)
@@ -59,6 +62,19 @@ def verify(build: Path) -> dict[str, Any]:
     if not isinstance(documents, list) or not documents:
         raise ValueError("publication manifest must contain at least one document")
 
+    inventory_bytes = inventory_path.read_bytes()
+    inventory = json.loads(inventory_bytes)
+    if inventory.get("schema_version") != "policy-nexus-source-inventory/v1":
+        raise ValueError("source inventory schema_version is invalid")
+    source_set_digest = inventory.get("source_set_digest", "")
+    if not HEX_DIGEST.fullmatch(source_set_digest):
+        raise ValueError("source inventory requires a valid source_set_digest")
+    inventoried_published = {
+        (source.get("source_repo"), source.get("source_path"))
+        for source in inventory.get("sources", [])
+        if source.get("disposition") == "published"
+    }
+
     verified: list[str] = []
     for document in documents:
         document_id = document.get("id", "<unknown>")
@@ -72,6 +88,8 @@ def verify(build: Path) -> dict[str, Any]:
             "review_due",
             "canonical_path",
             "revision_path",
+            "source_repo",
+            "source_path",
         ):
             if not document.get(field) or document.get(field) == "unknown":
                 raise ValueError(f"{document_id}: release metadata field {field} is required")
@@ -84,6 +102,9 @@ def verify(build: Path) -> dict[str, Any]:
             )
         if not HEX_DIGEST.fullmatch(source_digest):
             raise ValueError(f"{document_id}: invalid source_digest {source_digest!r}")
+        source_key = (document["source_repo"], document["source_path"])
+        if source_key not in inventoried_published:
+            raise ValueError(f"{document_id}: source is not published in source inventory")
 
         canonical = build / _safe_relative(document["canonical_path"])
         revision = build / _safe_relative(document["revision_path"])
@@ -105,6 +126,8 @@ def verify(build: Path) -> dict[str, Any]:
     return {
         "schema_version": "policy-nexus-release/v1",
         "publication_manifest_digest": hashlib.sha256(manifest_bytes).hexdigest(),
+        "source_inventory_digest": hashlib.sha256(inventory_bytes).hexdigest(),
+        "source_set_digest": source_set_digest,
         "generated_as_of": manifest["generated_as_of"],
         "documents": verified,
     }