Automate policy source freshness and inventory

tools/fetch_sources.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Fetch exact Forgejo source archives declared by the policy source inventory."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path, PurePosixPath
+import subprocess
+import tarfile
+import tempfile
+import urllib.request
+
+
+def _revision(remote: str, branch: str) -> str:
+    result = subprocess.run(
+        ["git", "ls-remote", remote, f"refs/heads/{branch}"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    revision = result.stdout.split(maxsplit=1)[0] if result.stdout.strip() else ""
+    if len(revision) != 40 or any(char not in "0123456789abcdef" for char in revision):
+        raise ValueError(f"{remote}: could not resolve a clean 40-hex {branch} revision")
+    return revision
+
+
+def _archive_url(remote: str, revision: str) -> str:
+    if not remote.startswith("https://") or not remote.endswith(".git"):
+        raise ValueError(f"archive source must be an HTTPS .git URL, got {remote!r}")
+    return f"{remote[:-4]}/archive/{revision[:7]}.tar.gz"
+
+
+def _extract(archive: Path, target: Path) -> None:
+    target.mkdir(parents=True, exist_ok=False)
+    with tarfile.open(archive, "r:gz") as bundle:
+        members = bundle.getmembers()
+        roots = {
+            PurePosixPath(member.name).parts[0]
+            for member in members
+            if PurePosixPath(member.name).parts
+        }
+        if len(roots) != 1:
+            raise ValueError(f"{archive}: expected exactly one archive root")
+        root = next(iter(roots))
+        for member in members:
+            path = PurePosixPath(member.name)
+            if not path.parts or path.parts[0] != root:
+                raise ValueError(f"{archive}: inconsistent archive root")
+            relative = PurePosixPath(*path.parts[1:])
+            if not relative.parts:
+                continue
+            if relative.is_absolute() or ".." in relative.parts:
+                raise ValueError(f"{archive}: unsafe member {member.name!r}")
+            destination = target.joinpath(*relative.parts)
+            if member.isdir():
+                destination.mkdir(parents=True, exist_ok=True)
+                continue
+            if not member.isfile():
+                # Links and special files are not part of the publishable source
+                # corpus. Skipping them avoids materializing archive links; if a
+                # selector ever names one, the inventory audit fails on absence.
+                continue
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            source = bundle.extractfile(member)
+            if source is None:
+                raise ValueError(f"{archive}: could not read {member.name!r}")
+            with destination.open("wb") as output:
+                while chunk := source.read(1024 * 1024):
+                    output.write(chunk)
+
+
+def fetch(config_path: Path, destination: Path, policy_revision: str) -> dict[str, object]:
+    if len(policy_revision) != 40 or any(
+        char not in "0123456789abcdef" for char in policy_revision
+    ):
+        raise ValueError("--policy-revision must be a clean 40-hex Git commit")
+    config = json.loads(config_path.read_text(encoding="utf-8"))
+    if config.get("schema_version") != 1:
+        raise ValueError("source inventory config schema_version must be 1")
+    destination.mkdir(parents=True, exist_ok=True)
+    if any(destination.iterdir()):
+        raise ValueError(f"destination must be empty: {destination}")
+
+    revisions: dict[str, str] = {}
+    repositories: dict[str, dict[str, str]] = {}
+    for name, repository in sorted(config["repositories"].items()):
+        if repository.get("local"):
+            revision = policy_revision
+            repositories[name] = {"revision": revision, "source": "workflow-checkout"}
+            revisions[name] = revision
+            continue
+        remote = repository["remote"]
+        branch = repository.get("branch", "main")
+        revision = _revision(remote, branch)
+        url = _archive_url(remote, revision)
+        with tempfile.NamedTemporaryFile(suffix=".tar.gz") as archive:
+            with urllib.request.urlopen(url, timeout=60) as response:
+                while chunk := response.read(1024 * 1024):
+                    archive.write(chunk)
+            archive.flush()
+            _extract(Path(archive.name), destination / name)
+        revisions[name] = revision
+        repositories[name] = {
+            "revision": revision,
+            "remote": remote,
+            "branch": branch,
+        }
+
+    canonical = json.dumps(revisions, sort_keys=True, separators=(",", ":")).encode()
+    lock: dict[str, object] = {
+        "schema_version": 1,
+        "source_set_digest": hashlib.sha256(canonical).hexdigest(),
+        "repositories": repositories,
+    }
+    (destination / "source-lock.json").write_text(
+        json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+    return lock
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--config", type=Path, default=Path("source-inventory.config.json"))
+    parser.add_argument("--destination", type=Path, required=True)
+    parser.add_argument("--policy-revision", required=True)
+    args = parser.parse_args()
+    lock = fetch(args.config.resolve(), args.destination.resolve(), args.policy_revision)
+    print(
+        json.dumps(
+            {
+                "source_set_digest": lock["source_set_digest"],
+                "repositories": len(lock["repositories"]),
+            },
+            sort_keys=True,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())