coulomb/repo-manager
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: publish classifications from Forgejo

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a053ff-1d6f-7fe2-ac1c-a6eb40a42a0c
This commit is contained in:
tegwick 2026-09-01 01:39:39 +02:00
parent 738d407bb7
commit fd624021ec
12 changed files with 2000 additions and 47 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/repo_manager/cli.py
View file

@ -549,6 +549,12 @@ def main(argv: list[str] | None = None) -> int:
"--cursor-secret", default=os.environ.get("REPO_MANAGER_CURSOR_SECRET")
)
p_publisher_snapshot.add_argument("--page-size", type=int, default=100)
p_publisher_snapshot.add_argument(
"--forgejo-base-url",
default=os.environ.get("REPO_MANAGER_FORGEJO_BASE_URL", "https://forgejo.coulomb.social"),
)
p_publisher_snapshot.add_argument("--forgejo-token-file", default=None)
p_publisher_snapshot.add_argument("--remote-workers", type=int, default=8)
p_publisher_import = publisher_sub.add_parser(
"import-registry",
help="Bootstrap the private local registry from primary State Hub identities",
@ -562,6 +568,9 @@ def main(argv: list[str] | None = None) -> int:
p_publisher_import.add_argument("--timeout", type=float, default=10.0)
p_publisher_import.add_argument("--workers", type=int, default=4)
p_publisher_import.add_argument("--retries", type=int, default=2)
p_publisher_import.add_argument(
"--source", choices=("local", "forgejo"), default="local"
)
p_publisher_import.add_argument("--force", action="store_true")
p_publisher_api = publisher_sub.add_parser(
"api", help="Run the read-only classification publisher API"
@ -1287,6 +1296,7 @@ def main(argv: list[str] | None = None) -> int:
timeout_seconds=args.timeout,
workers=args.workers,
retries=args.retries,
source=args.source,
)
except (OSError, ValueError) as exc:
print(json.dumps({"ok": False, "error": str(exc)}, indent=2))
@ -1322,12 +1332,21 @@ def main(argv: list[str] | None = None) -> int:
from repo_manager.repository_publisher import ClassificationPublisher
try:
forgejo_token = None
if args.forgejo_token_file:
forgejo_token = Path(args.forgejo_token_file).read_text(encoding="utf-8").strip()
if not forgejo_token:
raise ValueError("Forgejo token file is empty")
publisher = ClassificationPublisher(
Path(args.registry),
cursor_secret=args.cursor_secret,
page_size=args.page_size,
forgejo_base_url=args.forgejo_base_url,
forgejo_token=forgejo_token,
remote_workers=args.remote_workers,
)
result = publisher.fetch_page()
publisher.close()
except ValueError as exc:
print(json.dumps({"ok": False, "error": str(exc)}, indent=2))
return 1

240
src/repo_manager/repository_publisher.py
View file

@ -16,14 +16,14 @@ from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from urllib.parse import quote, urlparse
from uuid import UUID
import httpx
import yaml
from repo_manager import __version__
from repo_manager.classification import ClassificationError
from repo_manager.classification import ClassificationError, require_valid_classification
from repo_manager.observe import load_classification
from repo_manager.time import format_utc, utc_now
@ -49,7 +49,8 @@ class RepositoryRegistration:
repository_id: str
slug: str
lifecycle: str
path: Path
path: Path | None = None
forgejo_repository: str | None = None
@dataclass(frozen=True, slots=True)
@ -83,7 +84,8 @@ def load_repository_registry(path: Path) -> tuple[RepositoryRegistration, ...]:
repository_id = str(UUID(str(row["repository_id"])))
slug = str(row["slug"])
lifecycle = str(row["lifecycle"])
repo_path = Path(str(row["path"]))
raw_path = row.get("path")
raw_forgejo = row.get("forgejo_repository")
except (KeyError, TypeError, ValueError) as exc:
raise RegistryError(f"invalid repository registry row {number}: {exc}") from exc
if repository_id in ids:
@ -96,8 +98,18 @@ def load_repository_registry(path: Path) -> tuple[RepositoryRegistration, ...]:
raise RegistryError(
f"repository {slug!r} lifecycle must be one of {', '.join(sorted(LIFECYCLES))}"
)
if not repo_path.is_absolute():
if (raw_path is None) == (raw_forgejo is None):
raise RegistryError(
f"repository {slug!r} must declare exactly one of path or forgejo_repository"
)
repo_path = Path(str(raw_path)) if raw_path is not None else None
forgejo_repository = str(raw_forgejo) if raw_forgejo is not None else None
if repo_path is not None and not repo_path.is_absolute():
raise RegistryError(f"repository {slug!r} path must be absolute")
if forgejo_repository is not None and forgejo_repository != f"coulomb/{slug}":
raise RegistryError(
f"repository {slug!r} Forgejo identity must be exactly 'coulomb/{slug}'"
)
ids.add(repository_id)
slugs.add(slug)
registrations.append(
@ -106,6 +118,7 @@ def load_repository_registry(path: Path) -> tuple[RepositoryRegistration, ...]:
slug=slug,
lifecycle=lifecycle,
path=repo_path,
forgejo_repository=forgejo_repository,
)
)
return tuple(sorted(registrations, key=lambda item: item.repository_id))
@ -118,6 +131,7 @@ def import_state_hub_registry(
timeout_seconds: float = 10.0,
workers: int = 4,
retries: int = 2,
source: str = "local",
transport: httpx.BaseTransport | None = None,
) -> dict[str, Any]:
"""Bootstrap Repo Manager identities from the retiring authoritative registrar.
@ -132,6 +146,8 @@ def import_state_hub_registry(
raise ValueError("State Hub API base must be an absolute HTTP(S) origin")
if timeout_seconds <= 0 or workers < 1 or retries < 0:
raise ValueError("timeout_seconds/workers must be positive and retries non-negative")
if source not in {"local", "forgejo"}:
raise ValueError("registry import source must be 'local' or 'forgejo'")
paths = _discover_classified_repositories(fleet_root)
if not paths:
raise ValueError("fleet root contains no classified Git repositories")
@ -210,7 +226,11 @@ def import_state_hub_registry(
"repository_id": repository_id,
"slug": slug,
"lifecycle": lifecycle,
"path": str(local_path.resolve()),
**(
{"path": str(local_path.resolve())}
if source == "local"
else {"forgejo_repository": f"coulomb/{slug}"}
),
}
)
@ -241,10 +261,118 @@ def import_state_hub_registry(
}
class ForgejoClassificationClient:
"""Read exact default-branch classifications from the Forgejo API."""
def __init__(
self,
base_url: str,
*,
token: str | None = None,
timeout_seconds: float = 10.0,
retries: int = 2,
transport: httpx.BaseTransport | None = None,
) -> None:
parsed = urlparse(base_url)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("Forgejo base URL must be an absolute HTTP(S) origin")
if parsed.username or parsed.password or parsed.query or parsed.fragment:
raise ValueError("Forgejo base URL must not contain credentials, query, or fragment")
if timeout_seconds <= 0 or retries < 0:
raise ValueError("Forgejo timeout must be positive and retries non-negative")
headers = {"Authorization": f"token {token}"} if token else None
self._client = httpx.Client(
base_url=base_url.rstrip("/"),
headers=headers,
timeout=httpx.Timeout(timeout_seconds),
follow_redirects=True,
transport=transport,
)
self.retries = retries
def observe(
self, registration: RepositoryRegistration, *, observed_at: str
) -> dict[str, Any]:
repository = registration.forgejo_repository
if repository is None:
raise ValueError("registration has no Forgejo repository source")
owner, slug = repository.split("/", 1)
owner_path = quote(owner, safe="")
slug_path = quote(slug, safe="")
prefix = f"/api/v1/repos/{owner_path}/{slug_path}"
metadata = self._get_object(prefix)
if metadata.get("full_name") != repository:
raise ValueError("Forgejo repository identity does not match the registry")
branch = metadata.get("default_branch")
if not isinstance(branch, str) or not branch or branch.startswith("/"):
raise ValueError("Forgejo default branch is unavailable")
branch_payload = self._get_object(f"{prefix}/branches/{quote(branch, safe='')}")
commit = branch_payload.get("commit")
head_sha = commit.get("id") if isinstance(commit, dict) else None
if not isinstance(head_sha, str) or not re.fullmatch(r"[a-f0-9]{40,64}", head_sha):
raise ValueError("Forgejo default-branch revision is invalid")
content = self._get_object(
f"{prefix}/contents/.repo-classification.yaml",
params={"ref": head_sha},
)
if content.get("type", "file") != "file" or content.get("encoding") != "base64":
raise ValueError("Forgejo classification content has an unsupported representation")
encoded = content.get("content")
if not isinstance(encoded, str):
raise TypeError("Forgejo classification content is missing")
try:
raw = base64.b64decode(encoded, validate=True)
except (ValueError, TypeError) as exc:
raise ValueError("Forgejo classification content is not valid base64") from exc
if not raw or len(raw) > 131072:
raise ValueError("Forgejo classification content size is outside the safe bound")
classification = _classification_from_bytes(raw)
normalized = _normalize_classification(classification)
return {
"operation": "upsert",
"repository_id": registration.repository_id,
"slug": registration.slug,
"lifecycle": registration.lifecycle,
"classification": normalized,
"revision": {
"head_sha": head_sha,
"source_fingerprint": hashlib.sha256(raw).hexdigest(),
"observed_at": observed_at,
},
}
def close(self) -> None:
self._client.close()
def _get_object(
self, path: str, *, params: dict[str, str] | None = None
) -> dict[str, Any]:
for attempt in range(self.retries + 1):
try:
response = self._client.get(path, params=params)
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict):
raise TypeError("Forgejo response is not a JSON object")
return payload
except (httpx.TimeoutException, httpx.NetworkError):
if attempt == self.retries:
raise ValueError(f"Forgejo request failed after retries: {path}") from None
except httpx.HTTPStatusError as exc:
raise ValueError(
f"Forgejo request returned HTTP {exc.response.status_code}: {path}"
) from exc
except ValueError:
raise
raise AssertionError("unreachable retry loop")
def build_classification_snapshot(
registrations: tuple[RepositoryRegistration, ...],
*,
observed_at: datetime | None = None,
forgejo_client: ForgejoClassificationClient | None = None,
remote_workers: int = 8,
) -> ClassificationSnapshot:
"""Observe one atomic, full fleet generation without publishing host-local paths."""
instant = observed_at or utc_now()
@ -252,10 +380,38 @@ def build_classification_snapshot(
repositories: list[dict[str, Any]] = []
diagnostics: list[dict[str, Any]] = []
for registration in sorted(registrations, key=lambda item: item.repository_id):
def observe(registration: RepositoryRegistration) -> dict[str, Any]:
if registration.path is not None:
return _observe_registration(registration, observed_at=generated_at)
if forgejo_client is None:
raise ValueError("Forgejo registry source requires a configured Forgejo client")
return forgejo_client.observe(registration, observed_at=generated_at)
ordered = sorted(registrations, key=lambda item: item.repository_id)
local = [registration for registration in ordered if registration.path is not None]
remote = [registration for registration in ordered if registration.forgejo_repository is not None]
observations: dict[str, dict[str, Any]] = {}
errors: dict[str, Exception] = {}
for registration in local:
try:
change = _observe_registration(registration, observed_at=generated_at)
observations[registration.repository_id] = observe(registration)
except (ClassificationError, OSError, ValueError) as exc:
errors[registration.repository_id] = exc
if remote:
if remote_workers < 1:
raise ValueError("remote_workers must be positive")
with ThreadPoolExecutor(max_workers=remote_workers) as executor:
pending = {executor.submit(observe, registration): registration for registration in remote}
for future in as_completed(pending):
registration = pending[future]
try:
observations[registration.repository_id] = future.result()
except (ClassificationError, OSError, TypeError, ValueError) as exc:
errors[registration.repository_id] = exc
for registration in ordered:
if registration.repository_id in errors:
exc = errors[registration.repository_id]
diagnostics.append(
{
"severity": "error",
@ -264,8 +420,8 @@ def build_classification_snapshot(
"repository_id": registration.repository_id,
}
)
else:
repositories.append(change)
elif registration.repository_id in observations:
repositories.append(observations[registration.repository_id])
source_revision = _sha256(
{
@ -348,6 +504,12 @@ class ClassificationPublisher:
cursor_secret: str | bytes,
page_size: int = 100,
max_snapshots: int = 4,
forgejo_base_url: str | None = None,
forgejo_token: str | None = None,
forgejo_timeout_seconds: float = 10.0,
forgejo_retries: int = 2,
remote_workers: int = 8,
forgejo_transport: httpx.BaseTransport | None = None,
now: Callable[[], datetime] = utc_now,
) -> None:
secret = cursor_secret.encode() if isinstance(cursor_secret, str) else cursor_secret
@ -357,11 +519,25 @@ class ClassificationPublisher:
raise ValueError("page_size must be between 1 and 500")
if max_snapshots < 1:
raise ValueError("max_snapshots must be positive")
if remote_workers < 1:
raise ValueError("remote_workers must be positive")
self.registry_path = registry_path
self.cursor_secret = secret
self.page_size = page_size
self.max_snapshots = max_snapshots
self.remote_workers = remote_workers
self.now = now
self._forgejo_client = (
ForgejoClassificationClient(
forgejo_base_url,
token=forgejo_token,
timeout_seconds=forgejo_timeout_seconds,
retries=forgejo_retries,
transport=forgejo_transport,
)
if forgejo_base_url is not None
else None
)
self._snapshots: OrderedDict[str, ClassificationSnapshot] = OrderedDict()
self._lock = threading.Lock()
@ -372,7 +548,10 @@ class ClassificationPublisher:
def _fetch_page(self, cursor: str | None = None) -> dict[str, Any]:
if cursor is None:
snapshot = build_classification_snapshot(
load_repository_registry(self.registry_path), observed_at=self.now()
load_repository_registry(self.registry_path),
observed_at=self.now(),
forgejo_client=self._forgejo_client,
remote_workers=self.remote_workers,
)
self._snapshots[snapshot.snapshot_id] = snapshot
self._snapshots.move_to_end(snapshot.snapshot_id)
@ -399,6 +578,10 @@ class ClassificationPublisher:
page_cursor=cursor,
)
def close(self) -> None:
if self._forgejo_client is not None:
self._forgejo_client.close()
def encode_projection_cursor(
*, snapshot_id: str, offset: int, page_size: int, secret: bytes
@ -439,6 +622,8 @@ def _observe_registration(
registration: RepositoryRegistration, *, observed_at: str
) -> dict[str, Any]:
repo_root = registration.path
if repo_root is None:
raise ValueError("registration has no local checkout source")
if not repo_root.is_dir() or not (repo_root / ".git").exists():
raise ValueError("registered checkout is missing or is not a Git working copy")
classification_path = repo_root / ".repo-classification.yaml"
@ -459,14 +644,7 @@ def _observe_registration(
head_sha = _git_head(repo_root)
if head_sha is None:
raise ValueError("repository HEAD is unavailable")
normalized = {
"category": classification["category"],
"domain": classification["domain"],
"secondary_domains": sorted(classification.get("secondary_domains") or []),
"capability_tags": sorted(classification.get("capability_tags") or []),
"business_stake": sorted(classification.get("business_stake") or []),
"business_mechanics": sorted(classification.get("business_mechanics") or []),
}
normalized = _normalize_classification(classification)
return {
"operation": "upsert",
"repository_id": registration.repository_id,
@ -493,6 +671,30 @@ def _git_head(repo_root: Path) -> str | None:
return value if result.returncode == 0 and re.fullmatch(r"[a-f0-9]{40,64}", value) else None
def _classification_from_bytes(raw: bytes) -> dict[str, Any]:
try:
document = yaml.safe_load(raw.decode("utf-8")) or {}
except (UnicodeDecodeError, yaml.YAMLError) as exc:
raise ValueError(f"classification YAML is invalid: {exc}") from exc
if not isinstance(document, dict):
raise TypeError("classification document must be an object")
classification = document.get("repo_classification")
if not isinstance(classification, dict):
raise TypeError("classification document has no repo_classification object")
return require_valid_classification(classification)
def _normalize_classification(classification: dict[str, Any]) -> dict[str, Any]:
return {
"category": classification["category"],
"domain": classification["domain"],
"secondary_domains": sorted(classification.get("secondary_domains") or []),
"capability_tags": sorted(classification.get("capability_tags") or []),
"business_stake": sorted(classification.get("business_stake") or []),
"business_mechanics": sorted(classification.get("business_mechanics") or []),
}
def _discover_classified_repositories(root: Path) -> tuple[Path, ...]:
resolved = root.expanduser().resolve()
if (resolved / ".git").exists() and (resolved / ".repo-classification.yaml").is_file():

48
src/repo_manager/runtime.py
View file

@ -4,6 +4,8 @@ from __future__ import annotations
import hmac
import os
import secrets
from contextlib import asynccontextmanager
from dataclasses import dataclass
from pathlib import Path
@ -25,16 +27,43 @@ class PublisherSettings:
api_token: str | None = None
page_size: int = 100
max_snapshots: int = 4
forgejo_base_url: str | None = None
forgejo_token: str | None = None
forgejo_timeout_seconds: float = 10.0
forgejo_retries: int = 2
remote_workers: int = 8
@classmethod
def from_env(cls) -> PublisherSettings:
registry = os.getenv("REPO_MANAGER_REGISTRY_PATH")
token = os.getenv("REPO_MANAGER_FORGEJO_TOKEN")
token_file = os.getenv("REPO_MANAGER_FORGEJO_TOKEN_FILE")
if token and token_file:
raise ValueError(
"set only one of REPO_MANAGER_FORGEJO_TOKEN and REPO_MANAGER_FORGEJO_TOKEN_FILE"
)
if token_file:
try:
token = Path(token_file).read_text(encoding="utf-8").strip()
except OSError as exc:
raise ValueError(f"cannot read Forgejo token file: {exc}") from exc
if not token:
raise ValueError("Forgejo token file is empty")
return cls(
registry_path=Path(registry) if registry else None,
cursor_secret=os.getenv("REPO_MANAGER_CURSOR_SECRET"),
cursor_secret=os.getenv("REPO_MANAGER_CURSOR_SECRET") or secrets.token_urlsafe(32),
api_token=os.getenv("REPO_MANAGER_API_TOKEN"),
page_size=int(os.getenv("REPO_MANAGER_PROJECTION_PAGE_SIZE", "100")),
max_snapshots=int(os.getenv("REPO_MANAGER_PROJECTION_MAX_SNAPSHOTS", "4")),
forgejo_base_url=os.getenv(
"REPO_MANAGER_FORGEJO_BASE_URL", "https://forgejo.coulomb.social"
),
forgejo_token=token,
forgejo_timeout_seconds=float(
os.getenv("REPO_MANAGER_FORGEJO_TIMEOUT_SECONDS", "10")
),
forgejo_retries=int(os.getenv("REPO_MANAGER_FORGEJO_RETRIES", "2")),
remote_workers=int(os.getenv("REPO_MANAGER_FORGEJO_WORKERS", "8")),
)
@ -45,21 +74,32 @@ def create_app(*, settings: PublisherSettings | None = None) -> FastAPI:
try:
if resolved.registry_path is None:
raise ValueError("REPO_MANAGER_REGISTRY_PATH is required")
if resolved.cursor_secret is None:
raise ValueError("REPO_MANAGER_CURSOR_SECRET is required")
cursor_secret = resolved.cursor_secret or secrets.token_urlsafe(32)
publisher = ClassificationPublisher(
resolved.registry_path,
cursor_secret=resolved.cursor_secret,
cursor_secret=cursor_secret,
page_size=resolved.page_size,
max_snapshots=resolved.max_snapshots,
forgejo_base_url=resolved.forgejo_base_url,
forgejo_token=resolved.forgejo_token,
forgejo_timeout_seconds=resolved.forgejo_timeout_seconds,
forgejo_retries=resolved.forgejo_retries,
remote_workers=resolved.remote_workers,
)
except ValueError as exc:
configuration_error = str(exc)
@asynccontextmanager
async def lifespan(_app: FastAPI):
yield
if publisher is not None:
publisher.close()
app = FastAPI(
title="Repo Manager projection publisher",
version=__version__,
description="Read-only port.repo repository projection publication.",
lifespan=lifespan,
)
app.state.publisher = publisher
app.state.configuration_error = configuration_error