coulomb/sbom-nexus
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: establish sbom nexus extraction slice

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a028f0-a42f-7582-89a8-ebaad7343834
This commit is contained in:
tegwick 2026-08-22 12:50:08 +02:00
parent 79cd7dff06
commit d61698ea51
31 changed files with 3246 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/sbom_nexus/__init__.py Normal file
View file

@ -0,0 +1,3 @@
"""SBOM Nexus product package."""
__version__ = "0.1.0"

288
src/sbom_nexus/api.py Normal file
View file

@ -0,0 +1,288 @@
"""FastAPI product and State Hub compatibility surface."""
from __future__ import annotations
import os
from datetime import datetime
from pathlib import Path
from typing import Any, Literal
from fastapi import FastAPI, HTTPException, Query, Request
from pydantic import BaseModel, Field
from sbom_nexus.scanner import VALID_ECOSYSTEMS, scan_repository
from sbom_nexus.storage import Store
DEFAULT_DATABASE_PATH = os.getenv("SBOM_NEXUS_DATABASE_PATH", "sbom-nexus.db")
DEFAULT_STALE_DAYS = int(os.getenv("SBOM_NEXUS_STALE_DAYS", "30"))
class RepositoryUpsert(BaseModel):
checkout_path: str | None = None
active: bool = True
last_sbom_at: datetime | None = None
last_attempt_at: datetime | None = None
last_success_at: datetime | None = None
class EntryCreate(BaseModel):
package_name: str = Field(min_length=1, max_length=300)
package_version: str | None = Field(default=None, max_length=100)
ecosystem: str
license_spdx: str | None = Field(default=None, max_length=100)
is_direct: bool = True
is_dev: bool = False
source_path: str | None = None
class LegacyIngest(BaseModel):
repo_slug: str = Field(min_length=1)
entries: list[EntryCreate]
class HistoricalImport(BaseModel):
repo_slug: str = Field(min_length=1)
legacy_id: str = Field(min_length=1)
snapshot_at: datetime
source: str = "state-hub-import"
source_revision: str | None = None
entries: list[EntryCreate]
errors: list[dict[str, Any]] = Field(default_factory=list)
class SkipRequest(BaseModel):
reason: Literal["no-checkout", "no-manifest", "ingest-error"]
detail: str | None = None
def _store(request: Request) -> Store:
return request.app.state.store
def _not_found(repo_slug: str) -> HTTPException:
return HTTPException(status_code=404, detail=f"Repo '{repo_slug}' not found")
def _validate_entries(entries: list[EntryCreate]) -> list[dict[str, Any]]:
result: list[dict[str, Any]] = []
for entry in entries:
if entry.ecosystem not in VALID_ECOSYSTEMS:
raise HTTPException(
status_code=422,
detail=f"Unsupported ecosystem '{entry.ecosystem}'",
)
result.append(entry.model_dump())
return result
def create_app(database_path: str | Path | None = None) -> FastAPI:
application = FastAPI(
title="SBOM Nexus",
version="0.1.0",
description="SBOM capture, history, evaluation, and bounded catch-up service",
)
application.state.store = Store(database_path or DEFAULT_DATABASE_PATH)
application.state.store.init_schema()
@application.get("/state/health")
def health(request: Request) -> dict[str, str]:
_store(request).list_repositories()
return {"status": "ok", "store": "connected"}
@application.put("/repositories/{repo_slug}")
def upsert_repository(
repo_slug: str, body: RepositoryUpsert, request: Request
) -> dict[str, Any]:
legacy_time = body.last_sbom_at
return _store(request).upsert_repository(
repo_slug,
checkout_path=body.checkout_path,
active=body.active,
last_attempt_at=body.last_attempt_at or legacy_time,
last_success_at=body.last_success_at or legacy_time,
)
@application.get("/repositories/")
def list_repositories(request: Request) -> list[dict[str, Any]]:
return _store(request).list_repositories()
@application.get("/sbom/catch-up")
def catch_up(
request: Request,
limit: int = Query(default=3, ge=1, le=25),
stale_days: int = Query(default=DEFAULT_STALE_DAYS, ge=1, le=3650),
) -> dict[str, Any]:
return _store(request).catch_up(limit=limit, stale_days=stale_days)
@application.post("/sbom/ingest/")
def ingest_legacy(body: LegacyIngest, request: Request) -> dict[str, Any]:
store = _store(request)
if store.get_repository(body.repo_slug) is None:
raise _not_found(body.repo_slug)
snapshot, _created = store.record_snapshot(
body.repo_slug,
entries=_validate_entries(body.entries),
status="ingested",
source="manual",
)
return {
"repo_slug": body.repo_slug,
"snapshot_id": snapshot["id"],
"ingested": snapshot["entry_count"],
"snapshot_at": snapshot["snapshot_at"],
"status": snapshot["status"],
}
@application.post("/sbom/import/")
def import_historical(body: HistoricalImport, request: Request) -> dict[str, Any]:
store = _store(request)
if store.get_repository(body.repo_slug) is None:
store.upsert_repository(body.repo_slug)
snapshot, created = store.record_snapshot(
body.repo_slug,
entries=_validate_entries(body.entries),
status="imported",
source=body.source,
snapshot_at=body.snapshot_at,
source_revision=body.source_revision,
errors=body.errors,
snapshot_id=body.legacy_id,
legacy_id=body.legacy_id,
)
return {
"repo_slug": body.repo_slug,
"snapshot_id": snapshot["id"],
"imported": created,
"entry_count": snapshot["entry_count"],
"snapshot_at": snapshot["snapshot_at"],
}
@application.get("/sbom/snapshots/")
def list_snapshots(
request: Request, repo_slug: str | None = Query(default=None)
) -> list[dict[str, Any]]:
store = _store(request)
if repo_slug and store.get_repository(repo_slug) is None:
raise _not_found(repo_slug)
return store.list_snapshots(repo_slug)
@application.get("/sbom/snapshots/{snapshot_id}")
def get_snapshot(snapshot_id: str, request: Request) -> dict[str, Any]:
snapshot = _store(request).get_snapshot(snapshot_id)
if snapshot is None:
raise HTTPException(
status_code=404, detail=f"Snapshot '{snapshot_id}' not found"
)
return snapshot
@application.get("/sbom/report/licences/")
def licence_report(request: Request) -> dict[str, Any]:
return _store(request).licence_report()
@application.get("/sbom/")
def list_entries(
request: Request,
repo_slug: str | None = Query(default=None),
ecosystem: str | None = Query(default=None),
license_spdx: str | None = Query(default=None),
is_direct: bool | None = Query(default=None),
is_dev: bool | None = Query(default=None),
) -> list[dict[str, Any]]:
store = _store(request)
if repo_slug and store.get_repository(repo_slug) is None:
raise _not_found(repo_slug)
return store.latest_entries(
repo_slug=repo_slug,
ecosystem=ecosystem,
license_spdx=license_spdx,
is_direct=is_direct,
is_dev=is_dev,
)
@application.post("/sbom/{repo_slug}/ingest")
def ingest_repository(repo_slug: str, request: Request) -> dict[str, Any]:
store = _store(request)
repo = store.get_repository(repo_slug)
if repo is None:
raise _not_found(repo_slug)
checkout_path = repo["checkout_path"]
if not checkout_path or not Path(checkout_path).is_dir():
return _record_skip(store, repo_slug, "no-checkout")
scan = scan_repository(Path(checkout_path), slug=repo_slug)
if not scan["sources"] and not scan["errors"]:
return _record_skip(store, repo_slug, "no-manifest")
if scan["errors"]:
snapshot, _created = store.record_snapshot(
repo_slug,
entries=scan["entries"],
status="ingest-error",
source="ingest-error",
source_revision=scan["source_revision"],
sources=scan["sources"],
errors=scan["errors"],
)
return {
"repo_slug": repo_slug,
"snapshot_id": snapshot["id"],
"status": "skipped",
"reason": "ingest-error",
"entry_count": snapshot["entry_count"],
"errors": scan["errors"],
"snapshot_at": snapshot["snapshot_at"],
}
snapshot, _created = store.record_snapshot(
repo_slug,
entries=scan["entries"],
status="ingested",
source="repository-scan",
source_revision=scan["source_revision"],
sources=scan["sources"],
)
return {
"repo_slug": repo_slug,
"snapshot_id": snapshot["id"],
"status": "ingested",
"entry_count": snapshot["entry_count"],
"snapshot_at": snapshot["snapshot_at"],
"source_revision": snapshot["source_revision"],
}
@application.post("/sbom/{repo_slug}/skip")
def skip_repository(
repo_slug: str, body: SkipRequest, request: Request
) -> dict[str, Any]:
store = _store(request)
if store.get_repository(repo_slug) is None:
raise _not_found(repo_slug)
return _record_skip(store, repo_slug, body.reason, body.detail)
@application.get("/sbom/{repo_slug}")
def get_repo_sbom(repo_slug: str, request: Request) -> dict[str, Any]:
view = _store(request).repository_view(repo_slug)
if view is None:
raise _not_found(repo_slug)
return view
return application
def _record_skip(
store: Store, repo_slug: str, reason: str, detail: str | None = None
) -> dict[str, Any]:
errors = [{"reason": reason, "detail": detail}] if detail else []
snapshot, _created = store.record_snapshot(
repo_slug,
entries=[],
status=reason,
source=reason,
errors=errors,
)
return {
"repo_slug": repo_slug,
"snapshot_id": snapshot["id"],
"status": "skipped",
"reason": reason,
"entry_count": 0,
"snapshot_at": snapshot["snapshot_at"],
}

67
src/sbom_nexus/cli.py Normal file
View file

@ -0,0 +1,67 @@
"""Operator CLI for SBOM Nexus."""
from __future__ import annotations
import argparse
import json
import os
from pathlib import Path
from sbom_nexus.scanner import scan_repository
def _serve(args: argparse.Namespace) -> None:
if args.database:
os.environ["SBOM_NEXUS_DATABASE_PATH"] = str(Path(args.database).resolve())
import uvicorn
uvicorn.run(
"sbom_nexus.api:create_app",
host=args.host,
port=args.port,
reload=args.reload,
factory=True,
)
def _scan(args: argparse.Namespace) -> None:
result = scan_repository(Path(args.path), slug=args.slug)
rendered = json.dumps(result, indent=2, sort_keys=True)
if args.output:
output = Path(args.output)
if output.exists() and not args.force:
raise SystemExit(f"Refusing to overwrite {output}; pass --force")
output.write_text(rendered + "\n", encoding="utf-8")
else:
print(rendered)
if not result["ok"]:
raise SystemExit(1)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="sbom-nexus")
subparsers = parser.add_subparsers(dest="command", required=True)
serve = subparsers.add_parser("serve", help="Run the SBOM Nexus HTTP API")
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8010)
serve.add_argument("--database", help="SQLite database path")
serve.add_argument("--reload", action="store_true")
serve.set_defaults(handler=_serve)
scan = subparsers.add_parser("scan", help="Derive a snapshot from repository sources")
scan.add_argument("path", nargs="?", default=".")
scan.add_argument("--slug")
scan.add_argument("--output")
scan.add_argument("--force", action="store_true")
scan.set_defaults(handler=_scan)
return parser
def main() -> None:
args = build_parser().parse_args()
args.handler(args)
if __name__ == "__main__":
main()

364
src/sbom_nexus/scanner.py Normal file
View file

@ -0,0 +1,364 @@
"""Repository-derived SBOM scanning and licence triage.
The parser set is extracted from State Hub's ``scripts/ingest_sbom.py`` and the
provenance-aware receiving implementation in Repo Manager (RMGR-WP-0008).
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import subprocess
import tomllib
from collections import Counter
from collections.abc import Callable
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import yaml
SBOM_SCHEMA = "sbom-nexus.snapshot.v1"
COPYLEFT_MARKERS = frozenset({"GPL", "AGPL", "LGPL", "EUPL", "CDDL", "MPL"})
VALID_ECOSYSTEMS = frozenset(
{"python", "node", "rust", "go", "java", "terraform", "ansible", "tool", "other"}
)
SKIP_DIRECTORIES = frozenset(
{
".git",
".hg",
".svn",
".venv",
"venv",
".env",
"node_modules",
"__pycache__",
".mypy_cache",
".pytest_cache",
".ruff_cache",
"dist",
"build",
".build",
"target",
".tox",
".nox",
}
)
Entry = dict[str, Any]
Parser = Callable[[Path], list[Entry]]
def _entry(
name: str,
version: str | None,
ecosystem: str,
*,
license_spdx: str | None = None,
is_direct: bool = False,
is_dev: bool = False,
) -> Entry:
return {
"package_name": name,
"package_version": version,
"ecosystem": ecosystem,
"license_spdx": license_spdx,
"is_direct": is_direct,
"is_dev": is_dev,
}
def _parse_toml_packages(path: Path, ecosystem: str) -> list[Entry]:
data = tomllib.loads(path.read_text(encoding="utf-8"))
return [
_entry(str(item["name"]), str(item["version"]) if item.get("version") else None, ecosystem)
for item in data.get("package", [])
if isinstance(item, dict) and item.get("name")
]
def parse_uv_lock(path: Path) -> list[Entry]:
return _parse_toml_packages(path, "python")
def parse_cargo_lock(path: Path) -> list[Entry]:
return _parse_toml_packages(path, "rust")
def parse_requirements(path: Path) -> list[Entry]:
entries: list[Entry] = []
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith(("#", "-")):
continue
match = re.match(r"^([A-Za-z0-9_.-]+)(?:[>=<!~^]+([^\s;]+))?", line)
if match:
entries.append(_entry(match.group(1), match.group(2), "python", is_direct=True))
return entries
def parse_package_lock(path: Path) -> list[Entry]:
data = json.loads(path.read_text(encoding="utf-8"))
entries: list[Entry] = []
for package_path, item in (data.get("packages") or {}).items():
if not package_path or not isinstance(item, dict):
continue
name = item.get("name") or package_path.rsplit("node_modules/", 1)[-1]
entries.append(
_entry(
str(name),
str(item["version"]) if item.get("version") else None,
"node",
license_spdx=str(item["license"]) if item.get("license") else None,
is_direct=not bool(item.get("indirect", False)),
is_dev=bool(item.get("dev", False)),
)
)
return entries
def parse_yarn_lock(path: Path) -> list[Entry]:
entries: list[Entry] = []
names: list[str] = []
for raw in path.read_text(encoding="utf-8").splitlines():
stripped = raw.strip()
if raw and not raw.startswith((" ", "\t")) and stripped.endswith(":"):
names = []
for specifier in stripped.rstrip(":").split(","):
match = re.match(r'"?((?:@[^/" ]+/)?[^@" ]+)@', specifier.strip())
if match:
names.append(match.group(1))
elif stripped.startswith("version ") and names:
version_match = re.search(r'"([^"]+)"', stripped)
version = version_match.group(1) if version_match else None
entries.extend(_entry(name, version, "node") for name in names)
names = []
return entries
def _go_direct_modules(directory: Path) -> set[str]:
path = directory / "go.mod"
if not path.is_file():
return set()
direct: set[str] = set()
in_block = False
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if line == "require (":
in_block = True
continue
if in_block and line == ")":
in_block = False
continue
if in_block:
candidate = line
elif line.startswith("require "):
candidate = line.removeprefix("require ")
else:
candidate = ""
if candidate and "// indirect" not in candidate:
direct.add(candidate.split()[0])
return direct
def parse_go_sum(path: Path) -> list[Entry]:
direct = _go_direct_modules(path.parent)
seen: set[tuple[str, str]] = set()
entries: list[Entry] = []
for raw in path.read_text(encoding="utf-8").splitlines():
parts = raw.split()
if len(parts) < 2 or parts[1].endswith("/go.mod"):
continue
key = (parts[0], parts[1])
if key in seen:
continue
seen.add(key)
entries.append(_entry(key[0], key[1], "go", is_direct=parts[0] in direct))
return entries
def parse_terraform_lock(path: Path) -> list[Entry]:
entries: list[Entry] = []
provider: str | None = None
version: str | None = None
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
match = re.match(r'^provider\s+"([^"]+)"\s*\{', line)
if match:
provider, version = match.group(1), None
elif provider:
version_match = re.match(r'version\s*=\s*"([^"]+)"', line)
if version_match:
version = version_match.group(1)
elif line == "}":
entries.append(_entry(provider, version, "terraform", is_direct=True))
provider, version = None, None
return entries
def parse_ansible_requirements(path: Path) -> list[Entry]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict):
return []
entries: list[Entry] = []
for kind in ("collections", "roles"):
for item in data.get(kind, []) or []:
if isinstance(item, str):
name, version = item, None
elif isinstance(item, dict):
name = item.get("name") or item.get("src")
version = str(item["version"]) if item.get("version") else None
else:
continue
if name:
entries.append(_entry(str(name), version, "ansible", is_direct=True))
return entries
def parse_tools_manifest(path: Path) -> list[Entry]:
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict):
return []
entries: list[Entry] = []
for item in data.get("tools", []) or []:
if not isinstance(item, dict) or not item.get("name"):
continue
ecosystem = str(item.get("ecosystem") or "tool")
if ecosystem not in VALID_ECOSYSTEMS:
ecosystem = "tool"
raw_version = item.get("version")
version = str(raw_version) if raw_version not in {None, "unknown"} else None
entries.append(
_entry(
str(item["name"]),
version,
ecosystem,
license_spdx=str(item["license_spdx"]) if item.get("license_spdx") else None,
is_direct=bool(item.get("is_direct", True)),
is_dev=bool(item.get("is_dev", False)),
)
)
return entries
LOCKFILE_PARSERS: dict[str, Parser] = {
"uv.lock": parse_uv_lock,
"requirements.txt": parse_requirements,
"package-lock.json": parse_package_lock,
"yarn.lock": parse_yarn_lock,
"Cargo.lock": parse_cargo_lock,
".terraform.lock.hcl": parse_terraform_lock,
"go.sum": parse_go_sum,
}
def detect_sources(repo_root: Path) -> list[tuple[Path, Parser]]:
found: list[tuple[Path, Parser]] = []
seen: set[Path] = set()
for directory, directories, filenames in os.walk(repo_root):
directories[:] = sorted(name for name in directories if name not in SKIP_DIRECTORIES)
current = Path(directory)
for filename, parser in LOCKFILE_PARSERS.items():
if filename in filenames:
path = current / filename
found.append((path, parser))
seen.add(path)
if current.name == "ansible":
for filename in ("requirements.yml", "requirements.yaml"):
if filename in filenames:
path = current / filename
found.append((path, parse_ansible_requirements))
seen.add(path)
tools = repo_root / "sbom-tools.yaml"
if tools.is_file() and tools not in seen:
found.append((tools, parse_tools_manifest))
return sorted(found, key=lambda item: str(item[0]))
def is_copyleft(spdx: str | None) -> bool:
upper = (spdx or "").upper()
return any(marker in upper for marker in COPYLEFT_MARKERS)
def licence_report(entries: list[Entry]) -> dict[str, Any]:
counts = Counter(entry.get("license_spdx") for entry in entries)
groups = [
{"license_spdx": license_id, "count": count, "is_copyleft": is_copyleft(license_id)}
for license_id, count in sorted(counts.items(), key=lambda item: (-item[1], item[0] or ""))
]
risks = [
{
"package_name": entry["package_name"],
"package_version": entry.get("package_version"),
"license_spdx": entry.get("license_spdx"),
"source_path": entry.get("source_path"),
}
for entry in entries
if is_copyleft(entry.get("license_spdx"))
and entry.get("is_direct")
and not entry.get("is_dev")
]
return {
"groups": groups,
"copyleft_direct_prod": risks,
"copyleft_direct_count": len(risks),
}
def _head_sha(repo_root: Path) -> str | None:
try:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
)
except (OSError, subprocess.CalledProcessError):
return None
return result.stdout.strip() or None
def _utc_now_text() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
def scan_repository(repo_root: Path, *, slug: str | None = None) -> dict[str, Any]:
repo_root = repo_root.resolve()
sources: list[dict[str, Any]] = []
entries: list[Entry] = []
errors: list[dict[str, str]] = []
for path, parser in detect_sources(repo_root):
relative = str(path.relative_to(repo_root))
try:
parsed = parser(path)
except (
OSError,
ValueError,
TypeError,
json.JSONDecodeError,
tomllib.TOMLDecodeError,
yaml.YAMLError,
) as exc:
errors.append({"source_path": relative, "error": str(exc)})
continue
digest = hashlib.sha256(path.read_bytes()).hexdigest()
sources.append({"path": relative, "sha256": digest, "entry_count": len(parsed)})
for item in parsed:
entries.append({**item, "source_path": relative})
return {
"schema": SBOM_SCHEMA,
"ok": not errors,
"repo_slug": slug or repo_root.name,
"repo_path": str(repo_root),
"source_revision": _head_sha(repo_root),
"generated_at": _utc_now_text(),
"authority": "detected lockfiles and reviewed sbom-tools.yaml",
"sources": sources,
"entry_count": len(entries),
"entries": entries,
"licence_report": licence_report(entries),
"errors": errors,
}

518
src/sbom_nexus/storage.py Normal file
View file

@ -0,0 +1,518 @@
"""SQLite persistence for the extraction slice.
The store keeps the product behavior independent from FastAPI. PostgreSQL and
managed migrations are an explicit production-cutover gate in SBOM-WP-0001-T05.
"""
from __future__ import annotations
import json
import sqlite3
import uuid
from collections import defaultdict
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from sbom_nexus.scanner import is_copyleft
SUCCESS_STATUSES = frozenset({"ingested", "imported"})
def utc_now() -> datetime:
return datetime.now(UTC)
def datetime_text(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, str):
parsed = parse_datetime(value)
else:
parsed = value
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
def parse_datetime(value: str) -> datetime:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
class Store:
def __init__(self, database_path: str | Path) -> None:
self.database_path = str(database_path)
def connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.database_path, timeout=30)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
return connection
def init_schema(self) -> None:
with self.connect() as connection:
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS repositories (
id TEXT PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
checkout_path TEXT,
active INTEGER NOT NULL DEFAULT 1,
last_attempt_at TEXT,
last_success_at TEXT,
last_status TEXT,
last_source TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS snapshots (
id TEXT PRIMARY KEY,
repo_id TEXT NOT NULL REFERENCES repositories(id) ON DELETE RESTRICT,
snapshot_at TEXT NOT NULL,
source TEXT NOT NULL,
status TEXT NOT NULL,
entry_count INTEGER NOT NULL,
source_revision TEXT,
sources_json TEXT NOT NULL DEFAULT '[]',
errors_json TEXT NOT NULL DEFAULT '[]',
legacy_id TEXT UNIQUE,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS entries (
id TEXT PRIMARY KEY,
repo_id TEXT NOT NULL REFERENCES repositories(id) ON DELETE RESTRICT,
snapshot_id TEXT NOT NULL REFERENCES snapshots(id) ON DELETE RESTRICT,
package_name TEXT NOT NULL,
package_version TEXT,
ecosystem TEXT NOT NULL,
license_spdx TEXT,
is_direct INTEGER NOT NULL,
is_dev INTEGER NOT NULL,
source_path TEXT,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS ix_snapshots_repo_time
ON snapshots(repo_id, snapshot_at DESC);
CREATE INDEX IF NOT EXISTS ix_entries_snapshot ON entries(snapshot_id);
CREATE INDEX IF NOT EXISTS ix_entries_repo ON entries(repo_id);
CREATE INDEX IF NOT EXISTS ix_entries_license ON entries(license_spdx);
"""
)
def upsert_repository(
self,
slug: str,
*,
checkout_path: str | None = None,
active: bool = True,
last_attempt_at: datetime | str | None = None,
last_success_at: datetime | str | None = None,
) -> dict[str, Any]:
now = datetime_text(utc_now())
attempt = datetime_text(last_attempt_at)
success = datetime_text(last_success_at)
with self.connect() as connection:
existing = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
if existing:
connection.execute(
"""
UPDATE repositories
SET checkout_path = ?, active = ?,
last_attempt_at = COALESCE(?, last_attempt_at),
last_success_at = COALESCE(?, last_success_at),
updated_at = ?
WHERE slug = ?
""",
(checkout_path, int(active), attempt, success, now, slug),
)
else:
connection.execute(
"""
INSERT INTO repositories (
id, slug, checkout_path, active, last_attempt_at,
last_success_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
str(uuid.uuid4()),
slug,
checkout_path,
int(active),
attempt,
success,
now,
now,
),
)
row = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
return self._repository_dict(row)
def get_repository(self, slug: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
return self._repository_dict(row) if row else None
def list_repositories(self) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute("SELECT * FROM repositories ORDER BY slug").fetchall()
return [self._repository_dict(row) for row in rows]
def record_snapshot(
self,
repo_slug: str,
*,
entries: list[dict[str, Any]],
status: str,
source: str,
snapshot_at: datetime | str | None = None,
source_revision: str | None = None,
sources: list[dict[str, Any]] | None = None,
errors: list[dict[str, Any]] | None = None,
snapshot_id: str | None = None,
legacy_id: str | None = None,
) -> tuple[dict[str, Any], bool]:
timestamp = datetime_text(snapshot_at or utc_now())
created_at = datetime_text(utc_now())
with self.connect() as connection:
repo = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (repo_slug,)
).fetchone()
if repo is None:
raise KeyError(repo_slug)
if legacy_id:
existing = connection.execute(
"SELECT * FROM snapshots WHERE legacy_id = ?", (legacy_id,)
).fetchone()
if existing:
return self._snapshot_dict(existing), False
new_snapshot_id = snapshot_id or str(uuid.uuid4())
connection.execute(
"""
INSERT INTO snapshots (
id, repo_id, snapshot_at, source, status, entry_count,
source_revision, sources_json, errors_json, legacy_id, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
new_snapshot_id,
repo["id"],
timestamp,
source,
status,
len(entries),
source_revision,
json.dumps(sources or [], sort_keys=True),
json.dumps(errors or [], sort_keys=True),
legacy_id,
created_at,
),
)
for entry in entries:
connection.execute(
"""
INSERT INTO entries (
id, repo_id, snapshot_id, package_name, package_version,
ecosystem, license_spdx, is_direct, is_dev, source_path,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
str(uuid.uuid4()),
repo["id"],
new_snapshot_id,
entry["package_name"],
entry.get("package_version"),
entry["ecosystem"],
entry.get("license_spdx"),
int(bool(entry.get("is_direct", True))),
int(bool(entry.get("is_dev", False))),
entry.get("source_path"),
created_at,
),
)
previous_attempt = repo["last_attempt_at"]
is_latest = previous_attempt is None or parse_datetime(timestamp) >= parse_datetime(
previous_attempt
)
if is_latest:
last_success = repo["last_success_at"]
if status in SUCCESS_STATUSES:
last_success = timestamp
connection.execute(
"""
UPDATE repositories
SET last_attempt_at = ?, last_success_at = ?, last_status = ?,
last_source = ?, updated_at = ?
WHERE id = ?
""",
(timestamp, last_success, status, source, created_at, repo["id"]),
)
row = connection.execute(
"SELECT * FROM snapshots WHERE id = ?", (new_snapshot_id,)
).fetchone()
return self._snapshot_dict(row), True
def list_snapshots(self, repo_slug: str | None = None) -> list[dict[str, Any]]:
query = """
SELECT s.*, r.slug AS repo_slug
FROM snapshots s JOIN repositories r ON r.id = s.repo_id
"""
params: tuple[Any, ...] = ()
if repo_slug:
query += " WHERE r.slug = ?"
params = (repo_slug,)
query += " ORDER BY s.snapshot_at DESC, s.created_at DESC"
with self.connect() as connection:
rows = connection.execute(query, params).fetchall()
return [self._snapshot_dict(row) for row in rows]
def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"""
SELECT s.*, r.slug AS repo_slug
FROM snapshots s JOIN repositories r ON r.id = s.repo_id
WHERE s.id = ?
""",
(snapshot_id,),
).fetchone()
if row is None:
return None
entries = connection.execute(
"SELECT * FROM entries WHERE snapshot_id = ? ORDER BY package_name",
(snapshot_id,),
).fetchall()
result = self._snapshot_dict(row)
result["entries"] = []
for entry in entries:
rendered = self._entry_dict(entry)
rendered["snapshot_at"] = result["snapshot_at"]
result["entries"].append(rendered)
return result
def latest_entries(
self,
*,
repo_slug: str | None = None,
ecosystem: str | None = None,
license_spdx: str | None = None,
is_direct: bool | None = None,
is_dev: bool | None = None,
) -> list[dict[str, Any]]:
latest = self._latest_snapshot_rows(repo_slug)
if not latest:
return []
snapshot_ids = [row["id"] for row in latest]
placeholders = ",".join("?" for _ in snapshot_ids)
query = f"""
SELECT e.*, r.slug AS repo_slug, s.snapshot_at AS snapshot_at
FROM entries e
JOIN repositories r ON r.id = e.repo_id
JOIN snapshots s ON s.id = e.snapshot_id
WHERE e.snapshot_id IN ({placeholders})
"""
params: list[Any] = list(snapshot_ids)
for field, value in (
("ecosystem", ecosystem),
("license_spdx", license_spdx),
("is_direct", None if is_direct is None else int(is_direct)),
("is_dev", None if is_dev is None else int(is_dev)),
):
if value is not None:
query += f" AND e.{field} = ?"
params.append(value)
query += " ORDER BY e.package_name, r.slug"
with self.connect() as connection:
rows = connection.execute(query, params).fetchall()
return [self._entry_dict(row) for row in rows]
def repository_view(self, repo_slug: str) -> dict[str, Any] | None:
repo = self.get_repository(repo_slug)
if repo is None:
return None
entries = self.latest_entries(repo_slug=repo_slug)
snapshots = self.list_snapshots(repo_slug)
return {
"repo_slug": repo_slug,
"last_sbom_at": repo["last_attempt_at"],
"last_attempt_at": repo["last_attempt_at"],
"last_success_at": repo["last_success_at"],
"last_status": repo["last_status"],
"entry_count": len(entries),
"snapshot_id": snapshots[0]["id"] if snapshots else None,
"entries": entries,
}
def licence_report(self) -> dict[str, Any]:
entries = self.latest_entries()
groups: dict[str | None, dict[str, Any]] = defaultdict(
lambda: {"count": 0, "repos": set()}
)
risks: list[dict[str, Any]] = []
for entry in entries:
license_id = entry.get("license_spdx")
groups[license_id]["count"] += 1
groups[license_id]["repos"].add(entry["repo_slug"])
if is_copyleft(license_id) and entry["is_direct"] and not entry["is_dev"]:
risks.append(
{
"repo_slug": entry["repo_slug"],
"package_name": entry["package_name"],
"package_version": entry.get("package_version"),
"license_spdx": license_id,
"source_path": entry.get("source_path"),
}
)
sorted_groups = sorted(groups.items(), key=lambda item: (-item[1]["count"], item[0] or ""))
return {
"groups": [
{
"license_spdx": license_id,
"count": data["count"],
"repos": sorted(data["repos"]),
"is_copyleft": is_copyleft(license_id),
}
for license_id, data in sorted_groups
],
"copyleft_direct_prod": risks,
"copyleft_direct_count": len(risks),
"signal_qualification": "substring triage; not legal advice or full SPDX evaluation",
}
def catch_up(
self,
*,
limit: int = 3,
stale_days: int = 30,
now: datetime | None = None,
) -> dict[str, Any]:
effective_limit = max(1, min(25, int(limit)))
current = now or utc_now()
cutoff = current - timedelta(days=stale_days)
repositories = [repo for repo in self.list_repositories() if repo["active"]]
never = [repo for repo in repositories if repo["last_attempt_at"] is None]
stale = [
repo
for repo in repositories
if repo["last_attempt_at"] is None
or parse_datetime(repo["last_attempt_at"]) < cutoff
]
stale.sort(
key=lambda repo: (
repo["last_attempt_at"] is not None,
repo["last_attempt_at"] or "",
repo["slug"],
)
)
selected = stale[:effective_limit]
return {
"repos": [self._catch_up_repository(repo, current) for repo in selected],
"selected_count": len(selected),
"stale_count": len(stale),
"never_count": len(never),
"total_count": len(repositories),
"limit": effective_limit,
"stale_after_days": stale_days,
"evaluated_at": datetime_text(current),
}
def _latest_snapshot_rows(self, repo_slug: str | None) -> list[sqlite3.Row]:
query = """
SELECT s.*, r.slug AS repo_slug
FROM snapshots s
JOIN repositories r ON r.id = s.repo_id
WHERE s.id = (
SELECT inner_s.id FROM snapshots inner_s
WHERE inner_s.repo_id = s.repo_id
ORDER BY inner_s.snapshot_at DESC, inner_s.created_at DESC
LIMIT 1
)
"""
params: tuple[Any, ...] = ()
if repo_slug:
query += " AND r.slug = ?"
params = (repo_slug,)
with self.connect() as connection:
return connection.execute(query, params).fetchall()
@staticmethod
def _repository_dict(row: sqlite3.Row) -> dict[str, Any]:
return {
"id": row["id"],
"slug": row["slug"],
"checkout_path": row["checkout_path"],
"active": bool(row["active"]),
"last_attempt_at": row["last_attempt_at"],
"last_success_at": row["last_success_at"],
"last_status": row["last_status"],
"last_source": row["last_source"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
}
@staticmethod
def _snapshot_dict(row: sqlite3.Row) -> dict[str, Any]:
result = {
"id": row["id"],
"repo_id": row["repo_id"],
"snapshot_at": row["snapshot_at"],
"source": row["source"],
"status": row["status"],
"entry_count": row["entry_count"],
"source_revision": row["source_revision"],
"sources": json.loads(row["sources_json"]),
"errors": json.loads(row["errors_json"]),
"legacy_id": row["legacy_id"],
"created_at": row["created_at"],
}
if "repo_slug" in row.keys():
result["repo_slug"] = row["repo_slug"]
if "snapshot_at" in row.keys():
result["snapshot_at"] = row["snapshot_at"]
return result
@staticmethod
def _entry_dict(row: sqlite3.Row) -> dict[str, Any]:
result = {
"id": row["id"],
"repo_id": row["repo_id"],
"snapshot_id": row["snapshot_id"],
"package_name": row["package_name"],
"package_version": row["package_version"],
"ecosystem": row["ecosystem"],
"license_spdx": row["license_spdx"],
"is_direct": bool(row["is_direct"]),
"is_dev": bool(row["is_dev"]),
"source_path": row["source_path"],
"created_at": row["created_at"],
}
if "repo_slug" in row.keys():
result["repo_slug"] = row["repo_slug"]
return result
@staticmethod
def _catch_up_repository(repo: dict[str, Any], now: datetime) -> dict[str, Any]:
attempt = repo["last_attempt_at"]
age_days = 9999 if attempt is None else max(0, (now - parse_datetime(attempt)).days)
checkout_path = repo["checkout_path"]
return {
"repo_slug": repo["slug"],
"last_sbom_at": attempt,
"last_attempt_at": attempt,
"last_success_at": repo["last_success_at"],
"sbom_age_days": age_days,
"has_sbom": repo["last_success_at"] is not None,
"checkout_available": bool(checkout_path and Path(checkout_path).is_dir()),
"last_status": repo["last_status"],
}