coulomb/sbom-nexus
4
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: prepare postgres sbom cutover

Browse source 
Assistant: codex
Assistant-Model: gpt-5.6-sol
Assistant-Session: 01a028f0-a42f-7582-89a8-ebaad7343834
This commit is contained in:
tegwick 2026-08-22 13:14:24 +02:00
parent cf7e3acb78
commit ba535e1f8f
26 changed files with 1573 additions and 411 deletions
Download patch file Download diff file Expand all files Collapse all files

21
src/sbom_nexus/api.py
View file

@ -12,7 +12,9 @@ from pydantic import BaseModel, Field
from sbom_nexus.scanner import VALID_ECOSYSTEMS, scan_repository
from sbom_nexus.storage import Store
DEFAULT_DATABASE_PATH = os.getenv("SBOM_NEXUS_DATABASE_PATH", "sbom-nexus.db")
DEFAULT_DATABASE_TARGET = os.getenv("SBOM_NEXUS_DATABASE_URL") or os.getenv(
"SBOM_NEXUS_DATABASE_PATH", "sbom-nexus.db"
)
DEFAULT_STALE_DAYS = int(os.getenv("SBOM_NEXUS_STALE_DAYS", "30"))
@ -74,19 +76,28 @@ def _validate_entries(entries: list[EntryCreate]) -> list[dict[str, Any]]:
return result
def _auto_create(store: Store) -> bool:
configured = os.getenv("SBOM_NEXUS_AUTO_CREATE")
if configured is not None:
return configured.lower() in {"1", "true", "yes"}
return store.dialect == "sqlite"
def create_app(database_path: str | Path | None = None) -> FastAPI:
application = FastAPI(
title="SBOM Nexus",
version="0.1.0",
description="SBOM capture, history, evaluation, and bounded catch-up service",
)
application.state.store = Store(database_path or DEFAULT_DATABASE_PATH)
application.state.store.init_schema()
application.state.store = Store(database_path or DEFAULT_DATABASE_TARGET)
if _auto_create(application.state.store):
application.state.store.init_schema()
@application.get("/state/health")
def health(request: Request) -> dict[str, str]:
_store(request).list_repositories()
return {"status": "ok", "store": "connected"}
store = _store(request)
store.health()
return {"status": "ok", "store": "connected", "dialect": store.dialect}
@application.put("/repositories/{repo_slug}")
def upsert_repository(

5
src/sbom_nexus/cli.py
View file

@ -11,7 +11,9 @@ from sbom_nexus.scanner import scan_repository
def _serve(args: argparse.Namespace) -> None:
if args.database:
if args.database_url:
os.environ["SBOM_NEXUS_DATABASE_URL"] = args.database_url
elif args.database:
os.environ["SBOM_NEXUS_DATABASE_PATH"] = str(Path(args.database).resolve())
import uvicorn
@ -46,6 +48,7 @@ def build_parser() -> argparse.ArgumentParser:
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8010)
serve.add_argument("--database", help="SQLite database path")
serve.add_argument("--database-url", help="SQLAlchemy database URL (PostgreSQL in production)")
serve.add_argument("--reload", action="store_true")
serve.set_defaults(handler=_serve)

128
src/sbom_nexus/database.py Normal file
View file

@ -0,0 +1,128 @@
"""Portable SQLAlchemy schema shared by the runtime store and Alembic."""
from __future__ import annotations
from pathlib import Path
from sqlalchemy import (
JSON,
Boolean,
Column,
DateTime,
ForeignKey,
Index,
Integer,
MetaData,
String,
Table,
Text,
create_engine,
event,
)
from sqlalchemy.engine import Engine
NAMING_CONVENTION = {
"ix": "ix_%(table_name)s_%(column_0_name)s",
"uq": "uq_%(table_name)s_%(column_0_name)s",
"ck": "ck_%(table_name)s_%(constraint_name)s",
"fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
"pk": "pk_%(table_name)s",
}
metadata = MetaData(naming_convention=NAMING_CONVENTION)
repositories = Table(
"repositories",
metadata,
Column("id", String(36), primary_key=True),
Column("slug", String(200), nullable=False, unique=True),
Column("checkout_path", Text, nullable=True),
Column("active", Boolean, nullable=False),
Column("last_attempt_at", DateTime(timezone=True), nullable=True),
Column("last_success_at", DateTime(timezone=True), nullable=True),
Column("last_status", String(50), nullable=True),
Column("last_source", String(200), nullable=True),
Column("created_at", DateTime(timezone=True), nullable=False),
Column("updated_at", DateTime(timezone=True), nullable=False),
)
snapshots = Table(
"snapshots",
metadata,
Column("id", String(36), primary_key=True),
Column(
"repo_id",
String(36),
ForeignKey("repositories.id", ondelete="RESTRICT"),
nullable=False,
),
Column("snapshot_at", DateTime(timezone=True), nullable=False),
Column("source", String(200), nullable=False),
Column("status", String(50), nullable=False),
Column("entry_count", Integer, nullable=False),
Column("source_revision", String(200), nullable=True),
Column("sources_json", JSON, nullable=False),
Column("errors_json", JSON, nullable=False),
Column("legacy_id", String(100), nullable=True, unique=True),
Column("created_at", DateTime(timezone=True), nullable=False),
)
entries = Table(
"entries",
metadata,
Column("id", String(36), primary_key=True),
Column(
"repo_id",
String(36),
ForeignKey("repositories.id", ondelete="RESTRICT"),
nullable=False,
),
Column(
"snapshot_id",
String(36),
ForeignKey("snapshots.id", ondelete="RESTRICT"),
nullable=False,
),
Column("package_name", String(300), nullable=False),
Column("package_version", String(100), nullable=True),
Column("ecosystem", String(50), nullable=False),
Column("license_spdx", String(100), nullable=True),
Column("is_direct", Boolean, nullable=False),
Column("is_dev", Boolean, nullable=False),
Column("source_path", Text, nullable=True),
Column("created_at", DateTime(timezone=True), nullable=False),
)
Index("ix_snapshots_repo_time", snapshots.c.repo_id, snapshots.c.snapshot_at)
Index("ix_entries_snapshot", entries.c.snapshot_id)
Index("ix_entries_repo", entries.c.repo_id)
Index("ix_entries_license", entries.c.license_spdx)
def database_url(value: str | Path) -> str:
"""Normalize a filesystem path or supported URL into a SQLAlchemy URL."""
text = str(value)
if "://" not in text:
return f"sqlite:///{Path(text).resolve()}"
if text.startswith("postgres://"):
return text.replace("postgres://", "postgresql+psycopg://", 1)
if text.startswith("postgresql://"):
return text.replace("postgresql://", "postgresql+psycopg://", 1)
return text
def create_database_engine(value: str | Path) -> Engine:
url = database_url(value)
options: dict[str, object] = {"pool_pre_ping": True}
if url.startswith("sqlite:"):
options["connect_args"] = {"check_same_thread": False}
engine = create_engine(url, **options)
if engine.dialect.name == "sqlite":
@event.listens_for(engine, "connect")
def _enable_sqlite_foreign_keys(dbapi_connection, _connection_record) -> None:
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys = ON")
cursor.close()
return engine

197
src/sbom_nexus/importer.py Normal file
View file

@ -0,0 +1,197 @@
"""Idempotently import historical State Hub SBOM snapshots into SBOM Nexus."""
from __future__ import annotations
import argparse
import json
import socket
import urllib.error
import urllib.parse
import urllib.request
from collections import Counter
from typing import Any
def request_json(
base_url: str,
path: str,
*,
method: str = "GET",
body: dict[str, Any] | None = None,
) -> Any:
payload = json.dumps(body).encode() if body is not None else None
request = urllib.request.Request(
f"{base_url.rstrip('/')}{path}",
data=payload,
method=method,
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=30) as response:
return json.loads(response.read())
def checkout_path(repo: dict[str, Any]) -> str | None:
host_paths = repo.get("host_paths") or {}
return host_paths.get(socket.gethostname()) or repo.get("local_path")
def normalise_licence_groups(groups: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
"""Compare the report as a set of groups; API ordering is not contractual."""
return {
json.dumps(group.get("license_spdx"), sort_keys=True): {
"license_spdx": group.get("license_spdx"),
"count": group.get("count"),
"repos": sorted(group.get("repos") or []),
"is_copyleft": bool(group.get("is_copyleft")),
}
for group in groups
}
def import_history(source_url: str, target_url: str, *, dry_run: bool) -> dict[str, Any]:
repositories = request_json(source_url, "/repos/")
repo_by_id = {str(repo["id"]): repo for repo in repositories}
snapshots = request_json(source_url, "/sbom/snapshots/")
snapshots.sort(key=lambda item: (item["snapshot_at"], item["id"]))
results = Counter()
by_repo = Counter()
source_entries = 0
expected: dict[str, dict[str, Any]] = {}
for snapshot in snapshots:
repo = repo_by_id.get(str(snapshot["repo_id"]))
if repo is None:
results["missing_repo"] += 1
continue
repo_slug = repo["slug"]
detail = request_json(source_url, f"/sbom/snapshots/{snapshot['id']}")
entries = [
{
"package_name": entry["package_name"],
"package_version": entry.get("package_version"),
"ecosystem": entry["ecosystem"],
"license_spdx": entry.get("license_spdx"),
"is_direct": entry.get("is_direct", True),
"is_dev": entry.get("is_dev", False),
"source_path": entry.get("source_path"),
}
for entry in detail.get("entries", [])
]
source_entries += len(entries)
expected[str(snapshot["id"])] = {
"repo_slug": repo_slug,
"snapshot_at": snapshot["snapshot_at"],
"entry_count": len(entries),
}
if dry_run:
results["would_import"] += 1
by_repo[repo_slug] += 1
continue
request_json(
target_url,
f"/repositories/{urllib.parse.quote(repo_slug, safe='')}",
method="PUT",
body={
"checkout_path": checkout_path(repo),
"active": repo.get("status", "active") == "active",
},
)
imported = request_json(
target_url,
"/sbom/import/",
method="POST",
body={
"repo_slug": repo_slug,
"legacy_id": str(snapshot["id"]),
"snapshot_at": snapshot["snapshot_at"],
"source": f"state-hub:{snapshot.get('source') or 'manual'}",
"entries": entries,
},
)
results["imported" if imported["imported"] else "already_present"] += 1
by_repo[repo_slug] += 1
reconciliation: dict[str, Any] | None = None
licence_reconciliation: dict[str, Any] | None = None
if not dry_run:
target_snapshots = request_json(target_url, "/sbom/snapshots/")
imported = {
str(snapshot["legacy_id"]): snapshot
for snapshot in target_snapshots
if snapshot.get("legacy_id")
}
missing = sorted(set(expected) - set(imported))
mismatched = []
for legacy_id in sorted(set(expected) & set(imported)):
wanted = expected[legacy_id]
actual = imported[legacy_id]
differences = {
field: {"source": wanted[field], "target": actual.get(field)}
for field in ("repo_slug", "snapshot_at", "entry_count")
if wanted[field] != actual.get(field)
}
if differences:
mismatched.append({"legacy_id": legacy_id, "differences": differences})
reconciliation = {
"ok": not missing and not mismatched,
"expected_snapshot_count": len(expected),
"matched_snapshot_count": len(expected) - len(missing) - len(mismatched),
"expected_entry_count": source_entries,
"target_imported_entry_count": sum(
imported[legacy_id]["entry_count"]
for legacy_id in expected
if legacy_id in imported
),
"missing_legacy_ids": missing,
"mismatched_snapshots": mismatched,
}
source_licences = request_json(source_url, "/sbom/report/licences/")
target_licences = request_json(target_url, "/sbom/report/licences/")
source_groups = normalise_licence_groups(source_licences.get("groups", []))
target_groups = normalise_licence_groups(target_licences.get("groups", []))
licence_reconciliation = {
"ok": source_groups == target_groups
and source_licences.get("copyleft_direct_count")
== target_licences.get("copyleft_direct_count"),
"groups_match": source_groups == target_groups,
"source_copyleft_direct_count": source_licences.get("copyleft_direct_count"),
"target_copyleft_direct_count": target_licences.get("copyleft_direct_count"),
}
ok = results["missing_repo"] == 0
if reconciliation is not None:
ok = ok and reconciliation["ok"]
if licence_reconciliation is not None:
ok = ok and licence_reconciliation["ok"]
return {
"ok": ok,
"dry_run": dry_run,
"source_repo_count": len(by_repo),
"source_snapshot_count": len(snapshots),
"source_entry_count": source_entries,
"results": dict(results),
"snapshots_by_repo": dict(sorted(by_repo.items())),
"snapshot_reconciliation": reconciliation,
"licence_reconciliation": licence_reconciliation,
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-url", default="http://127.0.0.1:8000")
parser.add_argument("--target-url", default="http://127.0.0.1:8010")
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
try:
result = import_history(args.source_url, args.target_url, dry_run=args.dry_run)
except (urllib.error.URLError, json.JSONDecodeError) as exc:
raise SystemExit(f"Import failed: {exc}") from exc
print(json.dumps(result, indent=2, sort_keys=True))
if not result["ok"]:
raise SystemExit(1)
if __name__ == "__main__":
main()

468
src/sbom_nexus/storage.py
View file

@ -1,19 +1,23 @@
"""SQLite persistence for the extraction slice.
The store keeps the product behavior independent from FastAPI. PostgreSQL and
managed migrations are an explicit production-cutover gate in SBOM-WP-0001-T05.
"""
"""Transactional persistence for SQLite development and PostgreSQL production."""
from __future__ import annotations
import json
import sqlite3
import uuid
from collections import defaultdict
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from sqlalchemy import func, insert, select, update
from sqlalchemy.engine import RowMapping
from sbom_nexus.database import (
create_database_engine,
metadata,
repositories,
snapshots,
)
from sbom_nexus.database import entries as entry_table
from sbom_nexus.scanner import is_copyleft
SUCCESS_STATUSES = frozenset({"ingested", "imported"})
@ -23,84 +27,37 @@ def utc_now() -> datetime:
return datetime.now(UTC)
def parse_datetime(value: datetime | str) -> datetime:
parsed = (
datetime.fromisoformat(value.replace("Z", "+00:00"))
if isinstance(value, str)
else value
)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def datetime_text(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, str):
parsed = parse_datetime(value)
else:
parsed = value
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z")
def parse_datetime(value: str) -> datetime:
return datetime.fromisoformat(value.replace("Z", "+00:00"))
return parse_datetime(value).isoformat().replace("+00:00", "Z")
class Store:
def __init__(self, database_path: str | Path) -> None:
self.database_path = str(database_path)
def __init__(self, database_target: str | Path) -> None:
self.engine = create_database_engine(database_target)
def connect(self) -> sqlite3.Connection:
connection = sqlite3.connect(self.database_path, timeout=30)
connection.row_factory = sqlite3.Row
connection.execute("PRAGMA foreign_keys = ON")
return connection
@property
def dialect(self) -> str:
return self.engine.dialect.name
def init_schema(self) -> None:
with self.connect() as connection:
connection.executescript(
"""
CREATE TABLE IF NOT EXISTS repositories (
id TEXT PRIMARY KEY,
slug TEXT NOT NULL UNIQUE,
checkout_path TEXT,
active INTEGER NOT NULL DEFAULT 1,
last_attempt_at TEXT,
last_success_at TEXT,
last_status TEXT,
last_source TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
metadata.create_all(self.engine)
CREATE TABLE IF NOT EXISTS snapshots (
id TEXT PRIMARY KEY,
repo_id TEXT NOT NULL REFERENCES repositories(id) ON DELETE RESTRICT,
snapshot_at TEXT NOT NULL,
source TEXT NOT NULL,
status TEXT NOT NULL,
entry_count INTEGER NOT NULL,
source_revision TEXT,
sources_json TEXT NOT NULL DEFAULT '[]',
errors_json TEXT NOT NULL DEFAULT '[]',
legacy_id TEXT UNIQUE,
created_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS entries (
id TEXT PRIMARY KEY,
repo_id TEXT NOT NULL REFERENCES repositories(id) ON DELETE RESTRICT,
snapshot_id TEXT NOT NULL REFERENCES snapshots(id) ON DELETE RESTRICT,
package_name TEXT NOT NULL,
package_version TEXT,
ecosystem TEXT NOT NULL,
license_spdx TEXT,
is_direct INTEGER NOT NULL,
is_dev INTEGER NOT NULL,
source_path TEXT,
created_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS ix_snapshots_repo_time
ON snapshots(repo_id, snapshot_at DESC);
CREATE INDEX IF NOT EXISTS ix_entries_snapshot ON entries(snapshot_id);
CREATE INDEX IF NOT EXISTS ix_entries_repo ON entries(repo_id);
CREATE INDEX IF NOT EXISTS ix_entries_license ON entries(license_spdx);
"""
)
def health(self) -> None:
with self.engine.connect() as connection:
connection.execute(select(func.count()).select_from(repositories))
def upsert_repository(
self,
@ -111,59 +68,57 @@ class Store:
last_attempt_at: datetime | str | None = None,
last_success_at: datetime | str | None = None,
) -> dict[str, Any]:
now = datetime_text(utc_now())
attempt = datetime_text(last_attempt_at)
success = datetime_text(last_success_at)
with self.connect() as connection:
now = utc_now()
attempt = parse_datetime(last_attempt_at) if last_attempt_at else None
success = parse_datetime(last_success_at) if last_success_at else None
with self.engine.begin() as connection:
existing = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
select(repositories).where(repositories.c.slug == slug)
).mappings().one_or_none()
if existing:
connection.execute(
"""
UPDATE repositories
SET checkout_path = ?, active = ?,
last_attempt_at = COALESCE(?, last_attempt_at),
last_success_at = COALESCE(?, last_success_at),
updated_at = ?
WHERE slug = ?
""",
(checkout_path, int(active), attempt, success, now, slug),
update(repositories)
.where(repositories.c.slug == slug)
.values(
checkout_path=checkout_path,
active=active,
last_attempt_at=attempt or existing["last_attempt_at"],
last_success_at=success or existing["last_success_at"],
updated_at=now,
)
)
else:
connection.execute(
"""
INSERT INTO repositories (
id, slug, checkout_path, active, last_attempt_at,
last_success_at, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
str(uuid.uuid4()),
slug,
checkout_path,
int(active),
attempt,
success,
now,
now,
),
insert(repositories).values(
id=str(uuid.uuid4()),
slug=slug,
checkout_path=checkout_path,
active=active,
last_attempt_at=attempt,
last_success_at=success,
last_status=None,
last_source=None,
created_at=now,
updated_at=now,
)
)
row = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
select(repositories).where(repositories.c.slug == slug)
).mappings().one()
return self._repository_dict(row)
def get_repository(self, slug: str) -> dict[str, Any] | None:
with self.connect() as connection:
with self.engine.connect() as connection:
row = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (slug,)
).fetchone()
select(repositories).where(repositories.c.slug == slug)
).mappings().one_or_none()
return self._repository_dict(row) if row else None
def list_repositories(self) -> list[dict[str, Any]]:
with self.connect() as connection:
rows = connection.execute("SELECT * FROM repositories ORDER BY slug").fetchall()
with self.engine.connect() as connection:
rows = connection.execute(
select(repositories).order_by(repositories.c.slug)
).mappings().all()
return [self._repository_dict(row) for row in rows]
def record_snapshot(
@ -180,122 +135,110 @@ class Store:
snapshot_id: str | None = None,
legacy_id: str | None = None,
) -> tuple[dict[str, Any], bool]:
timestamp = datetime_text(snapshot_at or utc_now())
created_at = datetime_text(utc_now())
with self.connect() as connection:
timestamp = parse_datetime(snapshot_at or utc_now())
created_at = utc_now()
with self.engine.begin() as connection:
repo = connection.execute(
"SELECT * FROM repositories WHERE slug = ?", (repo_slug,)
).fetchone()
select(repositories).where(repositories.c.slug == repo_slug)
).mappings().one_or_none()
if repo is None:
raise KeyError(repo_slug)
if legacy_id:
existing = connection.execute(
"SELECT * FROM snapshots WHERE legacy_id = ?", (legacy_id,)
).fetchone()
select(snapshots).where(snapshots.c.legacy_id == legacy_id)
).mappings().one_or_none()
if existing:
return self._snapshot_dict(existing), False
new_snapshot_id = snapshot_id or str(uuid.uuid4())
connection.execute(
"""
INSERT INTO snapshots (
id, repo_id, snapshot_at, source, status, entry_count,
source_revision, sources_json, errors_json, legacy_id, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
new_snapshot_id,
repo["id"],
timestamp,
source,
status,
len(entries),
source_revision,
json.dumps(sources or [], sort_keys=True),
json.dumps(errors or [], sort_keys=True),
legacy_id,
created_at,
),
insert(snapshots).values(
id=new_snapshot_id,
repo_id=repo["id"],
snapshot_at=timestamp,
source=source,
status=status,
entry_count=len(entries),
source_revision=source_revision,
sources_json=sources or [],
errors_json=errors or [],
legacy_id=legacy_id,
created_at=created_at,
)
)
for entry in entries:
if entries:
connection.execute(
"""
INSERT INTO entries (
id, repo_id, snapshot_id, package_name, package_version,
ecosystem, license_spdx, is_direct, is_dev, source_path,
created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
str(uuid.uuid4()),
repo["id"],
new_snapshot_id,
entry["package_name"],
entry.get("package_version"),
entry["ecosystem"],
entry.get("license_spdx"),
int(bool(entry.get("is_direct", True))),
int(bool(entry.get("is_dev", False))),
entry.get("source_path"),
created_at,
),
insert(entry_table),
[
{
"id": str(uuid.uuid4()),
"repo_id": repo["id"],
"snapshot_id": new_snapshot_id,
"package_name": entry["package_name"],
"package_version": entry.get("package_version"),
"ecosystem": entry["ecosystem"],
"license_spdx": entry.get("license_spdx"),
"is_direct": bool(entry.get("is_direct", True)),
"is_dev": bool(entry.get("is_dev", False)),
"source_path": entry.get("source_path"),
"created_at": created_at,
}
for entry in entries
],
)
previous_attempt = repo["last_attempt_at"]
is_latest = previous_attempt is None or parse_datetime(timestamp) >= parse_datetime(
previous_attempt
)
is_latest = previous_attempt is None or timestamp >= parse_datetime(previous_attempt)
if is_latest:
last_success = repo["last_success_at"]
if status in SUCCESS_STATUSES:
last_success = timestamp
connection.execute(
"""
UPDATE repositories
SET last_attempt_at = ?, last_success_at = ?, last_status = ?,
last_source = ?, updated_at = ?
WHERE id = ?
""",
(timestamp, last_success, status, source, created_at, repo["id"]),
update(repositories)
.where(repositories.c.id == repo["id"])
.values(
last_attempt_at=timestamp,
last_success_at=last_success,
last_status=status,
last_source=source,
updated_at=created_at,
)
)
row = connection.execute(
"SELECT * FROM snapshots WHERE id = ?", (new_snapshot_id,)
).fetchone()
select(snapshots).where(snapshots.c.id == new_snapshot_id)
).mappings().one()
return self._snapshot_dict(row), True
def list_snapshots(self, repo_slug: str | None = None) -> list[dict[str, Any]]:
query = """
SELECT s.*, r.slug AS repo_slug
FROM snapshots s JOIN repositories r ON r.id = s.repo_id
"""
params: tuple[Any, ...] = ()
statement = (
select(snapshots, repositories.c.slug.label("repo_slug"))
.join(repositories, repositories.c.id == snapshots.c.repo_id)
.order_by(snapshots.c.snapshot_at.desc(), snapshots.c.created_at.desc())
)
if repo_slug:
query += " WHERE r.slug = ?"
params = (repo_slug,)
query += " ORDER BY s.snapshot_at DESC, s.created_at DESC"
with self.connect() as connection:
rows = connection.execute(query, params).fetchall()
statement = statement.where(repositories.c.slug == repo_slug)
with self.engine.connect() as connection:
rows = connection.execute(statement).mappings().all()
return [self._snapshot_dict(row) for row in rows]
def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
with self.connect() as connection:
row = connection.execute(
"""
SELECT s.*, r.slug AS repo_slug
FROM snapshots s JOIN repositories r ON r.id = s.repo_id
WHERE s.id = ?
""",
(snapshot_id,),
).fetchone()
statement = (
select(snapshots, repositories.c.slug.label("repo_slug"))
.join(repositories, repositories.c.id == snapshots.c.repo_id)
.where(snapshots.c.id == snapshot_id)
)
with self.engine.connect() as connection:
row = connection.execute(statement).mappings().one_or_none()
if row is None:
return None
entries = connection.execute(
"SELECT * FROM entries WHERE snapshot_id = ? ORDER BY package_name",
(snapshot_id,),
).fetchall()
entry_rows = connection.execute(
select(entry_table)
.where(entry_table.c.snapshot_id == snapshot_id)
.order_by(entry_table.c.package_name)
).mappings().all()
result = self._snapshot_dict(row)
result["entries"] = []
for entry in entries:
for entry in entry_rows:
rendered = self._entry_dict(entry)
rendered["snapshot_at"] = result["snapshot_at"]
result["entries"].append(rendered)
@ -310,57 +253,57 @@ class Store:
is_direct: bool | None = None,
is_dev: bool | None = None,
) -> list[dict[str, Any]]:
latest = self._latest_snapshot_rows(repo_slug)
if not latest:
return []
snapshot_ids = [row["id"] for row in latest]
placeholders = ",".join("?" for _ in snapshot_ids)
query = f"""
SELECT e.*, r.slug AS repo_slug, s.snapshot_at AS snapshot_at
FROM entries e
JOIN repositories r ON r.id = e.repo_id
JOIN snapshots s ON s.id = e.snapshot_id
WHERE e.snapshot_id IN ({placeholders})
"""
params: list[Any] = list(snapshot_ids)
for field, value in (
("ecosystem", ecosystem),
("license_spdx", license_spdx),
("is_direct", None if is_direct is None else int(is_direct)),
("is_dev", None if is_dev is None else int(is_dev)),
ranked = self._ranked_snapshots()
statement = (
select(
entry_table,
repositories.c.slug.label("repo_slug"),
snapshots.c.snapshot_at.label("entry_snapshot_at"),
)
.join(repositories, repositories.c.id == entry_table.c.repo_id)
.join(snapshots, snapshots.c.id == entry_table.c.snapshot_id)
.join(ranked, ranked.c.id == entry_table.c.snapshot_id)
.where(ranked.c.snapshot_rank == 1)
)
if repo_slug:
statement = statement.where(repositories.c.slug == repo_slug)
for column, value in (
(entry_table.c.ecosystem, ecosystem),
(entry_table.c.license_spdx, license_spdx),
(entry_table.c.is_direct, is_direct),
(entry_table.c.is_dev, is_dev),
):
if value is not None:
query += f" AND e.{field} = ?"
params.append(value)
query += " ORDER BY e.package_name, r.slug"
with self.connect() as connection:
rows = connection.execute(query, params).fetchall()
statement = statement.where(column == value)
statement = statement.order_by(entry_table.c.package_name, repositories.c.slug)
with self.engine.connect() as connection:
rows = connection.execute(statement).mappings().all()
return [self._entry_dict(row) for row in rows]
def repository_view(self, repo_slug: str) -> dict[str, Any] | None:
repo = self.get_repository(repo_slug)
if repo is None:
return None
entries = self.latest_entries(repo_slug=repo_slug)
snapshots = self.list_snapshots(repo_slug)
current_entries = self.latest_entries(repo_slug=repo_slug)
history = self.list_snapshots(repo_slug)
return {
"repo_slug": repo_slug,
"last_sbom_at": repo["last_attempt_at"],
"last_attempt_at": repo["last_attempt_at"],
"last_success_at": repo["last_success_at"],
"last_status": repo["last_status"],
"entry_count": len(entries),
"snapshot_id": snapshots[0]["id"] if snapshots else None,
"entries": entries,
"entry_count": len(current_entries),
"snapshot_id": history[0]["id"] if history else None,
"entries": current_entries,
}
def licence_report(self) -> dict[str, Any]:
entries = self.latest_entries()
current_entries = self.latest_entries()
groups: dict[str | None, dict[str, Any]] = defaultdict(
lambda: {"count": 0, "repos": set()}
)
risks: list[dict[str, Any]] = []
for entry in entries:
for entry in current_entries:
license_id = entry.get("license_spdx")
groups[license_id]["count"] += 1
groups[license_id]["repos"].add(entry["repo_slug"])
@ -374,7 +317,9 @@ class Store:
"source_path": entry.get("source_path"),
}
)
sorted_groups = sorted(groups.items(), key=lambda item: (-item[1]["count"], item[0] or ""))
sorted_groups = sorted(
groups.items(), key=lambda item: (-item[1]["count"], item[0] or "")
)
return {
"groups": [
{
@ -398,13 +343,13 @@ class Store:
now: datetime | None = None,
) -> dict[str, Any]:
effective_limit = max(1, min(25, int(limit)))
current = now or utc_now()
current = parse_datetime(now or utc_now())
cutoff = current - timedelta(days=stale_days)
repositories = [repo for repo in self.list_repositories() if repo["active"]]
never = [repo for repo in repositories if repo["last_attempt_at"] is None]
active_repositories = [repo for repo in self.list_repositories() if repo["active"]]
never = [repo for repo in active_repositories if repo["last_attempt_at"] is None]
stale = [
repo
for repo in repositories
for repo in active_repositories
if repo["last_attempt_at"] is None
or parse_datetime(repo["last_attempt_at"]) < cutoff
]
@ -421,69 +366,60 @@ class Store:
"selected_count": len(selected),
"stale_count": len(stale),
"never_count": len(never),
"total_count": len(repositories),
"total_count": len(active_repositories),
"limit": effective_limit,
"stale_after_days": stale_days,
"evaluated_at": datetime_text(current),
}
def _latest_snapshot_rows(self, repo_slug: str | None) -> list[sqlite3.Row]:
query = """
SELECT s.*, r.slug AS repo_slug
FROM snapshots s
JOIN repositories r ON r.id = s.repo_id
WHERE s.id = (
SELECT inner_s.id FROM snapshots inner_s
WHERE inner_s.repo_id = s.repo_id
ORDER BY inner_s.snapshot_at DESC, inner_s.created_at DESC
LIMIT 1
@staticmethod
def _ranked_snapshots():
return select(
snapshots.c.id,
func.row_number()
.over(
partition_by=snapshots.c.repo_id,
order_by=(snapshots.c.snapshot_at.desc(), snapshots.c.created_at.desc()),
)
"""
params: tuple[Any, ...] = ()
if repo_slug:
query += " AND r.slug = ?"
params = (repo_slug,)
with self.connect() as connection:
return connection.execute(query, params).fetchall()
.label("snapshot_rank"),
).subquery("ranked_snapshots")
@staticmethod
def _repository_dict(row: sqlite3.Row) -> dict[str, Any]:
def _repository_dict(row: RowMapping) -> dict[str, Any]:
return {
"id": row["id"],
"slug": row["slug"],
"checkout_path": row["checkout_path"],
"active": bool(row["active"]),
"last_attempt_at": row["last_attempt_at"],
"last_success_at": row["last_success_at"],
"last_attempt_at": datetime_text(row["last_attempt_at"]),
"last_success_at": datetime_text(row["last_success_at"]),
"last_status": row["last_status"],
"last_source": row["last_source"],
"created_at": row["created_at"],
"updated_at": row["updated_at"],
"created_at": datetime_text(row["created_at"]),
"updated_at": datetime_text(row["updated_at"]),
}
@staticmethod
def _snapshot_dict(row: sqlite3.Row) -> dict[str, Any]:
def _snapshot_dict(row: RowMapping) -> dict[str, Any]:
result = {
"id": row["id"],
"repo_id": row["repo_id"],
"snapshot_at": row["snapshot_at"],
"snapshot_at": datetime_text(row["snapshot_at"]),
"source": row["source"],
"status": row["status"],
"entry_count": row["entry_count"],
"source_revision": row["source_revision"],
"sources": json.loads(row["sources_json"]),
"errors": json.loads(row["errors_json"]),
"sources": row["sources_json"] or [],
"errors": row["errors_json"] or [],
"legacy_id": row["legacy_id"],
"created_at": row["created_at"],
"created_at": datetime_text(row["created_at"]),
}
if "repo_slug" in row.keys():
if "repo_slug" in row:
result["repo_slug"] = row["repo_slug"]
if "snapshot_at" in row.keys():
result["snapshot_at"] = row["snapshot_at"]
return result
@staticmethod
def _entry_dict(row: sqlite3.Row) -> dict[str, Any]:
def _entry_dict(row: RowMapping) -> dict[str, Any]:
result = {
"id": row["id"],
"repo_id": row["repo_id"],
@ -495,10 +431,12 @@ class Store:
"is_direct": bool(row["is_direct"]),
"is_dev": bool(row["is_dev"]),
"source_path": row["source_path"],
"created_at": row["created_at"],
"created_at": datetime_text(row["created_at"]),
}
if "repo_slug" in row.keys():
if "repo_slug" in row:
result["repo_slug"] = row["repo_slug"]
if "entry_snapshot_at" in row:
result["snapshot_at"] = datetime_text(row["entry_snapshot_at"])
return result
@staticmethod