content-explorer/ui/lib/catalog.ts

import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
import { uniqueOriginNotices, type SourceCredit } from "./attribution";
import { columnDefs, HOME_COLUMNS, mapBrowserRow, type BrowserRow, type ColumnDef } from "./browser-row";
import { memberLinks, partitionIncident, readingPathCrumbs, type GraphEdge, type RelationLabels } from "./graph";

type KindRecord = { id: string; label: string; plural: string };

type EntityRecord = {
  id: string;
  kindId: string;
  title: string;
  tldr: string;
  summary: string;
  tags?: string[];
  sourceRefs?: string[];
  claimRefs?: string[];
  properties?: unknown;
};

type SourceRecord = {
  id: string;
  title: string;
  license: string;
  uri?: string;
  attribution?: string;
  licenseUrl?: string;
  retrievedAt?: string;
  originSha?: string;
  modificationNotice?: string;
};

type ClaimRecord = { id: string; subjectId: string; key: string; value?: string; status: string };

type RelationRecord = { id: string; predicate: string; from: string; to: string };

type SnapshotFile = {
  dataspace: { id: string; title: string; description: string };
  kinds: KindRecord[];
  sources?: SourceRecord[];
  entities: EntityRecord[];
  relations?: RelationRecord[];
  claims?: ClaimRecord[];
};

type KindViewConfig = { columns?: string[]; facets?: string[] };

type ManifestFile = {
  relationLabels?: Record<string, { forward?: string; inverse?: string }>;
  kindViews?: Record<string, KindViewConfig>;
};

export type DataspaceNav = {
  id: string;
  title: string;
  kinds: { id: string; plural: string; href: string }[];
};

export type DataspaceView = {
  id: string;
  title: string;
  description: string;
  kinds: { id: string; label: string; plural: string; href: string }[];
  rows: BrowserRow[];
  columns: ColumnDef[];
};

export type KindView = {
  dataspaceId: string;
  dataspaceTitle: string;
  kindId: string;
  label: string;
  plural: string;
  rows: BrowserRow[];
  columns: ColumnDef[];
};

export type RelationView = {
  wording: string;
  direction: "out" | "in";
  otherTitle: string;
  otherHref: string | null;
};

export type EntityLink = {
  id: string;
  title: string;
  href: string | null;
  glossary: string | null;
  preview: { title: string; summary: string } | null;
};

export type RelationItem = {
  id: string;
  predicate: string;
  direction: "out" | "in";
  wording: string;
  other: EntityLink;
};

export type EntityView = {
  id: string;
  title: string;
  tldr: string;
  summary: string;
  tags: string[];
  dataspaceId: string;
  dataspaceTitle: string;
  kindId: string;
  kindLabel: string;
  kindHref: string;
  graphHref: string;
  glossary: string | null;
  sources: {
    title: string;
    license: string;
    uri?: string;
    attribution?: string;
    licenseUrl?: string;
    retrievedAt?: string;
    originSha?: string;
    modificationNotice?: string;
  }[];
  claims: { key: string; status: string; value: string | null }[];
  relations: RelationView[];
  outgoing: RelationItem[];
  incoming: RelationItem[];
  relationRemainder: number;
  collection: { title: string; members: EntityLink[] } | null;
  related: EntityLink[];
  readingPath: EntityLink[];
};

export type GraphNodeRecord = {
  id: string;
  title: string;
  summary: string;
  href: string;
};

export type GraphEdgeRecord = GraphEdge & {
  forward: string;
  inverse: string;
};

export type GraphDataset = {
  dataspaceId: string;
  dataspaceTitle: string;
  nodes: GraphNodeRecord[];
  edges: GraphEdgeRecord[];
  labels: RelationLabels;
  defaultFocusId: string;
};

type Loaded = {
  snapshot: SnapshotFile;
  labels: Map<string, { forward: string; inverse: string }>;
  views: Record<string, KindViewConfig>;
  byId: Map<string, EntityRecord>;
};

const ROOT = join(process.cwd(), "../examples/dataspaces");

export function dataspaceRoots(): string[] {
  const roots = [ROOT];
  if (process.env.CONTENT_EXPLORER_LICENSED === "1") {
    roots.push(join(process.cwd(), "../tests/onco-reference/local"));
  }
  return roots;
}

let cache: Map<string, Loaded> | null = null;

export function clearCatalogCache(): void {
  cache = null;
}

export function entitySlug(id: string, kindId: string): string {
  const marker = `:${kindId}:`;
  const at = id.lastIndexOf(marker);
  if (at >= 0) return id.slice(at + marker.length);
  const tail = id.split(":").pop();
  return tail && tail.length > 0 ? tail : id;
}

export function kindHref(dataspaceId: string, kindId: string): string {
  return `/${dataspaceId}/${kindId}/`;
}

export function entityHref(dataspaceId: string, kindId: string, id: string): string {
  return `/${dataspaceId}/${kindId}/${encodeURIComponent(entitySlug(id, kindId))}/`;
}

function loadAll(): Map<string, Loaded> {
  if (cache) return cache;
  const loaded = new Map<string, Loaded>();
  for (const root of dataspaceRoots()) {
    let names: string[] = [];
    try {
      names = readdirSync(/*turbopackIgnore: true*/ root);
    } catch {
      continue;
    }
    for (const name of names.sort()) {
    const snapshotPath = join(root, name, "snapshot.json");
    let snapshot: SnapshotFile;
    try {
      snapshot = JSON.parse(readFileSync(/*turbopackIgnore: true*/ snapshotPath, "utf8")) as SnapshotFile;
    } catch {
      continue;
    }
    if (!snapshot.dataspace?.id || !Array.isArray(snapshot.kinds) || !Array.isArray(snapshot.entities)) continue;
    let manifest: ManifestFile = {};
    try {
      manifest = JSON.parse(readFileSync(/*turbopackIgnore: true*/ join(root, name, "manifest.json"), "utf8")) as ManifestFile;
    } catch {
      manifest = {};
    }
    const labels = new Map<string, { forward: string; inverse: string }>();
    for (const [predicate, wording] of Object.entries(manifest.relationLabels ?? {})) {
      labels.set(predicate, {
        forward: wording.forward ?? predicate,
        inverse: wording.inverse ?? predicate,
      });
    }
    loaded.set(snapshot.dataspace.id, {
      snapshot,
      labels,
      views: manifest.kindViews ?? {},
      byId: new Map(snapshot.entities.map((entity) => [entity.id, entity])),
    });
    }
  }
  cache = loaded;
  return loaded;
}

function kindOf(loaded: Loaded, kindId: string): KindRecord | null {
  return loaded.snapshot.kinds.find((kind) => kind.id === kindId) ?? null;
}

function rowFor(loaded: Loaded, entity: EntityRecord): BrowserRow {
  const kind = kindOf(loaded, entity.kindId);
  const dataspaceId = loaded.snapshot.dataspace.id;
  return mapBrowserRow(
    {
      id: entity.id,
      title: entity.title,
      tldr: entity.tldr,
      tags: entity.tags,
      kindLabel: kind?.label ?? entity.kindId,
      route: entityHref(dataspaceId, entity.kindId, entity.id),
      kindHref: kindHref(dataspaceId, entity.kindId),
    },
    loaded.views[entity.kindId]?.facets ?? [],
  );
}

export function listNavigation(): DataspaceNav[] {
  return [...loadAll().values()].map((loaded) => ({
    id: loaded.snapshot.dataspace.id,
    title: loaded.snapshot.dataspace.title,
    kinds: loaded.snapshot.kinds.map((kind) => ({
      id: kind.id,
      plural: kind.plural,
      href: kindHref(loaded.snapshot.dataspace.id, kind.id),
    })),
  }));
}

function sourceCredit(source: SourceRecord): SourceCredit {
  return {
    title: source.title,
    license: source.license,
    uri: source.uri,
    attribution: source.attribution,
    licenseUrl: source.licenseUrl,
    retrievedAt: source.retrievedAt,
    originSha: source.originSha,
    modificationNotice: source.modificationNotice,
  };
}

export function originNotices(dataspaceId: string): SourceCredit[] {
  const loaded = loadAll().get(dataspaceId);
  if (!loaded) return [];
  return uniqueOriginNotices((loaded.snapshot.sources ?? []).map(sourceCredit));
}

export function findDataspace(id: string): DataspaceView | null {
  const loaded = loadAll().get(id);
  if (!loaded) return null;
  return {
    id: loaded.snapshot.dataspace.id,
    title: loaded.snapshot.dataspace.title,
    description: loaded.snapshot.dataspace.description,
    kinds: loaded.snapshot.kinds.map((kind) => ({
      id: kind.id,
      label: kind.label,
      plural: kind.plural,
      href: kindHref(loaded.snapshot.dataspace.id, kind.id),
    })),
    rows: loaded.snapshot.entities.map((entity) => rowFor(loaded, entity)),
    columns: columnDefs(HOME_COLUMNS),
  };
}

export function findKind(dataspaceId: string, kindId: string): KindView | null {
  const loaded = loadAll().get(dataspaceId);
  if (!loaded) return null;
  const kind = kindOf(loaded, kindId);
  if (!kind) return null;
  return {
    dataspaceId,
    dataspaceTitle: loaded.snapshot.dataspace.title,
    kindId: kind.id,
    label: kind.label,
    plural: kind.plural,
    rows: loaded.snapshot.entities.filter((entity) => entity.kindId === kind.id).map((entity) => rowFor(loaded, entity)),
    columns: columnDefs(loaded.views[kind.id]?.columns ?? ["name", "tldr", "tags"]),
  };
}

function hrefForEntity(loaded: Loaded, id: string): string | null {
  const entity = loaded.byId.get(id);
  if (!entity) return null;
  return entityHref(loaded.snapshot.dataspace.id, entity.kindId, entity.id);
}

function asRecord(value: unknown): Record<string, unknown> | null {
  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
  return value as Record<string, unknown>;
}

function asText(value: unknown): string | null {
  return typeof value === "string" && value.trim() ? value : null;
}

function asIdList(value: unknown): string[] {
  if (!Array.isArray(value)) return [];
  return value.filter((item): item is string => typeof item === "string" && item.length > 0);
}

function linkFor(loaded: Loaded, id: string): EntityLink {
  const entity = loaded.byId.get(id);
  if (!entity) return { id, title: id, href: null, glossary: null, preview: null };
  const props = asRecord(entity.properties);
  const preview = asRecord(props?.preview);
  const previewTitle = asText(preview?.title);
  const previewSummary = asText(preview?.summary);
  return {
    id: entity.id,
    title: entity.title,
    href: hrefForEntity(loaded, entity.id),
    glossary: asText(props?.glossary),
    preview: previewTitle && previewSummary ? { title: previewTitle, summary: previewSummary } : null,
  };
}

function labelRecord(loaded: Loaded): RelationLabels {
  const labels: Record<string, { forward: string; inverse: string }> = {};
  for (const [predicate, wording] of loaded.labels) labels[predicate] = wording;
  return labels;
}

export function findEntity(dataspaceId: string, kindId: string, slug: string): EntityView | null {
  const loaded = loadAll().get(dataspaceId);
  if (!loaded) return null;
  const kind = kindOf(loaded, kindId);
  if (!kind) return null;
  const matches = loaded.snapshot.entities.filter(
    (entity) => entity.kindId === kind.id && entitySlug(entity.id, kind.id) === slug,
  );
  if (matches.length !== 1) return null;
  const entity = matches[0];
  const sources = new Map((loaded.snapshot.sources ?? []).map((source) => [source.id, source]));
  const edges = loaded.snapshot.relations ?? [];
  const blocks = partitionIncident(entity.id, edges, labelRecord(loaded));
  const toItem = (relation: (typeof blocks.outgoing)[number]): RelationItem => ({
    id: relation.id,
    predicate: relation.predicate,
    direction: relation.direction,
    wording: relation.wording,
    other: linkFor(loaded, relation.otherId),
  });
  const outgoing = blocks.outgoing.map(toItem);
  const incoming = blocks.incoming.map(toItem);
  const relations: RelationView[] = [...outgoing, ...incoming].map((relation) => ({
    wording: relation.wording,
    direction: relation.direction,
    otherTitle: relation.other.title,
    otherHref: relation.other.href,
  }));
  const props = asRecord(entity.properties);
  const collection = asRecord(props?.collection);
  const collectionTitle = asText(collection?.title);
  const resolve = (id: string) => {
    const link = linkFor(loaded, id);
    return { title: link.title, href: link.href };
  };
  const readingIds = asIdList(props?.readingPath);
  const reading = readingPathCrumbs(readingIds, resolve).map((step) => linkFor(loaded, step.id));
  const related = memberLinks(asIdList(props?.related), resolve).map((step) => linkFor(loaded, step.id));
  const self = linkFor(loaded, entity.id);
  return {
    id: entity.id,
    title: entity.title,
    tldr: entity.tldr,
    summary: entity.summary,
    tags: entity.tags ?? [],
    dataspaceId,
    dataspaceTitle: loaded.snapshot.dataspace.title,
    kindId: kind.id,
    kindLabel: kind.label,
    kindHref: kindHref(dataspaceId, kind.id),
    graphHref: `/${dataspaceId}/graph/?${new URLSearchParams({ focus: entity.id }).toString()}`,
    glossary: self.glossary,
    sources: (entity.sourceRefs ?? []).map((id) => {
      const source = sources.get(id);
      return {
        title: source?.title ?? id,
        license: source?.license ?? "",
        uri: source?.uri,
        attribution: source?.attribution,
        licenseUrl: source?.licenseUrl,
        retrievedAt: source?.retrievedAt,
        originSha: source?.originSha,
        modificationNotice: source?.modificationNotice,
      };
    }),
    claims: (loaded.snapshot.claims ?? [])
      .filter((claim) => claim.subjectId === entity.id)
      .map((claim) => ({ key: claim.key, status: claim.status, value: claim.value ?? null })),
    relations,
    outgoing,
    incoming,
    relationRemainder: blocks.remainder,
    collection: collectionTitle
      ? { title: collectionTitle, members: memberLinks(asIdList(collection?.members), resolve).map((step) => linkFor(loaded, step.id)) }
      : null,
    related,
    readingPath: reading,
  };
}

export function findGraph(dataspaceId: string): GraphDataset | null {
  const loaded = loadAll().get(dataspaceId);
  if (!loaded || loaded.snapshot.entities.length === 0) return null;
  const labels = labelRecord(loaded);
  const nodes = loaded.snapshot.entities.map((entity) => ({
    id: entity.id,
    title: entity.title,
    summary: entity.summary,
    href: entityHref(loaded.snapshot.dataspace.id, entity.kindId, entity.id),
  }));
  const preferred = nodes.find((node) => node.id.endsWith(":feedback"));
  return {
    dataspaceId,
    dataspaceTitle: loaded.snapshot.dataspace.title,
    nodes,
    edges: (loaded.snapshot.relations ?? []).map((relation) => {
      const pair = labels[relation.predicate];
      return {
        id: relation.id,
        predicate: relation.predicate,
        from: relation.from,
        to: relation.to,
        forward: pair?.forward || relation.predicate,
        inverse: pair?.inverse || relation.predicate,
      };
    }),
    labels,
    defaultFocusId: preferred?.id ?? nodes[0].id,
  };
}