citation-evidence/src/source/pdf/extract.ts

/**
 * PDF text extraction → canonical text + PageMap + OffsetMap.
 *
 * Implements `wiki/ArchitectureOverview.md` §3.4 ("extract canonical text /
 * build format-specific maps") for the `pdf-text` representation
 * (`wiki/SharedContracts.md` §1, §3) and §6 (canonical normalization).
 *
 * Runtime independence: the PDF.js worker must be configured by the host
 * application (`GlobalWorkerOptions.workerSrc`) before this module is
 * called. In Vite/browser code the worker is bundled via the viewer; in
 * Node tests the test setup file points it at
 * `pdfjs-dist/legacy/build/pdf.worker.mjs`. No worker setup happens here
 * so the same module loads cleanly in both runtimes.
 *
 * Page boundary semantics: canonical text concatenates per-page normalized
 * text with a single "\n\n" paragraph separator. The separator is treated
 * as belonging to the *preceding* page in `OffsetMap`, so the map covers
 * `[0, canonicalText.length)` with no gaps. The last page has no trailing
 * separator. This means `pageLength = globalEnd - globalStart` for
 * every page; for non-last pages it equals (normalized page text length +
 * 2). See `PageOffsetRange` in `@shared/document.ts`.
 */

import { getDocument } from "pdfjs-dist";
import type { PDFPageProxy } from "pdfjs-dist";
import type {
  OffsetMap,
  PageInfo,
  PageMap,
  PageOffsetRange,
} from "@shared/document";
import { normalize } from "@shared/text/normalize";

const PAGE_SEPARATOR = "\n\n";

export interface PdfExtractionResult {
  readonly canonicalText: string;
  readonly pageMap: PageMap;
  readonly offsetMap: OffsetMap;
  readonly pageCount: number;
}

export async function extractPdf(bytes: Uint8Array): Promise<PdfExtractionResult> {
  // PDF.js mutates the bytes buffer (transfers ownership). Pass a fresh copy
  // so the caller's Uint8Array stays usable for fingerprinting after extract.
  const data = new Uint8Array(bytes);
  const loadingTask = getDocument({ data });
  const doc = await loadingTask.promise;

  try {
    const pageCount = doc.numPages;
    const pageInfos: PageInfo[] = [];
    const pageNormalizedTexts: string[] = [];

    for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
      const page = await doc.getPage(pageNumber);
      try {
        const viewport = page.getViewport({ scale: 1 });
        pageInfos.push({
          page: pageNumber,
          width: viewport.width,
          height: viewport.height,
        });

        const rawText = await extractPageText(page);
        pageNormalizedTexts.push(normalize(rawText).text);
      } finally {
        page.cleanup();
      }
    }

    const { canonicalText, offsetMap } = buildOffsetMap(pageNormalizedTexts);

    return {
      canonicalText,
      pageMap: pageInfos,
      offsetMap,
      pageCount,
    };
  } finally {
    await doc.destroy();
  }
}

async function extractPageText(page: PDFPageProxy): Promise<string> {
  const content = await page.getTextContent();
  // textContent.items are TextItem | TextMarkedContent. We want only the
  // TextItem strings (those have a `str` field); marked-content entries are
  // structural anchors and have no visible text.
  const parts: string[] = [];
  for (const item of content.items) {
    if ("str" in item) {
      parts.push(item.str);
      if (item.hasEOL) parts.push("\n");
    }
  }
  return parts.join("");
}

function buildOffsetMap(pageTexts: readonly string[]): {
  canonicalText: string;
  offsetMap: OffsetMap;
} {
  const ranges: PageOffsetRange[] = [];
  let offset = 0;
  for (let i = 0; i < pageTexts.length; i++) {
    const text = pageTexts[i]!;
    const isLast = i === pageTexts.length - 1;
    const segmentLength = text.length + (isLast ? 0 : PAGE_SEPARATOR.length);
    const globalStart = offset;
    const globalEnd = offset + segmentLength;
    ranges.push({
      page: i + 1,
      globalStart,
      globalEnd,
      pageLength: segmentLength,
    });
    offset = globalEnd;
  }
  const canonicalText = pageTexts.join(PAGE_SEPARATOR);
  return { canonicalText, offsetMap: ranges };
}