/** * PDF text extraction → canonical text + PageMap + OffsetMap. * * Implements `wiki/ArchitectureOverview.md` §3.4 ("extract canonical text / * build format-specific maps") for the `pdf-text` representation * (`wiki/SharedContracts.md` §1, §3) and §6 (canonical normalization). * * Runtime independence: the PDF.js worker must be configured by the host * application (`GlobalWorkerOptions.workerSrc`) before this module is * called. In Vite/browser code the worker is bundled via the viewer; in * Node tests the test setup file points it at * `pdfjs-dist/legacy/build/pdf.worker.mjs`. No worker setup happens here * so the same module loads cleanly in both runtimes. * * Page boundary semantics: canonical text concatenates per-page normalized * text with a single "\n\n" paragraph separator. The separator is treated * as belonging to the *preceding* page in `OffsetMap`, so the map covers * `[0, canonicalText.length)` with no gaps. The last page has no trailing * separator. This means `pageLength = globalEnd - globalStart` for * every page; for non-last pages it equals (normalized page text length + * 2). See `PageOffsetRange` in `@shared/document.ts`. */ import { getDocument } from "pdfjs-dist"; import type { PDFPageProxy } from "pdfjs-dist"; import type { OffsetMap, PageInfo, PageMap, PageOffsetRange, } from "@shared/document"; import { normalize } from "@shared/text/normalize"; const PAGE_SEPARATOR = "\n\n"; export interface PdfExtractionResult { readonly canonicalText: string; readonly pageMap: PageMap; readonly offsetMap: OffsetMap; readonly pageCount: number; } export async function extractPdf(bytes: Uint8Array): Promise { // PDF.js mutates the bytes buffer (transfers ownership). Pass a fresh copy // so the caller's Uint8Array stays usable for fingerprinting after extract. const data = new Uint8Array(bytes); const loadingTask = getDocument({ data }); const doc = await loadingTask.promise; try { const pageCount = doc.numPages; const pageInfos: PageInfo[] = []; const pageNormalizedTexts: string[] = []; for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { const page = await doc.getPage(pageNumber); try { const viewport = page.getViewport({ scale: 1 }); pageInfos.push({ page: pageNumber, width: viewport.width, height: viewport.height, }); const rawText = await extractPageText(page); pageNormalizedTexts.push(normalize(rawText).text); } finally { page.cleanup(); } } const { canonicalText, offsetMap } = buildOffsetMap(pageNormalizedTexts); return { canonicalText, pageMap: pageInfos, offsetMap, pageCount, }; } finally { await doc.destroy(); } } async function extractPageText(page: PDFPageProxy): Promise { const content = await page.getTextContent(); // textContent.items are TextItem | TextMarkedContent. We want only the // TextItem strings (those have a `str` field); marked-content entries are // structural anchors and have no visible text. const parts: string[] = []; for (const item of content.items) { if ("str" in item) { parts.push(item.str); if (item.hasEOL) parts.push("\n"); } } return parts.join(""); } function buildOffsetMap(pageTexts: readonly string[]): { canonicalText: string; offsetMap: OffsetMap; } { const ranges: PageOffsetRange[] = []; let offset = 0; for (let i = 0; i < pageTexts.length; i++) { const text = pageTexts[i]!; const isLast = i === pageTexts.length - 1; const segmentLength = text.length + (isLast ? 0 : PAGE_SEPARATOR.length); const globalStart = offset; const globalEnd = offset + segmentLength; ranges.push({ page: i + 1, globalStart, globalEnd, pageLength: segmentLength, }); offset = globalEnd; } const canonicalText = pageTexts.join(PAGE_SEPARATOR); return { canonicalText, offsetMap: ranges }; }