diff --git a/package.json b/package.json index 405be48..98c083d 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "@citation-evidence/engine": "link:../citation-engine", + "@citation-evidence/evidence-source": "link:../evidence-source", "jszip": "^3.10.1", "pdfjs-dist": "^4.4.168", "react": "^18.3.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4460989..0ba43ac 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@citation-evidence/engine': specifier: link:../citation-engine version: link:../citation-engine + '@citation-evidence/evidence-source': + specifier: link:../evidence-source + version: link:../evidence-source jszip: specifier: ^3.10.1 version: 3.10.1 diff --git a/src/source/README.md b/src/source/README.md index aaf527b..2362cf8 100644 --- a/src/source/README.md +++ b/src/source/README.md @@ -1,7 +1,21 @@ -# `src/source/` — ingest, fingerprint, representation extraction, recovery +# `src/source/` — façade over the extracted `evidence-source` package -Future home: `evidence-source`. -Owns: PDF/HTML/MD ingest, fingerprinting, page-/offset-map construction, -canonical-text extraction, and citation-recovery behavior. +The headless ingest core and browser upload helpers that used to live here +were extracted into the standalone `@citation-evidence/evidence-source` repo +(ESRC-WP-0001). This directory is now a thin façade: -May import from: `shared/`, `engine/` (`wiki/DependencyMap.md` §4). +- `index.ts` re-exports the ingest core (`ingestPdf`, `extractPdf`, + `fingerprintBytes`) from `@citation-evidence/evidence-source` and the browser + upload helpers (`createPdfByteStore`, `ingestPdfFromFile`) from + `@citation-evidence/evidence-source/browser`. +- `pdf/viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…` + URL convention and blob-vs-fixture fallback policy — viewer concerns, not + headless ingest (ESRC-WP-0001 ADR-0001, T04). It consumes the `PdfByteStore` + type from the extracted package. + +The source ↔ anchor round-trip integration test remains upstream in +`tests/integration/anchor-source-roundtrip.test.ts` and now exercises the +extracted package across the repo boundary. + +Package dependency: `@citation-evidence/evidence-source` via +`link:../evidence-source` (sibling checkout). diff --git a/src/source/index.ts b/src/source/index.ts index eedecd8..fdab42b 100644 --- a/src/source/index.ts +++ b/src/source/index.ts @@ -1,21 +1,39 @@ +/** + * `src/source/` — façade over the extracted `evidence-source` package. + * + * The headless PDF ingest core (ingest / extract / fingerprint) and the + * browser upload helpers (byte store / upload) were extracted into the + * standalone `@citation-evidence/evidence-source` repo (ESRC-WP-0001). This + * module re-exports them so the rest of the umbrella keeps importing from + * `@source/index` unchanged. + * + * `viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…` URL + * convention and blob-vs-fixture fallback policy, which are viewer concerns, + * not headless ingest (ESRC-WP-0001 ADR-0001, T04). + */ + +// Headless ingest core — from the extracted package root. export { ingestPdf, + extractPdf, + fingerprintBytes, type IngestPdfInput, type IngestPdfOptions, type IngestPdfResult, -} from "./pdf/ingest"; -export { extractPdf, type PdfExtractionResult } from "./pdf/extract"; -export { fingerprintBytes } from "./pdf/fingerprint"; + type PdfExtractionResult, +} from "@citation-evidence/evidence-source"; + +// Browser upload helpers — from the package's browser entry point. export { createPdfByteStore, + ingestPdfFromFile, type CreatePdfByteStoreOptions, type PdfByteRecord, type PdfByteStore, -} from "./pdf/byte-store"; -export { - ingestPdfFromFile, type IngestPdfFromFileOptions, -} from "./pdf/upload"; +} from "@citation-evidence/evidence-source/browser"; + +// Viewer URL resolution — stays local to this app. export { isEphemeralBlobUri, resolvePdfViewerUrl, diff --git a/src/source/pdf/byte-store.test.ts b/src/source/pdf/byte-store.test.ts deleted file mode 100644 index a797e51..0000000 --- a/src/source/pdf/byte-store.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import type { DocumentId } from "@shared/ids"; - -import { createPdfByteStore } from "./byte-store"; - -function stubUrlHelpers() { - let counter = 0; - const created: string[] = []; - const revoked: string[] = []; - const createObjectURL = vi.fn(() => { - const url = `blob:stub-${++counter}`; - created.push(url); - return url; - }); - const revokeObjectURL = vi.fn((url: string) => { - revoked.push(url); - }); - return { createObjectURL, revokeObjectURL, created, revoked }; -} - -describe("PdfByteStore", () => { - it("put / get round-trips bytes and exposes a blob URL", () => { - const helpers = stubUrlHelpers(); - const store = createPdfByteStore(helpers); - const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]); // %PDF - const record = store.put("doc_a" as DocumentId, bytes); - expect(record.blobUrl).toBe("blob:stub-1"); - expect(store.get("doc_a" as DocumentId)?.bytes).toBe(bytes); - expect(store.has("doc_a" as DocumentId)).toBe(true); - expect(store.list()).toEqual(["doc_a"]); - expect(store.size()).toBe(4); - }); - - it("put replaces an existing entry and revokes the old URL", () => { - const helpers = stubUrlHelpers(); - const store = createPdfByteStore(helpers); - const id = "doc_a" as DocumentId; - const first = store.put(id, new Uint8Array([1, 2])); - const second = store.put(id, new Uint8Array([3, 4, 5])); - expect(helpers.revoked).toEqual([first.blobUrl]); - expect(store.get(id)?.bytes).toHaveLength(3); - expect(second.blobUrl).not.toBe(first.blobUrl); - }); - - it("delete revokes the blob URL exactly once and is idempotent", () => { - const helpers = stubUrlHelpers(); - const store = createPdfByteStore(helpers); - const id = "doc_a" as DocumentId; - const record = store.put(id, new Uint8Array([1, 2, 3])); - expect(store.delete(id)).toBe(true); - expect(helpers.revoked).toEqual([record.blobUrl]); - expect(store.delete(id)).toBe(false); - // No additional revoke calls on the second delete. - expect(helpers.revoked).toHaveLength(1); - expect(store.get(id)).toBeNull(); - expect(store.has(id)).toBe(false); - }); - - it("clear revokes every URL and empties the store", () => { - const helpers = stubUrlHelpers(); - const store = createPdfByteStore(helpers); - const a = store.put("doc_a" as DocumentId, new Uint8Array([1])); - const b = store.put("doc_b" as DocumentId, new Uint8Array([2])); - store.clear(); - expect(helpers.revoked.sort()).toEqual([a.blobUrl, b.blobUrl].sort()); - expect(store.list()).toEqual([]); - expect(store.size()).toBe(0); - }); - - it("uses URL.createObjectURL by default when no override is supplied", () => { - const createObjectURL = vi.fn(() => "blob:built-in"); - const revokeObjectURL = vi.fn(); - const originalURL = globalThis.URL; - // happy-dom's URL has createObjectURL; node sometimes does not. Stub it. - Object.defineProperty(globalThis, "URL", { - configurable: true, - writable: true, - value: Object.assign(Object.create(originalURL.prototype as object), { - createObjectURL, - revokeObjectURL, - }), - }); - try { - const store = createPdfByteStore(); - const rec = store.put("doc_z" as DocumentId, new Uint8Array([9])); - expect(rec.blobUrl).toBe("blob:built-in"); - expect(createObjectURL).toHaveBeenCalledTimes(1); - store.delete("doc_z" as DocumentId); - expect(revokeObjectURL).toHaveBeenCalledWith("blob:built-in"); - } finally { - Object.defineProperty(globalThis, "URL", { - configurable: true, - writable: true, - value: originalURL, - }); - } - }); -}); diff --git a/src/source/pdf/byte-store.ts b/src/source/pdf/byte-store.ts deleted file mode 100644 index 81bcbbc..0000000 --- a/src/source/pdf/byte-store.ts +++ /dev/null @@ -1,113 +0,0 @@ -/** - * `PdfByteStore` — in-memory store for uploaded PDF bytes, keyed by - * `DocumentId`. - * - * CE-WP-0005 stores uploaded PDFs in memory only (per the workplan - * scoping decision). Bytes survive within a tab session; reloading the - * page loses them unless the user exported a ZIP. Re-importing the ZIP - * restores them. - * - * One store instance per active session. The session-management layer - * is responsible for swapping the active store when the user switches - * sessions. The store also owns a small registry of issued `blob:` - * URLs so it can revoke them on delete/clear — no cross-cutting - * cleanup at the app layer. - */ - -import type { DocumentId } from "@shared/ids"; - -export interface PdfByteRecord { - readonly bytes: Uint8Array; - /** A `blob:` URL the viewer can consume directly. */ - readonly blobUrl: string; -} - -export interface PdfByteStore { - put(documentId: DocumentId, bytes: Uint8Array): PdfByteRecord; - get(documentId: DocumentId): PdfByteRecord | null; - has(documentId: DocumentId): boolean; - delete(documentId: DocumentId): boolean; - list(): readonly DocumentId[]; - /** Revoke every blob URL and clear the store. */ - clear(): void; - /** Total bytes currently held — useful for UI dashboards. */ - size(): number; -} - -export interface CreatePdfByteStoreOptions { - /** - * Mint a URL for the given bytes. Defaults to `URL.createObjectURL` in - * environments that have it; tests can inject a deterministic stub. - */ - readonly createObjectURL?: (blob: Blob) => string; - /** Revoke a URL previously minted by `createObjectURL`. */ - readonly revokeObjectURL?: (url: string) => void; -} - -export function createPdfByteStore( - options: CreatePdfByteStoreOptions = {}, -): PdfByteStore { - const createUrl = - options.createObjectURL ?? - ((blob: Blob) => { - if (typeof URL === "undefined" || typeof URL.createObjectURL !== "function") { - throw new Error( - "createPdfByteStore: URL.createObjectURL not available — inject a stub via options", - ); - } - return URL.createObjectURL(blob); - }); - const revokeUrl = - options.revokeObjectURL ?? - ((url: string) => { - if (typeof URL !== "undefined" && typeof URL.revokeObjectURL === "function") { - URL.revokeObjectURL(url); - } - }); - - const records = new Map(); - - return { - put(documentId, bytes) { - // Replace previous record (revoking the prior URL) if any. - const prior = records.get(documentId); - if (prior) revokeUrl(prior.blobUrl); - // Cast: Blob() does accept Uint8Array at runtime, but TS narrows the - // buffer type to ArrayBufferLike (could be SharedArrayBuffer) and - // refuses without help. The bytes here always come from a fresh - // arrayBuffer() call, so a regular ArrayBuffer is guaranteed. - const blob = new Blob([bytes as unknown as ArrayBuffer], { - type: "application/pdf", - }); - const blobUrl = createUrl(blob); - const record: PdfByteRecord = { bytes, blobUrl }; - records.set(documentId, record); - return record; - }, - get(documentId) { - return records.get(documentId) ?? null; - }, - has(documentId) { - return records.has(documentId); - }, - delete(documentId) { - const record = records.get(documentId); - if (!record) return false; - revokeUrl(record.blobUrl); - records.delete(documentId); - return true; - }, - list() { - return [...records.keys()]; - }, - clear() { - for (const record of records.values()) revokeUrl(record.blobUrl); - records.clear(); - }, - size() { - let total = 0; - for (const r of records.values()) total += r.bytes.byteLength; - return total; - }, - }; -} diff --git a/src/source/pdf/extract.ts b/src/source/pdf/extract.ts deleted file mode 100644 index 63de6bc..0000000 --- a/src/source/pdf/extract.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * PDF text extraction → canonical text + PageMap + OffsetMap. - * - * Implements `wiki/ArchitectureOverview.md` §3.4 ("extract canonical text / - * build format-specific maps") for the `pdf-text` representation - * (`wiki/SharedContracts.md` §1, §3) and §6 (canonical normalization). - * - * Runtime independence: the PDF.js worker must be configured by the host - * application (`GlobalWorkerOptions.workerSrc`) before this module is - * called. In Vite/browser code the worker is bundled via the viewer; in - * Node tests the test setup file points it at - * `pdfjs-dist/legacy/build/pdf.worker.mjs`. No worker setup happens here - * so the same module loads cleanly in both runtimes. - * - * Page boundary semantics: canonical text concatenates per-page normalized - * text with a single "\n\n" paragraph separator. The separator is treated - * as belonging to the *preceding* page in `OffsetMap`, so the map covers - * `[0, canonicalText.length)` with no gaps. The last page has no trailing - * separator. This means `pageLength = globalEnd - globalStart` for - * every page; for non-last pages it equals (normalized page text length + - * 2). See `PageOffsetRange` in `@shared/document.ts`. - */ - -import { getDocument } from "pdfjs-dist"; -import type { PDFPageProxy } from "pdfjs-dist"; -import type { - OffsetMap, - PageInfo, - PageMap, - PageOffsetRange, -} from "@shared/document"; -import { normalize } from "@shared/text/normalize"; - -const PAGE_SEPARATOR = "\n\n"; - -export interface PdfExtractionResult { - readonly canonicalText: string; - readonly pageMap: PageMap; - readonly offsetMap: OffsetMap; - readonly pageCount: number; -} - -export async function extractPdf(bytes: Uint8Array): Promise { - // PDF.js mutates the bytes buffer (transfers ownership). Pass a fresh copy - // so the caller's Uint8Array stays usable for fingerprinting after extract. - const data = new Uint8Array(bytes); - const loadingTask = getDocument({ data }); - const doc = await loadingTask.promise; - - try { - const pageCount = doc.numPages; - const pageInfos: PageInfo[] = []; - const pageNormalizedTexts: string[] = []; - - for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) { - const page = await doc.getPage(pageNumber); - try { - const viewport = page.getViewport({ scale: 1 }); - pageInfos.push({ - page: pageNumber, - width: viewport.width, - height: viewport.height, - }); - - const rawText = await extractPageText(page); - pageNormalizedTexts.push(normalize(rawText).text); - } finally { - page.cleanup(); - } - } - - const { canonicalText, offsetMap } = buildOffsetMap(pageNormalizedTexts); - - return { - canonicalText, - pageMap: pageInfos, - offsetMap, - pageCount, - }; - } finally { - await doc.destroy(); - } -} - -async function extractPageText(page: PDFPageProxy): Promise { - const content = await page.getTextContent(); - // textContent.items are TextItem | TextMarkedContent. We want only the - // TextItem strings (those have a `str` field); marked-content entries are - // structural anchors and have no visible text. - const parts: string[] = []; - for (const item of content.items) { - if ("str" in item) { - parts.push(item.str); - if (item.hasEOL) parts.push("\n"); - } - } - return parts.join(""); -} - -function buildOffsetMap(pageTexts: readonly string[]): { - canonicalText: string; - offsetMap: OffsetMap; -} { - const ranges: PageOffsetRange[] = []; - let offset = 0; - for (let i = 0; i < pageTexts.length; i++) { - const text = pageTexts[i]!; - const isLast = i === pageTexts.length - 1; - const segmentLength = text.length + (isLast ? 0 : PAGE_SEPARATOR.length); - const globalStart = offset; - const globalEnd = offset + segmentLength; - ranges.push({ - page: i + 1, - globalStart, - globalEnd, - pageLength: segmentLength, - }); - offset = globalEnd; - } - const canonicalText = pageTexts.join(PAGE_SEPARATOR); - return { canonicalText, offsetMap: ranges }; -} diff --git a/src/source/pdf/fingerprint.ts b/src/source/pdf/fingerprint.ts deleted file mode 100644 index e5ee2e2..0000000 --- a/src/source/pdf/fingerprint.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * SHA-256 fingerprint of raw document bytes. - * - * Implements the fingerprint half of `wiki/ArchitectureOverview.md` §3.4 - * (the "compute fingerprint" pipeline step) and populates - * `Document.fingerprint` (`wiki/SharedContracts.md` §1). - * - * Uses Web Crypto's `crypto.subtle.digest`, which is available in browsers - * and in Node ≥ 20 (where it is exposed on `globalThis.crypto`). No - * platform branching — the API is the same in both environments. - */ - -export async function fingerprintBytes(bytes: Uint8Array): Promise { - // Copy into a fresh ArrayBuffer (not SharedArrayBuffer) so the digest call - // satisfies TS's updated `BufferSource` type, which excludes - // `SharedArrayBuffer`. The copy is O(n) — fine even for large PDFs since - // SHA-256 itself is already O(n). - const ab = new ArrayBuffer(bytes.byteLength); - new Uint8Array(ab).set(bytes); - const digest = await crypto.subtle.digest("SHA-256", ab); - return bytesToHex(new Uint8Array(digest)); -} - -function bytesToHex(bytes: Uint8Array): string { - let hex = ""; - for (let i = 0; i < bytes.length; i++) { - const b = bytes[i]!; - hex += (b < 0x10 ? "0" : "") + b.toString(16); - } - return hex; -} diff --git a/src/source/pdf/ingest.test.ts b/src/source/pdf/ingest.test.ts deleted file mode 100644 index 7b93c65..0000000 --- a/src/source/pdf/ingest.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Fixture-driven contract tests for the PDF ingest pipeline. - * - * For each fixture in `fixtures/pdfs/manifest.json`: - * 1. Read the PDF bytes from disk. - * 2. Run `ingestPdf` end-to-end. - * 3. Assert the resulting Document + DocumentRepresentation honour the - * manifest contract: media type is application/pdf, fingerprint is a - * 64-hex SHA-256, pageMap matches `page_count`, canonicalText - * contains `known_good_quote`, and the offsetMap covers - * `[0, canonicalText.length)` with no gaps. - * - * This is the verification gate for CE-WP-0002-T03. - */ - -import { readFileSync } from "node:fs"; -import { dirname, resolve } from "node:path"; -import { createRequire } from "node:module"; -import { fileURLToPath } from "node:url"; -import { beforeAll, describe, expect, it } from "vitest"; - -import { ingestPdf } from "./ingest"; -import { fingerprintBytes } from "./fingerprint"; -import manifest from "../../../fixtures/pdfs/manifest.json" with { type: "json" }; - -const __dirname = dirname(fileURLToPath(import.meta.url)); -const FIXTURE_DIR = resolve(__dirname, "../../../fixtures/pdfs"); - -interface Fixture { - id: string; - filename: string; - page_count: number; - known_good_quote: string; - known_good_quote_page: number; -} - -const FIXTURES: readonly Fixture[] = manifest.fixtures; - -beforeAll(async () => { - // PDF.js needs a workerSrc set. In Node tests we point it at the legacy - // worker bundle — the modern bundle uses APIs that aren't present in - // Node. The legacy worker is bundled as plain JS and runs through the - // fake-worker fallback that PDF.js spins up when no real Worker is - // available. - const pdfjs = await import("pdfjs-dist"); - const require = createRequire(import.meta.url); - pdfjs.GlobalWorkerOptions.workerSrc = require.resolve( - "pdfjs-dist/legacy/build/pdf.worker.mjs", - ); -}); - -describe("ingestPdf — fixture corpus", { timeout: 30_000 }, () => { - for (const fixture of FIXTURES) { - describe(fixture.id, () => { - const path = resolve(FIXTURE_DIR, fixture.filename); - const bytes = new Uint8Array(readFileSync(path)); - - it("produces a Document with PDF media type and SHA-256 fingerprint", async () => { - const { document } = await ingestPdf(bytes, { filename: fixture.filename }); - expect(document.mediaType).toBe("application/pdf"); - expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/); - expect(document.title).toBe(fixture.filename); - // Fingerprint must be deterministic across runs. - const expected = await fingerprintBytes(bytes); - expect(document.fingerprint).toBe(expected); - }); - - it("produces a pdf-text representation with the expected page count", async () => { - const { representation } = await ingestPdf(bytes); - expect(representation.representationType).toBe("pdf-text"); - expect(representation.pageMap?.length).toBe(fixture.page_count); - expect(representation.offsetMap?.length).toBe(fixture.page_count); - }); - - it("canonical text contains the manifest's known-good quote", async () => { - const { representation } = await ingestPdf(bytes); - const text = representation.canonicalText ?? ""; - expect(text).toContain(fixture.known_good_quote); - }); - - it("offsetMap is gap-free and covers [0, canonicalText.length)", async () => { - const { representation } = await ingestPdf(bytes); - const text = representation.canonicalText ?? ""; - const offsets = representation.offsetMap ?? []; - expect(offsets.length).toBeGreaterThan(0); - expect(offsets[0]!.globalStart).toBe(0); - expect(offsets.at(-1)!.globalEnd).toBe(text.length); - for (let i = 0; i < offsets.length; i++) { - const r = offsets[i]!; - expect(r.page).toBe(i + 1); - expect(r.globalEnd - r.globalStart).toBe(r.pageLength); - if (i > 0) expect(r.globalStart).toBe(offsets[i - 1]!.globalEnd); - } - }); - - it("pageMap entries have positive width and height in user-space points", async () => { - const { representation } = await ingestPdf(bytes); - const pages = representation.pageMap ?? []; - for (let i = 0; i < pages.length; i++) { - const p = pages[i]!; - expect(p.page).toBe(i + 1); - expect(p.width).toBeGreaterThan(0); - expect(p.height).toBeGreaterThan(0); - } - }); - }); - } -}); - -describe("ingestPdf — option handling", () => { - const fixture = FIXTURES[0]!; - const path = resolve(FIXTURE_DIR, fixture.filename); - const bytes = new Uint8Array(readFileSync(path)); - - it("uses explicit title over filename", async () => { - const { document } = await ingestPdf(bytes, { - filename: fixture.filename, - title: "Custom Title", - }); - expect(document.title).toBe("Custom Title"); - }); - - it("omits title entirely when neither filename nor title is supplied", async () => { - const { document } = await ingestPdf(bytes); - expect(document.title).toBeUndefined(); - }); - - it("propagates uri and metadata when supplied", async () => { - const { document } = await ingestPdf(bytes, { - uri: "file:///example.pdf", - metadata: { source: "test" }, - }); - expect(document.uri).toBe("file:///example.pdf"); - expect(document.metadata).toEqual({ source: "test" }); - }); - - it("accepts ArrayBuffer input", async () => { - const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - const { document } = await ingestPdf(ab); - expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/); - }); -}); diff --git a/src/source/pdf/ingest.ts b/src/source/pdf/ingest.ts deleted file mode 100644 index a473e3f..0000000 --- a/src/source/pdf/ingest.ts +++ /dev/null @@ -1,88 +0,0 @@ -/** - * PDF ingest pipeline → `{ document, representation }`. - * - * Implements `wiki/ArchitectureOverview.md` §3.4 ("Raw Source → identify - * media type → compute fingerprint → extract metadata → extract canonical - * text → build format-specific maps → persist Document + - * DocumentRepresentation") for the PDF source format. - * - * Ingest is a pure function over bytes: it does not persist anything. The - * caller (engine repositories in T05, app layer in T06) writes the returned - * Document + DocumentRepresentation into the chosen store. - */ - -import { - type Document, - type DocumentRepresentation, -} from "@shared/document"; -import { newId } from "@shared/ids"; -import { extractPdf } from "./extract"; -import { fingerprintBytes } from "./fingerprint"; - -const PDF_MEDIA_TYPE = "application/pdf"; - -export interface IngestPdfOptions { - /** Original filename, used as the default title when no title is given. */ - readonly filename?: string; - /** Optional pre-existing title (overrides filename). */ - readonly title?: string; - /** Optional source URI (e.g. file:// or https://). */ - readonly uri?: string; - /** Free-form metadata persisted on the Document record. */ - readonly metadata?: Readonly>; -} - -export interface IngestPdfResult { - readonly document: Document; - readonly representation: DocumentRepresentation; -} - -export type IngestPdfInput = Uint8Array | ArrayBuffer | Blob; - -export async function ingestPdf( - input: IngestPdfInput, - options: IngestPdfOptions = {}, -): Promise { - const bytes = await toBytes(input); - const [fingerprint, extraction] = await Promise.all([ - fingerprintBytes(bytes), - extractPdf(bytes), - ]); - - const now = new Date().toISOString(); - const documentId = newId("document"); - const representationId = newId("representation"); - const title = options.title ?? options.filename; - - const document: Document = { - id: documentId, - mediaType: PDF_MEDIA_TYPE, - fingerprint, - createdAt: now, - updatedAt: now, - ...(title !== undefined ? { title } : {}), - ...(options.uri !== undefined ? { uri: options.uri } : {}), - ...(options.metadata !== undefined ? { metadata: options.metadata } : {}), - }; - - const representation: DocumentRepresentation = { - id: representationId, - documentId, - representationType: "pdf-text", - contentHash: fingerprint, - canonicalText: extraction.canonicalText, - pageMap: extraction.pageMap, - offsetMap: extraction.offsetMap, - generatedAt: now, - }; - - return { document, representation }; -} - -async function toBytes(input: IngestPdfInput): Promise { - if (input instanceof Uint8Array) return input; - if (input instanceof ArrayBuffer) return new Uint8Array(input); - // Blob (covers `File` in browsers — File extends Blob). - const buf = await input.arrayBuffer(); - return new Uint8Array(buf); -} diff --git a/src/source/pdf/upload.test.ts b/src/source/pdf/upload.test.ts deleted file mode 100644 index f1b5165..0000000 --- a/src/source/pdf/upload.test.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * `ingestPdfFromFile` end-to-end: pipes a fixture PDF through the - * upload path, asserts the byte store keeps the bytes and the document - * record carries the minted `blob:` URL. - */ - -import { readFile } from "node:fs/promises"; -import { describe, expect, it, vi } from "vitest"; - -import { createPdfByteStore } from "./byte-store"; -import { ingestPdfFromFile } from "./upload"; - -const FIXTURE_PATH = new URL( - "../../../fixtures/pdfs/Fristsetzung zur Bezifferung GÜ an Gegenseite 3 Wochen.pdf", - import.meta.url, -); - -async function fixtureBytes(): Promise { - return new Uint8Array(await readFile(FIXTURE_PATH)); -} - -class FakeFile { - readonly name: string; - private readonly bytes: Uint8Array; - constructor(bytes: Uint8Array, name: string) { - this.bytes = bytes; - this.name = name; - } - async arrayBuffer(): Promise { - const out = new ArrayBuffer(this.bytes.byteLength); - new Uint8Array(out).set(this.bytes); - return out; - } -} - -describe("ingestPdfFromFile", () => { - it("round-trips a fixture PDF through ingest + byte store + blob URL", async () => { - const bytes = await fixtureBytes(); - const file = new FakeFile(bytes, "demo.pdf") as unknown as File; - let counter = 0; - const store = createPdfByteStore({ - createObjectURL: () => `blob:upload-stub-${++counter}`, - revokeObjectURL: () => {}, - }); - - const { document, representation } = await ingestPdfFromFile(file, store); - - // Bytes are stored, retrievable by document id. - const stored = store.get(document.id); - expect(stored).not.toBeNull(); - expect(stored!.bytes.byteLength).toBe(bytes.byteLength); - - // Document carries the blob URL minted by the store. - expect(document.uri).toBe(`blob:upload-stub-${counter}`); - expect(document.title).toBe("demo.pdf"); - expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/); - - // Representation is the standard pdf-text one. - expect(representation.representationType).toBe("pdf-text"); - expect((representation.canonicalText ?? "").length).toBeGreaterThan(0); - }, 30_000); - - it("falls through to ingestPdf with no filename when given a plain Blob", async () => { - const bytes = await fixtureBytes(); - const blob = { - async arrayBuffer() { - const out = new ArrayBuffer(bytes.byteLength); - new Uint8Array(out).set(bytes); - return out; - }, - } as Blob; - const store = createPdfByteStore({ - createObjectURL: () => "blob:no-name", - revokeObjectURL: () => {}, - }); - const { document } = await ingestPdfFromFile(blob, store); - expect(document.title).toBeUndefined(); - expect(document.uri).toBe("blob:no-name"); - }, 30_000); - - it("explicit title option overrides the filename", async () => { - const bytes = await fixtureBytes(); - const file = new FakeFile(bytes, "anonymous-name.pdf") as unknown as File; - const store = createPdfByteStore({ - createObjectURL: vi.fn(() => "blob:override"), - revokeObjectURL: vi.fn(), - }); - const { document } = await ingestPdfFromFile(file, store, { title: "Custom" }); - expect(document.title).toBe("Custom"); - }, 30_000); -}); diff --git a/src/source/pdf/upload.ts b/src/source/pdf/upload.ts deleted file mode 100644 index a72e44e..0000000 --- a/src/source/pdf/upload.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Upload-side ingest path. - * - * The fixture-loading path in `App.tsx` fetches a known URL and calls - * `ingestPdf` directly; that path stays untouched for the optional - * "Sample sessions" quick-start. Uploaded files flow through here - * instead: - * - * 1. Read `file.arrayBuffer()` once into a `Uint8Array`. - * 2. Run the existing `ingestPdf(bytes, { filename })` pipeline to - * produce `{document, representation}`. - * 3. Push the bytes into the per-session `PdfByteStore`, which mints - * a `blob:` URL and stamps it onto `document.uri` so the viewer - * adapter can mount the PDF directly. - * 4. Hand the engine inputs back to the caller, which wires them via - * `engine.documents.register(...)`. - * - * Keeping URL-minting inside the byte store (rather than at the call - * site) means there is exactly one place that creates `blob:` URLs and - * exactly one place that revokes them. - */ - -import { ingestPdf, type IngestPdfResult } from "./ingest"; -import type { PdfByteStore } from "./byte-store"; - -export interface IngestPdfFromFileOptions { - /** Override the filename used as the document title. */ - readonly title?: string; -} - -export async function ingestPdfFromFile( - file: File | Blob, - store: PdfByteStore, - options: IngestPdfFromFileOptions = {}, -): Promise { - const bytes = new Uint8Array(await file.arrayBuffer()); - const filename = "name" in file && typeof file.name === "string" ? file.name : undefined; - const ingested = await ingestPdf(bytes, { - ...(filename !== undefined ? { filename } : {}), - ...(options.title !== undefined ? { title: options.title } : {}), - }); - const record = store.put(ingested.document.id, bytes); - const document = { ...ingested.document, uri: record.blobUrl }; - return { document, representation: ingested.representation }; -} diff --git a/src/source/pdf/viewer-url.test.ts b/src/source/pdf/viewer-url.test.ts index 0168d3e..2e553ba 100644 --- a/src/source/pdf/viewer-url.test.ts +++ b/src/source/pdf/viewer-url.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import type { Document } from "@shared/document"; import type { DocumentId } from "@shared/ids"; -import { createPdfByteStore } from "./byte-store"; +import { createPdfByteStore } from "@citation-evidence/evidence-source/browser"; import { isEphemeralBlobUri, resolvePdfViewerUrl } from "./viewer-url"; function doc(overrides: Partial = {}): Document { diff --git a/src/source/pdf/viewer-url.ts b/src/source/pdf/viewer-url.ts index a4d5d4c..4768b0f 100644 --- a/src/source/pdf/viewer-url.ts +++ b/src/source/pdf/viewer-url.ts @@ -9,7 +9,10 @@ import type { Document } from "@shared/document"; import type { DocumentId } from "@shared/ids"; -import type { PdfByteStore } from "./byte-store"; +// PdfByteStore was extracted into the standalone evidence-source package +// (ESRC-WP-0001). viewer-url stays local to this app (T04) but consumes the +// store type from the package. +import type { PdfByteStore } from "@citation-evidence/evidence-source/browser"; export function isEphemeralBlobUri(uri: string | undefined): boolean { return typeof uri === "string" && uri.startsWith("blob:"); diff --git a/tests/integration/anchor-source-roundtrip.test.ts b/tests/integration/anchor-source-roundtrip.test.ts index d4f264d..289cf43 100644 --- a/tests/integration/anchor-source-roundtrip.test.ts +++ b/tests/integration/anchor-source-roundtrip.test.ts @@ -17,7 +17,7 @@ import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, it } from "vitest"; -import { ingestPdf } from "@source/pdf/ingest"; +import { ingestPdf } from "@citation-evidence/evidence-source"; import { createSelectors, resolveSelectors } from "@anchor/selectors"; import type { PdfSelectionCapture } from "@anchor/types"; import manifest from "../../fixtures/pdfs/manifest.json" with { type: "json" };