refactor(source): consume extracted evidence-source package (ESRC-WP-0001-T06)
Replace the local src/source PDF slice with the standalone @citation-evidence/evidence-source package (link:../evidence-source). - src/source/index.ts is now a thin façade re-exporting the ingest core and browser upload helpers from the package - viewer-url stays local (viewer concern) but takes PdfByteStore from the package - extracted ingest/extract/fingerprint/byte-store/upload files removed - anchor-source round-trip integration test now exercises the package across the repo boundary Full suite (125 tests), typecheck, and lint all green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
54b585197b
commit
4ede97983a
15 changed files with 54 additions and 746 deletions
|
|
@ -20,6 +20,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@citation-evidence/engine": "link:../citation-engine",
|
||||
"@citation-evidence/evidence-source": "link:../evidence-source",
|
||||
"jszip": "^3.10.1",
|
||||
"pdfjs-dist": "^4.4.168",
|
||||
"react": "^18.3.1",
|
||||
|
|
|
|||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
|
|
@ -11,6 +11,9 @@ importers:
|
|||
'@citation-evidence/engine':
|
||||
specifier: link:../citation-engine
|
||||
version: link:../citation-engine
|
||||
'@citation-evidence/evidence-source':
|
||||
specifier: link:../evidence-source
|
||||
version: link:../evidence-source
|
||||
jszip:
|
||||
specifier: ^3.10.1
|
||||
version: 3.10.1
|
||||
|
|
|
|||
|
|
@ -1,7 +1,21 @@
|
|||
# `src/source/` — ingest, fingerprint, representation extraction, recovery
|
||||
# `src/source/` — façade over the extracted `evidence-source` package
|
||||
|
||||
Future home: `evidence-source`.
|
||||
Owns: PDF/HTML/MD ingest, fingerprinting, page-/offset-map construction,
|
||||
canonical-text extraction, and citation-recovery behavior.
|
||||
The headless ingest core and browser upload helpers that used to live here
|
||||
were extracted into the standalone `@citation-evidence/evidence-source` repo
|
||||
(ESRC-WP-0001). This directory is now a thin façade:
|
||||
|
||||
May import from: `shared/`, `engine/` (`wiki/DependencyMap.md` §4).
|
||||
- `index.ts` re-exports the ingest core (`ingestPdf`, `extractPdf`,
|
||||
`fingerprintBytes`) from `@citation-evidence/evidence-source` and the browser
|
||||
upload helpers (`createPdfByteStore`, `ingestPdfFromFile`) from
|
||||
`@citation-evidence/evidence-source/browser`.
|
||||
- `pdf/viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…`
|
||||
URL convention and blob-vs-fixture fallback policy — viewer concerns, not
|
||||
headless ingest (ESRC-WP-0001 ADR-0001, T04). It consumes the `PdfByteStore`
|
||||
type from the extracted package.
|
||||
|
||||
The source ↔ anchor round-trip integration test remains upstream in
|
||||
`tests/integration/anchor-source-roundtrip.test.ts` and now exercises the
|
||||
extracted package across the repo boundary.
|
||||
|
||||
Package dependency: `@citation-evidence/evidence-source` via
|
||||
`link:../evidence-source` (sibling checkout).
|
||||
|
|
|
|||
|
|
@ -1,21 +1,39 @@
|
|||
/**
|
||||
* `src/source/` — façade over the extracted `evidence-source` package.
|
||||
*
|
||||
* The headless PDF ingest core (ingest / extract / fingerprint) and the
|
||||
* browser upload helpers (byte store / upload) were extracted into the
|
||||
* standalone `@citation-evidence/evidence-source` repo (ESRC-WP-0001). This
|
||||
* module re-exports them so the rest of the umbrella keeps importing from
|
||||
* `@source/index` unchanged.
|
||||
*
|
||||
* `viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…` URL
|
||||
* convention and blob-vs-fixture fallback policy, which are viewer concerns,
|
||||
* not headless ingest (ESRC-WP-0001 ADR-0001, T04).
|
||||
*/
|
||||
|
||||
// Headless ingest core — from the extracted package root.
|
||||
export {
|
||||
ingestPdf,
|
||||
extractPdf,
|
||||
fingerprintBytes,
|
||||
type IngestPdfInput,
|
||||
type IngestPdfOptions,
|
||||
type IngestPdfResult,
|
||||
} from "./pdf/ingest";
|
||||
export { extractPdf, type PdfExtractionResult } from "./pdf/extract";
|
||||
export { fingerprintBytes } from "./pdf/fingerprint";
|
||||
type PdfExtractionResult,
|
||||
} from "@citation-evidence/evidence-source";
|
||||
|
||||
// Browser upload helpers — from the package's browser entry point.
|
||||
export {
|
||||
createPdfByteStore,
|
||||
ingestPdfFromFile,
|
||||
type CreatePdfByteStoreOptions,
|
||||
type PdfByteRecord,
|
||||
type PdfByteStore,
|
||||
} from "./pdf/byte-store";
|
||||
export {
|
||||
ingestPdfFromFile,
|
||||
type IngestPdfFromFileOptions,
|
||||
} from "./pdf/upload";
|
||||
} from "@citation-evidence/evidence-source/browser";
|
||||
|
||||
// Viewer URL resolution — stays local to this app.
|
||||
export {
|
||||
isEphemeralBlobUri,
|
||||
resolvePdfViewerUrl,
|
||||
|
|
|
|||
|
|
@ -1,99 +0,0 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { DocumentId } from "@shared/ids";
|
||||
|
||||
import { createPdfByteStore } from "./byte-store";
|
||||
|
||||
function stubUrlHelpers() {
|
||||
let counter = 0;
|
||||
const created: string[] = [];
|
||||
const revoked: string[] = [];
|
||||
const createObjectURL = vi.fn(() => {
|
||||
const url = `blob:stub-${++counter}`;
|
||||
created.push(url);
|
||||
return url;
|
||||
});
|
||||
const revokeObjectURL = vi.fn((url: string) => {
|
||||
revoked.push(url);
|
||||
});
|
||||
return { createObjectURL, revokeObjectURL, created, revoked };
|
||||
}
|
||||
|
||||
describe("PdfByteStore", () => {
|
||||
it("put / get round-trips bytes and exposes a blob URL", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]); // %PDF
|
||||
const record = store.put("doc_a" as DocumentId, bytes);
|
||||
expect(record.blobUrl).toBe("blob:stub-1");
|
||||
expect(store.get("doc_a" as DocumentId)?.bytes).toBe(bytes);
|
||||
expect(store.has("doc_a" as DocumentId)).toBe(true);
|
||||
expect(store.list()).toEqual(["doc_a"]);
|
||||
expect(store.size()).toBe(4);
|
||||
});
|
||||
|
||||
it("put replaces an existing entry and revokes the old URL", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const id = "doc_a" as DocumentId;
|
||||
const first = store.put(id, new Uint8Array([1, 2]));
|
||||
const second = store.put(id, new Uint8Array([3, 4, 5]));
|
||||
expect(helpers.revoked).toEqual([first.blobUrl]);
|
||||
expect(store.get(id)?.bytes).toHaveLength(3);
|
||||
expect(second.blobUrl).not.toBe(first.blobUrl);
|
||||
});
|
||||
|
||||
it("delete revokes the blob URL exactly once and is idempotent", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const id = "doc_a" as DocumentId;
|
||||
const record = store.put(id, new Uint8Array([1, 2, 3]));
|
||||
expect(store.delete(id)).toBe(true);
|
||||
expect(helpers.revoked).toEqual([record.blobUrl]);
|
||||
expect(store.delete(id)).toBe(false);
|
||||
// No additional revoke calls on the second delete.
|
||||
expect(helpers.revoked).toHaveLength(1);
|
||||
expect(store.get(id)).toBeNull();
|
||||
expect(store.has(id)).toBe(false);
|
||||
});
|
||||
|
||||
it("clear revokes every URL and empties the store", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const a = store.put("doc_a" as DocumentId, new Uint8Array([1]));
|
||||
const b = store.put("doc_b" as DocumentId, new Uint8Array([2]));
|
||||
store.clear();
|
||||
expect(helpers.revoked.sort()).toEqual([a.blobUrl, b.blobUrl].sort());
|
||||
expect(store.list()).toEqual([]);
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("uses URL.createObjectURL by default when no override is supplied", () => {
|
||||
const createObjectURL = vi.fn(() => "blob:built-in");
|
||||
const revokeObjectURL = vi.fn();
|
||||
const originalURL = globalThis.URL;
|
||||
// happy-dom's URL has createObjectURL; node sometimes does not. Stub it.
|
||||
Object.defineProperty(globalThis, "URL", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: Object.assign(Object.create(originalURL.prototype as object), {
|
||||
createObjectURL,
|
||||
revokeObjectURL,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
const store = createPdfByteStore();
|
||||
const rec = store.put("doc_z" as DocumentId, new Uint8Array([9]));
|
||||
expect(rec.blobUrl).toBe("blob:built-in");
|
||||
expect(createObjectURL).toHaveBeenCalledTimes(1);
|
||||
store.delete("doc_z" as DocumentId);
|
||||
expect(revokeObjectURL).toHaveBeenCalledWith("blob:built-in");
|
||||
} finally {
|
||||
Object.defineProperty(globalThis, "URL", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalURL,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1,113 +0,0 @@
|
|||
/**
|
||||
* `PdfByteStore` — in-memory store for uploaded PDF bytes, keyed by
|
||||
* `DocumentId`.
|
||||
*
|
||||
* CE-WP-0005 stores uploaded PDFs in memory only (per the workplan
|
||||
* scoping decision). Bytes survive within a tab session; reloading the
|
||||
* page loses them unless the user exported a ZIP. Re-importing the ZIP
|
||||
* restores them.
|
||||
*
|
||||
* One store instance per active session. The session-management layer
|
||||
* is responsible for swapping the active store when the user switches
|
||||
* sessions. The store also owns a small registry of issued `blob:`
|
||||
* URLs so it can revoke them on delete/clear — no cross-cutting
|
||||
* cleanup at the app layer.
|
||||
*/
|
||||
|
||||
import type { DocumentId } from "@shared/ids";
|
||||
|
||||
export interface PdfByteRecord {
|
||||
readonly bytes: Uint8Array;
|
||||
/** A `blob:` URL the viewer can consume directly. */
|
||||
readonly blobUrl: string;
|
||||
}
|
||||
|
||||
export interface PdfByteStore {
|
||||
put(documentId: DocumentId, bytes: Uint8Array): PdfByteRecord;
|
||||
get(documentId: DocumentId): PdfByteRecord | null;
|
||||
has(documentId: DocumentId): boolean;
|
||||
delete(documentId: DocumentId): boolean;
|
||||
list(): readonly DocumentId[];
|
||||
/** Revoke every blob URL and clear the store. */
|
||||
clear(): void;
|
||||
/** Total bytes currently held — useful for UI dashboards. */
|
||||
size(): number;
|
||||
}
|
||||
|
||||
export interface CreatePdfByteStoreOptions {
|
||||
/**
|
||||
* Mint a URL for the given bytes. Defaults to `URL.createObjectURL` in
|
||||
* environments that have it; tests can inject a deterministic stub.
|
||||
*/
|
||||
readonly createObjectURL?: (blob: Blob) => string;
|
||||
/** Revoke a URL previously minted by `createObjectURL`. */
|
||||
readonly revokeObjectURL?: (url: string) => void;
|
||||
}
|
||||
|
||||
export function createPdfByteStore(
|
||||
options: CreatePdfByteStoreOptions = {},
|
||||
): PdfByteStore {
|
||||
const createUrl =
|
||||
options.createObjectURL ??
|
||||
((blob: Blob) => {
|
||||
if (typeof URL === "undefined" || typeof URL.createObjectURL !== "function") {
|
||||
throw new Error(
|
||||
"createPdfByteStore: URL.createObjectURL not available — inject a stub via options",
|
||||
);
|
||||
}
|
||||
return URL.createObjectURL(blob);
|
||||
});
|
||||
const revokeUrl =
|
||||
options.revokeObjectURL ??
|
||||
((url: string) => {
|
||||
if (typeof URL !== "undefined" && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
});
|
||||
|
||||
const records = new Map<DocumentId, PdfByteRecord>();
|
||||
|
||||
return {
|
||||
put(documentId, bytes) {
|
||||
// Replace previous record (revoking the prior URL) if any.
|
||||
const prior = records.get(documentId);
|
||||
if (prior) revokeUrl(prior.blobUrl);
|
||||
// Cast: Blob() does accept Uint8Array at runtime, but TS narrows the
|
||||
// buffer type to ArrayBufferLike (could be SharedArrayBuffer) and
|
||||
// refuses without help. The bytes here always come from a fresh
|
||||
// arrayBuffer() call, so a regular ArrayBuffer is guaranteed.
|
||||
const blob = new Blob([bytes as unknown as ArrayBuffer], {
|
||||
type: "application/pdf",
|
||||
});
|
||||
const blobUrl = createUrl(blob);
|
||||
const record: PdfByteRecord = { bytes, blobUrl };
|
||||
records.set(documentId, record);
|
||||
return record;
|
||||
},
|
||||
get(documentId) {
|
||||
return records.get(documentId) ?? null;
|
||||
},
|
||||
has(documentId) {
|
||||
return records.has(documentId);
|
||||
},
|
||||
delete(documentId) {
|
||||
const record = records.get(documentId);
|
||||
if (!record) return false;
|
||||
revokeUrl(record.blobUrl);
|
||||
records.delete(documentId);
|
||||
return true;
|
||||
},
|
||||
list() {
|
||||
return [...records.keys()];
|
||||
},
|
||||
clear() {
|
||||
for (const record of records.values()) revokeUrl(record.blobUrl);
|
||||
records.clear();
|
||||
},
|
||||
size() {
|
||||
let total = 0;
|
||||
for (const r of records.values()) total += r.bytes.byteLength;
|
||||
return total;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,122 +0,0 @@
|
|||
/**
|
||||
* PDF text extraction → canonical text + PageMap + OffsetMap.
|
||||
*
|
||||
* Implements `wiki/ArchitectureOverview.md` §3.4 ("extract canonical text /
|
||||
* build format-specific maps") for the `pdf-text` representation
|
||||
* (`wiki/SharedContracts.md` §1, §3) and §6 (canonical normalization).
|
||||
*
|
||||
* Runtime independence: the PDF.js worker must be configured by the host
|
||||
* application (`GlobalWorkerOptions.workerSrc`) before this module is
|
||||
* called. In Vite/browser code the worker is bundled via the viewer; in
|
||||
* Node tests the test setup file points it at
|
||||
* `pdfjs-dist/legacy/build/pdf.worker.mjs`. No worker setup happens here
|
||||
* so the same module loads cleanly in both runtimes.
|
||||
*
|
||||
* Page boundary semantics: canonical text concatenates per-page normalized
|
||||
* text with a single "\n\n" paragraph separator. The separator is treated
|
||||
* as belonging to the *preceding* page in `OffsetMap`, so the map covers
|
||||
* `[0, canonicalText.length)` with no gaps. The last page has no trailing
|
||||
* separator. This means `pageLength = globalEnd - globalStart` for
|
||||
* every page; for non-last pages it equals (normalized page text length +
|
||||
* 2). See `PageOffsetRange` in `@shared/document.ts`.
|
||||
*/
|
||||
|
||||
import { getDocument } from "pdfjs-dist";
|
||||
import type { PDFPageProxy } from "pdfjs-dist";
|
||||
import type {
|
||||
OffsetMap,
|
||||
PageInfo,
|
||||
PageMap,
|
||||
PageOffsetRange,
|
||||
} from "@shared/document";
|
||||
import { normalize } from "@shared/text/normalize";
|
||||
|
||||
const PAGE_SEPARATOR = "\n\n";
|
||||
|
||||
export interface PdfExtractionResult {
|
||||
readonly canonicalText: string;
|
||||
readonly pageMap: PageMap;
|
||||
readonly offsetMap: OffsetMap;
|
||||
readonly pageCount: number;
|
||||
}
|
||||
|
||||
export async function extractPdf(bytes: Uint8Array): Promise<PdfExtractionResult> {
|
||||
// PDF.js mutates the bytes buffer (transfers ownership). Pass a fresh copy
|
||||
// so the caller's Uint8Array stays usable for fingerprinting after extract.
|
||||
const data = new Uint8Array(bytes);
|
||||
const loadingTask = getDocument({ data });
|
||||
const doc = await loadingTask.promise;
|
||||
|
||||
try {
|
||||
const pageCount = doc.numPages;
|
||||
const pageInfos: PageInfo[] = [];
|
||||
const pageNormalizedTexts: string[] = [];
|
||||
|
||||
for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
|
||||
const page = await doc.getPage(pageNumber);
|
||||
try {
|
||||
const viewport = page.getViewport({ scale: 1 });
|
||||
pageInfos.push({
|
||||
page: pageNumber,
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
});
|
||||
|
||||
const rawText = await extractPageText(page);
|
||||
pageNormalizedTexts.push(normalize(rawText).text);
|
||||
} finally {
|
||||
page.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
const { canonicalText, offsetMap } = buildOffsetMap(pageNormalizedTexts);
|
||||
|
||||
return {
|
||||
canonicalText,
|
||||
pageMap: pageInfos,
|
||||
offsetMap,
|
||||
pageCount,
|
||||
};
|
||||
} finally {
|
||||
await doc.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
async function extractPageText(page: PDFPageProxy): Promise<string> {
|
||||
const content = await page.getTextContent();
|
||||
// textContent.items are TextItem | TextMarkedContent. We want only the
|
||||
// TextItem strings (those have a `str` field); marked-content entries are
|
||||
// structural anchors and have no visible text.
|
||||
const parts: string[] = [];
|
||||
for (const item of content.items) {
|
||||
if ("str" in item) {
|
||||
parts.push(item.str);
|
||||
if (item.hasEOL) parts.push("\n");
|
||||
}
|
||||
}
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function buildOffsetMap(pageTexts: readonly string[]): {
|
||||
canonicalText: string;
|
||||
offsetMap: OffsetMap;
|
||||
} {
|
||||
const ranges: PageOffsetRange[] = [];
|
||||
let offset = 0;
|
||||
for (let i = 0; i < pageTexts.length; i++) {
|
||||
const text = pageTexts[i]!;
|
||||
const isLast = i === pageTexts.length - 1;
|
||||
const segmentLength = text.length + (isLast ? 0 : PAGE_SEPARATOR.length);
|
||||
const globalStart = offset;
|
||||
const globalEnd = offset + segmentLength;
|
||||
ranges.push({
|
||||
page: i + 1,
|
||||
globalStart,
|
||||
globalEnd,
|
||||
pageLength: segmentLength,
|
||||
});
|
||||
offset = globalEnd;
|
||||
}
|
||||
const canonicalText = pageTexts.join(PAGE_SEPARATOR);
|
||||
return { canonicalText, offsetMap: ranges };
|
||||
}
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
/**
|
||||
* SHA-256 fingerprint of raw document bytes.
|
||||
*
|
||||
* Implements the fingerprint half of `wiki/ArchitectureOverview.md` §3.4
|
||||
* (the "compute fingerprint" pipeline step) and populates
|
||||
* `Document.fingerprint` (`wiki/SharedContracts.md` §1).
|
||||
*
|
||||
* Uses Web Crypto's `crypto.subtle.digest`, which is available in browsers
|
||||
* and in Node ≥ 20 (where it is exposed on `globalThis.crypto`). No
|
||||
* platform branching — the API is the same in both environments.
|
||||
*/
|
||||
|
||||
export async function fingerprintBytes(bytes: Uint8Array): Promise<string> {
|
||||
// Copy into a fresh ArrayBuffer (not SharedArrayBuffer) so the digest call
|
||||
// satisfies TS's updated `BufferSource` type, which excludes
|
||||
// `SharedArrayBuffer`. The copy is O(n) — fine even for large PDFs since
|
||||
// SHA-256 itself is already O(n).
|
||||
const ab = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(ab).set(bytes);
|
||||
const digest = await crypto.subtle.digest("SHA-256", ab);
|
||||
return bytesToHex(new Uint8Array(digest));
|
||||
}
|
||||
|
||||
function bytesToHex(bytes: Uint8Array): string {
|
||||
let hex = "";
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
const b = bytes[i]!;
|
||||
hex += (b < 0x10 ? "0" : "") + b.toString(16);
|
||||
}
|
||||
return hex;
|
||||
}
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
/**
|
||||
* Fixture-driven contract tests for the PDF ingest pipeline.
|
||||
*
|
||||
* For each fixture in `fixtures/pdfs/manifest.json`:
|
||||
* 1. Read the PDF bytes from disk.
|
||||
* 2. Run `ingestPdf` end-to-end.
|
||||
* 3. Assert the resulting Document + DocumentRepresentation honour the
|
||||
* manifest contract: media type is application/pdf, fingerprint is a
|
||||
* 64-hex SHA-256, pageMap matches `page_count`, canonicalText
|
||||
* contains `known_good_quote`, and the offsetMap covers
|
||||
* `[0, canonicalText.length)` with no gaps.
|
||||
*
|
||||
* This is the verification gate for CE-WP-0002-T03.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
import { ingestPdf } from "./ingest";
|
||||
import { fingerprintBytes } from "./fingerprint";
|
||||
import manifest from "../../../fixtures/pdfs/manifest.json" with { type: "json" };
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURE_DIR = resolve(__dirname, "../../../fixtures/pdfs");
|
||||
|
||||
interface Fixture {
|
||||
id: string;
|
||||
filename: string;
|
||||
page_count: number;
|
||||
known_good_quote: string;
|
||||
known_good_quote_page: number;
|
||||
}
|
||||
|
||||
const FIXTURES: readonly Fixture[] = manifest.fixtures;
|
||||
|
||||
beforeAll(async () => {
|
||||
// PDF.js needs a workerSrc set. In Node tests we point it at the legacy
|
||||
// worker bundle — the modern bundle uses APIs that aren't present in
|
||||
// Node. The legacy worker is bundled as plain JS and runs through the
|
||||
// fake-worker fallback that PDF.js spins up when no real Worker is
|
||||
// available.
|
||||
const pdfjs = await import("pdfjs-dist");
|
||||
const require = createRequire(import.meta.url);
|
||||
pdfjs.GlobalWorkerOptions.workerSrc = require.resolve(
|
||||
"pdfjs-dist/legacy/build/pdf.worker.mjs",
|
||||
);
|
||||
});
|
||||
|
||||
describe("ingestPdf — fixture corpus", { timeout: 30_000 }, () => {
|
||||
for (const fixture of FIXTURES) {
|
||||
describe(fixture.id, () => {
|
||||
const path = resolve(FIXTURE_DIR, fixture.filename);
|
||||
const bytes = new Uint8Array(readFileSync(path));
|
||||
|
||||
it("produces a Document with PDF media type and SHA-256 fingerprint", async () => {
|
||||
const { document } = await ingestPdf(bytes, { filename: fixture.filename });
|
||||
expect(document.mediaType).toBe("application/pdf");
|
||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(document.title).toBe(fixture.filename);
|
||||
// Fingerprint must be deterministic across runs.
|
||||
const expected = await fingerprintBytes(bytes);
|
||||
expect(document.fingerprint).toBe(expected);
|
||||
});
|
||||
|
||||
it("produces a pdf-text representation with the expected page count", async () => {
|
||||
const { representation } = await ingestPdf(bytes);
|
||||
expect(representation.representationType).toBe("pdf-text");
|
||||
expect(representation.pageMap?.length).toBe(fixture.page_count);
|
||||
expect(representation.offsetMap?.length).toBe(fixture.page_count);
|
||||
});
|
||||
|
||||
it("canonical text contains the manifest's known-good quote", async () => {
|
||||
const { representation } = await ingestPdf(bytes);
|
||||
const text = representation.canonicalText ?? "";
|
||||
expect(text).toContain(fixture.known_good_quote);
|
||||
});
|
||||
|
||||
it("offsetMap is gap-free and covers [0, canonicalText.length)", async () => {
|
||||
const { representation } = await ingestPdf(bytes);
|
||||
const text = representation.canonicalText ?? "";
|
||||
const offsets = representation.offsetMap ?? [];
|
||||
expect(offsets.length).toBeGreaterThan(0);
|
||||
expect(offsets[0]!.globalStart).toBe(0);
|
||||
expect(offsets.at(-1)!.globalEnd).toBe(text.length);
|
||||
for (let i = 0; i < offsets.length; i++) {
|
||||
const r = offsets[i]!;
|
||||
expect(r.page).toBe(i + 1);
|
||||
expect(r.globalEnd - r.globalStart).toBe(r.pageLength);
|
||||
if (i > 0) expect(r.globalStart).toBe(offsets[i - 1]!.globalEnd);
|
||||
}
|
||||
});
|
||||
|
||||
it("pageMap entries have positive width and height in user-space points", async () => {
|
||||
const { representation } = await ingestPdf(bytes);
|
||||
const pages = representation.pageMap ?? [];
|
||||
for (let i = 0; i < pages.length; i++) {
|
||||
const p = pages[i]!;
|
||||
expect(p.page).toBe(i + 1);
|
||||
expect(p.width).toBeGreaterThan(0);
|
||||
expect(p.height).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe("ingestPdf — option handling", () => {
|
||||
const fixture = FIXTURES[0]!;
|
||||
const path = resolve(FIXTURE_DIR, fixture.filename);
|
||||
const bytes = new Uint8Array(readFileSync(path));
|
||||
|
||||
it("uses explicit title over filename", async () => {
|
||||
const { document } = await ingestPdf(bytes, {
|
||||
filename: fixture.filename,
|
||||
title: "Custom Title",
|
||||
});
|
||||
expect(document.title).toBe("Custom Title");
|
||||
});
|
||||
|
||||
it("omits title entirely when neither filename nor title is supplied", async () => {
|
||||
const { document } = await ingestPdf(bytes);
|
||||
expect(document.title).toBeUndefined();
|
||||
});
|
||||
|
||||
it("propagates uri and metadata when supplied", async () => {
|
||||
const { document } = await ingestPdf(bytes, {
|
||||
uri: "file:///example.pdf",
|
||||
metadata: { source: "test" },
|
||||
});
|
||||
expect(document.uri).toBe("file:///example.pdf");
|
||||
expect(document.metadata).toEqual({ source: "test" });
|
||||
});
|
||||
|
||||
it("accepts ArrayBuffer input", async () => {
|
||||
const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
||||
const { document } = await ingestPdf(ab);
|
||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
/**
|
||||
* PDF ingest pipeline → `{ document, representation }`.
|
||||
*
|
||||
* Implements `wiki/ArchitectureOverview.md` §3.4 ("Raw Source → identify
|
||||
* media type → compute fingerprint → extract metadata → extract canonical
|
||||
* text → build format-specific maps → persist Document +
|
||||
* DocumentRepresentation") for the PDF source format.
|
||||
*
|
||||
* Ingest is a pure function over bytes: it does not persist anything. The
|
||||
* caller (engine repositories in T05, app layer in T06) writes the returned
|
||||
* Document + DocumentRepresentation into the chosen store.
|
||||
*/
|
||||
|
||||
import {
|
||||
type Document,
|
||||
type DocumentRepresentation,
|
||||
} from "@shared/document";
|
||||
import { newId } from "@shared/ids";
|
||||
import { extractPdf } from "./extract";
|
||||
import { fingerprintBytes } from "./fingerprint";
|
||||
|
||||
const PDF_MEDIA_TYPE = "application/pdf";
|
||||
|
||||
export interface IngestPdfOptions {
|
||||
/** Original filename, used as the default title when no title is given. */
|
||||
readonly filename?: string;
|
||||
/** Optional pre-existing title (overrides filename). */
|
||||
readonly title?: string;
|
||||
/** Optional source URI (e.g. file:// or https://). */
|
||||
readonly uri?: string;
|
||||
/** Free-form metadata persisted on the Document record. */
|
||||
readonly metadata?: Readonly<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface IngestPdfResult {
|
||||
readonly document: Document;
|
||||
readonly representation: DocumentRepresentation;
|
||||
}
|
||||
|
||||
export type IngestPdfInput = Uint8Array | ArrayBuffer | Blob;
|
||||
|
||||
export async function ingestPdf(
|
||||
input: IngestPdfInput,
|
||||
options: IngestPdfOptions = {},
|
||||
): Promise<IngestPdfResult> {
|
||||
const bytes = await toBytes(input);
|
||||
const [fingerprint, extraction] = await Promise.all([
|
||||
fingerprintBytes(bytes),
|
||||
extractPdf(bytes),
|
||||
]);
|
||||
|
||||
const now = new Date().toISOString();
|
||||
const documentId = newId("document");
|
||||
const representationId = newId("representation");
|
||||
const title = options.title ?? options.filename;
|
||||
|
||||
const document: Document = {
|
||||
id: documentId,
|
||||
mediaType: PDF_MEDIA_TYPE,
|
||||
fingerprint,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
...(title !== undefined ? { title } : {}),
|
||||
...(options.uri !== undefined ? { uri: options.uri } : {}),
|
||||
...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
|
||||
};
|
||||
|
||||
const representation: DocumentRepresentation = {
|
||||
id: representationId,
|
||||
documentId,
|
||||
representationType: "pdf-text",
|
||||
contentHash: fingerprint,
|
||||
canonicalText: extraction.canonicalText,
|
||||
pageMap: extraction.pageMap,
|
||||
offsetMap: extraction.offsetMap,
|
||||
generatedAt: now,
|
||||
};
|
||||
|
||||
return { document, representation };
|
||||
}
|
||||
|
||||
async function toBytes(input: IngestPdfInput): Promise<Uint8Array> {
|
||||
if (input instanceof Uint8Array) return input;
|
||||
if (input instanceof ArrayBuffer) return new Uint8Array(input);
|
||||
// Blob (covers `File` in browsers — File extends Blob).
|
||||
const buf = await input.arrayBuffer();
|
||||
return new Uint8Array(buf);
|
||||
}
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
/**
|
||||
* `ingestPdfFromFile` end-to-end: pipes a fixture PDF through the
|
||||
* upload path, asserts the byte store keeps the bytes and the document
|
||||
* record carries the minted `blob:` URL.
|
||||
*/
|
||||
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createPdfByteStore } from "./byte-store";
|
||||
import { ingestPdfFromFile } from "./upload";
|
||||
|
||||
const FIXTURE_PATH = new URL(
|
||||
"../../../fixtures/pdfs/Fristsetzung zur Bezifferung GÜ an Gegenseite 3 Wochen.pdf",
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
async function fixtureBytes(): Promise<Uint8Array> {
|
||||
return new Uint8Array(await readFile(FIXTURE_PATH));
|
||||
}
|
||||
|
||||
class FakeFile {
|
||||
readonly name: string;
|
||||
private readonly bytes: Uint8Array;
|
||||
constructor(bytes: Uint8Array, name: string) {
|
||||
this.bytes = bytes;
|
||||
this.name = name;
|
||||
}
|
||||
async arrayBuffer(): Promise<ArrayBuffer> {
|
||||
const out = new ArrayBuffer(this.bytes.byteLength);
|
||||
new Uint8Array(out).set(this.bytes);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
describe("ingestPdfFromFile", () => {
|
||||
it("round-trips a fixture PDF through ingest + byte store + blob URL", async () => {
|
||||
const bytes = await fixtureBytes();
|
||||
const file = new FakeFile(bytes, "demo.pdf") as unknown as File;
|
||||
let counter = 0;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: () => `blob:upload-stub-${++counter}`,
|
||||
revokeObjectURL: () => {},
|
||||
});
|
||||
|
||||
const { document, representation } = await ingestPdfFromFile(file, store);
|
||||
|
||||
// Bytes are stored, retrievable by document id.
|
||||
const stored = store.get(document.id);
|
||||
expect(stored).not.toBeNull();
|
||||
expect(stored!.bytes.byteLength).toBe(bytes.byteLength);
|
||||
|
||||
// Document carries the blob URL minted by the store.
|
||||
expect(document.uri).toBe(`blob:upload-stub-${counter}`);
|
||||
expect(document.title).toBe("demo.pdf");
|
||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
||||
|
||||
// Representation is the standard pdf-text one.
|
||||
expect(representation.representationType).toBe("pdf-text");
|
||||
expect((representation.canonicalText ?? "").length).toBeGreaterThan(0);
|
||||
}, 30_000);
|
||||
|
||||
it("falls through to ingestPdf with no filename when given a plain Blob", async () => {
|
||||
const bytes = await fixtureBytes();
|
||||
const blob = {
|
||||
async arrayBuffer() {
|
||||
const out = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(out).set(bytes);
|
||||
return out;
|
||||
},
|
||||
} as Blob;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: () => "blob:no-name",
|
||||
revokeObjectURL: () => {},
|
||||
});
|
||||
const { document } = await ingestPdfFromFile(blob, store);
|
||||
expect(document.title).toBeUndefined();
|
||||
expect(document.uri).toBe("blob:no-name");
|
||||
}, 30_000);
|
||||
|
||||
it("explicit title option overrides the filename", async () => {
|
||||
const bytes = await fixtureBytes();
|
||||
const file = new FakeFile(bytes, "anonymous-name.pdf") as unknown as File;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: vi.fn(() => "blob:override"),
|
||||
revokeObjectURL: vi.fn(),
|
||||
});
|
||||
const { document } = await ingestPdfFromFile(file, store, { title: "Custom" });
|
||||
expect(document.title).toBe("Custom");
|
||||
}, 30_000);
|
||||
});
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/**
|
||||
* Upload-side ingest path.
|
||||
*
|
||||
* The fixture-loading path in `App.tsx` fetches a known URL and calls
|
||||
* `ingestPdf` directly; that path stays untouched for the optional
|
||||
* "Sample sessions" quick-start. Uploaded files flow through here
|
||||
* instead:
|
||||
*
|
||||
* 1. Read `file.arrayBuffer()` once into a `Uint8Array`.
|
||||
* 2. Run the existing `ingestPdf(bytes, { filename })` pipeline to
|
||||
* produce `{document, representation}`.
|
||||
* 3. Push the bytes into the per-session `PdfByteStore`, which mints
|
||||
* a `blob:` URL and stamps it onto `document.uri` so the viewer
|
||||
* adapter can mount the PDF directly.
|
||||
* 4. Hand the engine inputs back to the caller, which wires them via
|
||||
* `engine.documents.register(...)`.
|
||||
*
|
||||
* Keeping URL-minting inside the byte store (rather than at the call
|
||||
* site) means there is exactly one place that creates `blob:` URLs and
|
||||
* exactly one place that revokes them.
|
||||
*/
|
||||
|
||||
import { ingestPdf, type IngestPdfResult } from "./ingest";
|
||||
import type { PdfByteStore } from "./byte-store";
|
||||
|
||||
export interface IngestPdfFromFileOptions {
|
||||
/** Override the filename used as the document title. */
|
||||
readonly title?: string;
|
||||
}
|
||||
|
||||
export async function ingestPdfFromFile(
|
||||
file: File | Blob,
|
||||
store: PdfByteStore,
|
||||
options: IngestPdfFromFileOptions = {},
|
||||
): Promise<IngestPdfResult> {
|
||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
||||
const filename = "name" in file && typeof file.name === "string" ? file.name : undefined;
|
||||
const ingested = await ingestPdf(bytes, {
|
||||
...(filename !== undefined ? { filename } : {}),
|
||||
...(options.title !== undefined ? { title: options.title } : {}),
|
||||
});
|
||||
const record = store.put(ingested.document.id, bytes);
|
||||
const document = { ...ingested.document, uri: record.blobUrl };
|
||||
return { document, representation: ingested.representation };
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
|
|||
import type { Document } from "@shared/document";
|
||||
import type { DocumentId } from "@shared/ids";
|
||||
|
||||
import { createPdfByteStore } from "./byte-store";
|
||||
import { createPdfByteStore } from "@citation-evidence/evidence-source/browser";
|
||||
import { isEphemeralBlobUri, resolvePdfViewerUrl } from "./viewer-url";
|
||||
|
||||
function doc(overrides: Partial<Document> = {}): Document {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@
|
|||
import type { Document } from "@shared/document";
|
||||
import type { DocumentId } from "@shared/ids";
|
||||
|
||||
import type { PdfByteStore } from "./byte-store";
|
||||
// PdfByteStore was extracted into the standalone evidence-source package
|
||||
// (ESRC-WP-0001). viewer-url stays local to this app (T04) but consumes the
|
||||
// store type from the package.
|
||||
import type { PdfByteStore } from "@citation-evidence/evidence-source/browser";
|
||||
|
||||
export function isEphemeralBlobUri(uri: string | undefined): boolean {
|
||||
return typeof uri === "string" && uri.startsWith("blob:");
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import { createRequire } from "node:module";
|
|||
import { fileURLToPath } from "node:url";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
import { ingestPdf } from "@source/pdf/ingest";
|
||||
import { ingestPdf } from "@citation-evidence/evidence-source";
|
||||
import { createSelectors, resolveSelectors } from "@anchor/selectors";
|
||||
import type { PdfSelectionCapture } from "@anchor/types";
|
||||
import manifest from "../../fixtures/pdfs/manifest.json" with { type: "json" };
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue