refactor(source): consume extracted evidence-source package (ESRC-WP-0001-T06)
Replace the local src/source PDF slice with the standalone @citation-evidence/evidence-source package (link:../evidence-source). - src/source/index.ts is now a thin façade re-exporting the ingest core and browser upload helpers from the package - viewer-url stays local (viewer concern) but takes PdfByteStore from the package - extracted ingest/extract/fingerprint/byte-store/upload files removed - anchor-source round-trip integration test now exercises the package across the repo boundary Full suite (125 tests), typecheck, and lint all green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
54b585197b
commit
4ede97983a
15 changed files with 54 additions and 746 deletions
|
|
@ -20,6 +20,7 @@
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@citation-evidence/engine": "link:../citation-engine",
|
"@citation-evidence/engine": "link:../citation-engine",
|
||||||
|
"@citation-evidence/evidence-source": "link:../evidence-source",
|
||||||
"jszip": "^3.10.1",
|
"jszip": "^3.10.1",
|
||||||
"pdfjs-dist": "^4.4.168",
|
"pdfjs-dist": "^4.4.168",
|
||||||
"react": "^18.3.1",
|
"react": "^18.3.1",
|
||||||
|
|
|
||||||
3
pnpm-lock.yaml
generated
3
pnpm-lock.yaml
generated
|
|
@ -11,6 +11,9 @@ importers:
|
||||||
'@citation-evidence/engine':
|
'@citation-evidence/engine':
|
||||||
specifier: link:../citation-engine
|
specifier: link:../citation-engine
|
||||||
version: link:../citation-engine
|
version: link:../citation-engine
|
||||||
|
'@citation-evidence/evidence-source':
|
||||||
|
specifier: link:../evidence-source
|
||||||
|
version: link:../evidence-source
|
||||||
jszip:
|
jszip:
|
||||||
specifier: ^3.10.1
|
specifier: ^3.10.1
|
||||||
version: 3.10.1
|
version: 3.10.1
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,21 @@
|
||||||
# `src/source/` — ingest, fingerprint, representation extraction, recovery
|
# `src/source/` — façade over the extracted `evidence-source` package
|
||||||
|
|
||||||
Future home: `evidence-source`.
|
The headless ingest core and browser upload helpers that used to live here
|
||||||
Owns: PDF/HTML/MD ingest, fingerprinting, page-/offset-map construction,
|
were extracted into the standalone `@citation-evidence/evidence-source` repo
|
||||||
canonical-text extraction, and citation-recovery behavior.
|
(ESRC-WP-0001). This directory is now a thin façade:
|
||||||
|
|
||||||
May import from: `shared/`, `engine/` (`wiki/DependencyMap.md` §4).
|
- `index.ts` re-exports the ingest core (`ingestPdf`, `extractPdf`,
|
||||||
|
`fingerprintBytes`) from `@citation-evidence/evidence-source` and the browser
|
||||||
|
upload helpers (`createPdfByteStore`, `ingestPdfFromFile`) from
|
||||||
|
`@citation-evidence/evidence-source/browser`.
|
||||||
|
- `pdf/viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…`
|
||||||
|
URL convention and blob-vs-fixture fallback policy — viewer concerns, not
|
||||||
|
headless ingest (ESRC-WP-0001 ADR-0001, T04). It consumes the `PdfByteStore`
|
||||||
|
type from the extracted package.
|
||||||
|
|
||||||
|
The source ↔ anchor round-trip integration test remains upstream in
|
||||||
|
`tests/integration/anchor-source-roundtrip.test.ts` and now exercises the
|
||||||
|
extracted package across the repo boundary.
|
||||||
|
|
||||||
|
Package dependency: `@citation-evidence/evidence-source` via
|
||||||
|
`link:../evidence-source` (sibling checkout).
|
||||||
|
|
|
||||||
|
|
@ -1,21 +1,39 @@
|
||||||
|
/**
|
||||||
|
* `src/source/` — façade over the extracted `evidence-source` package.
|
||||||
|
*
|
||||||
|
* The headless PDF ingest core (ingest / extract / fingerprint) and the
|
||||||
|
* browser upload helpers (byte store / upload) were extracted into the
|
||||||
|
* standalone `@citation-evidence/evidence-source` repo (ESRC-WP-0001). This
|
||||||
|
* module re-exports them so the rest of the umbrella keeps importing from
|
||||||
|
* `@source/index` unchanged.
|
||||||
|
*
|
||||||
|
* `viewer-url.ts` stays local: it encodes this app's `/fixtures/pdfs/…` URL
|
||||||
|
* convention and blob-vs-fixture fallback policy, which are viewer concerns,
|
||||||
|
* not headless ingest (ESRC-WP-0001 ADR-0001, T04).
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Headless ingest core — from the extracted package root.
|
||||||
export {
|
export {
|
||||||
ingestPdf,
|
ingestPdf,
|
||||||
|
extractPdf,
|
||||||
|
fingerprintBytes,
|
||||||
type IngestPdfInput,
|
type IngestPdfInput,
|
||||||
type IngestPdfOptions,
|
type IngestPdfOptions,
|
||||||
type IngestPdfResult,
|
type IngestPdfResult,
|
||||||
} from "./pdf/ingest";
|
type PdfExtractionResult,
|
||||||
export { extractPdf, type PdfExtractionResult } from "./pdf/extract";
|
} from "@citation-evidence/evidence-source";
|
||||||
export { fingerprintBytes } from "./pdf/fingerprint";
|
|
||||||
|
// Browser upload helpers — from the package's browser entry point.
|
||||||
export {
|
export {
|
||||||
createPdfByteStore,
|
createPdfByteStore,
|
||||||
|
ingestPdfFromFile,
|
||||||
type CreatePdfByteStoreOptions,
|
type CreatePdfByteStoreOptions,
|
||||||
type PdfByteRecord,
|
type PdfByteRecord,
|
||||||
type PdfByteStore,
|
type PdfByteStore,
|
||||||
} from "./pdf/byte-store";
|
|
||||||
export {
|
|
||||||
ingestPdfFromFile,
|
|
||||||
type IngestPdfFromFileOptions,
|
type IngestPdfFromFileOptions,
|
||||||
} from "./pdf/upload";
|
} from "@citation-evidence/evidence-source/browser";
|
||||||
|
|
||||||
|
// Viewer URL resolution — stays local to this app.
|
||||||
export {
|
export {
|
||||||
isEphemeralBlobUri,
|
isEphemeralBlobUri,
|
||||||
resolvePdfViewerUrl,
|
resolvePdfViewerUrl,
|
||||||
|
|
|
||||||
|
|
@ -1,99 +0,0 @@
|
||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
|
|
||||||
import type { DocumentId } from "@shared/ids";
|
|
||||||
|
|
||||||
import { createPdfByteStore } from "./byte-store";
|
|
||||||
|
|
||||||
function stubUrlHelpers() {
|
|
||||||
let counter = 0;
|
|
||||||
const created: string[] = [];
|
|
||||||
const revoked: string[] = [];
|
|
||||||
const createObjectURL = vi.fn(() => {
|
|
||||||
const url = `blob:stub-${++counter}`;
|
|
||||||
created.push(url);
|
|
||||||
return url;
|
|
||||||
});
|
|
||||||
const revokeObjectURL = vi.fn((url: string) => {
|
|
||||||
revoked.push(url);
|
|
||||||
});
|
|
||||||
return { createObjectURL, revokeObjectURL, created, revoked };
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("PdfByteStore", () => {
|
|
||||||
it("put / get round-trips bytes and exposes a blob URL", () => {
|
|
||||||
const helpers = stubUrlHelpers();
|
|
||||||
const store = createPdfByteStore(helpers);
|
|
||||||
const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]); // %PDF
|
|
||||||
const record = store.put("doc_a" as DocumentId, bytes);
|
|
||||||
expect(record.blobUrl).toBe("blob:stub-1");
|
|
||||||
expect(store.get("doc_a" as DocumentId)?.bytes).toBe(bytes);
|
|
||||||
expect(store.has("doc_a" as DocumentId)).toBe(true);
|
|
||||||
expect(store.list()).toEqual(["doc_a"]);
|
|
||||||
expect(store.size()).toBe(4);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("put replaces an existing entry and revokes the old URL", () => {
|
|
||||||
const helpers = stubUrlHelpers();
|
|
||||||
const store = createPdfByteStore(helpers);
|
|
||||||
const id = "doc_a" as DocumentId;
|
|
||||||
const first = store.put(id, new Uint8Array([1, 2]));
|
|
||||||
const second = store.put(id, new Uint8Array([3, 4, 5]));
|
|
||||||
expect(helpers.revoked).toEqual([first.blobUrl]);
|
|
||||||
expect(store.get(id)?.bytes).toHaveLength(3);
|
|
||||||
expect(second.blobUrl).not.toBe(first.blobUrl);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("delete revokes the blob URL exactly once and is idempotent", () => {
|
|
||||||
const helpers = stubUrlHelpers();
|
|
||||||
const store = createPdfByteStore(helpers);
|
|
||||||
const id = "doc_a" as DocumentId;
|
|
||||||
const record = store.put(id, new Uint8Array([1, 2, 3]));
|
|
||||||
expect(store.delete(id)).toBe(true);
|
|
||||||
expect(helpers.revoked).toEqual([record.blobUrl]);
|
|
||||||
expect(store.delete(id)).toBe(false);
|
|
||||||
// No additional revoke calls on the second delete.
|
|
||||||
expect(helpers.revoked).toHaveLength(1);
|
|
||||||
expect(store.get(id)).toBeNull();
|
|
||||||
expect(store.has(id)).toBe(false);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("clear revokes every URL and empties the store", () => {
|
|
||||||
const helpers = stubUrlHelpers();
|
|
||||||
const store = createPdfByteStore(helpers);
|
|
||||||
const a = store.put("doc_a" as DocumentId, new Uint8Array([1]));
|
|
||||||
const b = store.put("doc_b" as DocumentId, new Uint8Array([2]));
|
|
||||||
store.clear();
|
|
||||||
expect(helpers.revoked.sort()).toEqual([a.blobUrl, b.blobUrl].sort());
|
|
||||||
expect(store.list()).toEqual([]);
|
|
||||||
expect(store.size()).toBe(0);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("uses URL.createObjectURL by default when no override is supplied", () => {
|
|
||||||
const createObjectURL = vi.fn(() => "blob:built-in");
|
|
||||||
const revokeObjectURL = vi.fn();
|
|
||||||
const originalURL = globalThis.URL;
|
|
||||||
// happy-dom's URL has createObjectURL; node sometimes does not. Stub it.
|
|
||||||
Object.defineProperty(globalThis, "URL", {
|
|
||||||
configurable: true,
|
|
||||||
writable: true,
|
|
||||||
value: Object.assign(Object.create(originalURL.prototype as object), {
|
|
||||||
createObjectURL,
|
|
||||||
revokeObjectURL,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
try {
|
|
||||||
const store = createPdfByteStore();
|
|
||||||
const rec = store.put("doc_z" as DocumentId, new Uint8Array([9]));
|
|
||||||
expect(rec.blobUrl).toBe("blob:built-in");
|
|
||||||
expect(createObjectURL).toHaveBeenCalledTimes(1);
|
|
||||||
store.delete("doc_z" as DocumentId);
|
|
||||||
expect(revokeObjectURL).toHaveBeenCalledWith("blob:built-in");
|
|
||||||
} finally {
|
|
||||||
Object.defineProperty(globalThis, "URL", {
|
|
||||||
configurable: true,
|
|
||||||
writable: true,
|
|
||||||
value: originalURL,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,113 +0,0 @@
|
||||||
/**
|
|
||||||
* `PdfByteStore` — in-memory store for uploaded PDF bytes, keyed by
|
|
||||||
* `DocumentId`.
|
|
||||||
*
|
|
||||||
* CE-WP-0005 stores uploaded PDFs in memory only (per the workplan
|
|
||||||
* scoping decision). Bytes survive within a tab session; reloading the
|
|
||||||
* page loses them unless the user exported a ZIP. Re-importing the ZIP
|
|
||||||
* restores them.
|
|
||||||
*
|
|
||||||
* One store instance per active session. The session-management layer
|
|
||||||
* is responsible for swapping the active store when the user switches
|
|
||||||
* sessions. The store also owns a small registry of issued `blob:`
|
|
||||||
* URLs so it can revoke them on delete/clear — no cross-cutting
|
|
||||||
* cleanup at the app layer.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import type { DocumentId } from "@shared/ids";
|
|
||||||
|
|
||||||
export interface PdfByteRecord {
|
|
||||||
readonly bytes: Uint8Array;
|
|
||||||
/** A `blob:` URL the viewer can consume directly. */
|
|
||||||
readonly blobUrl: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PdfByteStore {
|
|
||||||
put(documentId: DocumentId, bytes: Uint8Array): PdfByteRecord;
|
|
||||||
get(documentId: DocumentId): PdfByteRecord | null;
|
|
||||||
has(documentId: DocumentId): boolean;
|
|
||||||
delete(documentId: DocumentId): boolean;
|
|
||||||
list(): readonly DocumentId[];
|
|
||||||
/** Revoke every blob URL and clear the store. */
|
|
||||||
clear(): void;
|
|
||||||
/** Total bytes currently held — useful for UI dashboards. */
|
|
||||||
size(): number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface CreatePdfByteStoreOptions {
|
|
||||||
/**
|
|
||||||
* Mint a URL for the given bytes. Defaults to `URL.createObjectURL` in
|
|
||||||
* environments that have it; tests can inject a deterministic stub.
|
|
||||||
*/
|
|
||||||
readonly createObjectURL?: (blob: Blob) => string;
|
|
||||||
/** Revoke a URL previously minted by `createObjectURL`. */
|
|
||||||
readonly revokeObjectURL?: (url: string) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function createPdfByteStore(
|
|
||||||
options: CreatePdfByteStoreOptions = {},
|
|
||||||
): PdfByteStore {
|
|
||||||
const createUrl =
|
|
||||||
options.createObjectURL ??
|
|
||||||
((blob: Blob) => {
|
|
||||||
if (typeof URL === "undefined" || typeof URL.createObjectURL !== "function") {
|
|
||||||
throw new Error(
|
|
||||||
"createPdfByteStore: URL.createObjectURL not available — inject a stub via options",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return URL.createObjectURL(blob);
|
|
||||||
});
|
|
||||||
const revokeUrl =
|
|
||||||
options.revokeObjectURL ??
|
|
||||||
((url: string) => {
|
|
||||||
if (typeof URL !== "undefined" && typeof URL.revokeObjectURL === "function") {
|
|
||||||
URL.revokeObjectURL(url);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
const records = new Map<DocumentId, PdfByteRecord>();
|
|
||||||
|
|
||||||
return {
|
|
||||||
put(documentId, bytes) {
|
|
||||||
// Replace previous record (revoking the prior URL) if any.
|
|
||||||
const prior = records.get(documentId);
|
|
||||||
if (prior) revokeUrl(prior.blobUrl);
|
|
||||||
// Cast: Blob() does accept Uint8Array at runtime, but TS narrows the
|
|
||||||
// buffer type to ArrayBufferLike (could be SharedArrayBuffer) and
|
|
||||||
// refuses without help. The bytes here always come from a fresh
|
|
||||||
// arrayBuffer() call, so a regular ArrayBuffer is guaranteed.
|
|
||||||
const blob = new Blob([bytes as unknown as ArrayBuffer], {
|
|
||||||
type: "application/pdf",
|
|
||||||
});
|
|
||||||
const blobUrl = createUrl(blob);
|
|
||||||
const record: PdfByteRecord = { bytes, blobUrl };
|
|
||||||
records.set(documentId, record);
|
|
||||||
return record;
|
|
||||||
},
|
|
||||||
get(documentId) {
|
|
||||||
return records.get(documentId) ?? null;
|
|
||||||
},
|
|
||||||
has(documentId) {
|
|
||||||
return records.has(documentId);
|
|
||||||
},
|
|
||||||
delete(documentId) {
|
|
||||||
const record = records.get(documentId);
|
|
||||||
if (!record) return false;
|
|
||||||
revokeUrl(record.blobUrl);
|
|
||||||
records.delete(documentId);
|
|
||||||
return true;
|
|
||||||
},
|
|
||||||
list() {
|
|
||||||
return [...records.keys()];
|
|
||||||
},
|
|
||||||
clear() {
|
|
||||||
for (const record of records.values()) revokeUrl(record.blobUrl);
|
|
||||||
records.clear();
|
|
||||||
},
|
|
||||||
size() {
|
|
||||||
let total = 0;
|
|
||||||
for (const r of records.values()) total += r.bytes.byteLength;
|
|
||||||
return total;
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -1,122 +0,0 @@
|
||||||
/**
|
|
||||||
* PDF text extraction → canonical text + PageMap + OffsetMap.
|
|
||||||
*
|
|
||||||
* Implements `wiki/ArchitectureOverview.md` §3.4 ("extract canonical text /
|
|
||||||
* build format-specific maps") for the `pdf-text` representation
|
|
||||||
* (`wiki/SharedContracts.md` §1, §3) and §6 (canonical normalization).
|
|
||||||
*
|
|
||||||
* Runtime independence: the PDF.js worker must be configured by the host
|
|
||||||
* application (`GlobalWorkerOptions.workerSrc`) before this module is
|
|
||||||
* called. In Vite/browser code the worker is bundled via the viewer; in
|
|
||||||
* Node tests the test setup file points it at
|
|
||||||
* `pdfjs-dist/legacy/build/pdf.worker.mjs`. No worker setup happens here
|
|
||||||
* so the same module loads cleanly in both runtimes.
|
|
||||||
*
|
|
||||||
* Page boundary semantics: canonical text concatenates per-page normalized
|
|
||||||
* text with a single "\n\n" paragraph separator. The separator is treated
|
|
||||||
* as belonging to the *preceding* page in `OffsetMap`, so the map covers
|
|
||||||
* `[0, canonicalText.length)` with no gaps. The last page has no trailing
|
|
||||||
* separator. This means `pageLength = globalEnd - globalStart` for
|
|
||||||
* every page; for non-last pages it equals (normalized page text length +
|
|
||||||
* 2). See `PageOffsetRange` in `@shared/document.ts`.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { getDocument } from "pdfjs-dist";
|
|
||||||
import type { PDFPageProxy } from "pdfjs-dist";
|
|
||||||
import type {
|
|
||||||
OffsetMap,
|
|
||||||
PageInfo,
|
|
||||||
PageMap,
|
|
||||||
PageOffsetRange,
|
|
||||||
} from "@shared/document";
|
|
||||||
import { normalize } from "@shared/text/normalize";
|
|
||||||
|
|
||||||
const PAGE_SEPARATOR = "\n\n";
|
|
||||||
|
|
||||||
export interface PdfExtractionResult {
|
|
||||||
readonly canonicalText: string;
|
|
||||||
readonly pageMap: PageMap;
|
|
||||||
readonly offsetMap: OffsetMap;
|
|
||||||
readonly pageCount: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function extractPdf(bytes: Uint8Array): Promise<PdfExtractionResult> {
|
|
||||||
// PDF.js mutates the bytes buffer (transfers ownership). Pass a fresh copy
|
|
||||||
// so the caller's Uint8Array stays usable for fingerprinting after extract.
|
|
||||||
const data = new Uint8Array(bytes);
|
|
||||||
const loadingTask = getDocument({ data });
|
|
||||||
const doc = await loadingTask.promise;
|
|
||||||
|
|
||||||
try {
|
|
||||||
const pageCount = doc.numPages;
|
|
||||||
const pageInfos: PageInfo[] = [];
|
|
||||||
const pageNormalizedTexts: string[] = [];
|
|
||||||
|
|
||||||
for (let pageNumber = 1; pageNumber <= pageCount; pageNumber++) {
|
|
||||||
const page = await doc.getPage(pageNumber);
|
|
||||||
try {
|
|
||||||
const viewport = page.getViewport({ scale: 1 });
|
|
||||||
pageInfos.push({
|
|
||||||
page: pageNumber,
|
|
||||||
width: viewport.width,
|
|
||||||
height: viewport.height,
|
|
||||||
});
|
|
||||||
|
|
||||||
const rawText = await extractPageText(page);
|
|
||||||
pageNormalizedTexts.push(normalize(rawText).text);
|
|
||||||
} finally {
|
|
||||||
page.cleanup();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const { canonicalText, offsetMap } = buildOffsetMap(pageNormalizedTexts);
|
|
||||||
|
|
||||||
return {
|
|
||||||
canonicalText,
|
|
||||||
pageMap: pageInfos,
|
|
||||||
offsetMap,
|
|
||||||
pageCount,
|
|
||||||
};
|
|
||||||
} finally {
|
|
||||||
await doc.destroy();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function extractPageText(page: PDFPageProxy): Promise<string> {
|
|
||||||
const content = await page.getTextContent();
|
|
||||||
// textContent.items are TextItem | TextMarkedContent. We want only the
|
|
||||||
// TextItem strings (those have a `str` field); marked-content entries are
|
|
||||||
// structural anchors and have no visible text.
|
|
||||||
const parts: string[] = [];
|
|
||||||
for (const item of content.items) {
|
|
||||||
if ("str" in item) {
|
|
||||||
parts.push(item.str);
|
|
||||||
if (item.hasEOL) parts.push("\n");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parts.join("");
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildOffsetMap(pageTexts: readonly string[]): {
|
|
||||||
canonicalText: string;
|
|
||||||
offsetMap: OffsetMap;
|
|
||||||
} {
|
|
||||||
const ranges: PageOffsetRange[] = [];
|
|
||||||
let offset = 0;
|
|
||||||
for (let i = 0; i < pageTexts.length; i++) {
|
|
||||||
const text = pageTexts[i]!;
|
|
||||||
const isLast = i === pageTexts.length - 1;
|
|
||||||
const segmentLength = text.length + (isLast ? 0 : PAGE_SEPARATOR.length);
|
|
||||||
const globalStart = offset;
|
|
||||||
const globalEnd = offset + segmentLength;
|
|
||||||
ranges.push({
|
|
||||||
page: i + 1,
|
|
||||||
globalStart,
|
|
||||||
globalEnd,
|
|
||||||
pageLength: segmentLength,
|
|
||||||
});
|
|
||||||
offset = globalEnd;
|
|
||||||
}
|
|
||||||
const canonicalText = pageTexts.join(PAGE_SEPARATOR);
|
|
||||||
return { canonicalText, offsetMap: ranges };
|
|
||||||
}
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
/**
|
|
||||||
* SHA-256 fingerprint of raw document bytes.
|
|
||||||
*
|
|
||||||
* Implements the fingerprint half of `wiki/ArchitectureOverview.md` §3.4
|
|
||||||
* (the "compute fingerprint" pipeline step) and populates
|
|
||||||
* `Document.fingerprint` (`wiki/SharedContracts.md` §1).
|
|
||||||
*
|
|
||||||
* Uses Web Crypto's `crypto.subtle.digest`, which is available in browsers
|
|
||||||
* and in Node ≥ 20 (where it is exposed on `globalThis.crypto`). No
|
|
||||||
* platform branching — the API is the same in both environments.
|
|
||||||
*/
|
|
||||||
|
|
||||||
export async function fingerprintBytes(bytes: Uint8Array): Promise<string> {
|
|
||||||
// Copy into a fresh ArrayBuffer (not SharedArrayBuffer) so the digest call
|
|
||||||
// satisfies TS's updated `BufferSource` type, which excludes
|
|
||||||
// `SharedArrayBuffer`. The copy is O(n) — fine even for large PDFs since
|
|
||||||
// SHA-256 itself is already O(n).
|
|
||||||
const ab = new ArrayBuffer(bytes.byteLength);
|
|
||||||
new Uint8Array(ab).set(bytes);
|
|
||||||
const digest = await crypto.subtle.digest("SHA-256", ab);
|
|
||||||
return bytesToHex(new Uint8Array(digest));
|
|
||||||
}
|
|
||||||
|
|
||||||
function bytesToHex(bytes: Uint8Array): string {
|
|
||||||
let hex = "";
|
|
||||||
for (let i = 0; i < bytes.length; i++) {
|
|
||||||
const b = bytes[i]!;
|
|
||||||
hex += (b < 0x10 ? "0" : "") + b.toString(16);
|
|
||||||
}
|
|
||||||
return hex;
|
|
||||||
}
|
|
||||||
|
|
@ -1,142 +0,0 @@
|
||||||
/**
|
|
||||||
* Fixture-driven contract tests for the PDF ingest pipeline.
|
|
||||||
*
|
|
||||||
* For each fixture in `fixtures/pdfs/manifest.json`:
|
|
||||||
* 1. Read the PDF bytes from disk.
|
|
||||||
* 2. Run `ingestPdf` end-to-end.
|
|
||||||
* 3. Assert the resulting Document + DocumentRepresentation honour the
|
|
||||||
* manifest contract: media type is application/pdf, fingerprint is a
|
|
||||||
* 64-hex SHA-256, pageMap matches `page_count`, canonicalText
|
|
||||||
* contains `known_good_quote`, and the offsetMap covers
|
|
||||||
* `[0, canonicalText.length)` with no gaps.
|
|
||||||
*
|
|
||||||
* This is the verification gate for CE-WP-0002-T03.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { readFileSync } from "node:fs";
|
|
||||||
import { dirname, resolve } from "node:path";
|
|
||||||
import { createRequire } from "node:module";
|
|
||||||
import { fileURLToPath } from "node:url";
|
|
||||||
import { beforeAll, describe, expect, it } from "vitest";
|
|
||||||
|
|
||||||
import { ingestPdf } from "./ingest";
|
|
||||||
import { fingerprintBytes } from "./fingerprint";
|
|
||||||
import manifest from "../../../fixtures/pdfs/manifest.json" with { type: "json" };
|
|
||||||
|
|
||||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
||||||
const FIXTURE_DIR = resolve(__dirname, "../../../fixtures/pdfs");
|
|
||||||
|
|
||||||
interface Fixture {
|
|
||||||
id: string;
|
|
||||||
filename: string;
|
|
||||||
page_count: number;
|
|
||||||
known_good_quote: string;
|
|
||||||
known_good_quote_page: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
const FIXTURES: readonly Fixture[] = manifest.fixtures;
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
// PDF.js needs a workerSrc set. In Node tests we point it at the legacy
|
|
||||||
// worker bundle — the modern bundle uses APIs that aren't present in
|
|
||||||
// Node. The legacy worker is bundled as plain JS and runs through the
|
|
||||||
// fake-worker fallback that PDF.js spins up when no real Worker is
|
|
||||||
// available.
|
|
||||||
const pdfjs = await import("pdfjs-dist");
|
|
||||||
const require = createRequire(import.meta.url);
|
|
||||||
pdfjs.GlobalWorkerOptions.workerSrc = require.resolve(
|
|
||||||
"pdfjs-dist/legacy/build/pdf.worker.mjs",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("ingestPdf — fixture corpus", { timeout: 30_000 }, () => {
|
|
||||||
for (const fixture of FIXTURES) {
|
|
||||||
describe(fixture.id, () => {
|
|
||||||
const path = resolve(FIXTURE_DIR, fixture.filename);
|
|
||||||
const bytes = new Uint8Array(readFileSync(path));
|
|
||||||
|
|
||||||
it("produces a Document with PDF media type and SHA-256 fingerprint", async () => {
|
|
||||||
const { document } = await ingestPdf(bytes, { filename: fixture.filename });
|
|
||||||
expect(document.mediaType).toBe("application/pdf");
|
|
||||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
|
||||||
expect(document.title).toBe(fixture.filename);
|
|
||||||
// Fingerprint must be deterministic across runs.
|
|
||||||
const expected = await fingerprintBytes(bytes);
|
|
||||||
expect(document.fingerprint).toBe(expected);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("produces a pdf-text representation with the expected page count", async () => {
|
|
||||||
const { representation } = await ingestPdf(bytes);
|
|
||||||
expect(representation.representationType).toBe("pdf-text");
|
|
||||||
expect(representation.pageMap?.length).toBe(fixture.page_count);
|
|
||||||
expect(representation.offsetMap?.length).toBe(fixture.page_count);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("canonical text contains the manifest's known-good quote", async () => {
|
|
||||||
const { representation } = await ingestPdf(bytes);
|
|
||||||
const text = representation.canonicalText ?? "";
|
|
||||||
expect(text).toContain(fixture.known_good_quote);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("offsetMap is gap-free and covers [0, canonicalText.length)", async () => {
|
|
||||||
const { representation } = await ingestPdf(bytes);
|
|
||||||
const text = representation.canonicalText ?? "";
|
|
||||||
const offsets = representation.offsetMap ?? [];
|
|
||||||
expect(offsets.length).toBeGreaterThan(0);
|
|
||||||
expect(offsets[0]!.globalStart).toBe(0);
|
|
||||||
expect(offsets.at(-1)!.globalEnd).toBe(text.length);
|
|
||||||
for (let i = 0; i < offsets.length; i++) {
|
|
||||||
const r = offsets[i]!;
|
|
||||||
expect(r.page).toBe(i + 1);
|
|
||||||
expect(r.globalEnd - r.globalStart).toBe(r.pageLength);
|
|
||||||
if (i > 0) expect(r.globalStart).toBe(offsets[i - 1]!.globalEnd);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it("pageMap entries have positive width and height in user-space points", async () => {
|
|
||||||
const { representation } = await ingestPdf(bytes);
|
|
||||||
const pages = representation.pageMap ?? [];
|
|
||||||
for (let i = 0; i < pages.length; i++) {
|
|
||||||
const p = pages[i]!;
|
|
||||||
expect(p.page).toBe(i + 1);
|
|
||||||
expect(p.width).toBeGreaterThan(0);
|
|
||||||
expect(p.height).toBeGreaterThan(0);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("ingestPdf — option handling", () => {
|
|
||||||
const fixture = FIXTURES[0]!;
|
|
||||||
const path = resolve(FIXTURE_DIR, fixture.filename);
|
|
||||||
const bytes = new Uint8Array(readFileSync(path));
|
|
||||||
|
|
||||||
it("uses explicit title over filename", async () => {
|
|
||||||
const { document } = await ingestPdf(bytes, {
|
|
||||||
filename: fixture.filename,
|
|
||||||
title: "Custom Title",
|
|
||||||
});
|
|
||||||
expect(document.title).toBe("Custom Title");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("omits title entirely when neither filename nor title is supplied", async () => {
|
|
||||||
const { document } = await ingestPdf(bytes);
|
|
||||||
expect(document.title).toBeUndefined();
|
|
||||||
});
|
|
||||||
|
|
||||||
it("propagates uri and metadata when supplied", async () => {
|
|
||||||
const { document } = await ingestPdf(bytes, {
|
|
||||||
uri: "file:///example.pdf",
|
|
||||||
metadata: { source: "test" },
|
|
||||||
});
|
|
||||||
expect(document.uri).toBe("file:///example.pdf");
|
|
||||||
expect(document.metadata).toEqual({ source: "test" });
|
|
||||||
});
|
|
||||||
|
|
||||||
it("accepts ArrayBuffer input", async () => {
|
|
||||||
const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
|
|
||||||
const { document } = await ingestPdf(ab);
|
|
||||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
@ -1,88 +0,0 @@
|
||||||
/**
|
|
||||||
* PDF ingest pipeline → `{ document, representation }`.
|
|
||||||
*
|
|
||||||
* Implements `wiki/ArchitectureOverview.md` §3.4 ("Raw Source → identify
|
|
||||||
* media type → compute fingerprint → extract metadata → extract canonical
|
|
||||||
* text → build format-specific maps → persist Document +
|
|
||||||
* DocumentRepresentation") for the PDF source format.
|
|
||||||
*
|
|
||||||
* Ingest is a pure function over bytes: it does not persist anything. The
|
|
||||||
* caller (engine repositories in T05, app layer in T06) writes the returned
|
|
||||||
* Document + DocumentRepresentation into the chosen store.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import {
|
|
||||||
type Document,
|
|
||||||
type DocumentRepresentation,
|
|
||||||
} from "@shared/document";
|
|
||||||
import { newId } from "@shared/ids";
|
|
||||||
import { extractPdf } from "./extract";
|
|
||||||
import { fingerprintBytes } from "./fingerprint";
|
|
||||||
|
|
||||||
const PDF_MEDIA_TYPE = "application/pdf";
|
|
||||||
|
|
||||||
export interface IngestPdfOptions {
|
|
||||||
/** Original filename, used as the default title when no title is given. */
|
|
||||||
readonly filename?: string;
|
|
||||||
/** Optional pre-existing title (overrides filename). */
|
|
||||||
readonly title?: string;
|
|
||||||
/** Optional source URI (e.g. file:// or https://). */
|
|
||||||
readonly uri?: string;
|
|
||||||
/** Free-form metadata persisted on the Document record. */
|
|
||||||
readonly metadata?: Readonly<Record<string, unknown>>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface IngestPdfResult {
|
|
||||||
readonly document: Document;
|
|
||||||
readonly representation: DocumentRepresentation;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type IngestPdfInput = Uint8Array | ArrayBuffer | Blob;
|
|
||||||
|
|
||||||
export async function ingestPdf(
|
|
||||||
input: IngestPdfInput,
|
|
||||||
options: IngestPdfOptions = {},
|
|
||||||
): Promise<IngestPdfResult> {
|
|
||||||
const bytes = await toBytes(input);
|
|
||||||
const [fingerprint, extraction] = await Promise.all([
|
|
||||||
fingerprintBytes(bytes),
|
|
||||||
extractPdf(bytes),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const now = new Date().toISOString();
|
|
||||||
const documentId = newId("document");
|
|
||||||
const representationId = newId("representation");
|
|
||||||
const title = options.title ?? options.filename;
|
|
||||||
|
|
||||||
const document: Document = {
|
|
||||||
id: documentId,
|
|
||||||
mediaType: PDF_MEDIA_TYPE,
|
|
||||||
fingerprint,
|
|
||||||
createdAt: now,
|
|
||||||
updatedAt: now,
|
|
||||||
...(title !== undefined ? { title } : {}),
|
|
||||||
...(options.uri !== undefined ? { uri: options.uri } : {}),
|
|
||||||
...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
|
|
||||||
};
|
|
||||||
|
|
||||||
const representation: DocumentRepresentation = {
|
|
||||||
id: representationId,
|
|
||||||
documentId,
|
|
||||||
representationType: "pdf-text",
|
|
||||||
contentHash: fingerprint,
|
|
||||||
canonicalText: extraction.canonicalText,
|
|
||||||
pageMap: extraction.pageMap,
|
|
||||||
offsetMap: extraction.offsetMap,
|
|
||||||
generatedAt: now,
|
|
||||||
};
|
|
||||||
|
|
||||||
return { document, representation };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function toBytes(input: IngestPdfInput): Promise<Uint8Array> {
|
|
||||||
if (input instanceof Uint8Array) return input;
|
|
||||||
if (input instanceof ArrayBuffer) return new Uint8Array(input);
|
|
||||||
// Blob (covers `File` in browsers — File extends Blob).
|
|
||||||
const buf = await input.arrayBuffer();
|
|
||||||
return new Uint8Array(buf);
|
|
||||||
}
|
|
||||||
|
|
@ -1,91 +0,0 @@
|
||||||
/**
|
|
||||||
* `ingestPdfFromFile` end-to-end: pipes a fixture PDF through the
|
|
||||||
* upload path, asserts the byte store keeps the bytes and the document
|
|
||||||
* record carries the minted `blob:` URL.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { readFile } from "node:fs/promises";
|
|
||||||
import { describe, expect, it, vi } from "vitest";
|
|
||||||
|
|
||||||
import { createPdfByteStore } from "./byte-store";
|
|
||||||
import { ingestPdfFromFile } from "./upload";
|
|
||||||
|
|
||||||
const FIXTURE_PATH = new URL(
|
|
||||||
"../../../fixtures/pdfs/Fristsetzung zur Bezifferung GÜ an Gegenseite 3 Wochen.pdf",
|
|
||||||
import.meta.url,
|
|
||||||
);
|
|
||||||
|
|
||||||
async function fixtureBytes(): Promise<Uint8Array> {
|
|
||||||
return new Uint8Array(await readFile(FIXTURE_PATH));
|
|
||||||
}
|
|
||||||
|
|
||||||
class FakeFile {
|
|
||||||
readonly name: string;
|
|
||||||
private readonly bytes: Uint8Array;
|
|
||||||
constructor(bytes: Uint8Array, name: string) {
|
|
||||||
this.bytes = bytes;
|
|
||||||
this.name = name;
|
|
||||||
}
|
|
||||||
async arrayBuffer(): Promise<ArrayBuffer> {
|
|
||||||
const out = new ArrayBuffer(this.bytes.byteLength);
|
|
||||||
new Uint8Array(out).set(this.bytes);
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
describe("ingestPdfFromFile", () => {
|
|
||||||
it("round-trips a fixture PDF through ingest + byte store + blob URL", async () => {
|
|
||||||
const bytes = await fixtureBytes();
|
|
||||||
const file = new FakeFile(bytes, "demo.pdf") as unknown as File;
|
|
||||||
let counter = 0;
|
|
||||||
const store = createPdfByteStore({
|
|
||||||
createObjectURL: () => `blob:upload-stub-${++counter}`,
|
|
||||||
revokeObjectURL: () => {},
|
|
||||||
});
|
|
||||||
|
|
||||||
const { document, representation } = await ingestPdfFromFile(file, store);
|
|
||||||
|
|
||||||
// Bytes are stored, retrievable by document id.
|
|
||||||
const stored = store.get(document.id);
|
|
||||||
expect(stored).not.toBeNull();
|
|
||||||
expect(stored!.bytes.byteLength).toBe(bytes.byteLength);
|
|
||||||
|
|
||||||
// Document carries the blob URL minted by the store.
|
|
||||||
expect(document.uri).toBe(`blob:upload-stub-${counter}`);
|
|
||||||
expect(document.title).toBe("demo.pdf");
|
|
||||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
|
||||||
|
|
||||||
// Representation is the standard pdf-text one.
|
|
||||||
expect(representation.representationType).toBe("pdf-text");
|
|
||||||
expect((representation.canonicalText ?? "").length).toBeGreaterThan(0);
|
|
||||||
}, 30_000);
|
|
||||||
|
|
||||||
it("falls through to ingestPdf with no filename when given a plain Blob", async () => {
|
|
||||||
const bytes = await fixtureBytes();
|
|
||||||
const blob = {
|
|
||||||
async arrayBuffer() {
|
|
||||||
const out = new ArrayBuffer(bytes.byteLength);
|
|
||||||
new Uint8Array(out).set(bytes);
|
|
||||||
return out;
|
|
||||||
},
|
|
||||||
} as Blob;
|
|
||||||
const store = createPdfByteStore({
|
|
||||||
createObjectURL: () => "blob:no-name",
|
|
||||||
revokeObjectURL: () => {},
|
|
||||||
});
|
|
||||||
const { document } = await ingestPdfFromFile(blob, store);
|
|
||||||
expect(document.title).toBeUndefined();
|
|
||||||
expect(document.uri).toBe("blob:no-name");
|
|
||||||
}, 30_000);
|
|
||||||
|
|
||||||
it("explicit title option overrides the filename", async () => {
|
|
||||||
const bytes = await fixtureBytes();
|
|
||||||
const file = new FakeFile(bytes, "anonymous-name.pdf") as unknown as File;
|
|
||||||
const store = createPdfByteStore({
|
|
||||||
createObjectURL: vi.fn(() => "blob:override"),
|
|
||||||
revokeObjectURL: vi.fn(),
|
|
||||||
});
|
|
||||||
const { document } = await ingestPdfFromFile(file, store, { title: "Custom" });
|
|
||||||
expect(document.title).toBe("Custom");
|
|
||||||
}, 30_000);
|
|
||||||
});
|
|
||||||
|
|
@ -1,45 +0,0 @@
|
||||||
/**
|
|
||||||
* Upload-side ingest path.
|
|
||||||
*
|
|
||||||
* The fixture-loading path in `App.tsx` fetches a known URL and calls
|
|
||||||
* `ingestPdf` directly; that path stays untouched for the optional
|
|
||||||
* "Sample sessions" quick-start. Uploaded files flow through here
|
|
||||||
* instead:
|
|
||||||
*
|
|
||||||
* 1. Read `file.arrayBuffer()` once into a `Uint8Array`.
|
|
||||||
* 2. Run the existing `ingestPdf(bytes, { filename })` pipeline to
|
|
||||||
* produce `{document, representation}`.
|
|
||||||
* 3. Push the bytes into the per-session `PdfByteStore`, which mints
|
|
||||||
* a `blob:` URL and stamps it onto `document.uri` so the viewer
|
|
||||||
* adapter can mount the PDF directly.
|
|
||||||
* 4. Hand the engine inputs back to the caller, which wires them via
|
|
||||||
* `engine.documents.register(...)`.
|
|
||||||
*
|
|
||||||
* Keeping URL-minting inside the byte store (rather than at the call
|
|
||||||
* site) means there is exactly one place that creates `blob:` URLs and
|
|
||||||
* exactly one place that revokes them.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { ingestPdf, type IngestPdfResult } from "./ingest";
|
|
||||||
import type { PdfByteStore } from "./byte-store";
|
|
||||||
|
|
||||||
export interface IngestPdfFromFileOptions {
|
|
||||||
/** Override the filename used as the document title. */
|
|
||||||
readonly title?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function ingestPdfFromFile(
|
|
||||||
file: File | Blob,
|
|
||||||
store: PdfByteStore,
|
|
||||||
options: IngestPdfFromFileOptions = {},
|
|
||||||
): Promise<IngestPdfResult> {
|
|
||||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
|
||||||
const filename = "name" in file && typeof file.name === "string" ? file.name : undefined;
|
|
||||||
const ingested = await ingestPdf(bytes, {
|
|
||||||
...(filename !== undefined ? { filename } : {}),
|
|
||||||
...(options.title !== undefined ? { title: options.title } : {}),
|
|
||||||
});
|
|
||||||
const record = store.put(ingested.document.id, bytes);
|
|
||||||
const document = { ...ingested.document, uri: record.blobUrl };
|
|
||||||
return { document, representation: ingested.representation };
|
|
||||||
}
|
|
||||||
|
|
@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest";
|
||||||
import type { Document } from "@shared/document";
|
import type { Document } from "@shared/document";
|
||||||
import type { DocumentId } from "@shared/ids";
|
import type { DocumentId } from "@shared/ids";
|
||||||
|
|
||||||
import { createPdfByteStore } from "./byte-store";
|
import { createPdfByteStore } from "@citation-evidence/evidence-source/browser";
|
||||||
import { isEphemeralBlobUri, resolvePdfViewerUrl } from "./viewer-url";
|
import { isEphemeralBlobUri, resolvePdfViewerUrl } from "./viewer-url";
|
||||||
|
|
||||||
function doc(overrides: Partial<Document> = {}): Document {
|
function doc(overrides: Partial<Document> = {}): Document {
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,10 @@
|
||||||
import type { Document } from "@shared/document";
|
import type { Document } from "@shared/document";
|
||||||
import type { DocumentId } from "@shared/ids";
|
import type { DocumentId } from "@shared/ids";
|
||||||
|
|
||||||
import type { PdfByteStore } from "./byte-store";
|
// PdfByteStore was extracted into the standalone evidence-source package
|
||||||
|
// (ESRC-WP-0001). viewer-url stays local to this app (T04) but consumes the
|
||||||
|
// store type from the package.
|
||||||
|
import type { PdfByteStore } from "@citation-evidence/evidence-source/browser";
|
||||||
|
|
||||||
export function isEphemeralBlobUri(uri: string | undefined): boolean {
|
export function isEphemeralBlobUri(uri: string | undefined): boolean {
|
||||||
return typeof uri === "string" && uri.startsWith("blob:");
|
return typeof uri === "string" && uri.startsWith("blob:");
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ import { createRequire } from "node:module";
|
||||||
import { fileURLToPath } from "node:url";
|
import { fileURLToPath } from "node:url";
|
||||||
import { beforeAll, describe, expect, it } from "vitest";
|
import { beforeAll, describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import { ingestPdf } from "@source/pdf/ingest";
|
import { ingestPdf } from "@citation-evidence/evidence-source";
|
||||||
import { createSelectors, resolveSelectors } from "@anchor/selectors";
|
import { createSelectors, resolveSelectors } from "@anchor/selectors";
|
||||||
import type { PdfSelectionCapture } from "@anchor/types";
|
import type { PdfSelectionCapture } from "@anchor/types";
|
||||||
import manifest from "../../fixtures/pdfs/manifest.json" with { type: "json" };
|
import manifest from "../../fixtures/pdfs/manifest.json" with { type: "json" };
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue