feat(pdf): extract standalone PDF ingest package (ESRC-WP-0001)
Bootstrap evidence-source from the citation-evidence src/source PDF slice. - Headless core (src/pdf): ingest/extract/fingerprint, importing domain contracts from @citation-evidence/engine/shared (no local copies) - Browser upload helpers isolated under src/browser behind a ./browser entry point, with an eslint boundary keeping the core browser-free - pnpm/TS/vitest/eslint scaffold; 52 tests (contract + determinism) - Fixtures resolved from the sibling citation-evidence checkout, not duplicated (real PII) — see docs/ADR-0002; suites skip when absent - Boundary + fixture decisions recorded as docs/ADR-0001 / ADR-0002 - README/SCOPE rewritten; capability.infotech.pdf-evidence-ingest registered, NO_CAPABILITIES removed - Follow-on workplans ESRC-WP-0002..0004 queued; ESRC-WP-0001 finished Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
2fd715ba45
commit
cb93c322c0
31 changed files with 5066 additions and 105 deletions
97
src/browser/byte-store.test.ts
Normal file
97
src/browser/byte-store.test.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import type { DocumentId } from "@citation-evidence/engine/shared";
|
||||
|
||||
import { createPdfByteStore } from "./byte-store";
|
||||
|
||||
function stubUrlHelpers() {
|
||||
let counter = 0;
|
||||
const created: string[] = [];
|
||||
const revoked: string[] = [];
|
||||
const createObjectURL = vi.fn(() => {
|
||||
const url = `blob:stub-${++counter}`;
|
||||
created.push(url);
|
||||
return url;
|
||||
});
|
||||
const revokeObjectURL = vi.fn((url: string) => {
|
||||
revoked.push(url);
|
||||
});
|
||||
return { createObjectURL, revokeObjectURL, created, revoked };
|
||||
}
|
||||
|
||||
describe("PdfByteStore", () => {
|
||||
it("put / get round-trips bytes and exposes a blob URL", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46]); // %PDF
|
||||
const record = store.put("doc_a" as DocumentId, bytes);
|
||||
expect(record.blobUrl).toBe("blob:stub-1");
|
||||
expect(store.get("doc_a" as DocumentId)?.bytes).toBe(bytes);
|
||||
expect(store.has("doc_a" as DocumentId)).toBe(true);
|
||||
expect(store.list()).toEqual(["doc_a"]);
|
||||
expect(store.size()).toBe(4);
|
||||
});
|
||||
|
||||
it("put replaces an existing entry and revokes the old URL", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const id = "doc_a" as DocumentId;
|
||||
const first = store.put(id, new Uint8Array([1, 2]));
|
||||
const second = store.put(id, new Uint8Array([3, 4, 5]));
|
||||
expect(helpers.revoked).toEqual([first.blobUrl]);
|
||||
expect(store.get(id)?.bytes).toHaveLength(3);
|
||||
expect(second.blobUrl).not.toBe(first.blobUrl);
|
||||
});
|
||||
|
||||
it("delete revokes the blob URL exactly once and is idempotent", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const id = "doc_a" as DocumentId;
|
||||
const record = store.put(id, new Uint8Array([1, 2, 3]));
|
||||
expect(store.delete(id)).toBe(true);
|
||||
expect(helpers.revoked).toEqual([record.blobUrl]);
|
||||
expect(store.delete(id)).toBe(false);
|
||||
expect(helpers.revoked).toHaveLength(1);
|
||||
expect(store.get(id)).toBeNull();
|
||||
expect(store.has(id)).toBe(false);
|
||||
});
|
||||
|
||||
it("clear revokes every URL and empties the store", () => {
|
||||
const helpers = stubUrlHelpers();
|
||||
const store = createPdfByteStore(helpers);
|
||||
const a = store.put("doc_a" as DocumentId, new Uint8Array([1]));
|
||||
const b = store.put("doc_b" as DocumentId, new Uint8Array([2]));
|
||||
store.clear();
|
||||
expect(helpers.revoked.sort()).toEqual([a.blobUrl, b.blobUrl].sort());
|
||||
expect(store.list()).toEqual([]);
|
||||
expect(store.size()).toBe(0);
|
||||
});
|
||||
|
||||
it("uses URL.createObjectURL by default when no override is supplied", () => {
|
||||
const createObjectURL = vi.fn(() => "blob:built-in");
|
||||
const revokeObjectURL = vi.fn();
|
||||
const originalURL = globalThis.URL;
|
||||
Object.defineProperty(globalThis, "URL", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: Object.assign(Object.create(originalURL.prototype as object), {
|
||||
createObjectURL,
|
||||
revokeObjectURL,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
const store = createPdfByteStore();
|
||||
const rec = store.put("doc_z" as DocumentId, new Uint8Array([9]));
|
||||
expect(rec.blobUrl).toBe("blob:built-in");
|
||||
expect(createObjectURL).toHaveBeenCalledTimes(1);
|
||||
store.delete("doc_z" as DocumentId);
|
||||
expect(revokeObjectURL).toHaveBeenCalledWith("blob:built-in");
|
||||
} finally {
|
||||
Object.defineProperty(globalThis, "URL", {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: originalURL,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
115
src/browser/byte-store.ts
Normal file
115
src/browser/byte-store.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
/**
|
||||
* `PdfByteStore` — in-memory store for uploaded PDF bytes, keyed by
|
||||
* `DocumentId`.
|
||||
*
|
||||
* BROWSER SURFACE — not part of the headless ingest core. This mints and
|
||||
* revokes `blob:` URLs and is meant to back an interactive upload/viewer
|
||||
* session. Headless callers should use `src/pdf/` directly and never import
|
||||
* this module.
|
||||
*
|
||||
* Uploaded PDFs are stored in memory only. Bytes survive within a tab
|
||||
* session; reloading the page loses them unless the session was exported.
|
||||
*
|
||||
* One store instance per active session. The session-management layer is
|
||||
* responsible for swapping the active store when the user switches sessions.
|
||||
* The store also owns a small registry of issued `blob:` URLs so it can
|
||||
* revoke them on delete/clear — no cross-cutting cleanup at the app layer.
|
||||
*/
|
||||
|
||||
import type { DocumentId } from "@citation-evidence/engine/shared";
|
||||
|
||||
export interface PdfByteRecord {
|
||||
readonly bytes: Uint8Array;
|
||||
/** A `blob:` URL the viewer can consume directly. */
|
||||
readonly blobUrl: string;
|
||||
}
|
||||
|
||||
export interface PdfByteStore {
|
||||
put(documentId: DocumentId, bytes: Uint8Array): PdfByteRecord;
|
||||
get(documentId: DocumentId): PdfByteRecord | null;
|
||||
has(documentId: DocumentId): boolean;
|
||||
delete(documentId: DocumentId): boolean;
|
||||
list(): readonly DocumentId[];
|
||||
/** Revoke every blob URL and clear the store. */
|
||||
clear(): void;
|
||||
/** Total bytes currently held — useful for UI dashboards. */
|
||||
size(): number;
|
||||
}
|
||||
|
||||
export interface CreatePdfByteStoreOptions {
|
||||
/**
|
||||
* Mint a URL for the given bytes. Defaults to `URL.createObjectURL` in
|
||||
* environments that have it; tests can inject a deterministic stub.
|
||||
*/
|
||||
readonly createObjectURL?: (blob: Blob) => string;
|
||||
/** Revoke a URL previously minted by `createObjectURL`. */
|
||||
readonly revokeObjectURL?: (url: string) => void;
|
||||
}
|
||||
|
||||
export function createPdfByteStore(
|
||||
options: CreatePdfByteStoreOptions = {},
|
||||
): PdfByteStore {
|
||||
const createUrl =
|
||||
options.createObjectURL ??
|
||||
((blob: Blob) => {
|
||||
if (typeof URL === "undefined" || typeof URL.createObjectURL !== "function") {
|
||||
throw new Error(
|
||||
"createPdfByteStore: URL.createObjectURL not available — inject a stub via options",
|
||||
);
|
||||
}
|
||||
return URL.createObjectURL(blob);
|
||||
});
|
||||
const revokeUrl =
|
||||
options.revokeObjectURL ??
|
||||
((url: string) => {
|
||||
if (typeof URL !== "undefined" && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
});
|
||||
|
||||
const records = new Map<DocumentId, PdfByteRecord>();
|
||||
|
||||
return {
|
||||
put(documentId, bytes) {
|
||||
// Replace previous record (revoking the prior URL) if any.
|
||||
const prior = records.get(documentId);
|
||||
if (prior) revokeUrl(prior.blobUrl);
|
||||
// Cast: Blob() does accept Uint8Array at runtime, but TS narrows the
|
||||
// buffer type to ArrayBufferLike (could be SharedArrayBuffer) and
|
||||
// refuses without help. The bytes here always come from a fresh
|
||||
// arrayBuffer() call, so a regular ArrayBuffer is guaranteed.
|
||||
const blob = new Blob([bytes as unknown as ArrayBuffer], {
|
||||
type: "application/pdf",
|
||||
});
|
||||
const blobUrl = createUrl(blob);
|
||||
const record: PdfByteRecord = { bytes, blobUrl };
|
||||
records.set(documentId, record);
|
||||
return record;
|
||||
},
|
||||
get(documentId) {
|
||||
return records.get(documentId) ?? null;
|
||||
},
|
||||
has(documentId) {
|
||||
return records.has(documentId);
|
||||
},
|
||||
delete(documentId) {
|
||||
const record = records.get(documentId);
|
||||
if (!record) return false;
|
||||
revokeUrl(record.blobUrl);
|
||||
records.delete(documentId);
|
||||
return true;
|
||||
},
|
||||
list() {
|
||||
return [...records.keys()];
|
||||
},
|
||||
clear() {
|
||||
for (const record of records.values()) revokeUrl(record.blobUrl);
|
||||
records.clear();
|
||||
},
|
||||
size() {
|
||||
let total = 0;
|
||||
for (const r of records.values()) total += r.bytes.byteLength;
|
||||
return total;
|
||||
},
|
||||
};
|
||||
}
|
||||
20
src/browser/index.ts
Normal file
20
src/browser/index.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
/**
|
||||
* `evidence-source/browser` — browser-facing upload helpers.
|
||||
*
|
||||
* BROWSER SURFACE. These helpers assume a browser session: they mint and
|
||||
* revoke `blob:` URLs and hold uploaded bytes in memory for a viewer to
|
||||
* mount. They are deliberately separated from the headless core (the package
|
||||
* root, `src/index.ts`) so that headless consumers never pull in
|
||||
* `URL.createObjectURL`/`Blob` upload machinery.
|
||||
*/
|
||||
|
||||
export {
|
||||
createPdfByteStore,
|
||||
type CreatePdfByteStoreOptions,
|
||||
type PdfByteRecord,
|
||||
type PdfByteStore,
|
||||
} from "./byte-store";
|
||||
export {
|
||||
ingestPdfFromFile,
|
||||
type IngestPdfFromFileOptions,
|
||||
} from "./upload";
|
||||
82
src/browser/upload.test.ts
Normal file
82
src/browser/upload.test.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* `ingestPdfFromFile` end-to-end: pipes a fixture PDF through the upload path,
|
||||
* asserts the byte store keeps the bytes and the document record carries the
|
||||
* minted `blob:` URL. Corpus resolution/skip lives in `tests/fixtures.ts`.
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { createPdfByteStore } from "./byte-store";
|
||||
import { ingestPdfFromFile } from "./upload";
|
||||
import { fixtureBytes, fixtureCorpusAvailable } from "../../tests/fixtures";
|
||||
|
||||
const hasCorpus = fixtureCorpusAvailable();
|
||||
const FIXTURE_FILE = "Fristsetzung zur Bezifferung GÜ an Gegenseite 3 Wochen.pdf";
|
||||
|
||||
class FakeFile {
|
||||
readonly name: string;
|
||||
private readonly bytes: Uint8Array;
|
||||
constructor(bytes: Uint8Array, name: string) {
|
||||
this.bytes = bytes;
|
||||
this.name = name;
|
||||
}
|
||||
async arrayBuffer(): Promise<ArrayBuffer> {
|
||||
const out = new ArrayBuffer(this.bytes.byteLength);
|
||||
new Uint8Array(out).set(this.bytes);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(!hasCorpus)("ingestPdfFromFile", () => {
|
||||
it("round-trips a fixture PDF through ingest + byte store + blob URL", async () => {
|
||||
const bytes = fixtureBytes(FIXTURE_FILE);
|
||||
const file = new FakeFile(bytes, "demo.pdf") as unknown as File;
|
||||
let counter = 0;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: () => `blob:upload-stub-${++counter}`,
|
||||
revokeObjectURL: () => {},
|
||||
});
|
||||
|
||||
const { document, representation } = await ingestPdfFromFile(file, store);
|
||||
|
||||
const stored = store.get(document.id);
|
||||
expect(stored).not.toBeNull();
|
||||
expect(stored!.bytes.byteLength).toBe(bytes.byteLength);
|
||||
|
||||
expect(document.uri).toBe(`blob:upload-stub-${counter}`);
|
||||
expect(document.title).toBe("demo.pdf");
|
||||
expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
|
||||
|
||||
expect(representation.representationType).toBe("pdf-text");
|
||||
expect((representation.canonicalText ?? "").length).toBeGreaterThan(0);
|
||||
}, 30_000);
|
||||
|
||||
it("falls through to ingestPdf with no filename when given a plain Blob", async () => {
|
||||
const bytes = fixtureBytes(FIXTURE_FILE);
|
||||
const blob = {
|
||||
async arrayBuffer() {
|
||||
const out = new ArrayBuffer(bytes.byteLength);
|
||||
new Uint8Array(out).set(bytes);
|
||||
return out;
|
||||
},
|
||||
} as Blob;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: () => "blob:no-name",
|
||||
revokeObjectURL: () => {},
|
||||
});
|
||||
const { document } = await ingestPdfFromFile(blob, store);
|
||||
expect(document.title).toBeUndefined();
|
||||
expect(document.uri).toBe("blob:no-name");
|
||||
}, 30_000);
|
||||
|
||||
it("explicit title option overrides the filename", async () => {
|
||||
const bytes = fixtureBytes(FIXTURE_FILE);
|
||||
const file = new FakeFile(bytes, "anonymous-name.pdf") as unknown as File;
|
||||
const store = createPdfByteStore({
|
||||
createObjectURL: vi.fn(() => "blob:override"),
|
||||
revokeObjectURL: vi.fn(),
|
||||
});
|
||||
const { document } = await ingestPdfFromFile(file, store, { title: "Custom" });
|
||||
expect(document.title).toBe("Custom");
|
||||
}, 30_000);
|
||||
});
|
||||
43
src/browser/upload.ts
Normal file
43
src/browser/upload.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Upload-side ingest path.
|
||||
*
|
||||
* BROWSER SURFACE — not part of the headless ingest core. This is a thin
|
||||
* wrapper over the headless `ingestPdf` that additionally pushes the bytes
|
||||
* into a per-session `PdfByteStore`, which mints a `blob:` URL and stamps it
|
||||
* onto `document.uri` so a viewer adapter can mount the PDF directly.
|
||||
*
|
||||
* 1. Read `file.arrayBuffer()` once into a `Uint8Array`.
|
||||
* 2. Run the existing `ingestPdf(bytes, { filename })` pipeline to produce
|
||||
* `{document, representation}`.
|
||||
* 3. Push the bytes into the per-session `PdfByteStore`.
|
||||
* 4. Hand the engine inputs back to the caller, which wires them via
|
||||
* `engine.documents.register(...)`.
|
||||
*
|
||||
* Keeping URL-minting inside the byte store (rather than at the call site)
|
||||
* means there is exactly one place that creates `blob:` URLs and exactly one
|
||||
* place that revokes them.
|
||||
*/
|
||||
|
||||
import { ingestPdf, type IngestPdfResult } from "../pdf/ingest";
|
||||
import type { PdfByteStore } from "./byte-store";
|
||||
|
||||
export interface IngestPdfFromFileOptions {
|
||||
/** Override the filename used as the document title. */
|
||||
readonly title?: string;
|
||||
}
|
||||
|
||||
export async function ingestPdfFromFile(
|
||||
file: File | Blob,
|
||||
store: PdfByteStore,
|
||||
options: IngestPdfFromFileOptions = {},
|
||||
): Promise<IngestPdfResult> {
|
||||
const bytes = new Uint8Array(await file.arrayBuffer());
|
||||
const filename = "name" in file && typeof file.name === "string" ? file.name : undefined;
|
||||
const ingested = await ingestPdf(bytes, {
|
||||
...(filename !== undefined ? { filename } : {}),
|
||||
...(options.title !== undefined ? { title: options.title } : {}),
|
||||
});
|
||||
const record = store.put(ingested.document.id, bytes);
|
||||
const document = { ...ingested.document, uri: record.blobUrl };
|
||||
return { document, representation: ingested.representation };
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue