Implement ESRC-WP-0002/0003/0004: HTML/MD ingest, metadata, recovery

Add ingestHtml and ingestMarkdown with ADR-0003 pageless offset semantics,
PDF intrinsic metadata extraction with caller-wins merge (WP-0003), and
citation recovery primitives including re-ingest reconcile, local quote
search, and pluggable discovery hooks (ADR-0004). Mark all three workplans
finished with contract tests (80 passing).

src/markdown/extract.ts

@@ -0,0 +1,53 @@
+/**
+ * Markdown text extraction → canonical text + gap-free offset map.
+ *
+ * Implements ADR-0003 for the `markdown-rendered` representation. Markup is
+ * stripped to plain text before `normalize()` is applied.
+ */
+
+import { normalize, type OffsetMap } from "@citation-evidence/engine/shared";
+import { buildWholeTextOffsetMap } from "../shared/offset-map";
+
+const FENCED_CODE = /```[\s\S]*?```/g;
+const INLINE_CODE = /`([^`]+)`/g;
+const IMAGE = /!\[([^\]]*)\]\([^)]+\)/g;
+const LINK = /\[([^\]]+)\]\([^)]+\)/g;
+const HEADING = /^#{1,6}\s+/gm;
+const BLOCKQUOTE = /^>\s?/gm;
+const LIST_MARKER = /^[\s]*[-*+]\s+/gm;
+const ORDERED_LIST = /^[\s]*\d+\.\s+/gm;
+const EMPHASIS = /(\*\*|__)(.*?)\1/g;
+const ITALIC = /(\*|_)([^*_]+)\1/g;
+const STRIKETHROUGH = /~~(.*?)~~/g;
+const HORIZONTAL_RULE = /^[-*_]{3,}\s*$/gm;
+
+export interface MarkdownExtractionResult {
+  readonly canonicalText: string;
+  readonly offsetMap: OffsetMap;
+}
+
+export function extractMarkdown(source: string): MarkdownExtractionResult {
+  const rawText = markdownToPlainText(source);
+  const canonicalText = normalize(rawText).text;
+  return {
+    canonicalText,
+    offsetMap: buildWholeTextOffsetMap(canonicalText),
+  };
+}
+
+function markdownToPlainText(source: string): string {
+  let text = source;
+  text = text.replace(FENCED_CODE, "\n\n");
+  text = text.replace(INLINE_CODE, "$1");
+  text = text.replace(IMAGE, "$1");
+  text = text.replace(LINK, "$1");
+  text = text.replace(HEADING, "");
+  text = text.replace(BLOCKQUOTE, "");
+  text = text.replace(LIST_MARKER, "");
+  text = text.replace(ORDERED_LIST, "");
+  text = text.replace(EMPHASIS, "$2");
+  text = text.replace(ITALIC, "$2");
+  text = text.replace(STRIKETHROUGH, "$1");
+  text = text.replace(HORIZONTAL_RULE, "\n\n");
+  return text;
+}

src/markdown/ingest.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+
+import { fingerprintBytes } from "../pdf/fingerprint";
+import { MARKDOWN_FIXTURE } from "../../tests/fixtures-text";
+import { ingestMarkdown } from "./ingest";
+import { extractMarkdown } from "./extract";
+
+describe("extractMarkdown", () => {
+  it("strips markup and preserves the known-good quote", () => {
+    const { canonicalText } = extractMarkdown(MARKDOWN_FIXTURE);
+    expect(canonicalText).toContain("Known good quote for Markdown ingest testing.");
+    expect(canonicalText).toContain("bold");
+    expect(canonicalText).toContain("italic");
+    expect(canonicalText).not.toContain("**");
+  });
+});
+
+describe("ingestMarkdown", () => {
+  const bytes = new TextEncoder().encode(MARKDOWN_FIXTURE);
+
+  it("produces a Document with text/markdown media type and SHA-256 fingerprint", async () => {
+    const { document } = await ingestMarkdown(bytes, { filename: "sample.md" });
+    expect(document.mediaType).toBe("text/markdown");
+    expect(document.fingerprint).toMatch(/^[0-9a-f]{64}$/);
+    expect(document.title).toBe("sample.md");
+    expect(document.fingerprint).toBe(await fingerprintBytes(bytes));
+  });
+
+  it("produces a markdown-rendered representation with a gap-free offset map", async () => {
+    const { representation } = await ingestMarkdown(MARKDOWN_FIXTURE);
+    const text = representation.canonicalText ?? "";
+    const offsets = representation.offsetMap ?? [];
+    expect(representation.representationType).toBe("markdown-rendered");
+    expect(representation.pageMap).toBeUndefined();
+    expect(offsets).toHaveLength(1);
+    expect(offsets[0]!.page).toBe(1);
+    expect(offsets[0]!.globalStart).toBe(0);
+    expect(offsets[0]!.globalEnd).toBe(text.length);
+    expect(text).toContain("Known good quote for Markdown ingest testing.");
+  });
+
+  it("uses explicit title over filename", async () => {
+    const { document } = await ingestMarkdown(bytes, {
+      filename: "sample.md",
+      title: "Custom Title",
+    });
+    expect(document.title).toBe("Custom Title");
+  });
+});

src/markdown/ingest.ts

@@ -0,0 +1,81 @@
+/**
+ * Markdown ingest pipeline → `{ document, representation }`.
+ *
+ * Implements ADR-0003 for the `markdown-rendered` representation type.
+ */
+
+import {
+  newId,
+  type Document,
+  type DocumentRepresentation,
+} from "@citation-evidence/engine/shared";
+import { fingerprintBytes } from "../pdf/fingerprint";
+import { toBytes, type ByteInput } from "../shared/to-bytes";
+import { extractMarkdown } from "./extract";
+
+const MARKDOWN_MEDIA_TYPE = "text/markdown";
+
+export interface IngestMarkdownOptions {
+  readonly filename?: string;
+  readonly title?: string;
+  readonly uri?: string;
+  readonly metadata?: Readonly<Record<string, unknown>>;
+}
+
+export interface IngestMarkdownResult {
+  readonly document: Document;
+  readonly representation: DocumentRepresentation;
+}
+
+export type IngestMarkdownInput = ByteInput | string;
+
+export async function ingestMarkdown(
+  input: IngestMarkdownInput,
+  options: IngestMarkdownOptions = {},
+): Promise<IngestMarkdownResult> {
+  const { bytes, source } = await resolveInput(input);
+  const [fingerprint, extraction] = await Promise.all([
+    fingerprintBytes(bytes),
+    Promise.resolve(extractMarkdown(source)),
+  ]);
+
+  const now = new Date().toISOString();
+  const documentId = newId("document");
+  const representationId = newId("representation");
+  const title = options.title ?? options.filename;
+
+  const document: Document = {
+    id: documentId,
+    mediaType: MARKDOWN_MEDIA_TYPE,
+    fingerprint,
+    createdAt: now,
+    updatedAt: now,
+    ...(title !== undefined ? { title } : {}),
+    ...(options.uri !== undefined ? { uri: options.uri } : {}),
+    ...(options.metadata !== undefined ? { metadata: options.metadata } : {}),
+  };
+
+  const representation: DocumentRepresentation = {
+    id: representationId,
+    documentId,
+    representationType: "markdown-rendered",
+    contentHash: fingerprint,
+    canonicalText: extraction.canonicalText,
+    offsetMap: extraction.offsetMap,
+    generatedAt: now,
+  };
+
+  return { document, representation };
+}
+
+async function resolveInput(
+  input: IngestMarkdownInput,
+): Promise<{ bytes: Uint8Array; source: string }> {
+  if (typeof input === "string") {
+    const bytes = new TextEncoder().encode(input);
+    return { bytes, source: input };
+  }
+  const bytes = await toBytes(input);
+  const source = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+  return { bytes, source };
+}