Implement ESRC-WP-0002/0003/0004: HTML/MD ingest, metadata, recovery

Add ingestHtml and ingestMarkdown with ADR-0003 pageless offset semantics,
PDF intrinsic metadata extraction with caller-wins merge (WP-0003), and
citation recovery primitives including re-ingest reconcile, local quote
search, and pluggable discovery hooks (ADR-0004). Mark all three workplans
finished with contract tests (80 passing).

src/recovery/attempt.ts

@@ -0,0 +1,39 @@
+import { newId } from "@citation-evidence/engine/shared";
+
+import type { CitationClue, CitationRecoveryState } from "./types";
+
+export interface RecoveryAttempt {
+  readonly id: string;
+  readonly state: CitationRecoveryState;
+  readonly clue: CitationClue;
+  readonly createdAt: string;
+  readonly updatedAt: string;
+  readonly documentId?: string;
+  readonly notes: readonly string[];
+}
+
+export function createRecoveryAttempt(clue: CitationClue): RecoveryAttempt {
+  const now = new Date().toISOString();
+  return {
+    id: newId("citation-recovery"),
+    state: "created",
+    clue,
+    createdAt: now,
+    updatedAt: now,
+    notes: [],
+  };
+}
+
+export function withRecoveryState(
+  attempt: RecoveryAttempt,
+  state: CitationRecoveryState,
+  note?: string,
+): RecoveryAttempt {
+  const now = new Date().toISOString();
+  return {
+    ...attempt,
+    state,
+    updatedAt: now,
+    notes: note === undefined ? attempt.notes : [...attempt.notes, note],
+  };
+}

src/recovery/discovery.ts

@@ -0,0 +1,37 @@
+import type { CitationClue } from "./types";
+
+export interface DiscoveryCandidate {
+  readonly uri: string;
+  readonly title?: string;
+  readonly confidence: number;
+  readonly source: "local" | "external";
+  readonly metadata?: Readonly<Record<string, unknown>>;
+}
+
+/** Pluggable discovery provider — implementations may perform network I/O. */
+export interface SourceDiscoveryHook {
+  readonly name: string;
+  discover(clue: CitationClue): Promise<readonly DiscoveryCandidate[]>;
+}
+
+export interface SourceDiscoveryRegistry {
+  register(hook: SourceDiscoveryHook): void;
+  discover(clue: CitationClue): Promise<readonly DiscoveryCandidate[]>;
+}
+
+export function createSourceDiscoveryRegistry(): SourceDiscoveryRegistry {
+  const hooks: SourceDiscoveryHook[] = [];
+
+  return {
+    register(hook: SourceDiscoveryHook) {
+      hooks.push(hook);
+    },
+
+    async discover(clue: CitationClue): Promise<readonly DiscoveryCandidate[]> {
+      const results = await Promise.all(hooks.map((hook) => hook.discover(clue)));
+      return results
+        .flat()
+        .sort((a, b) => b.confidence - a.confidence);
+    },
+  };
+}

src/recovery/index.ts

@@ -0,0 +1,26 @@
+export type { CitationClue, CitationRecoveryState } from "./types";
+export {
+  createRecoveryAttempt,
+  withRecoveryState,
+  type RecoveryAttempt,
+} from "./attempt";
+export { reconcileFingerprint, type ReconcileResult } from "./reconcile";
+export {
+  reIngestAndCompare,
+  type IngestFn,
+  type IngestOutcome,
+  type ReIngestInput,
+  type ReIngestResult,
+} from "./re-ingest";
+export {
+  applyLocalQuoteSearch,
+  recoveryStateAfterLocalSearch,
+  searchCanonicalQuote,
+  type QuoteMatch,
+} from "./local-search";
+export {
+  createSourceDiscoveryRegistry,
+  type DiscoveryCandidate,
+  type SourceDiscoveryHook,
+  type SourceDiscoveryRegistry,
+} from "./discovery";

src/recovery/local-search.ts

@@ -0,0 +1,65 @@
+import { normalize, type DocumentRepresentation } from "@citation-evidence/engine/shared";
+
+import type { CitationRecoveryState } from "./types";
+import { withRecoveryState, type RecoveryAttempt } from "./attempt";
+
+export interface QuoteMatch {
+  readonly start: number;
+  readonly end: number;
+  readonly matchedText: string;
+}
+
+export function searchCanonicalQuote(
+  representation: Pick<DocumentRepresentation, "canonicalText">,
+  quote: string,
+): readonly QuoteMatch[] {
+  const canonicalText = representation.canonicalText ?? "";
+  const normalizedQuote = normalize(quote).text;
+  if (normalizedQuote.length === 0 || canonicalText.length === 0) return [];
+
+  const normalizedCanonical = normalize(canonicalText).text;
+  const matches: QuoteMatch[] = [];
+  let from = 0;
+  while (from <= normalizedCanonical.length) {
+    const index = normalizedCanonical.indexOf(normalizedQuote, from);
+    if (index === -1) break;
+    matches.push({
+      start: index,
+      end: index + normalizedQuote.length,
+      matchedText: normalizedCanonical.slice(index, index + normalizedQuote.length),
+    });
+    from = index + 1;
+  }
+  return matches;
+}
+
+export function applyLocalQuoteSearch(
+  attempt: RecoveryAttempt,
+  representation: Pick<DocumentRepresentation, "canonicalText">,
+): RecoveryAttempt {
+  const quote = attempt.clue.quote;
+  if (quote === undefined || quote.trim().length === 0) {
+    return withRecoveryState(attempt, "quote-not-found", "No quote in clue");
+  }
+
+  const matches = searchCanonicalQuote(representation, quote);
+  if (matches.length === 1) {
+    return withRecoveryState(attempt, "quote-found", "Exact quote match");
+  }
+  if (matches.length > 1) {
+    return withRecoveryState(
+      attempt,
+      "candidate-passages-found",
+      `${matches.length} quote matches`,
+    );
+  }
+  return withRecoveryState(attempt, "quote-not-found", "Quote not found locally");
+}
+
+export function recoveryStateAfterLocalSearch(
+  matchCount: number,
+): CitationRecoveryState {
+  if (matchCount === 1) return "quote-found";
+  if (matchCount > 1) return "candidate-passages-found";
+  return "quote-not-found";
+}

src/recovery/re-ingest.ts

@@ -0,0 +1,33 @@
+import type { Document, DocumentRepresentation } from "@citation-evidence/engine/shared";
+
+import { reconcileFingerprint, type ReconcileResult } from "./reconcile";
+
+export interface IngestOutcome {
+  readonly document: Document;
+  readonly representation: DocumentRepresentation;
+}
+
+export type IngestFn<TOptions = void> = (
+  bytes: Uint8Array,
+  options?: TOptions,
+) => Promise<IngestOutcome>;
+
+export interface ReIngestInput<TOptions = void> {
+  readonly previous: Document;
+  readonly bytes: Uint8Array;
+  readonly options?: TOptions;
+}
+
+export interface ReIngestResult {
+  readonly outcome: IngestOutcome;
+  readonly reconcile: ReconcileResult;
+}
+
+export async function reIngestAndCompare<TOptions>(
+  input: ReIngestInput<TOptions>,
+  ingest: IngestFn<TOptions>,
+): Promise<ReIngestResult> {
+  const outcome = await ingest(input.bytes, input.options);
+  const reconcile = reconcileFingerprint(input.previous, outcome.document);
+  return { outcome, reconcile };
+}

src/recovery/reconcile.ts

@@ -0,0 +1,27 @@
+import type { Document } from "@citation-evidence/engine/shared";
+
+export interface ReconcileResult {
+  readonly unchanged: boolean;
+  readonly previousFingerprint?: string;
+  readonly currentFingerprint: string;
+  readonly requiresReanchor: boolean;
+}
+
+export function reconcileFingerprint(
+  previous: Pick<Document, "fingerprint">,
+  current: Pick<Document, "fingerprint">,
+): ReconcileResult {
+  const previousFingerprint = previous.fingerprint;
+  const currentFingerprint = current.fingerprint ?? "";
+  const unchanged =
+    previousFingerprint !== undefined &&
+    currentFingerprint.length > 0 &&
+    previousFingerprint === currentFingerprint;
+
+  return {
+    unchanged,
+    previousFingerprint,
+    currentFingerprint,
+    requiresReanchor: !unchanged,
+  };
+}

src/recovery/recovery.test.ts

@@ -0,0 +1,97 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { HTML_FIXTURE } from "../../tests/fixtures-text";
+import { ingestHtml } from "../html/ingest";
+import {
+  applyLocalQuoteSearch,
+  createRecoveryAttempt,
+  createSourceDiscoveryRegistry,
+  reconcileFingerprint,
+  reIngestAndCompare,
+  recoveryStateAfterLocalSearch,
+  searchCanonicalQuote,
+} from "./index";
+
+describe("reconcileFingerprint", () => {
+  it("flags re-anchor when fingerprints differ", () => {
+    const result = reconcileFingerprint(
+      { fingerprint: "aaa" },
+      { fingerprint: "bbb" },
+    );
+    expect(result.unchanged).toBe(false);
+    expect(result.requiresReanchor).toBe(true);
+  });
+
+  it("reports unchanged when fingerprints match", () => {
+    const fp = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+    const result = reconcileFingerprint({ fingerprint: fp }, { fingerprint: fp });
+    expect(result.unchanged).toBe(true);
+    expect(result.requiresReanchor).toBe(false);
+  });
+});
+
+describe("searchCanonicalQuote", () => {
+  it("finds normalized quote matches in canonical text", async () => {
+    const { representation } = await ingestHtml(HTML_FIXTURE);
+    const matches = searchCanonicalQuote(
+      representation,
+      "Known good quote for HTML ingest testing.",
+    );
+    expect(matches).toHaveLength(1);
+    expect(matches[0]!.matchedText).toContain("Known good quote");
+  });
+
+  it("maps match counts to recovery states", () => {
+    expect(recoveryStateAfterLocalSearch(1)).toBe("quote-found");
+    expect(recoveryStateAfterLocalSearch(2)).toBe("candidate-passages-found");
+    expect(recoveryStateAfterLocalSearch(0)).toBe("quote-not-found");
+  });
+});
+
+describe("applyLocalQuoteSearch", () => {
+  it("updates attempt state from local quote search", async () => {
+    const { representation } = await ingestHtml(HTML_FIXTURE);
+    const attempt = createRecoveryAttempt({
+      quote: "Known good quote for HTML ingest testing.",
+    });
+    const updated = applyLocalQuoteSearch(attempt, representation);
+    expect(updated.state).toBe("quote-found");
+  });
+});
+
+describe("reIngestAndCompare", () => {
+  it("detects unchanged document on identical bytes", async () => {
+    const bytes = new TextEncoder().encode(HTML_FIXTURE);
+    const first = await ingestHtml(bytes);
+    const { reconcile } = await reIngestAndCompare(
+      { previous: first.document, bytes },
+      ingestHtml,
+    );
+    expect(reconcile.unchanged).toBe(true);
+    expect(reconcile.requiresReanchor).toBe(false);
+  });
+});
+
+describe("createSourceDiscoveryRegistry", () => {
+  it("aggregates provider results sorted by confidence", async () => {
+    const registry = createSourceDiscoveryRegistry();
+    registry.register({
+      name: "low",
+      discover: vi.fn(async () => [
+        { uri: "file:///a", confidence: 0.2, source: "local" },
+      ]),
+    });
+    registry.register({
+      name: "high",
+      discover: vi.fn(async () => [
+        { uri: "https://example.com/doc", confidence: 0.9, source: "external" },
+      ]),
+    });
+
+    const candidates = await registry.discover({ title: "Example" });
+    expect(candidates).toHaveLength(2);
+    expect(candidates[0]!.confidence).toBe(0.9);
+    expect(candidates[1]!.confidence).toBe(0.2);
+  });
+});
+

src/recovery/types.ts

@@ -0,0 +1,31 @@
+/**
+ * Citation recovery vocabulary aligned with citation-engine SharedContracts §2.6.
+ *
+ * The canonical `CitationRecoveryAttempt` type will live in citation-engine;
+ * this repo uses the same state strings for local attempt records.
+ */
+
+export type CitationRecoveryState =
+  | "created"
+  | "source-found-fulltext"
+  | "source-found-preview-only"
+  | "source-found-metadata-only"
+  | "source-not-found"
+  | "quote-found"
+  | "quote-not-found"
+  | "candidate-passages-found"
+  | "manual-confirmation-needed"
+  | "confirmed"
+  | "annotation-created"
+  | "failed";
+
+/** Clue supplied when starting or continuing a recovery attempt. */
+export interface CitationClue {
+  readonly quote?: string;
+  readonly prefix?: string;
+  readonly suffix?: string;
+  readonly title?: string;
+  readonly uri?: string;
+  readonly doi?: string;
+  readonly metadata?: Readonly<Record<string, unknown>>;
+}