tegwick
97437dfa18
Add ingestHtml and ingestMarkdown with ADR-0003 pageless offset semantics, PDF intrinsic metadata extraction with caller-wins merge (WP-0003), and citation recovery primitives including re-ingest reconcile, local quote search, and pluggable discovery hooks (ADR-0004). Mark all three workplans finished with contract tests (80 passing).
53 lines
No EOL
1.7 KiB
TypeScript
53 lines
No EOL
1.7 KiB
TypeScript
/**
|
|
* Markdown text extraction → canonical text + gap-free offset map.
|
|
*
|
|
* Implements ADR-0003 for the `markdown-rendered` representation. Markup is
|
|
* stripped to plain text before `normalize()` is applied.
|
|
*/
|
|
|
|
import { normalize, type OffsetMap } from "@citation-evidence/engine/shared";
|
|
import { buildWholeTextOffsetMap } from "../shared/offset-map";
|
|
|
|
const FENCED_CODE = /```[\s\S]*?```/g;
|
|
const INLINE_CODE = /`([^`]+)`/g;
|
|
const IMAGE = /!\[([^\]]*)\]\([^)]+\)/g;
|
|
const LINK = /\[([^\]]+)\]\([^)]+\)/g;
|
|
const HEADING = /^#{1,6}\s+/gm;
|
|
const BLOCKQUOTE = /^>\s?/gm;
|
|
const LIST_MARKER = /^[\s]*[-*+]\s+/gm;
|
|
const ORDERED_LIST = /^[\s]*\d+\.\s+/gm;
|
|
const EMPHASIS = /(\*\*|__)(.*?)\1/g;
|
|
const ITALIC = /(\*|_)([^*_]+)\1/g;
|
|
const STRIKETHROUGH = /~~(.*?)~~/g;
|
|
const HORIZONTAL_RULE = /^[-*_]{3,}\s*$/gm;
|
|
|
|
export interface MarkdownExtractionResult {
|
|
readonly canonicalText: string;
|
|
readonly offsetMap: OffsetMap;
|
|
}
|
|
|
|
export function extractMarkdown(source: string): MarkdownExtractionResult {
|
|
const rawText = markdownToPlainText(source);
|
|
const canonicalText = normalize(rawText).text;
|
|
return {
|
|
canonicalText,
|
|
offsetMap: buildWholeTextOffsetMap(canonicalText),
|
|
};
|
|
}
|
|
|
|
function markdownToPlainText(source: string): string {
|
|
let text = source;
|
|
text = text.replace(FENCED_CODE, "\n\n");
|
|
text = text.replace(INLINE_CODE, "$1");
|
|
text = text.replace(IMAGE, "$1");
|
|
text = text.replace(LINK, "$1");
|
|
text = text.replace(HEADING, "");
|
|
text = text.replace(BLOCKQUOTE, "");
|
|
text = text.replace(LIST_MARKER, "");
|
|
text = text.replace(ORDERED_LIST, "");
|
|
text = text.replace(EMPHASIS, "$2");
|
|
text = text.replace(ITALIC, "$2");
|
|
text = text.replace(STRIKETHROUGH, "$1");
|
|
text = text.replace(HORIZONTAL_RULE, "\n\n");
|
|
return text;
|
|
} |