import { JSDOM } from 'jsdom'; import TurndownService from 'turndown'; import { URL } from 'url'; interface HtmlProcessorOptions { baseUrl: string; includeImages?: boolean; stripTrackingParams?: boolean; } export class HtmlToMarkdownProcessor { private turndownService: TurndownService; private options: HtmlProcessorOptions; constructor(options: HtmlProcessorOptions) { this.options = options; this.turndownService = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', hr: '---', }); // Custom rules for turndown if needed, e.g., to handle specific elements // or to modify how links/images are rendered. this.turndownService.addRule('anchor', { filter: ['a'], replacement: (content, node) => { const anchor = node as HTMLAnchorElement; let href = anchor.href; if (href) { try { const resolvedUrl = new URL(href, this.options.baseUrl); if (this.options.stripTrackingParams) { // Simple stripping example resolvedUrl.searchParams.forEach((_value, key) => { if (key.startsWith('utm_') || key.startsWith('gclid')) { resolvedUrl.searchParams.delete(key); } }); } // Ensure absolute URL href = resolvedUrl.toString(); } catch (e) { // Handle malformed URLs if necessary } } return `[${content}](${href || ''})`; } }); if (!this.options.includeImages) { this.turndownService.remove('img'); } else { this.turndownService.addRule('image', { filter: ['img'], replacement: (content, node) => { const img = node as HTMLImageElement; let src = img.src; if (src) { try { const resolvedUrl = new URL(src, this.options.baseUrl); src = resolvedUrl.toString(); } catch (e) { // Handle malformed URLs } } return `![${img.alt || ''}](${src || ''})`; } }); } } public process(html: string): string { const dom = new JSDOM(html); const document = dom.window.document; // A simplistic main content extraction for demonstration. // In a real library, this would involve sophisticated algorithms. let mainContentElement = document.querySelector('main') || document.body; // Remove common noisy elements from the main content (or entire document) ['nav', 'header', 'footer', 'aside', 'script', 'style', 'iframe'].forEach(tag => { document.querySelectorAll(tag).forEach(el => el.remove()); }); // Further heuristic-based removal could target ad divs, social buttons, etc. return this.turndownService.turndown(mainContentElement.innerHTML); } } // Example usage: const htmlContent = ` Product Page

Site Header

Product Name

This is a great product. More info

Product Image
`; const processor = new HtmlToMarkdownProcessor({ baseUrl: 'https://example.com', includeImages: true, stripTrackingParams: true, }); const markdown = processor.process(htmlContent); console.log(markdown); /* Expected output (simplified): ## Product Name This is a great product. [More info](https://example.com/details?id=123) ![Product Image](https://example.com/assets/product.jpg) */