Product Name
This is a great product. More info
import { JSDOM } from 'jsdom'; import TurndownService from 'turndown'; import { URL } from 'url'; interface HtmlProcessorOptions { baseUrl: string; includeImages?: boolean; stripTrackingParams?: boolean; } export class HtmlToMarkdownProcessor { private turndownService: TurndownService; private options: HtmlProcessorOptions; constructor(options: HtmlProcessorOptions) { this.options = options; this.turndownService = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', hr: '---', }); // Custom rules for turndown if needed, e.g., to handle specific elements // or to modify how links/images are rendered. this.turndownService.addRule('anchor', { filter: ['a'], replacement: (content, node) => { const anchor = node as HTMLAnchorElement; let href = anchor.href; if (href) { try { const resolvedUrl = new URL(href, this.options.baseUrl); if (this.options.stripTrackingParams) { // Simple stripping example resolvedUrl.searchParams.forEach((_value, key) => { if (key.startsWith('utm_') || key.startsWith('gclid')) { resolvedUrl.searchParams.delete(key); } }); } // Ensure absolute URL href = resolvedUrl.toString(); } catch (e) { // Handle malformed URLs if necessary } } return `[${content}](${href || ''})`; } }); if (!this.options.includeImages) { this.turndownService.remove('img'); } else { this.turndownService.addRule('image', { filter: ['img'], replacement: (content, node) => { const img = node as HTMLImageElement; let src = img.src; if (src) { try { const resolvedUrl = new URL(src, this.options.baseUrl); src = resolvedUrl.toString(); } catch (e) { // Handle malformed URLs } } return ``; } }); } } public process(html: string): string { const dom = new JSDOM(html); const document = dom.window.document; // A simplistic main content extraction for demonstration. // In a real library, this would involve sophisticated algorithms. let mainContentElement = document.querySelector('main') || document.body; // Remove common noisy elements from the main content (or entire document) ['nav', 'header', 'footer', 'aside', 'script', 'style', 'iframe'].forEach(tag => { document.querySelectorAll(tag).forEach(el => el.remove()); }); // Further heuristic-based removal could target ad divs, social buttons, etc. return this.turndownService.turndown(mainContentElement.innerHTML); } } // Example usage: const htmlContent = `
This is a great product. More info