|
|
@@ -21,6 +21,38 @@ export class ContentExtractor {
|
|
|
// /araink/ price table on artdent was being saved as a flat list of
|
|
|
// numbers, which makes it useless for RAG.
|
|
|
this.turndownService.use(turndownGfm.tables);
|
|
|
+ // GFM strikethrough handles <s>, <del>, <strike>. Without it the slash
|
|
|
+ // information is silently dropped, which on artdent's discounted prices
|
|
|
+ // (`<s>10.000 Ft</s> 7.000 Ft`) leaves both prices side by side with
|
|
|
+ // no indication which is the old one.
|
|
|
+ this.turndownService.use(turndownGfm.strikethrough);
|
|
|
+
|
|
|
+ // WordPress WYSIWYG editors and Divi/Elementor often emit emphasis as
|
|
|
+ // <span style="..."> rather than semantic tags. Map the common ones to
|
|
|
+ // markdown so the formatting reaches the RAG corpus.
|
|
|
+ this.turndownService.addRule('inlineStyleSpan', {
|
|
|
+ filter: (node: any) => {
|
|
|
+ if (node.nodeName !== 'SPAN') return false;
|
|
|
+ const style = (node.getAttribute('style') || '').toLowerCase();
|
|
|
+ return /text-decoration\s*:\s*[^;]*line-through/.test(style)
|
|
|
+ || /font-weight\s*:\s*(bold|[6-9]00)/.test(style)
|
|
|
+ || /font-style\s*:\s*italic/.test(style);
|
|
|
+ },
|
|
|
+ replacement: (content: string, node: any) => {
|
|
|
+ const style = (node.getAttribute('style') || '').toLowerCase();
|
|
|
+ let out = content;
|
|
|
+ // Apply in inside-out order so the final string looks like ~~**X**~~.
|
|
|
+ if (/font-style\s*:\s*italic/.test(style)) out = `_${out}_`;
|
|
|
+ if (/font-weight\s*:\s*(bold|[6-9]00)/.test(style)) out = `**${out}**`;
|
|
|
+ if (/text-decoration\s*:\s*[^;]*line-through/.test(style)) out = `~~${out}~~`;
|
|
|
+ return out;
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // Markdown has no native syntax for these inline elements. Keep them as
|
|
|
+ // raw HTML so the semantic info isn't lost (most markdown renderers and
|
|
|
+ // embedding tokenizers handle inline HTML fine).
|
|
|
+ this.turndownService.keep(['u', 'mark', 'sup', 'sub', 'kbd']);
|
|
|
}
|
|
|
|
|
|
/**
|