|
|
@@ -1,6 +1,8 @@
|
|
|
import axios from 'axios';
|
|
|
import * as cheerio from 'cheerio';
|
|
|
import TurndownService from 'turndown';
|
|
|
+// turndown-plugin-gfm ships no type definitions; CommonJS interop import.
|
|
|
+import turndownGfm = require('turndown-plugin-gfm');
|
|
|
import { logger } from '../utils/logger';
|
|
|
import { getUserAgent } from '../version';
|
|
|
|
|
|
@@ -14,6 +16,11 @@ export class ContentExtractor {
|
|
|
hr: '---',
|
|
|
bulletListMarker: '-'
|
|
|
});
|
|
|
+ // Without the GFM tables plugin, Turndown flattens <table> into one
|
|
|
+ // cell-per-line text block — the row/column structure is lost. The
|
|
|
+ // /araink/ price table on artdent was being saved as a flat list of
|
|
|
+ // numbers, which makes it useless for RAG.
|
|
|
+ this.turndownService.use(turndownGfm.tables);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
@@ -50,6 +57,25 @@ export class ContentExtractor {
|
|
|
$('.advertisement').remove();
|
|
|
$('.ad').remove();
|
|
|
|
|
|
+ // Promote each table's first row to a header so the GFM tables plugin
|
|
|
+ // matches. WordPress/Divi tables come through as all-<td> bodies with no
|
|
|
+ // <thead>; without a heading row, turndown-plugin-gfm explicitly keeps
|
|
|
+ // the table as raw HTML (see plugin source: `keep(node => TABLE && !isHeadingRow(rows[0]))`).
|
|
|
+ // We don't have semantic header info on these tables anyway, so treating
|
|
|
+ // the first row as a label row is the closest we get to a useful table.
|
|
|
+ $('table').each((_, tableEl) => {
|
|
|
+ const $table = $(tableEl);
|
|
|
+ if ($table.find('th').length > 0) return; // already has headers
|
|
|
+ const $firstRow = $table.find('tr').first();
|
|
|
+ if ($firstRow.length === 0) return;
|
|
|
+ $firstRow.find('td').each((__, td) => {
|
|
|
+ const $td = $(td);
|
|
|
+ const $th = $('<th></th>');
|
|
|
+ $th.html($td.html() || '');
|
|
|
+ $td.replaceWith($th);
|
|
|
+ });
|
|
|
+ });
|
|
|
+
|
|
|
// Try to find main content area
|
|
|
let mainContent = '';
|
|
|
|
