fix: convert HTML tables to markdown tables when extracting content

The Turndown configuration in ContentExtractor flattens every <table> into
a list of cell-per-line text — rows/columns are lost, which makes price
sheets and feature tables useless for RAG. On artdent's /araink/ page the
GFM tables plugin wasn't enough on its own: those tables are Yoast-style
all-<td> bodies with no <thead>, and turndown-plugin-gfm explicitly keeps
headerless tables as raw HTML (see the plugin's `keep` rule).

This wires the plugin in and preprocesses tables in cheerio first: any
<table> without existing <th>/thead has its first <tr>'s <td> cells
promoted to <th>. Verified on /araink/: 0 unconverted <table> tags, 71
markdown-table rows produced.

Ambient declaration added for turndown-plugin-gfm since the package ships
no types and there's no @types/turndown-plugin-gfm on npm.

package-lock.json

@@ -21,6 +21,7 @@
         "node-cron": "^4.2.1",
         "openai": "^6.9.1",
         "turndown": "^7.1.2",
+        "turndown-plugin-gfm": "^1.0.2",
         "uuid": "^9.0.1",
         "xml2js": "^0.6.2"
       },
@@ -545,7 +546,6 @@
       "version": "20.19.25",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
       "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
-      "peer": true,
       "dependencies": {
         "undici-types": "~6.21.0"
       }
@@ -1313,7 +1313,6 @@
       "version": "4.21.2",
       "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
       "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
-      "peer": true,
       "dependencies": {
         "accepts": "~1.3.8",
         "array-flatten": "1.1.1",
@@ -2489,6 +2488,11 @@
         "@mixmark-io/domino": "^2.2.0"
       }
     },
+    "node_modules/turndown-plugin-gfm": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
+      "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg=="
+    },
     "node_modules/type-is": {
       "version": "1.6.18",
       "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
@@ -2505,7 +2509,6 @@
       "version": "5.9.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
       "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
-      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -2647,7 +2650,6 @@
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
       "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
       "license": "MIT",
-      "peer": true,
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
       }

package.json

@@ -35,6 +35,7 @@
     "node-cron": "^4.2.1",
     "openai": "^6.9.1",
     "turndown": "^7.1.2",
+    "turndown-plugin-gfm": "^1.0.2",
     "uuid": "^9.0.1",
     "xml2js": "^0.6.2"
   },

src/scraper/ContentExtractor.ts

@@ -1,6 +1,8 @@
 import axios from 'axios';
 import * as cheerio from 'cheerio';
 import TurndownService from 'turndown';
+// turndown-plugin-gfm ships no type definitions; CommonJS interop import.
+import turndownGfm = require('turndown-plugin-gfm');
 import { logger } from '../utils/logger';
 import { getUserAgent } from '../version';
 
@@ -14,6 +16,11 @@ export class ContentExtractor {
       hr: '---',
       bulletListMarker: '-'
     });
+    // Without the GFM tables plugin, Turndown flattens <table> into one
+    // cell-per-line text block — the row/column structure is lost. The
+    // /araink/ price table on artdent was being saved as a flat list of
+    // numbers, which makes it useless for RAG.
+    this.turndownService.use(turndownGfm.tables);
   }
 
   /**
@@ -50,6 +57,25 @@ export class ContentExtractor {
       $('.advertisement').remove();
       $('.ad').remove();
 
+      // Promote each table's first row to a header so the GFM tables plugin
+      // matches. WordPress/Divi tables come through as all-<td> bodies with no
+      // <thead>; without a heading row, turndown-plugin-gfm explicitly keeps
+      // the table as raw HTML (see plugin source: `keep(node => TABLE && !isHeadingRow(rows[0]))`).
+      // We don't have semantic header info on these tables anyway, so treating
+      // the first row as a label row is the closest we get to a useful table.
+      $('table').each((_, tableEl) => {
+        const $table = $(tableEl);
+        if ($table.find('th').length > 0) return; // already has headers
+        const $firstRow = $table.find('tr').first();
+        if ($firstRow.length === 0) return;
+        $firstRow.find('td').each((__, td) => {
+          const $td = $(td);
+          const $th = $('<th></th>');
+          $th.html($td.html() || '');
+          $td.replaceWith($th);
+        });
+      });
+
       // Try to find main content area
       let mainContent = '';
 

src/types/turndown-plugin-gfm.d.ts

@@ -0,0 +1,9 @@
+// Minimal ambient declaration for turndown-plugin-gfm (no @types package exists).
+declare module 'turndown-plugin-gfm' {
+  import TurndownService from 'turndown';
+  export function gfm(turndownService: TurndownService): void;
+  export function tables(turndownService: TurndownService): void;
+  export function strikethrough(turndownService: TurndownService): void;
+  export function taskListItems(turndownService: TurndownService): void;
+  export function highlightedCodeBlock(turndownService: TurndownService): void;
+}