feat: expand categories to 13, add link discovery and URL exclusions

- Expand ContentType from 7 to 13 categories (services, about, team, blog, pricing, testimonials, gallery, landing)
- Add LinkDiscovery engine with BFS crawl for supplementing/replacing sitemap URLs
- Add URL exclusions table and API endpoints for glob-pattern based filtering
- Integrate link discovery and exclusions into WebScraper scrape flow
- Fall back to link discovery when no sitemap is available

src/api/components/ContentEndpoint.ts

@@ -6,6 +6,7 @@ import { QdrantService } from '../../services/QdrantService';
 import { ContentExtractor } from '../../scraper/ContentExtractor';
 import { QdrantEmbeddingWorkflow } from '../../services/QdrantEmbeddingWorkflow';
 import { createHash } from 'crypto';
+import { ContentType } from '../../types';
 import type { AppConfig } from '../../config';
 
 export class ContentEndpoint extends BaseEndpointComponent {
@@ -261,7 +262,7 @@ export class ContentEndpoint extends BaseEndpointComponent {
           // Save the scraped content
           const saveResult = this.db.saveContent(
             site.id,
-            content.content_type as 'shipping' | 'contacts' | 'terms' | 'faq',
+            content.content_type as ContentType,
             content.url,
             extractionResult.content,
             extractionResult.title
@@ -273,7 +274,7 @@ export class ContentEndpoint extends BaseEndpointComponent {
             await this.embeddingWorkflow.processContentForEmbedding(
               site.id,
               saveResult.contentId,
-              content.content_type as 'shipping' | 'contacts' | 'terms' | 'faq',
+              content.content_type as ContentType,
               content.url,
               extractionResult.content,
               extractionResult.title,

src/api/components/CustomUrlsEndpoint.ts

@@ -42,7 +42,7 @@ export class CustomUrlsEndpoint extends BaseEndpointComponent {
                 },
                 content_type: {
                   type: 'string',
-                  enum: ['shipping', 'contacts', 'terms', 'faq'],
+                  enum: ['shipping', 'contacts', 'terms', 'faq', 'services', 'about', 'team', 'blog', 'pricing', 'testimonials', 'gallery', 'landing', 'other'],
                   description: 'Type of content to scrape from this URL'
                 }
               },
@@ -256,10 +256,10 @@ export class CustomUrlsEndpoint extends BaseEndpointComponent {
       }
 
       // Validate content_type
-      const validContentTypes = ['shipping', 'contacts', 'terms', 'faq'];
+      const validContentTypes = ['shipping', 'contacts', 'terms', 'faq', 'services', 'about', 'team', 'blog', 'pricing', 'testimonials', 'gallery', 'landing', 'other'];
       if (!validContentTypes.includes(content_type)) {
         res.status(400).json({
-          error: 'content_type must be one of: shipping, contacts, terms, faq'
+          error: 'content_type must be one of: ' + validContentTypes.join(', ')
         });
         return;
       }

src/api/components/QdrantEndpoint.ts

@@ -8,6 +8,7 @@ import { QdrantCleanupService } from '../../services/QdrantCleanupService';
 import { QdrantEmbeddingWorkflow } from '../../services/QdrantEmbeddingWorkflow';
 import { QdrantMigrationService } from '../../services/QdrantMigrationService';
 import { SchedulerService } from '../../services/SchedulerService';
+import { ContentType, ALL_CONTENT_TYPES } from '../../types';
 import type { AppConfig } from '../../config';
 
 export class QdrantEndpoint extends BaseEndpointComponent {
@@ -439,7 +440,7 @@ export class QdrantEndpoint extends BaseEndpointComponent {
               properties: {
                 content_type: {
                   type: 'string',
-                  enum: ['shipping', 'contacts', 'terms', 'faq'],
+                  enum: ['shipping', 'contacts', 'terms', 'faq', 'services', 'about', 'team', 'blog', 'pricing', 'testimonials', 'gallery', 'landing', 'other'],
                   description: 'Optional: re-embed only specific content type'
                 }
               }
@@ -1253,7 +1254,7 @@ export class QdrantEndpoint extends BaseEndpointComponent {
 
       // Optimized: Process content in chunks to avoid loading everything into memory
       const CHUNK_SIZE = 50;
-      const contentTypes: Array<'shipping' | 'contacts' | 'terms' | 'faq'> = ['shipping', 'contacts', 'terms', 'faq'];
+      const contentTypes: Array<ContentType> = [...ALL_CONTENT_TYPES];
       let totalItemsQueued = 0;
 
       // Start background processing without blocking the response

src/api/components/SearchEndpoint.ts

@@ -7,8 +7,9 @@ import { EmbeddingService } from '../../services/EmbeddingService';
 import { McpLogger } from '../../mcp/McpLogger';
 import type { AppConfig } from '../../config';
 
-type Category = 'shipping' | 'contacts' | 'terms' | 'faq';
-const CATEGORIES: Category[] = ['shipping', 'contacts', 'terms', 'faq'];
+import { ContentType, ALL_CONTENT_TYPES } from '../../types';
+type Category = ContentType;
+const CATEGORIES: Category[] = [...ALL_CONTENT_TYPES];
 
 export class SearchEndpoint extends BaseEndpointComponent {
   private qdrantService?: QdrantService;
@@ -69,7 +70,7 @@ export class SearchEndpoint extends BaseEndpointComponent {
                 },
                 category: {
                   type: 'string',
-                  description: 'Optional category filter: shipping | contacts | terms | faq',
+                  description: 'Optional category filter: shipping | contacts | terms | faq | services | about | team | blog | pricing | testimonials | gallery | landing | other',
                 },
                 limit: {
                   type: 'number',

src/api/components/SitesEndpoint.ts

@@ -166,7 +166,7 @@ export class SitesEndpoint extends BaseEndpointComponent {
               name: 'content_type',
               in: 'query',
               type: 'string',
-              description: 'Filter by content type: shipping, contacts, terms, faq'
+              description: 'Filter by content type: shipping, contacts, terms, faq, services, about, team, blog, pricing, testimonials, gallery, landing, other'
             }
           ],
           responses: {

src/api/components/UrlExclusionsEndpoint.ts

@@ -0,0 +1,247 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { SiteDatabase } from '../../database/Database';
+
+export class UrlExclusionsEndpoint extends BaseEndpointComponent {
+  constructor(private db: SiteDatabase) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'POST',
+        '/api/sites/:siteId/url-exclusions',
+        this.addUrlExclusion.bind(this),
+        {
+          summary: 'Add a URL exclusion pattern for a site',
+          description: 'Adds a glob pattern to exclude matching URLs from scraping for the specified site.',
+          tags: ['URL Exclusions'],
+          parameters: [
+            {
+              name: 'siteId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Site ID (internal UUID or custom UUID)'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                pattern: {
+                  type: 'string',
+                  description: 'Glob pattern to exclude URLs (e.g. "*/admin/*")'
+                }
+              },
+              required: ['pattern']
+            }
+          },
+          responses: {
+            '201': {
+              description: 'URL exclusion created successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  id: { type: 'string' },
+                  site_id: { type: 'string' },
+                  pattern: { type: 'string' },
+                  created_at: { type: 'string' }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid request (empty pattern)'
+            },
+            '404': {
+              description: 'Site not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/sites/:siteId/url-exclusions',
+        this.getUrlExclusions.bind(this),
+        {
+          summary: 'List all URL exclusion patterns for a site',
+          description: 'Retrieves all URL exclusion glob patterns configured for a site.',
+          tags: ['URL Exclusions'],
+          parameters: [
+            {
+              name: 'siteId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Site ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'List of URL exclusion patterns',
+              schema: {
+                type: 'object',
+                properties: {
+                  site_id: { type: 'string' },
+                  exclusions: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        site_id: { type: 'string' },
+                        pattern: { type: 'string' },
+                        created_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            '404': {
+              description: 'Site not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'DELETE',
+        '/api/sites/:siteId/url-exclusions/:exclusionId',
+        this.deleteUrlExclusion.bind(this),
+        {
+          summary: 'Delete a URL exclusion pattern',
+          description: 'Permanently removes a URL exclusion pattern from the site configuration.',
+          tags: ['URL Exclusions'],
+          parameters: [
+            {
+              name: 'siteId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Site ID (internal UUID or custom UUID)'
+            },
+            {
+              name: 'exclusionId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'URL exclusion ID'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'URL exclusion deleted successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Site not found'
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async addUrlExclusion(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const siteId = req.params.siteId || req.params.id;
+      const { pattern } = req.body;
+
+      if (!pattern || typeof pattern !== 'string' || pattern.trim() === '') {
+        res.status(400).json({ error: 'pattern field is required and must be a non-empty string' });
+        return;
+      }
+
+      const site = this.db.getSiteByAnyId(siteId);
+      if (!site) {
+        res.status(404).json({ error: 'Site not found' });
+        return;
+      }
+
+      try {
+        const id = this.db.createUrlExclusion(site.id, pattern.trim());
+        const exclusions = this.db.getUrlExclusions(site.id);
+        const created = exclusions.find(e => e.id === id);
+
+        res.status(201).json(created || { id, site_id: site.id, pattern: pattern.trim(), created_at: new Date().toISOString() });
+      } catch (error: any) {
+        if (error.code === 'SQLITE_CONSTRAINT_UNIQUE' || error.message?.includes('UNIQUE constraint')) {
+          res.status(409).json({ error: 'This exclusion pattern already exists for this site' });
+          return;
+        }
+        throw error;
+      }
+    } catch (error) {
+      logger.error('Error creating URL exclusion', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getUrlExclusions(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const siteId = req.params.siteId || req.params.id;
+      const site = this.db.getSiteByAnyId(siteId);
+
+      if (!site) {
+        res.status(404).json({ error: 'Site not found' });
+        return;
+      }
+
+      const exclusions = this.db.getUrlExclusions(site.id);
+
+      res.json({
+        site_id: site.id,
+        exclusions
+      });
+    } catch (error) {
+      logger.error('Error fetching URL exclusions', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async deleteUrlExclusion(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const siteId = req.params.siteId || req.params.id;
+      const exclusionId = req.params.exclusionId;
+
+      const site = this.db.getSiteByAnyId(siteId);
+      if (!site) {
+        res.status(404).json({ error: 'Site not found' });
+        return;
+      }
+
+      this.db.deleteUrlExclusion(exclusionId);
+
+      res.json({ message: 'URL exclusion deleted' });
+    } catch (error) {
+      logger.error('Error deleting URL exclusion', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/routes.ts

@@ -17,6 +17,7 @@ import { HealthEndpoint } from './components/HealthEndpoint';
 import { QdrantEndpoint } from './components/QdrantEndpoint';
 import { McpEndpoint } from './components/McpEndpoint';
 import { SearchEndpoint } from './components/SearchEndpoint';
+import { UrlExclusionsEndpoint } from './components/UrlExclusionsEndpoint';
 import type { AppConfig } from '../config';
 
 export function createRouter(jobQueue: JobQueue, apiKey: string, db?: SiteDatabase, config?: AppConfig): Router {
@@ -39,7 +40,8 @@ export function createRouter(jobQueue: JobQueue, apiKey: string, db?: SiteDataba
       new WebhooksEndpoint(db),
       new CustomUrlsEndpoint(db, jobQueue),
       new ContentEndpoint(db, config),
-      new McpEndpoint(db)
+      new McpEndpoint(db),
+      new UrlExclusionsEndpoint(db)
     );
 
     // Register Qdrant endpoints if config is available

src/database/Database.ts

@@ -1,6 +1,7 @@
 import Database from 'better-sqlite3';
 import { v4 as uuidv4 } from 'uuid';
 import { logger } from '../utils/logger';
+import { ContentType } from '../types';
 import path from 'path';
 
 export interface Site {
@@ -44,7 +45,7 @@ export interface ScrapeHistory {
 export interface SiteContent {
   id: string;
   site_id: string;
-  content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+  content_type: ContentType;
   url: string;
   content: string;
   content_hash: string; // MD5 hash of content to detect changes
@@ -95,7 +96,7 @@ export interface CustomUrl {
   id: string;
   site_id: string;
   url: string;
-  content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+  content_type: ContentType;
   enabled: boolean;
   created_at: string;
 }
@@ -110,6 +111,13 @@ export interface SiteEmbedding {
   updated_at: string;
 }
 
+export interface UrlExclusion {
+  id: string;
+  site_id: string;
+  pattern: string;
+  created_at: string;
+}
+
 export interface QdrantDeletionQueue {
   id: string;
   site_id: string;
@@ -506,6 +514,18 @@ export class SiteDatabase {
       ON mcp_logs(site_id, created_at)
     `);
 
+    // URL exclusions table (glob patterns to exclude URLs from scraping)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS url_exclusions (
+        id TEXT PRIMARY KEY,
+        site_id TEXT NOT NULL,
+        pattern TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE,
+        UNIQUE(site_id, pattern)
+      )
+    `);
+
     logger.info('Database tables initialized');
   }
 
@@ -902,7 +922,7 @@ export class SiteDatabase {
   // Content operations
   saveContent(
     siteId: string,
-    contentType: 'shipping' | 'contacts' | 'terms' | 'faq',
+    contentType: ContentType,
     url: string,
     content: string,
     title?: string
@@ -1489,7 +1509,7 @@ export class SiteDatabase {
   createCustomUrl(
     siteIdentifier: string,
     url: string,
-    contentType: 'shipping' | 'contacts' | 'terms' | 'faq'
+    contentType: ContentType
   ): CustomUrl {
     const site = this.getSiteByAnyId(siteIdentifier);
     if (!site) {
@@ -1596,6 +1616,55 @@ export class SiteDatabase {
     logger.info(`Deleted custom URL ${id}`);
   }
 
+  // URL Exclusion operations
+  createUrlExclusion(siteId: string, pattern: string): string {
+    const id = uuidv4();
+    const now = new Date().toISOString();
+
+    this.db.prepare(`
+      INSERT INTO url_exclusions (id, site_id, pattern, created_at)
+      VALUES (?, ?, ?, ?)
+    `).run(id, siteId, pattern, now);
+
+    logger.info(`Created URL exclusion ${id} for site ${siteId}: ${pattern}`);
+    return id;
+  }
+
+  getUrlExclusions(siteId: string): UrlExclusion[] {
+    const stmt = this.db.prepare(`
+      SELECT * FROM url_exclusions
+      WHERE site_id = ?
+      ORDER BY created_at DESC
+    `);
+    const results = stmt.all(siteId) as any[];
+
+    return results.map(r => ({
+      id: r.id,
+      site_id: r.site_id,
+      pattern: r.pattern,
+      created_at: r.created_at
+    }));
+  }
+
+  deleteUrlExclusion(id: string): void {
+    this.db.prepare('DELETE FROM url_exclusions WHERE id = ?').run(id);
+    logger.info(`Deleted URL exclusion ${id}`);
+  }
+
+  isUrlExcluded(siteId: string, url: string): boolean {
+    const exclusions = this.getUrlExclusions(siteId);
+    for (const exclusion of exclusions) {
+      // Convert glob pattern to regex: escape regex special chars, then convert * to .*
+      const escaped = exclusion.pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
+      const regexStr = escaped.replace(/\*/g, '.*');
+      const regex = new RegExp(`^${regexStr}$`);
+      if (regex.test(url)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   // Check if a URL exists in shop_content (already scraped)
   isUrlAlreadyScraped(siteIdentifier: string, url: string): boolean {
     const site = this.getSiteByAnyId(siteIdentifier);

src/mcp/tools/BaseTool.ts

@@ -2,6 +2,7 @@ import { SiteDatabase } from '../../database/Database';
 import { QdrantService } from '../../services/QdrantService';
 import { EmbeddingService } from '../../services/EmbeddingService';
 import { McpLogger } from '../McpLogger';
+import { ContentType } from '../../types';
 import { logger } from '../../utils/logger';
 
 export interface ToolResult {
@@ -23,14 +24,14 @@ export interface SearchResult {
 }
 
 export abstract class BaseTool {
-  protected contentType: 'shipping' | 'contacts' | 'terms' | 'faq';
+  protected contentType: ContentType;
 
   constructor(
     protected database: SiteDatabase,
     protected qdrantService: QdrantService,
     protected embeddingService: EmbeddingService,
     protected mcpLogger: McpLogger,
-    contentType: 'shipping' | 'contacts' | 'terms' | 'faq'
+    contentType: ContentType
   ) {
     this.contentType = contentType;
   }

src/mcp/tools/ListContentsTool.ts

@@ -1,4 +1,5 @@
 import { SiteDatabase } from '../../database/Database';
+import { ALL_CONTENT_TYPES } from '../../types';
 import { logger } from '../../utils/logger';
 
 export interface ToolResult {
@@ -15,6 +16,15 @@ const CONTENT_TYPE_TO_TOOL: Record<string, string> = {
   contacts: 'contacts_search',
   terms: 'terms_search',
   faq: 'faq_search',
+  services: 'services_search',
+  about: 'about_search',
+  team: 'team_search',
+  blog: 'blog_search',
+  pricing: 'pricing_search',
+  testimonials: 'testimonials_search',
+  gallery: 'gallery_search',
+  landing: 'landing_search',
+  other: 'other_search',
 };
 
 export class ListContentsTool {
@@ -54,20 +64,12 @@ export class ListContentsTool {
       }
 
       // Group content by type and get unique titles
-      const contentByType: Record<string, string[]> = {
-        shipping: [],
-        contacts: [],
-        terms: [],
-        faq: [],
-      };
-
-      // Use a Set to track unique URLs per type (to avoid duplicates from versioning)
-      const seenUrls: Record<string, Set<string>> = {
-        shipping: new Set(),
-        contacts: new Set(),
-        terms: new Set(),
-        faq: new Set(),
-      };
+      const contentByType: Record<string, string[]> = {};
+      const seenUrls: Record<string, Set<string>> = {};
+      for (const ct of ALL_CONTENT_TYPES) {
+        contentByType[ct] = [];
+        seenUrls[ct] = new Set();
+      }
 
       for (const content of allContent) {
         const type = content.content_type;

src/scraper/LinkDiscovery.ts

@@ -0,0 +1,156 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { logger } from '../utils/logger';
+
+export interface DiscoveredUrl {
+  url: string;
+  depth: number;
+  foundOn: string;
+}
+
+const TRACKING_PARAMS = new Set([
+  'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
+  'fbclid', 'gclid', 'ref',
+]);
+
+const SKIP_EXTENSIONS = new Set([
+  '.js', '.css', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.pdf',
+  '.zip', '.woff', '.woff2', '.ttf', '.eot', '.mp4', '.mp3', '.webp',
+  '.avif', '.map',
+]);
+
+const SKIP_PATH_PATTERNS = [
+  '/wp-admin/', '/wp-json/', '/wp-content/uploads/', '/wp-includes/',
+  '/feed/', '/xmlrpc.php', '/cart', '/checkout', '/my-account',
+  '/login', '/register', '/admin/', '/.well-known/',
+];
+
+const PAGINATION_RE = /\/page\/\d+/;
+
+export class LinkDiscovery {
+  private normalizeUrl(raw: string, origin: string): string | null {
+    let parsed: URL;
+    try {
+      parsed = new URL(raw, origin);
+    } catch {
+      return null;
+    }
+
+    if (parsed.origin !== origin) {
+      return null;
+    }
+
+    // Strip fragment
+    parsed.hash = '';
+
+    // Remove tracking params
+    for (const param of TRACKING_PARAMS) {
+      parsed.searchParams.delete(param);
+    }
+
+    // Sort remaining params for consistency
+    parsed.searchParams.sort();
+
+    let href = parsed.href;
+
+    // Collapse trailing slashes but keep root /
+    if (href !== origin + '/' && href.endsWith('/')) {
+      href = href.replace(/\/+$/, '');
+    }
+
+    return href;
+  }
+
+  private shouldSkip(url: string): boolean {
+    const pathname = new URL(url).pathname.toLowerCase();
+
+    // Skip static asset extensions
+    for (const ext of SKIP_EXTENSIONS) {
+      if (pathname.endsWith(ext)) {
+        return true;
+      }
+    }
+
+    // Skip known non-content paths
+    for (const pattern of SKIP_PATH_PATTERNS) {
+      if (pathname.includes(pattern)) {
+        return true;
+      }
+    }
+
+    // Skip pagination
+    if (PAGINATION_RE.test(pathname)) {
+      return true;
+    }
+
+    return false;
+  }
+
+  async discoverLinks(
+    baseUrl: string,
+    maxDepth: number = 3,
+    maxPages: number = 200,
+  ): Promise<DiscoveredUrl[]> {
+    const origin = new URL(baseUrl).origin;
+    const startUrl = this.normalizeUrl(baseUrl, origin) ?? baseUrl;
+
+    logger.info(
+      `LinkDiscovery: starting BFS crawl from ${origin}, maxDepth=${maxDepth}, maxPages=${maxPages}`,
+    );
+
+    const visited = new Set<string>();
+    const results: DiscoveredUrl[] = [];
+
+    // BFS queue: [url, depth, foundOn]
+    const queue: Array<[string, number, string]> = [[startUrl, 0, '']];
+
+    while (queue.length > 0) {
+      const [currentUrl, depth, foundOn] = queue.shift()!;
+
+      if (visited.has(currentUrl)) continue;
+      if (results.length >= maxPages) break;
+      if (depth > maxDepth) continue;
+      if (this.shouldSkip(currentUrl)) continue;
+
+      visited.add(currentUrl);
+      results.push({ url: currentUrl, depth, foundOn });
+
+      if (depth >= maxDepth) continue;
+
+      // Fetch and extract links
+      let html: string;
+      try {
+        const response = await axios.get(currentUrl, {
+          timeout: 15000,
+          maxRedirects: 3,
+          headers: { 'User-Agent': 'WebScraper/1.0' },
+          responseType: 'text',
+        });
+        html = response.data as string;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        logger.warn(`LinkDiscovery: failed to fetch ${currentUrl}: ${message}`);
+        continue;
+      }
+
+      const $ = cheerio.load(html);
+      $('a[href]').each((_, el) => {
+        const href = $(el).attr('href');
+        if (!href) return;
+
+        const normalized = this.normalizeUrl(href, origin);
+        if (!normalized) return;
+        if (visited.has(normalized)) return;
+        if (this.shouldSkip(normalized)) return;
+
+        queue.push([normalized, depth + 1, currentUrl]);
+      });
+    }
+
+    logger.info(
+      `LinkDiscovery: found ${results.length} pages (visited ${visited.size} URLs)`,
+    );
+
+    return results;
+  }
+}

src/scraper/SitemapParser.ts

@@ -1,7 +1,7 @@
 import axios from 'axios';
 import { parseString } from 'xml2js';
 import { promisify } from 'util';
-import { SitemapUrl, SitePlatform } from '../types';
+import { SitemapUrl, SitePlatform, ContentType, ALL_CONTENT_TYPES } from '../types';
 import { logger } from '../utils/logger';
 import { getUserAgent } from '../version';
 
@@ -198,15 +198,10 @@ export class SitemapParser {
   }
 
   /**
-   * Filter URLs to find specific page types
+   * Keyword arrays for URL classification by content type
    */
-  static filterUrlsByType(urls: SitemapUrl[]): {
-    shipping: SitemapUrl[];
-    contacts: SitemapUrl[];
-    terms: SitemapUrl[];
-    faq: SitemapUrl[];
-  } {
-    const shippingKeywords = [
+  private static readonly keywordMap: Record<string, string[]> = {
+    shipping: [
       // English terms
       'shipping', 'delivery', 'ship', 'deliver', 'postage', 'courier',
       'logistics', 'transport', 'freight', 'dispatch', 'fulfillment',
@@ -223,31 +218,27 @@ export class SitemapParser {
       'futarszolgalat', 'futarszallitas', 'hazhozszallitas', 'ingyenes-szallitas',
       'expressz-szallitas', 'gyors-szallitas', 'nemzetkozi-szallitas', 'hazai-szallitas',
       'postai-szallitas', 'csomagpont', 'csomagautomata', 'pickup-pont'
-    ];
+    ],
 
-    const contactKeywords = [
+    contacts: [
       // English terms
       'contact', 'contact-us', 'contact-info', 'contact-page', 'get-in-touch',
       'reach-us', 'find-us', 'locate-us', 'visit-us', 'where-to-find-us',
-      'about', 'about-us', 'about-company', 'who-we-are', 'our-story',
-      'company-info', 'business-info', 'team', 'our-team', 'meet-the-team',
       'email', 'phone', 'telephone', 'address', 'location', 'directions',
       'office', 'store-location', 'headquarters', 'branch', 'outlet',
-      'customer-service', 'customer-support', 'support', 'help-center',
+      'customer-service', 'customer-support',
       'impressum', 'imprint', 'legal-notice', 'company-details',
 
       // Hungarian terms
       'kapcsolat', 'kapcsolatfelvetel', 'elerhetoseg', 'elerheto', 'kontakt',
       'kapcsolatok', 'kapcsolat-felvetel', 'elerhetosegek', 'megkozelites',
-      'rolunk', 'magunkrol', 'cegunkrol', 'tortenelem', 'cegtortenet',
-      'bemutatkozas', 'csapat', 'csapatunk', 'munkatarsak', 'kollektiva',
       'telefon', 'telefonszam', 'email', 'e-mail', 'cim', 'lakcim', 'szekhely',
       'iroda', 'uzlet', 'bolt', 'telephelyunk', 'fiokunk', 'kepviselet',
-      'ugyfelszolgalat', 'vevoszolgalat', 'segitseg', 'tamogatas', 'support',
+      'ugyfelszolgalat', 'vevoszolgalat',
       'impresszum', 'jogi-nyilatkozat', 'cegadatok', 'vallalkozasi-adatok'
-    ];
+    ],
 
-    const termsKeywords = [
+    terms: [
       // English terms
       'terms', 'conditions', 'terms-and-conditions', 'terms-of-service', 'terms-of-use',
       'tos', 'toc', 'tc', 'user-agreement', 'user-terms', 'service-terms',
@@ -270,9 +261,9 @@ export class SitemapParser {
       'visszaterites', 'visszavételar', 'cserepolitika', 'reklamacio',
       'szerzoi-jog', 'vedjegy', 'jogok', 'impresszum', 'impressum',
       'gdpr', 'adatvedelmi-rendelet', 'megfeleloseg', 'jogszabaly'
-    ];
+    ],
 
-    const faqKeywords = [
+    faq: [
       // English terms
       'faq', 'frequently-asked-questions', 'frequently-asked', 'f-a-q',
       'questions', 'common-questions', 'popular-questions', 'most-asked',
@@ -294,18 +285,97 @@ export class SitemapParser {
       'utmutato', 'utmutatók', 'kezikonyv', 'manual', 'dokumentacio',
       'felhasznaloi-utmutato', 'hogyan', 'hibaelharitas', 'problema',
       'megoldas', 'valaszok', 'kozosseg', 'forum', 'megbeszeles'
-    ];
+    ],
+
+    services: [
+      'services', 'szolgaltatas', 'treatments', 'kezeles', 'offerings',
+      'megoldasok', 'solutions', 'leistungen'
+    ],
+
+    about: [
+      'about', 'rolunk', 'history', 'our-story', 'tortenet',
+      'ars-poetica', 'uber-uns', 'cegunkrol'
+    ],
+
+    team: [
+      'team', 'csapat', 'munkatars', 'our-team', 'doctors',
+      'staff', 'orvos', 'specialist', 'mitarbeiter'
+    ],
+
+    blog: [
+      'blog', 'articles', 'hirek', 'cikkek', 'posts',
+      'news', 'magazin', 'artikel'
+    ],
+
+    pricing: [
+      'pricing', 'prices', 'araink', 'arajanlat', 'plans',
+      'packages', 'fees', 'dijszabas', 'preise', 'price-list'
+    ],
+
+    testimonials: [
+      'testimonials', 'reviews', 'velemenyek', 'mondtak', 'kamera-elott',
+      'feedback', 'ertekelesek', 'bewertungen'
+    ],
+
+    gallery: [
+      'gallery', 'kepek', 'photos', 'pictures', 'portfolio',
+      'esetbemutatas', 'case-stud', 'album', 'galerie'
+    ],
+
+    landing: [
+      'akcio', 'promo', 'campaign', 'landing', 'offer',
+      'ajanlat', 'special', 'deal', 'angebot'
+    ]
+  };
+
+  private static containsKeyword(url: string, keywords: string[]): boolean {
+    const lowerUrl = url.toLowerCase();
+    return keywords.some(keyword => lowerUrl.includes(keyword));
+  }
 
-    const containsKeyword = (url: string, keywords: string[]): boolean => {
-      const lowerUrl = url.toLowerCase();
-      return keywords.some(keyword => lowerUrl.includes(keyword));
-    };
+  /**
+   * Classify a single URL into a content type
+   */
+  static classifyUrl(url: string): ContentType {
+    for (const [type, keywords] of Object.entries(this.keywordMap)) {
+      if (this.containsKeyword(url, keywords)) {
+        return type as ContentType;
+      }
+    }
+    return 'other';
+  }
 
-    return {
-      shipping: urls.filter(u => containsKeyword(u.loc, shippingKeywords)),
-      contacts: urls.filter(u => containsKeyword(u.loc, contactKeywords)),
-      terms: urls.filter(u => containsKeyword(u.loc, termsKeywords)),
-      faq: urls.filter(u => containsKeyword(u.loc, faqKeywords))
-    };
+  /**
+   * Classify multiple URLs, grouping them by content type
+   */
+  static classifyUrls(urls: string[]): Map<ContentType, string[]> {
+    const result = new Map<ContentType, string[]>();
+    for (const ct of ALL_CONTENT_TYPES) {
+      result.set(ct, []);
+    }
+    for (const url of urls) {
+      const type = this.classifyUrl(url);
+      result.get(type)!.push(url);
+    }
+    return result;
+  }
+
+  /**
+   * Filter URLs to find specific page types
+   */
+  static filterUrlsByType(urls: SitemapUrl[]): Record<ContentType, SitemapUrl[]> {
+    const result: Record<string, SitemapUrl[]> = {};
+    for (const ct of ALL_CONTENT_TYPES) {
+      result[ct] = [];
+    }
+
+    for (const u of urls) {
+      const type = this.classifyUrl(u.loc);
+      if (type !== 'other') {
+        result[type].push(u);
+      }
+    }
+
+    return result as Record<ContentType, SitemapUrl[]>;
   }
 }

src/scraper/WebScraper.ts

@@ -1,5 +1,6 @@
-import { ScraperResult, PageContent } from '../types';
+import { ScraperResult, PageContent, ContentType, ALL_CONTENT_TYPES } from '../types';
 import { SitemapParser, resolveSitemapUrl } from './SitemapParser';
+import { LinkDiscovery } from './LinkDiscovery';
 import { ContentExtractor } from './ContentExtractor';
 import { logger } from '../utils/logger';
 import { SiteDatabase } from '../database/Database';
@@ -56,7 +57,61 @@ export class WebScraper {
       }
 
       // Parse sitemap
-      const urls = await SitemapParser.parseSitemap(sitemapUrl);
+      let urls: any[] = [];
+      let sitemapAvailable = true;
+      try {
+        urls = await SitemapParser.parseSitemap(sitemapUrl);
+      } catch (error) {
+        logger.warn(`Failed to parse sitemap at ${sitemapUrl}: ${error}`);
+        urls = [];
+        sitemapAvailable = false;
+      }
+
+      // Run link discovery to supplement (or replace) sitemap URLs
+      const discovery = new LinkDiscovery();
+      let maxDepth = 3;
+      let maxPages = 200;
+      if (this.db && siteId) {
+        const siteRecord = this.db.getSiteByAnyId(siteId);
+        if (siteRecord) {
+          maxDepth = siteRecord.max_crawl_depth ?? 3;
+          maxPages = siteRecord.max_pages ?? 200;
+        }
+      }
+
+      if (urls.length === 0) {
+        logger.info('No sitemap found, using link discovery as primary URL source');
+      }
+
+      const discoveredUrls = await discovery.discoverLinks(baseUrl, maxDepth, maxPages);
+
+      // Merge sitemap URLs with discovered URLs (deduplicate by URL string)
+      const existingUrlSet = new Set(urls.map((u: any) => u.loc));
+      for (const discovered of discoveredUrls) {
+        if (!existingUrlSet.has(discovered.url)) {
+          urls.push({
+            loc: discovered.url,
+            lastmod: undefined,
+            changefreq: undefined,
+            priority: undefined
+          });
+          existingUrlSet.add(discovered.url);
+        }
+      }
+
+      if (discoveredUrls.length > 0) {
+        logger.info(`Link discovery found ${discoveredUrls.length} URLs, ${urls.length} total after merge with sitemap`);
+      }
+
+      // Remove excluded URLs before classification
+      if (this.db && siteId) {
+        const beforeCount = urls.length;
+        urls = urls.filter((u: any) => !this.db!.isUrlExcluded(siteId!, u.loc));
+        const excludedCount = beforeCount - urls.length;
+        if (excludedCount > 0) {
+          logger.info(`Excluded ${excludedCount} URLs via URL exclusion rules`);
+        }
+      }
 
       // Get the first URL to extract changefreq and lastmod for scheduling
       const firstUrl = urls.length > 0 ? urls[0] : null;
@@ -71,10 +126,9 @@ export class WebScraper {
 
         // Remove disabled URLs from each category
         if (disabledUrlSet.size > 0) {
-          filteredUrls.shipping = filteredUrls.shipping.filter((u: any) => !disabledUrlSet.has(u.loc));
-          filteredUrls.contacts = filteredUrls.contacts.filter((u: any) => !disabledUrlSet.has(u.loc));
-          filteredUrls.terms = filteredUrls.terms.filter((u: any) => !disabledUrlSet.has(u.loc));
-          filteredUrls.faq = filteredUrls.faq.filter((u: any) => !disabledUrlSet.has(u.loc));
+          for (const ct of ALL_CONTENT_TYPES) {
+            filteredUrls[ct] = filteredUrls[ct].filter((u: any) => !disabledUrlSet.has(u.loc));
+          }
           logger.info(`Filtered out ${disabledUrlSet.size} disabled URLs from scraping`);
         }
       }
@@ -84,23 +138,18 @@ export class WebScraper {
         const customUrls = this.db.getEnabledCustomUrls(siteId);
 
         // Group custom URLs by content type
-        const customUrlsByType: { [key: string]: any[] } = {
-          shipping: [],
-          contacts: [],
-          terms: [],
-          faq: []
-        };
+        const customUrlsByType: Record<string, any[]> = {};
+        for (const ct of ALL_CONTENT_TYPES) {
+          customUrlsByType[ct] = [];
+        }
 
         for (const customUrl of customUrls) {
-          const contentTypeKey = customUrl.content_type === 'shipping' ? 'shipping' :
-                                customUrl.content_type === 'contacts' ? 'contacts' :
-                                customUrl.content_type === 'terms' ? 'terms' :
-                                'faq';
+          const contentTypeKey = customUrl.content_type as ContentType;
 
           // Check if this URL already exists in sitemap URLs
-          const alreadyExists = filteredUrls[contentTypeKey].some((u: any) => u.loc === customUrl.url);
+          const alreadyExists = filteredUrls[contentTypeKey]?.some((u: any) => u.loc === customUrl.url);
 
-          if (!alreadyExists) {
+          if (!alreadyExists && customUrlsByType[contentTypeKey]) {
             // Add custom URL in the same format as sitemap URLs
             customUrlsByType[contentTypeKey].push({
               loc: customUrl.url,
@@ -115,168 +164,63 @@ export class WebScraper {
         }
 
         // Merge custom URLs with sitemap URLs
-        filteredUrls.shipping = [...filteredUrls.shipping, ...customUrlsByType.shipping];
-        filteredUrls.contacts = [...filteredUrls.contacts, ...customUrlsByType.contacts];
-        filteredUrls.terms = [...filteredUrls.terms, ...customUrlsByType.terms];
-        filteredUrls.faq = [...filteredUrls.faq, ...customUrlsByType.faq];
+        for (const ct of ALL_CONTENT_TYPES) {
+          filteredUrls[ct] = [...filteredUrls[ct], ...customUrlsByType[ct]];
+        }
       }
 
-      logger.info('Filtered URLs', {
-        shipping: filteredUrls.shipping.length,
-        contacts: filteredUrls.contacts.length,
-        terms: filteredUrls.terms.length,
-        faq: filteredUrls.faq.length
-      });
+      const urlCounts: Record<string, number> = {};
+      for (const ct of ALL_CONTENT_TYPES) {
+        urlCounts[ct] = filteredUrls[ct].length;
+      }
+      logger.info('Filtered URLs', urlCounts);
 
       // Extract content from each page type - now supporting multiple pages per category
-      const shippingContent = await this.extractAllPageContent(filteredUrls.shipping);
-      const contactsContent = await this.extractAllPageContent(filteredUrls.contacts);
-      const termsContent = await this.extractAllPageContent(filteredUrls.terms);
-      const faqContent = await this.extractAllPageContent(filteredUrls.faq);
+      const contentByType: Record<ContentType, PageContent[]> = {} as any;
+      for (const ct of ALL_CONTENT_TYPES) {
+        contentByType[ct] = await this.extractAllPageContent(filteredUrls[ct]);
+      }
 
       // Save content to database if available
       if (this.db && siteId) {
         // Collect all content for batch embedding processing
         const embeddingBatch: any[] = [];
 
-        // Save all shipping pages
-        for (const content of shippingContent) {
-          const result = this.db.saveContent(siteId, 'shipping', content.url, content.content, content.title);
-
-          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
-          if (result.skipped) continue;
-
-          // Add to embedding batch if workflow is available
-          if (this.embeddingWorkflow) {
-            embeddingBatch.push({
-              contentId: result.contentId,
-              contentType: 'shipping',
-              url: content.url,
-              content: content.content,
-              title: content.title,
-              contentHash: this.calculateContentHash(content.content),
-              contentChanged: result.changed
-            });
-          }
-
-          // Trigger webhook if content changed
-          if (result.changed && this.webhookManager) {
-            const previous = this.db.getContentHistory(siteId, 'shipping', 2);
-            const oldHash = previous.length > 1 ? previous[1].content_hash : '';
-            const newHash = previous.length > 0 ? previous[0].content_hash : '';
-
-            await this.webhookManager.notifyContentChanged(
-              siteId,
-              'shipping',
-              content.url,
-              oldHash,
-              newHash
-            );
-          }
-        }
-        // Save all contact pages
-        for (const content of contactsContent) {
-          const result = this.db.saveContent(siteId, 'contacts', content.url, content.content, content.title);
-
-          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
-          if (result.skipped) continue;
-
-          // Add to embedding batch if workflow is available
-          if (this.embeddingWorkflow) {
-            embeddingBatch.push({
-              contentId: result.contentId,
-              contentType: 'contacts',
-              url: content.url,
-              content: content.content,
-              title: content.title,
-              contentHash: this.calculateContentHash(content.content),
-              contentChanged: result.changed
-            });
-          }
-
-          // Trigger webhook if content changed
-          if (result.changed && this.webhookManager) {
-            const previous = this.db.getContentHistory(siteId, 'contacts', 2);
-            const oldHash = previous.length > 1 ? previous[1].content_hash : '';
-            const newHash = previous.length > 0 ? previous[0].content_hash : '';
-
-            await this.webhookManager.notifyContentChanged(
-              siteId,
-              'contacts',
-              content.url,
-              oldHash,
-              newHash
-            );
-          }
-        }
-        // Save all terms pages
-        for (const content of termsContent) {
-          const result = this.db.saveContent(siteId, 'terms', content.url, content.content, content.title);
-
-          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
-          if (result.skipped) continue;
-
-          // Add to embedding batch if workflow is available
-          if (this.embeddingWorkflow) {
-            embeddingBatch.push({
-              contentId: result.contentId,
-              contentType: 'terms',
-              url: content.url,
-              content: content.content,
-              title: content.title,
-              contentHash: this.calculateContentHash(content.content),
-              contentChanged: result.changed
-            });
-          }
-
-          // Trigger webhook if content changed
-          if (result.changed && this.webhookManager) {
-            const previous = this.db.getContentHistory(siteId, 'terms', 2);
-            const oldHash = previous.length > 1 ? previous[1].content_hash : '';
-            const newHash = previous.length > 0 ? previous[0].content_hash : '';
-
-            await this.webhookManager.notifyContentChanged(
-              siteId,
-              'terms',
-              content.url,
-              oldHash,
-              newHash
-            );
-          }
-        }
-        // Save all FAQ pages
-        for (const content of faqContent) {
-          const result = this.db.saveContent(siteId, 'faq', content.url, content.content, content.title);
-
-          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
-          if (result.skipped) continue;
-
-          // Add to embedding batch if workflow is available
-          if (this.embeddingWorkflow) {
-            embeddingBatch.push({
-              contentId: result.contentId,
-              contentType: 'faq',
-              url: content.url,
-              content: content.content,
-              title: content.title,
-              contentHash: this.calculateContentHash(content.content),
-              contentChanged: result.changed
-            });
-          }
+        // Save all pages for each content type
+        for (const ct of ALL_CONTENT_TYPES) {
+          for (const content of contentByType[ct]) {
+            const result = this.db.saveContent(siteId, ct, content.url, content.content, content.title);
+
+            // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
+            if (result.skipped) continue;
+
+            // Add to embedding batch if workflow is available
+            if (this.embeddingWorkflow) {
+              embeddingBatch.push({
+                contentId: result.contentId,
+                contentType: ct,
+                url: content.url,
+                content: content.content,
+                title: content.title,
+                contentHash: this.calculateContentHash(content.content),
+                contentChanged: result.changed
+              });
+            }
 
-          // Trigger webhook if content changed
-          if (result.changed && this.webhookManager) {
-            const previous = this.db.getContentHistory(siteId, 'faq', 2);
-            const oldHash = previous.length > 1 ? previous[1].content_hash : '';
-            const newHash = previous.length > 0 ? previous[0].content_hash : '';
-
-            await this.webhookManager.notifyContentChanged(
-              siteId,
-              'faq',
-              content.url,
-              oldHash,
-              newHash
-            );
+            // Trigger webhook if content changed
+            if (result.changed && this.webhookManager) {
+              const previous = this.db.getContentHistory(siteId, ct, 2);
+              const oldHash = previous.length > 1 ? previous[1].content_hash : '';
+              const newHash = previous.length > 0 ? previous[0].content_hash : '';
+
+              await this.webhookManager.notifyContentChanged(
+                siteId,
+                ct,
+                content.url,
+                oldHash,
+                newHash
+              );
+            }
           }
         }
 
@@ -303,7 +247,10 @@ export class WebScraper {
 
         // Update analytics
         const scrapeTimeMs = Date.now() - startTime;
-        const pageCount = shippingContent.length + contactsContent.length + termsContent.length + faqContent.length;
+        let pageCount = 0;
+        for (const ct of ALL_CONTENT_TYPES) {
+          pageCount += contentByType[ct].length;
+        }
 
         this.db.updateAnalytics(siteId, {
           scrapeTimeMs,
@@ -321,10 +268,19 @@ export class WebScraper {
 
       const result: ScraperResult = {
         initial_sitemap: sitemapUrl,
-        shipping_informations: shippingContent,
-        contacts: contactsContent,
-        terms_of_conditions: termsContent,
-        faq: faqContent
+        shipping_informations: contentByType.shipping,
+        contacts: contentByType.contacts,
+        terms_of_conditions: contentByType.terms,
+        faq: contentByType.faq,
+        services: contentByType.services,
+        about: contentByType.about,
+        team: contentByType.team,
+        blog: contentByType.blog,
+        pricing: contentByType.pricing,
+        testimonials: contentByType.testimonials,
+        gallery: contentByType.gallery,
+        landing: contentByType.landing,
+        other: contentByType.other
       };
 
       logger.info('Scraping completed successfully');

src/services/EmbeddingService.ts

@@ -1,12 +1,13 @@
 import OpenAI from 'openai';
 import { logger } from '../utils/logger';
+import { ContentType } from '../types';
 import type { AppConfig } from '../config';
 
 export interface EmbeddingRequest {
   text: string;
   contentId: string;
   siteId: string;
-  contentType: 'shipping' | 'contacts' | 'terms' | 'faq';
+  contentType: ContentType;
 }
 
 export interface EmbeddingResponse {

src/services/QdrantEmbeddingWorkflow.ts

@@ -7,6 +7,7 @@ import { EmbeddingService } from './EmbeddingService';
 import { QdrantCleanupService } from './QdrantCleanupService';
 import { ChunkingService } from './ChunkingService';
 import { WebhookManager } from '../webhooks/WebhookManager';
+import { ContentType } from '../types';
 import type { AppConfig } from '../config';
 
 export interface EmbeddingWorkflowResult {
@@ -53,7 +54,7 @@ export class QdrantEmbeddingWorkflow {
   async processContentForEmbedding(
     siteId: string,
     contentId: string,
-    contentType: 'shipping' | 'contacts' | 'terms' | 'faq',
+    contentType: ContentType,
     url: string,
     content: string,
     title: string | null,
@@ -281,7 +282,7 @@ export class QdrantEmbeddingWorkflow {
     siteId: string,
     contentItems: Array<{
       contentId: string;
-      contentType: 'shipping' | 'contacts' | 'terms' | 'faq';
+      contentType: ContentType;
       url: string;
       content: string;
       title: string | null;

src/services/QdrantService.ts

@@ -1,6 +1,7 @@
 import { QdrantClient } from '@qdrant/js-client-rest';
 import { createHash } from 'crypto';
 import { logger } from '../utils/logger';
+import { ContentType } from '../types';
 import type { AppConfig } from '../config';
 
 export interface QdrantPoint {
@@ -9,7 +10,7 @@ export interface QdrantPoint {
   payload: {
     site_id: string;
     content_id: string;
-    content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+    content_type: ContentType;
     url: string;
     title?: string;
     content_hash: string;

src/types/index.ts

@@ -16,6 +16,15 @@ export interface ScraperResult {
   contacts: PageContent[];
   terms_of_conditions: PageContent[];
   faq: PageContent[];
+  services: PageContent[];
+  about: PageContent[];
+  team: PageContent[];
+  blog: PageContent[];
+  pricing: PageContent[];
+  testimonials: PageContent[];
+  gallery: PageContent[];
+  landing: PageContent[];
+  other: PageContent[];
 }
 
 export interface PageContent {
@@ -24,6 +33,19 @@ export interface PageContent {
   title?: string;
 }
 
+export type ContentType =
+  | 'shipping' | 'contacts' | 'terms' | 'faq'
+  | 'services' | 'about' | 'team' | 'blog'
+  | 'pricing' | 'testimonials' | 'gallery' | 'landing'
+  | 'other';
+
+export const ALL_CONTENT_TYPES: ContentType[] = [
+  'shipping', 'contacts', 'terms', 'faq',
+  'services', 'about', 'team', 'blog',
+  'pricing', 'testimonials', 'gallery', 'landing',
+  'other'
+];
+
 export type SitePlatform = 'shoprenter' | 'woocommerce' | 'shopify' | 'generic';
 
 export interface SitemapUrl {

src/webhooks/WebhookManager.ts

@@ -1,6 +1,7 @@
 import axios, { AxiosError } from 'axios';
 import { logger } from '../utils/logger';
 import { SiteDatabase, Webhook } from '../database/Database';
+import { ContentType } from '../types';
 import crypto from 'crypto';
 
 export type WebhookEvent =
@@ -246,7 +247,7 @@ export class WebhookManager {
     siteId: string,
     data: {
       content_id: string;
-      content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+      content_type: ContentType;
       collection_name: string;
       point_id: string;
       processing_time_ms?: number;
@@ -262,7 +263,7 @@ export class WebhookManager {
     siteId: string,
     data: {
       content_id: string;
-      content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+      content_type: ContentType;
       error: string;
       retry_count?: number;
     }
@@ -277,7 +278,7 @@ export class WebhookManager {
     siteId: string,
     data: {
       collection_name: string;
-      content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+      content_type: ContentType;
       vector_size: number;
       distance_metric: string;
     }
@@ -292,7 +293,7 @@ export class WebhookManager {
     siteId: string,
     data: {
       collection_name: string;
-      content_type?: 'shipping' | 'contacts' | 'terms' | 'faq';
+      content_type?: ContentType;
       vectors_deleted: number;
       reason: 'manual' | 'shop_deletion' | 'cleanup';
     }