feat: add site types, platform detection, and custom sitemap support

Add site_platform, custom_sitemap_url, max_crawl_depth, max_pages columns
to the sites table with migration from legacy site_type values. Add
detectPlatform() and resolveSitemapUrl() to SitemapParser. Add PATCH
/api/sites/:id/settings and POST /api/sites/detect endpoints. Update
WebScraper to resolve sitemaps using custom_sitemap_url from site record.

src/api/components/JobsEndpoint.ts

@@ -154,7 +154,7 @@ export class JobsEndpoint extends BaseEndpointComponent {
 
   private async createJob(req: Request, res: Response): Promise<void> {
     try {
-      const { url, custom_id } = req.body;
+      const { url, custom_id, site_type, site_platform, custom_sitemap_url } = req.body;
 
       if (!url) {
         res.status(400).json({ error: 'URL is required' });
@@ -173,8 +173,24 @@ export class JobsEndpoint extends BaseEndpointComponent {
 
       // If database is available, create or find site
       if (this.db) {
-        const { SitemapParser } = require('../../scraper/SitemapParser');
-        const { sitemapUrl, sitePlatform } = SitemapParser.getSitemapUrl(url);
+        const { SitemapParser, detectPlatform } = require('../../scraper/SitemapParser');
+        const { sitemapUrl, sitePlatform: detectedPlatform } = SitemapParser.getSitemapUrl(url);
+
+        // Use provided values or auto-detect
+        let finalPlatform = site_platform || detectedPlatform;
+        let finalSiteType = site_type;
+
+        // If neither was provided, try auto-detection
+        if (!site_platform && !site_type) {
+          try {
+            const detected = await detectPlatform(url);
+            finalPlatform = detected.sitePlatform;
+            finalSiteType = detected.siteType;
+          } catch {
+            // Fall back to SitemapParser detection
+            finalPlatform = detectedPlatform;
+          }
+        }
 
         let site = this.db.getSiteByUrl(url);
         if (!site) {
@@ -190,7 +206,7 @@ export class JobsEndpoint extends BaseEndpointComponent {
             return;
           }
 
-          site = this.db.createSite(url, sitemapUrl, sitePlatform, custom_id);
+          site = this.db.createSite(url, sitemapUrl, finalPlatform, custom_id, undefined, finalSiteType, custom_sitemap_url);
           logger.info(`Created new site: ${site.id}${site.custom_id ? ` with custom ID ${site.custom_id}` : ''}`);
         }
         siteId = site.id;

src/api/components/SitesEndpoint.ts

@@ -2,6 +2,7 @@ import { Request, Response } from 'express';
 import { BaseEndpointComponent, EndpointMethod } from '../core/types';
 import { logger } from '../../utils/logger';
 import { SiteDatabase } from '../../database/Database';
+import { detectPlatform, resolveSitemapUrl } from '../../scraper/SitemapParser';
 
 export class SitesEndpoint extends BaseEndpointComponent {
   constructor(private db: SiteDatabase) {
@@ -350,6 +351,78 @@ export class SitesEndpoint extends BaseEndpointComponent {
           requiresAuth: true
         }
       ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/sites/:id/settings',
+        this.updateSiteSettings.bind(this),
+        {
+          summary: 'Update site settings',
+          description: 'Updates site type, platform, sitemap URL, crawl depth, and max pages',
+          tags: ['Sites'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Site ID (internal UUID or custom UUID)'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                site_type: { type: 'string', description: 'Site type: webshop or website' },
+                site_platform: { type: 'string', description: 'Platform: shoprenter, woocommerce, shopify, or generic' },
+                custom_sitemap_url: { type: 'string', nullable: true, description: 'Custom sitemap URL or null' },
+                max_crawl_depth: { type: 'number', description: 'Maximum crawl depth' },
+                max_pages: { type: 'number', description: 'Maximum pages to crawl' }
+              }
+            }
+          },
+          responses: {
+            '200': { description: 'Settings updated' },
+            '404': { description: 'Site not found' }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'POST',
+        '/api/sites/detect',
+        this.detectSite.bind(this),
+        {
+          summary: 'Detect site type and platform',
+          description: 'Analyzes a URL to detect site type, platform, and sitemap URL',
+          tags: ['Sites'],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                url: { type: 'string', format: 'url', description: 'URL to analyze' }
+              },
+              required: ['url']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Detection result',
+              schema: {
+                type: 'object',
+                properties: {
+                  site_type: { type: 'string' },
+                  site_platform: { type: 'string' },
+                  sitemap_url: { type: 'string', nullable: true }
+                }
+              }
+            },
+            '400': { description: 'Invalid URL' }
+          },
+          requiresAuth: true
+        }
+      ),
       this.createEndpoint(
         'DELETE',
         '/api/sites/:id',
@@ -550,6 +623,10 @@ export class SitesEndpoint extends BaseEndpointComponent {
           url: site.url,
           sitemap_url: site.sitemap_url,
           site_type: site.site_type,
+          site_platform: site.site_platform,
+          custom_sitemap_url: site.custom_sitemap_url,
+          max_crawl_depth: site.max_crawl_depth,
+          max_pages: site.max_pages,
           qdrant_enabled: site.qdrant_enabled,
           created_at: site.created_at,
           updated_at: site.updated_at
@@ -789,6 +866,72 @@ export class SitesEndpoint extends BaseEndpointComponent {
     }
   }
 
+  private async updateSiteSettings(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const site = this.db.getSiteByAnyId(id);
+      if (!site) {
+        res.status(404).json({ error: 'Site not found' });
+        return;
+      }
+
+      const { site_type, site_platform, custom_sitemap_url, max_crawl_depth, max_pages } = req.body;
+
+      const settings: any = {};
+      if (site_type !== undefined) settings.site_type = site_type;
+      if (site_platform !== undefined) settings.site_platform = site_platform;
+      if (custom_sitemap_url !== undefined) settings.custom_sitemap_url = custom_sitemap_url;
+      if (max_crawl_depth !== undefined) settings.max_crawl_depth = max_crawl_depth;
+      if (max_pages !== undefined) settings.max_pages = max_pages;
+
+      this.db.updateSiteSettings(site.id, settings);
+
+      res.json({
+        site_id: site.id,
+        message: 'Site settings updated successfully',
+        settings
+      });
+    } catch (error) {
+      logger.error('Error updating site settings', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async detectSite(req: Request, res: Response): Promise<void> {
+    try {
+      const { url } = req.body;
+
+      if (!url) {
+        res.status(400).json({ error: 'URL is required' });
+        return;
+      }
+
+      try {
+        new URL(url);
+      } catch {
+        res.status(400).json({ error: 'Invalid URL format' });
+        return;
+      }
+
+      const { siteType, sitePlatform } = await detectPlatform(url);
+      const sitemapUrl = await resolveSitemapUrl(url, null, siteType);
+
+      res.json({
+        site_type: siteType,
+        site_platform: sitePlatform,
+        sitemap_url: sitemapUrl
+      });
+    } catch (error) {
+      logger.error('Error detecting site', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
   private async deleteShop(req: Request, res: Response): Promise<void> {
     try {
       if (!this.db) {
@@ -842,7 +985,7 @@ export class SitesEndpoint extends BaseEndpointComponent {
       const { id } = req.params;
 
       // Find site without filtering deleted_at
-      const stmt = (this.db as any).db.prepare('SELECT * FROM shops WHERE id = ? OR custom_id = ?');
+      const stmt = (this.db as any).db.prepare('SELECT * FROM sites WHERE id = ? OR custom_id = ?');
       const result = stmt.get(id, id) as any;
       const site = result ? { ...result, qdrant_enabled: Boolean(result.qdrant_enabled) } : null;
 

src/database/Database.ts

@@ -9,6 +9,10 @@ export interface Site {
   url: string;
   sitemap_url: string;
   site_type: string;
+  site_platform: string;
+  custom_sitemap_url: string | null;
+  max_crawl_depth: number;
+  max_pages: number;
   qdrant_enabled: boolean;
   deleted_at: string | null; // Soft delete timestamp
   created_at: string;
@@ -152,6 +156,7 @@ export class SiteDatabase {
 
     this.db = new Database(dbPath);
     this.migrateShopsToSites();
+    this.migrateSiteTypeToPlatform();
     this.initializeTables();
     logger.info(`Database initialized at ${dbPath}`);
   }
@@ -185,6 +190,33 @@ export class SiteDatabase {
     logger.info('Database migration complete: shops to sites');
   }
 
+  private migrateSiteTypeToPlatform(): void {
+    // Check if site_platform column already exists
+    const columns = this.db.pragma('table_info(sites)') as any[];
+    const hasSitePlatform = columns.some((col: any) => col.name === 'site_platform');
+    if (hasSitePlatform) return;
+
+    logger.info('Migrating database: adding site_platform and related columns');
+
+    this.db.exec(`ALTER TABLE sites ADD COLUMN site_platform TEXT NOT NULL DEFAULT 'generic'`);
+    this.db.exec(`ALTER TABLE sites ADD COLUMN custom_sitemap_url TEXT`);
+    this.db.exec(`ALTER TABLE sites ADD COLUMN max_crawl_depth INTEGER NOT NULL DEFAULT 3`);
+    this.db.exec(`ALTER TABLE sites ADD COLUMN max_pages INTEGER NOT NULL DEFAULT 200`);
+
+    // Move platform values from site_type to site_platform
+    this.db.exec(`
+      UPDATE sites SET site_platform = site_type, site_type = 'webshop'
+      WHERE site_type IN ('shoprenter', 'woocommerce', 'shopify')
+    `);
+    // Remaining rows get site_type = 'website' where site_platform is still 'generic'
+    this.db.exec(`
+      UPDATE sites SET site_type = 'website'
+      WHERE site_platform = 'generic' AND site_type NOT IN ('webshop', 'website')
+    `);
+
+    logger.info('Database migration complete: site_platform columns added');
+  }
+
   private initializeTables(): void {
     // Sites table
     this.db.exec(`
@@ -194,6 +226,10 @@ export class SiteDatabase {
         url TEXT NOT NULL UNIQUE,
         sitemap_url TEXT NOT NULL,
         site_type TEXT NOT NULL,
+        site_platform TEXT NOT NULL DEFAULT 'generic',
+        custom_sitemap_url TEXT,
+        max_crawl_depth INTEGER NOT NULL DEFAULT 3,
+        max_pages INTEGER NOT NULL DEFAULT 200,
         qdrant_enabled INTEGER DEFAULT 0,
         deleted_at TEXT,
         created_at TEXT NOT NULL,
@@ -474,18 +510,28 @@ export class SiteDatabase {
   }
 
   // Site operations
-  createSite(url: string, sitemapUrl: string, sitePlatform: string, customId?: string, qdrantEnabled?: boolean): Site {
+  createSite(
+    url: string,
+    sitemapUrl: string,
+    sitePlatform: string,
+    customId?: string,
+    qdrantEnabled?: boolean,
+    siteType?: string,
+    customSitemapUrl?: string | null
+  ): Site {
     const id = uuidv4();
     const now = new Date().toISOString();
     const custom_id = customId || null;
     const qdrant_enabled = qdrantEnabled !== undefined ? qdrantEnabled : true; // Default true for new sites
+    const resolvedSiteType = siteType || (sitePlatform === 'generic' ? 'website' : 'webshop');
+    const resolvedCustomSitemap = customSitemapUrl ?? null;
 
     const stmt = this.db.prepare(`
-      INSERT INTO sites (id, custom_id, url, sitemap_url, site_type, qdrant_enabled, deleted_at, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
+      INSERT INTO sites (id, custom_id, url, sitemap_url, site_type, site_platform, custom_sitemap_url, max_crawl_depth, max_pages, qdrant_enabled, deleted_at, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?, 3, 200, ?, NULL, ?, ?)
     `);
 
-    stmt.run(id, custom_id, url, sitemapUrl, sitePlatform, qdrant_enabled ? 1 : 0, now, now);
+    stmt.run(id, custom_id, url, sitemapUrl, resolvedSiteType, sitePlatform, resolvedCustomSitemap, qdrant_enabled ? 1 : 0, now, now);
 
     // Initialize analytics for this site
     this.db.prepare(`
@@ -500,7 +546,11 @@ export class SiteDatabase {
       custom_id,
       url,
       sitemap_url: sitemapUrl,
-      site_type: sitePlatform,
+      site_type: resolvedSiteType,
+      site_platform: sitePlatform,
+      custom_sitemap_url: resolvedCustomSitemap,
+      max_crawl_depth: 3,
+      max_pages: 200,
       qdrant_enabled,
       deleted_at: null,
       created_at: now,
@@ -551,6 +601,10 @@ export class SiteDatabase {
         s.url,
         s.sitemap_url,
         s.site_type,
+        s.site_platform,
+        s.custom_sitemap_url,
+        s.max_crawl_depth,
+        s.max_pages,
         s.qdrant_enabled,
         s.deleted_at,
         s.created_at,
@@ -589,6 +643,10 @@ export class SiteDatabase {
       url: row.url,
       sitemap_url: row.sitemap_url,
       site_type: row.site_type,
+      site_platform: row.site_platform,
+      custom_sitemap_url: row.custom_sitemap_url,
+      max_crawl_depth: row.max_crawl_depth,
+      max_pages: row.max_pages,
       qdrant_enabled: Boolean(row.qdrant_enabled),
       deleted_at: row.deleted_at,
       created_at: row.created_at,
@@ -623,6 +681,47 @@ export class SiteDatabase {
     `).run(...values);
   }
 
+  updateSiteSettings(siteId: string, settings: {
+    site_type?: string;
+    site_platform?: string;
+    custom_sitemap_url?: string | null;
+    max_crawl_depth?: number;
+    max_pages?: number;
+  }): void {
+    const now = new Date().toISOString();
+    const setClauses: string[] = ['updated_at = ?'];
+    const values: any[] = [now];
+
+    if (settings.site_type !== undefined) {
+      setClauses.push('site_type = ?');
+      values.push(settings.site_type);
+    }
+    if (settings.site_platform !== undefined) {
+      setClauses.push('site_platform = ?');
+      values.push(settings.site_platform);
+    }
+    if (settings.custom_sitemap_url !== undefined) {
+      setClauses.push('custom_sitemap_url = ?');
+      values.push(settings.custom_sitemap_url);
+    }
+    if (settings.max_crawl_depth !== undefined) {
+      setClauses.push('max_crawl_depth = ?');
+      values.push(settings.max_crawl_depth);
+    }
+    if (settings.max_pages !== undefined) {
+      setClauses.push('max_pages = ?');
+      values.push(settings.max_pages);
+    }
+
+    values.push(siteId);
+
+    this.db.prepare(`
+      UPDATE sites SET ${setClauses.join(', ')} WHERE id = ?
+    `).run(...values);
+
+    logger.info(`Updated settings for site ${siteId}`);
+  }
+
   // Method to set/update custom_id for a site
   setCustomId(identifier: string, customId: string | null): boolean {
     const site = this.getSiteByAnyId(identifier);

src/scraper/SitemapParser.ts

@@ -7,6 +7,113 @@ import { getUserAgent } from '../version';
 
 const parseXml = promisify(parseString);
 
+/**
+ * Detect platform type from URL by checking hostname patterns and homepage content
+ */
+export async function detectPlatform(url: string): Promise<{ siteType: string; sitePlatform: string }> {
+  try {
+    const parsedUrl = new URL(url);
+    const hostname = parsedUrl.hostname.toLowerCase();
+
+    // ShopRenter detection by hostname
+    if (hostname.includes('shoprenter') || hostname.endsWith('.sr.hu')) {
+      return { siteType: 'webshop', sitePlatform: 'shoprenter' };
+    }
+
+    // Shopify detection by hostname
+    if (hostname.includes('myshopify.com') || hostname.includes('.shopify.')) {
+      return { siteType: 'webshop', sitePlatform: 'shopify' };
+    }
+
+    // Fetch homepage to detect WooCommerce
+    try {
+      const response = await axios.get(url, {
+        timeout: 15000,
+        headers: { 'User-Agent': getUserAgent() },
+        maxRedirects: 5
+      });
+      const html = typeof response.data === 'string' ? response.data : '';
+      if (
+        html.includes('woocommerce') ||
+        html.includes('wc-block') ||
+        html.includes('wp-content/plugins/woocommerce') ||
+        html.includes('is-type-product')
+      ) {
+        return { siteType: 'webshop', sitePlatform: 'woocommerce' };
+      }
+    } catch {
+      // Homepage fetch failed, fall through to default
+    }
+
+    return { siteType: 'website', sitePlatform: 'generic' };
+  } catch {
+    return { siteType: 'website', sitePlatform: 'generic' };
+  }
+}
+
+/**
+ * Resolve the best sitemap URL for a site
+ */
+export async function resolveSitemapUrl(
+  baseUrl: string,
+  customSitemapUrl: string | null,
+  _siteType: string
+): Promise<string | null> {
+  const headers = { 'User-Agent': getUserAgent() };
+
+  // 1. If custom sitemap URL is provided, validate it
+  if (customSitemapUrl) {
+    try {
+      const res = await axios.head(customSitemapUrl, { timeout: 10000, headers });
+      if (res.status === 200) return customSitemapUrl;
+    } catch {
+      logger.warn(`Custom sitemap URL not reachable: ${customSitemapUrl}`);
+    }
+  }
+
+  const origin = new URL(baseUrl).origin;
+
+  // 2. Try robots.txt for Sitemap directives
+  try {
+    const robotsRes = await axios.get(`${origin}/robots.txt`, { timeout: 10000, headers });
+    if (typeof robotsRes.data === 'string') {
+      const lines = robotsRes.data.split('\n');
+      for (const line of lines) {
+        const match = line.match(/^Sitemap:\s*(.+)/i);
+        if (match) {
+          const sitemapUrl = match[1].trim();
+          try {
+            const headRes = await axios.head(sitemapUrl, { timeout: 10000, headers });
+            if (headRes.status === 200) return sitemapUrl;
+          } catch {
+            // Try next Sitemap directive
+          }
+        }
+      }
+    }
+  } catch {
+    // robots.txt not found
+  }
+
+  // 3. Try /sitemap.xml
+  try {
+    const res = await axios.head(`${origin}/sitemap.xml`, { timeout: 10000, headers });
+    if (res.status === 200) return `${origin}/sitemap.xml`;
+  } catch {
+    // Not found
+  }
+
+  // 4. Try /sitemap_index.xml
+  try {
+    const res = await axios.head(`${origin}/sitemap_index.xml`, { timeout: 10000, headers });
+    if (res.status === 200) return `${origin}/sitemap_index.xml`;
+  } catch {
+    // Not found
+  }
+
+  return null;
+}
+
 export class SitemapParser {
   /**
    * Detect webshop type from base URL and get sitemap URL

src/scraper/WebScraper.ts

@@ -1,5 +1,5 @@
 import { ScraperResult, PageContent } from '../types';
-import { SitemapParser } from './SitemapParser';
+import { SitemapParser, resolveSitemapUrl } from './SitemapParser';
 import { ContentExtractor } from './ContentExtractor';
 import { logger } from '../utils/logger';
 import { SiteDatabase } from '../database/Database';
@@ -29,19 +29,32 @@ export class WebScraper {
 
     try {
       // Get sitemap URL based on webshop type
-      const { sitemapUrl, sitePlatform } = SitemapParser.getSitemapUrl(baseUrl);
-      logger.info(`Detected site type: ${sitePlatform}, sitemap: ${sitemapUrl}`);
+      const { sitemapUrl: defaultSitemapUrl, sitePlatform } = SitemapParser.getSitemapUrl(baseUrl);
+      logger.info(`Detected site type: ${sitePlatform}, default sitemap: ${defaultSitemapUrl}`);
 
       // If database is available and no siteId provided, find or create site
       if (this.db && !siteId) {
         let site = this.db.getSiteByUrl(baseUrl);
         if (!site) {
-          site = this.db.createSite(baseUrl, sitemapUrl, sitePlatform);
+          site = this.db.createSite(baseUrl, defaultSitemapUrl, sitePlatform);
           logger.info(`Created new site record with ID: ${site.id}`);
         }
         siteId = site.id;
       }
 
+      // Resolve the actual sitemap URL, considering custom_sitemap_url from site record
+      let sitemapUrl = defaultSitemapUrl;
+      if (this.db && siteId) {
+        const siteRecord = this.db.getSiteById(siteId);
+        if (siteRecord) {
+          const resolved = await resolveSitemapUrl(baseUrl, siteRecord.custom_sitemap_url, siteRecord.site_type);
+          if (resolved) {
+            sitemapUrl = resolved;
+            logger.info(`Resolved sitemap URL: ${sitemapUrl}`);
+          }
+        }
+      }
+
       // Parse sitemap
       const urls = await SitemapParser.parseSitemap(sitemapUrl);