feat: implement component-based endpoint system with automatic documentation

- Replace monolithic routes.ts with component-based endpoint architecture
- Add EndpointRegistry for centralized endpoint management and documentation
- Create endpoint components for modular API organization:
  - JobsEndpoint: Job creation and management
  - ShopsEndpoint: Shop CRUD and analytics
  - WebhooksEndpoint: Webhook configuration
  - CustomUrlsEndpoint: Custom URL management
  - HealthEndpoint: System health checks
  - DocumentationEndpoint: Auto-generated API docs
- Implement self-documenting endpoints with comprehensive metadata
- Add automatic documentation generation at /doc endpoints:
  - /doc: Interactive HTML documentation
  - /doc/openapi: OpenAPI 3.0 specification
  - /doc/endpoints: JSON endpoint listing
- Authentication handled per-endpoint based on documentation metadata
- Full backward compatibility with existing API endpoints
- Remove old monolithic route handlers completely

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/components/CustomUrlsEndpoint.ts

@@ -0,0 +1,440 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+
+export class CustomUrlsEndpoint extends BaseEndpointComponent {
+  constructor(private db: ShopDatabase) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'POST',
+        '/api/shops/:id/custom-urls',
+        this.addCustomUrl.bind(this),
+        {
+          summary: 'Add a custom URL to scrape for a shop',
+          description: 'Adds a custom URL to be scraped for specific content type',
+          tags: ['Custom URLs'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                url: {
+                  type: 'string',
+                  format: 'url',
+                  description: 'URL to scrape (must belong to the same domain as the shop)'
+                },
+                content_type: {
+                  type: 'string',
+                  enum: ['shipping', 'contacts', 'terms', 'faq'],
+                  description: 'Type of content to scrape from this URL'
+                }
+              },
+              required: ['url', 'content_type']
+            }
+          },
+          responses: {
+            '201': {
+              description: 'Custom URL added successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  custom_url: {
+                    type: 'object',
+                    properties: {
+                      id: { type: 'string' },
+                      shop_id: { type: 'string' },
+                      url: { type: 'string' },
+                      content_type: { type: 'string' },
+                      enabled: { type: 'boolean' },
+                      created_at: { type: 'string' }
+                    }
+                  }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid request data or URL not in same domain'
+            },
+            '404': {
+              description: 'Shop not found'
+            },
+            '409': {
+              description: 'URL already exists or already scraped'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:id/custom-urls',
+        this.getCustomUrls.bind(this),
+        {
+          summary: 'List all custom URLs for a shop',
+          description: 'Retrieves all custom URLs configured for a shop',
+          tags: ['Custom URLs'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'List of custom URLs',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  custom_urls: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        url: { type: 'string' },
+                        content_type: { type: 'string' },
+                        enabled: { type: 'boolean' },
+                        created_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/shops/:id/custom-urls/:customUrlId',
+        this.updateCustomUrl.bind(this),
+        {
+          summary: 'Enable or disable a custom URL',
+          description: 'Updates the enabled status of a custom URL',
+          tags: ['Custom URLs'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            },
+            {
+              name: 'customUrlId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Custom URL ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                enabled: {
+                  type: 'boolean',
+                  description: 'Whether to enable or disable the custom URL'
+                }
+              },
+              required: ['enabled']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Custom URL status updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  custom_url_id: { type: 'string' },
+                  enabled: { type: 'boolean' }
+                }
+              }
+            },
+            '403': {
+              description: 'Custom URL does not belong to this shop'
+            },
+            '404': {
+              description: 'Shop or custom URL not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'DELETE',
+        '/api/shops/:id/custom-urls/:customUrlId',
+        this.deleteCustomUrl.bind(this),
+        {
+          summary: 'Delete a custom URL',
+          description: 'Permanently removes a custom URL from the shop configuration',
+          tags: ['Custom URLs'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            },
+            {
+              name: 'customUrlId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Custom URL ID'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Custom URL deleted successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  custom_url_id: { type: 'string' }
+                }
+              }
+            },
+            '403': {
+              description: 'Custom URL does not belong to this shop'
+            },
+            '404': {
+              description: 'Shop or custom URL not found'
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async addCustomUrl(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { url, content_type } = req.body;
+
+      // Validate required fields
+      if (!url || !content_type) {
+        res.status(400).json({ error: 'url and content_type fields are required' });
+        return;
+      }
+
+      // Validate content_type
+      const validContentTypes = ['shipping', 'contacts', 'terms', 'faq'];
+      if (!validContentTypes.includes(content_type)) {
+        res.status(400).json({
+          error: 'content_type must be one of: shipping, contacts, terms, faq'
+        });
+        return;
+      }
+
+      // Validate URL format
+      let parsedUrl: URL;
+      try {
+        parsedUrl = new URL(url);
+      } catch (error) {
+        res.status(400).json({ error: 'Invalid URL format' });
+        return;
+      }
+
+      // Check if shop exists
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Validate that the URL belongs to the same domain as the shop
+      const shopUrl = new URL(shop.url);
+      if (parsedUrl.hostname !== shopUrl.hostname) {
+        res.status(400).json({
+          error: `URL must belong to the same domain as the shop (${shopUrl.hostname})`
+        });
+        return;
+      }
+
+      // Check if URL was already scraped
+      const alreadyScraped = this.db.isUrlAlreadyScraped(shop.id, url);
+      if (alreadyScraped) {
+        res.status(409).json({
+          error: 'This URL has already been scraped from the sitemap',
+          message: 'URL already exists in scraped content'
+        });
+        return;
+      }
+
+      // Create custom URL
+      try {
+        const customUrl = this.db.createCustomUrl(shop.id, url, content_type);
+
+        res.status(201).json({
+          message: 'Custom URL added successfully',
+          custom_url: {
+            id: customUrl.id,
+            shop_id: customUrl.shop_id,
+            url: customUrl.url,
+            content_type: customUrl.content_type,
+            enabled: customUrl.enabled,
+            created_at: customUrl.created_at
+          }
+        });
+      } catch (error: any) {
+        if (error.message?.includes('already been added')) {
+          res.status(409).json({ error: error.message });
+          return;
+        }
+        throw error;
+      }
+    } catch (error) {
+      logger.error('Error creating custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getCustomUrls(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrls = this.db.getCustomUrls(shop.id);
+
+      res.json({
+        shop_id: shop.id,
+        custom_urls: customUrls.map(cu => ({
+          id: cu.id,
+          url: cu.url,
+          content_type: cu.content_type,
+          enabled: cu.enabled,
+          created_at: cu.created_at
+        }))
+      });
+    } catch (error) {
+      logger.error('Error fetching custom URLs', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateCustomUrl(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id, customUrlId } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrl = this.db.getCustomUrlById(customUrlId);
+      if (!customUrl) {
+        res.status(404).json({ error: 'Custom URL not found' });
+        return;
+      }
+
+      if (customUrl.shop_id !== shop.id) {
+        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
+        return;
+      }
+
+      this.db.setCustomUrlEnabled(customUrlId, enabled);
+
+      res.json({
+        message: `Custom URL ${enabled ? 'enabled' : 'disabled'} successfully`,
+        custom_url_id: customUrlId,
+        enabled
+      });
+    } catch (error) {
+      logger.error('Error updating custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async deleteCustomUrl(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id, customUrlId } = req.params;
+
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrl = this.db.getCustomUrlById(customUrlId);
+      if (!customUrl) {
+        res.status(404).json({ error: 'Custom URL not found' });
+        return;
+      }
+
+      if (customUrl.shop_id !== shop.id) {
+        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
+        return;
+      }
+
+      this.db.deleteCustomUrl(customUrlId);
+
+      res.json({
+        message: 'Custom URL deleted successfully',
+        custom_url_id: customUrlId
+      });
+    } catch (error) {
+      logger.error('Error deleting custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/components/DocumentationEndpoint.ts

@@ -0,0 +1,130 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { EndpointRegistry } from '../core/EndpointRegistry';
+
+export class DocumentationEndpoint extends BaseEndpointComponent {
+  constructor(private registry: EndpointRegistry) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'GET',
+        '/doc',
+        this.getDocumentation.bind(this),
+        {
+          summary: 'Get API Documentation',
+          description: 'Returns interactive HTML documentation for all registered API endpoints',
+          tags: ['Documentation'],
+          responses: {
+            '200': {
+              description: 'HTML documentation page',
+              schema: {
+                type: 'string',
+                format: 'html'
+              }
+            }
+          },
+          requiresAuth: false
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/doc/openapi',
+        this.getOpenApiDoc.bind(this),
+        {
+          summary: 'Get OpenAPI Specification',
+          description: 'Returns the OpenAPI 3.0 specification for the API in JSON format',
+          tags: ['Documentation'],
+          responses: {
+            '200': {
+              description: 'OpenAPI specification',
+              schema: {
+                type: 'object'
+              }
+            }
+          },
+          requiresAuth: false
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/doc/endpoints',
+        this.getEndpointsList.bind(this),
+        {
+          summary: 'List All Endpoints',
+          description: 'Returns a JSON list of all registered endpoints with their metadata',
+          tags: ['Documentation'],
+          responses: {
+            '200': {
+              description: 'List of endpoints',
+              schema: {
+                type: 'object',
+                properties: {
+                  endpoints: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        method: { type: 'string' },
+                        path: { type: 'string' },
+                        summary: { type: 'string' },
+                        tags: { type: 'array', items: { type: 'string' } },
+                        requiresAuth: { type: 'boolean' }
+                      }
+                    }
+                  },
+                  total: { type: 'number' }
+                }
+              }
+            }
+          },
+          requiresAuth: false
+        }
+      )
+    ];
+  }
+
+  private async getDocumentation(req: Request, res: Response): Promise<void> {
+    try {
+      const html = this.registry.generateHtmlDoc();
+      res.setHeader('Content-Type', 'text/html');
+      res.send(html);
+    } catch (error) {
+      console.error('Error generating documentation:', error);
+      res.status(500).json({ error: 'Failed to generate documentation' });
+    }
+  }
+
+  private async getOpenApiDoc(req: Request, res: Response): Promise<void> {
+    try {
+      const openApiDoc = this.registry.generateOpenApiDoc();
+      res.json(openApiDoc);
+    } catch (error) {
+      console.error('Error generating OpenAPI documentation:', error);
+      res.status(500).json({ error: 'Failed to generate OpenAPI documentation' });
+    }
+  }
+
+  private async getEndpointsList(req: Request, res: Response): Promise<void> {
+    try {
+      const docs = this.registry.getDocumentation();
+      const endpoints = docs.map(doc => ({
+        method: doc.method,
+        path: doc.path,
+        summary: doc.summary,
+        tags: doc.tags || [],
+        requiresAuth: doc.requiresAuth || false
+      }));
+
+      res.json({
+        endpoints,
+        total: endpoints.length
+      });
+    } catch (error) {
+      console.error('Error listing endpoints:', error);
+      res.status(500).json({ error: 'Failed to list endpoints' });
+    }
+  }
+}

src/api/components/HealthEndpoint.ts

@@ -0,0 +1,63 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { JobQueue } from '../../queue/JobQueue';
+
+export class HealthEndpoint extends BaseEndpointComponent {
+  constructor(private jobQueue: JobQueue) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'GET',
+        '/health',
+        this.getHealth.bind(this),
+        {
+          summary: 'Health check endpoint',
+          description: 'Returns the health status of the application and queue statistics',
+          tags: ['Health'],
+          responses: {
+            '200': {
+              description: 'Application health status',
+              schema: {
+                type: 'object',
+                properties: {
+                  status: {
+                    type: 'string',
+                    enum: ['ok'],
+                    description: 'Health status of the application'
+                  },
+                  timestamp: {
+                    type: 'string',
+                    format: 'date-time',
+                    description: 'Current server timestamp'
+                  },
+                  queue_stats: {
+                    type: 'object',
+                    properties: {
+                      total: { type: 'number' },
+                      pending: { type: 'number' },
+                      processing: { type: 'number' },
+                      completed: { type: 'number' },
+                      failed: { type: 'number' }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          requiresAuth: false
+        }
+      )
+    ];
+  }
+
+  private async getHealth(req: Request, res: Response): Promise<void> {
+    res.json({
+      status: 'ok',
+      timestamp: new Date().toISOString(),
+      queue_stats: this.jobQueue.getStats()
+    });
+  }
+}

src/api/components/JobsEndpoint.ts

@@ -0,0 +1,277 @@
+import { Request, Response } from 'express';
+import { v4 as uuidv4 } from 'uuid';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { JobQueue } from '../../queue/JobQueue';
+import { ScraperJob } from '../../types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+
+export class JobsEndpoint extends BaseEndpointComponent {
+  constructor(
+    private jobQueue: JobQueue,
+    private db?: ShopDatabase
+  ) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'POST',
+        '/api/jobs',
+        this.createJob.bind(this),
+        {
+          summary: 'Create a new scraping job',
+          description: 'Creates a new job to scrape a webshop. If database is available, creates or finds the shop record.',
+          tags: ['Jobs'],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                url: {
+                  type: 'string',
+                  format: 'url',
+                  description: 'The URL of the webshop to scrape'
+                },
+                custom_id: {
+                  type: 'string',
+                  pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$',
+                  description: 'Optional custom UUID for the shop'
+                }
+              },
+              required: ['url']
+            }
+          },
+          responses: {
+            '201': {
+              description: 'Job created successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  job_id: { type: 'string' },
+                  shop_id: { type: 'string' },
+                  status: { type: 'string' },
+                  message: { type: 'string' }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid request data'
+            },
+            '409': {
+              description: 'Custom ID already in use'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/jobs/:id',
+        this.getJob.bind(this),
+        {
+          summary: 'Get job status and result',
+          description: 'Retrieves the status and result of a specific job',
+          tags: ['Jobs'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Job ID'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Job details',
+              schema: {
+                type: 'object',
+                properties: {
+                  job_id: { type: 'string' },
+                  status: { type: 'string', enum: ['pending', 'processing', 'completed', 'failed'] },
+                  created_at: { type: 'string' },
+                  updated_at: { type: 'string' },
+                  result: { type: 'object' },
+                  error: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Job not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/jobs',
+        this.getAllJobs.bind(this),
+        {
+          summary: 'List all jobs',
+          description: 'Retrieves a list of all jobs with their current status and queue statistics',
+          tags: ['Jobs'],
+          responses: {
+            '200': {
+              description: 'List of jobs',
+              schema: {
+                type: 'object',
+                properties: {
+                  stats: {
+                    type: 'object',
+                    properties: {
+                      total: { type: 'number' },
+                      pending: { type: 'number' },
+                      processing: { type: 'number' },
+                      completed: { type: 'number' },
+                      failed: { type: 'number' }
+                    }
+                  },
+                  jobs: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        job_id: { type: 'string' },
+                        status: { type: 'string' },
+                        sitemap_url: { type: 'string' },
+                        created_at: { type: 'string' },
+                        updated_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async createJob(req: Request, res: Response): Promise<void> {
+    try {
+      const { url, custom_id } = req.body;
+
+      if (!url) {
+        res.status(400).json({ error: 'URL is required' });
+        return;
+      }
+
+      // Validate URL format
+      try {
+        new URL(url);
+      } catch (error) {
+        res.status(400).json({ error: 'Invalid URL format' });
+        return;
+      }
+
+      let shopId: string | undefined;
+
+      // If database is available, create or find shop
+      if (this.db) {
+        const { SitemapParser } = require('../../scraper/SitemapParser');
+        const { sitemapUrl, webshopType } = SitemapParser.getSitemapUrl(url);
+
+        let shop = this.db.getShopByUrl(url);
+        if (!shop) {
+          // Validate custom_id format if provided
+          if (custom_id && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(custom_id)) {
+            res.status(400).json({ error: 'custom_id must be a valid UUID format' });
+            return;
+          }
+
+          // Check if custom_id is already in use
+          if (custom_id && this.db.getShopByCustomId(custom_id)) {
+            res.status(409).json({ error: 'custom_id is already in use' });
+            return;
+          }
+
+          shop = this.db.createShop(url, sitemapUrl, webshopType, custom_id);
+          logger.info(`Created new shop: ${shop.id}${shop.custom_id ? ` with custom ID ${shop.custom_id}` : ''}`);
+        }
+        shopId = shop.id;
+      }
+
+      // Create new job
+      const job: ScraperJob = {
+        id: uuidv4(),
+        sitemapUrl: url,
+        status: 'pending',
+        createdAt: new Date(),
+        updatedAt: new Date(),
+        shopId
+      };
+
+      this.jobQueue.addJob(job);
+
+      logger.info(`New job created: ${job.id} for shop: ${shopId || 'N/A'}`);
+
+      res.status(201).json({
+        job_id: job.id,
+        shop_id: shopId,
+        status: job.status,
+        message: 'Job created successfully'
+      });
+    } catch (error) {
+      logger.error('Error creating job', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getJob(req: Request, res: Response): Promise<void> {
+    try {
+      const { id } = req.params;
+      const job = this.jobQueue.getJob(id);
+
+      if (!job) {
+        res.status(404).json({ error: 'Job not found' });
+        return;
+      }
+
+      const response: any = {
+        job_id: job.id,
+        status: job.status,
+        created_at: job.createdAt,
+        updated_at: job.updatedAt
+      };
+
+      if (job.status === 'completed' && job.result) {
+        response.result = job.result;
+      }
+
+      if (job.status === 'failed' && job.error) {
+        response.error = job.error;
+      }
+
+      res.json(response);
+    } catch (error) {
+      logger.error('Error fetching job', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getAllJobs(req: Request, res: Response): Promise<void> {
+    try {
+      const jobs = this.jobQueue.getAllJobs();
+      const stats = this.jobQueue.getStats();
+
+      res.json({
+        stats,
+        jobs: jobs.map(job => ({
+          job_id: job.id,
+          status: job.status,
+          sitemap_url: job.sitemapUrl,
+          created_at: job.createdAt,
+          updated_at: job.updatedAt
+        }))
+      });
+    } catch (error) {
+      logger.error('Error listing jobs', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/components/ShopsEndpoint.ts

@@ -0,0 +1,624 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+
+export class ShopsEndpoint extends BaseEndpointComponent {
+  constructor(private db: ShopDatabase) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'GET',
+        '/api/shops',
+        this.getAllShops.bind(this),
+        {
+          summary: 'List all shops',
+          description: 'Retrieves a list of all shops with their analytics',
+          tags: ['Shops'],
+          responses: {
+            '200': {
+              description: 'List of shops with analytics',
+              schema: {
+                type: 'object',
+                properties: {
+                  shops: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        custom_id: { type: 'string', nullable: true },
+                        url: { type: 'string' },
+                        sitemap_url: { type: 'string' },
+                        webshop_type: { type: 'string' },
+                        created_at: { type: 'string' },
+                        updated_at: { type: 'string' },
+                        analytics: {
+                          type: 'object',
+                          properties: {
+                            total_scrapes: { type: 'number' },
+                            last_scraped_at: { type: 'string', nullable: true },
+                            next_scrape_at: { type: 'string', nullable: true },
+                            total_urls_found: { type: 'number' },
+                            average_scrape_time: { type: 'number', nullable: true },
+                            average_page_scrape_time: { type: 'number', nullable: true }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            '503': {
+              description: 'Database not available'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:id',
+        this.getShop.bind(this),
+        {
+          summary: 'Get detailed shop information',
+          description: 'Retrieves detailed information about a specific shop including analytics, content metadata, scrape history, and scheduled jobs',
+          tags: ['Shops'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Detailed shop information',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop: { type: 'object' },
+                  analytics: { type: 'object' },
+                  content_metadata: { type: 'object' },
+                  scrape_history: { type: 'array' },
+                  scheduled_jobs: { type: 'array' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:id/results',
+        this.getShopResults.bind(this),
+        {
+          summary: 'Get shop results with full content',
+          description: 'Retrieves scraped content for a shop with optional filtering',
+          tags: ['Shops'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            },
+            {
+              name: 'limit',
+              in: 'query',
+              type: 'number',
+              description: 'Maximum number of results to return'
+            },
+            {
+              name: 'date_from',
+              in: 'query',
+              type: 'string',
+              description: 'Filter results from this date (ISO format)'
+            },
+            {
+              name: 'date_to',
+              in: 'query',
+              type: 'string',
+              description: 'Filter results up to this date (ISO format)'
+            },
+            {
+              name: 'content_type',
+              in: 'query',
+              type: 'string',
+              description: 'Filter by content type: shipping, contacts, terms, faq'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Shop results with content',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  filters: { type: 'object' },
+                  results: {
+                    type: 'object',
+                    properties: {
+                      shipping_informations: { type: 'array' },
+                      contacts: { type: 'array' },
+                      terms_of_conditions: { type: 'array' },
+                      faq: { type: 'array' }
+                    }
+                  }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/shops/:id/schedule',
+        this.updateSchedule.bind(this),
+        {
+          summary: 'Enable or disable scheduled scraping',
+          description: 'Updates the scheduling status for a shop',
+          tags: ['Shops'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                enabled: {
+                  type: 'boolean',
+                  description: 'Whether to enable or disable scheduling'
+                }
+              },
+              required: ['enabled']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Schedule status updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  schedule_enabled: { type: 'boolean' },
+                  message: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/shops/:id/custom-id',
+        this.updateCustomId.bind(this),
+        {
+          summary: 'Set or update custom ID for a shop',
+          description: 'Updates the custom UUID identifier for a shop',
+          tags: ['Shops'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or current custom UUID)'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                custom_id: {
+                  type: 'string',
+                  nullable: true,
+                  pattern: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$',
+                  description: 'Custom UUID or null to remove'
+                }
+              },
+              required: ['custom_id']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Custom ID updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  custom_id: { type: 'string', nullable: true },
+                  message: { type: 'string' }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid custom ID format'
+            },
+            '404': {
+              description: 'Shop not found'
+            },
+            '409': {
+              description: 'Custom ID already in use'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'DELETE',
+        '/api/shops/:id',
+        this.deleteShop.bind(this),
+        {
+          summary: 'Delete a shop and all related data',
+          description: 'Permanently removes a shop and all associated data including scrape history, scheduled jobs, and content',
+          tags: ['Shops'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Shop deleted successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  shop_id: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async getAllShops(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const shops = this.db.getAllShops();
+      const shopsWithAnalytics = shops.map(shop => {
+        const analytics = this.db.getShopAnalytics(shop.id);
+        return {
+          ...shop,
+          analytics
+        };
+      });
+
+      res.json({ shops: shopsWithAnalytics });
+    } catch (error) {
+      logger.error('Error listing shops', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getShop(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Get analytics
+      const analytics = this.db.getShopAnalytics(shop.id);
+
+      // Get scrape history
+      const scrapeHistory = this.db.getScrapeHistory(shop.id, 10);
+
+      // Get latest content (without full content, just metadata)
+      const latestContent = this.db.getLatestContentByType(shop.id);
+
+      // Get scheduled jobs
+      const scheduledJobs = this.db.getScheduledJobs(shop.id);
+
+      // Build response with content change markers and URLs only (no content)
+      const contentMetadata: any = {
+        shipping_informations: latestContent.shipping.map(c => ({
+          url: c.url,
+          changed: Boolean(c.changed),
+          last_updated: c.updated_at
+        })),
+        contacts: latestContent.contacts.map(c => ({
+          url: c.url,
+          changed: Boolean(c.changed),
+          last_updated: c.updated_at
+        })),
+        terms_of_conditions: latestContent.terms.map(c => ({
+          url: c.url,
+          changed: Boolean(c.changed),
+          last_updated: c.updated_at
+        })),
+        faq: latestContent.faq.map(c => ({
+          url: c.url,
+          changed: Boolean(c.changed),
+          last_updated: c.updated_at
+        }))
+      };
+
+      res.json({
+        shop: {
+          id: shop.id,
+          custom_id: shop.custom_id,
+          url: shop.url,
+          sitemap_url: shop.sitemap_url,
+          webshop_type: shop.webshop_type,
+          created_at: shop.created_at,
+          updated_at: shop.updated_at
+        },
+        analytics: {
+          total_scrapes: analytics?.total_scrapes || 0,
+          last_scraped_at: analytics?.last_scraped_at || null,
+          next_scrape_at: analytics?.next_scrape_at || null,
+          total_urls_found: analytics?.total_urls_found || 0,
+          average_scrape_time: analytics?.average_scrape_time || null,
+          average_page_scrape_time: analytics?.average_page_scrape_time || null
+        },
+        content_metadata: contentMetadata,
+        scrape_history: scrapeHistory.map(h => ({
+          id: h.id,
+          job_id: h.job_id,
+          started_at: h.started_at,
+          completed_at: h.completed_at,
+          status: h.status,
+          error: h.error,
+          scrape_time_ms: h.scrape_time_ms,
+          urls_found: h.urls_found
+        })),
+        scheduled_jobs: scheduledJobs.map(j => ({
+          id: j.id,
+          next_run_at: j.next_run_at,
+          frequency: j.frequency,
+          last_modified: j.last_modified,
+          status: j.status,
+          enabled: j.enabled,
+          created_at: j.created_at
+        }))
+      });
+    } catch (error) {
+      logger.error('Error fetching shop details', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getShopResults(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Parse query parameters
+      const limit = req.query.limit ? parseInt(req.query.limit as string) : undefined;
+      const dateFrom = req.query.date_from as string | undefined;
+      const dateTo = req.query.date_to as string | undefined;
+      const contentType = req.query.content_type as string | undefined;
+
+      // Get content with filters
+      const allContent = this.db.getAllContent(shop.id, {
+        limit,
+        dateFrom,
+        dateTo,
+        contentType
+      });
+
+      // Group by type and URL (latest version of each URL)
+      const grouped: { [key: string]: any[] } = {
+        shipping_informations: [],
+        contacts: [],
+        terms_of_conditions: [],
+        faq: []
+      };
+
+      const urlMap = new Map<string, any>();
+      for (const content of allContent) {
+        const key = `${content.content_type}:${content.url}`;
+        if (!urlMap.has(key)) {
+          urlMap.set(key, {
+            url: content.url,
+            content: content.content,
+            changed: Boolean(content.changed),
+            last_updated: content.updated_at
+          });
+        }
+      }
+
+      // Organize by type
+      for (const content of allContent) {
+        const key = `${content.content_type}:${content.url}`;
+        const urlKey = urlMap.get(key);
+
+        if (!urlKey) continue;
+
+        const typeKey = content.content_type === 'shipping' ? 'shipping_informations' :
+                       content.content_type === 'contacts' ? 'contacts' :
+                       content.content_type === 'terms' ? 'terms_of_conditions' :
+                       'faq';
+
+        // Only add if not already added
+        if (!grouped[typeKey].find((c: any) => c.url === urlKey.url)) {
+          grouped[typeKey].push(urlKey);
+        }
+      }
+
+      res.json({
+        shop_id: shop.id,
+        filters: {
+          limit: limit || null,
+          date_from: dateFrom || null,
+          date_to: dateTo || null,
+          content_type: contentType || null
+        },
+        results: grouped
+      });
+    } catch (error) {
+      logger.error('Error fetching shop results', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateSchedule(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      this.db.setScheduleEnabled(shop.id, enabled);
+
+      res.json({
+        shop_id: shop.id,
+        schedule_enabled: enabled,
+        message: `Schedule ${enabled ? 'enabled' : 'disabled'} successfully`
+      });
+    } catch (error) {
+      logger.error('Error updating schedule status', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateCustomId(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { custom_id } = req.body;
+
+      const shop = this.db.getShopByAnyId(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Validate custom_id format if provided
+      if (custom_id !== null && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(custom_id)) {
+        res.status(400).json({ error: 'custom_id must be a valid UUID format or null' });
+        return;
+      }
+
+      // Check if custom_id is already in use by another shop
+      if (custom_id) {
+        const existingShop = this.db.getShopByCustomId(custom_id);
+        if (existingShop && existingShop.id !== shop.id) {
+          res.status(409).json({ error: 'custom_id is already in use' });
+          return;
+        }
+      }
+
+      const success = this.db.setCustomId(shop.id, custom_id);
+      if (!success) {
+        res.status(500).json({ error: 'Failed to update custom_id' });
+        return;
+      }
+
+      res.json({
+        shop_id: shop.id,
+        custom_id: custom_id,
+        message: custom_id ? `Custom ID set to ${custom_id}` : 'Custom ID removed'
+      });
+    } catch (error) {
+      logger.error('Error updating custom ID', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async deleteShop(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      this.db.deleteShop(shop.id);
+
+      res.json({
+        message: 'Shop and all related data deleted successfully',
+        shop_id: shop.id
+      });
+    } catch (error) {
+      logger.error('Error deleting shop', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/components/WebhooksEndpoint.ts

@@ -0,0 +1,392 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+
+export class WebhooksEndpoint extends BaseEndpointComponent {
+  constructor(private db: ShopDatabase) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'POST',
+        '/api/shops/:id/webhooks',
+        this.createWebhook.bind(this),
+        {
+          summary: 'Create or replace webhook for a shop',
+          description: 'Creates a new webhook or updates an existing one for the specified shop',
+          tags: ['Webhooks'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                url: {
+                  type: 'string',
+                  format: 'url',
+                  description: 'Webhook URL'
+                },
+                secret: {
+                  type: 'string',
+                  description: 'Optional webhook secret for verification'
+                }
+              },
+              required: ['url']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Webhook updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  webhook: { type: 'object' }
+                }
+              }
+            },
+            '201': {
+              description: 'Webhook created',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  webhook: { type: 'object' }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid webhook URL'
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:id/webhooks',
+        this.getWebhook.bind(this),
+        {
+          summary: 'Get webhook configuration for a shop',
+          description: 'Retrieves webhook configuration and recent delivery history',
+          tags: ['Webhooks'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Webhook configuration and delivery history',
+              schema: {
+                type: 'object',
+                properties: {
+                  webhook: { type: 'object' },
+                  recent_deliveries: { type: 'array' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop or webhook not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/shops/:id/webhooks',
+        this.updateWebhook.bind(this),
+        {
+          summary: 'Enable or disable webhook',
+          description: 'Updates the enabled status of a webhook',
+          tags: ['Webhooks'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                enabled: {
+                  type: 'boolean',
+                  description: 'Whether to enable or disable the webhook'
+                }
+              },
+              required: ['enabled']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Webhook status updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  webhook_enabled: { type: 'boolean' },
+                  message: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop or webhook not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'DELETE',
+        '/api/shops/:id/webhooks',
+        this.deleteWebhook.bind(this),
+        {
+          summary: 'Delete webhook for a shop',
+          description: 'Permanently removes the webhook configuration for a shop',
+          tags: ['Webhooks'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Webhook deleted successfully',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  shop_id: { type: 'string' }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop or webhook not found'
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async createWebhook(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { url, secret } = req.body;
+
+      if (!url) {
+        res.status(400).json({ error: 'url field is required' });
+        return;
+      }
+
+      // Validate URL format
+      try {
+        new URL(url);
+      } catch (error) {
+        res.status(400).json({ error: 'Invalid URL format' });
+        return;
+      }
+
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Check if webhook already exists
+      const existingWebhook = this.db.getWebhook(shop.id);
+      if (existingWebhook) {
+        // Update existing webhook
+        this.db.updateWebhook(shop.id, url, secret);
+        const updated = this.db.getWebhook(shop.id);
+
+        res.json({
+          message: 'Webhook updated successfully',
+          webhook: {
+            id: updated!.id,
+            shop_id: updated!.shop_id,
+            url: updated!.url,
+            enabled: updated!.enabled,
+            created_at: updated!.created_at,
+            updated_at: updated!.updated_at
+          }
+        });
+      } else {
+        // Create new webhook
+        const webhookId = this.db.createWebhook(shop.id, url, secret);
+        const webhook = this.db.getWebhook(shop.id);
+
+        res.status(201).json({
+          message: 'Webhook created successfully',
+          webhook: {
+            id: webhook!.id,
+            shop_id: webhook!.shop_id,
+            url: webhook!.url,
+            enabled: webhook!.enabled,
+            created_at: webhook!.created_at,
+            updated_at: webhook!.updated_at
+          }
+        });
+      }
+    } catch (error) {
+      logger.error('Error creating/updating webhook', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getWebhook(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const webhook = this.db.getWebhook(shop.id);
+
+      if (!webhook) {
+        res.status(404).json({ error: 'No webhook configured for this shop' });
+        return;
+      }
+
+      // Get recent deliveries
+      const deliveries = this.db.getWebhookDeliveries(shop.id, 10);
+
+      res.json({
+        webhook: {
+          id: webhook.id,
+          shop_id: webhook.shop_id,
+          url: webhook.url,
+          enabled: webhook.enabled,
+          created_at: webhook.created_at,
+          updated_at: webhook.updated_at
+        },
+        recent_deliveries: deliveries.map(d => ({
+          id: d.id,
+          event: d.event,
+          status: d.status,
+          response_code: d.response_code,
+          error: d.error,
+          attempted_at: d.attempted_at
+        }))
+      });
+    } catch (error) {
+      logger.error('Error fetching webhook', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateWebhook(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      const shop = this.db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const webhook = this.db.getWebhook(shop.id);
+      if (!webhook && enabled) {
+        res.status(404).json({ error: 'No webhook configured for this shop' });
+        return;
+      }
+
+      this.db.setWebhookEnabled(shop.id, enabled);
+
+      res.json({
+        shop_id: shop.id,
+        webhook_enabled: enabled,
+        message: `Webhook ${enabled ? 'enabled' : 'disabled'} successfully`
+      });
+    } catch (error) {
+      logger.error('Error updating webhook status', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async deleteWebhook(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = this.db.getShopByAnyId(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const webhook = this.db.getWebhook(shop.id);
+      if (!webhook) {
+        res.status(404).json({ error: 'No webhook configured for this shop' });
+        return;
+      }
+
+      this.db.deleteWebhook(shop.id);
+
+      res.json({
+        message: 'Webhook deleted successfully',
+        shop_id: shop.id
+      });
+    } catch (error) {
+      logger.error('Error deleting webhook', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/core/EndpointRegistry.ts

@@ -0,0 +1,279 @@
+import { Router, Request, Response, NextFunction } from 'express';
+import { EndpointComponent, EndpointDocumentation, EndpointMethod, HttpMethod } from './types';
+
+export class EndpointRegistry {
+  private endpoints: Map<string, EndpointComponent> = new Map();
+  private documentation: Map<string, EndpointDocumentation> = new Map();
+
+  /**
+   * Register an endpoint component
+   */
+  register(component: EndpointComponent): void {
+    const key = `${component.method}:${component.path}`;
+
+    if (this.endpoints.has(key)) {
+      throw new Error(`Endpoint ${component.method} ${component.path} is already registered`);
+    }
+
+    this.endpoints.set(key, component);
+    this.documentation.set(key, component.documentation);
+  }
+
+  /**
+   * Get all registered endpoints
+   */
+  getAllEndpoints(): EndpointComponent[] {
+    return Array.from(this.endpoints.values());
+  }
+
+  /**
+   * Get endpoint documentation
+   */
+  getDocumentation(): EndpointDocumentation[] {
+    return Array.from(this.documentation.values());
+  }
+
+  /**
+   * Build Express router from registered endpoints
+   */
+  buildRouter(): Router {
+    const router = Router();
+
+    for (const endpoint of this.endpoints.values()) {
+      const { method, path, handler, middleware = [] } = endpoint;
+
+      // Create the Express route
+      const expressMethod = method.toLowerCase() as Lowercase<HttpMethod>;
+
+      // Apply middleware chain and handler
+      const handlers: Array<(req: Request, res: Response, next?: NextFunction) => void> = [
+        ...middleware,
+        async (req: Request, res: Response, next?: NextFunction) => {
+          try {
+            await handler(req, res, next);
+          } catch (error) {
+            console.error(`Error in ${method} ${path}:`, error);
+            if (!res.headersSent) {
+              res.status(500).json({ error: 'Internal server error' });
+            }
+          }
+        }
+      ];
+
+      switch (expressMethod) {
+        case 'get':
+          router.get(path, ...handlers);
+          break;
+        case 'post':
+          router.post(path, ...handlers);
+          break;
+        case 'put':
+          router.put(path, ...handlers);
+          break;
+        case 'patch':
+          router.patch(path, ...handlers);
+          break;
+        case 'delete':
+          router.delete(path, ...handlers);
+          break;
+        default:
+          throw new Error(`Unsupported HTTP method: ${method}`);
+      }
+    }
+
+    return router;
+  }
+
+  /**
+   * Generate OpenAPI-style documentation
+   */
+  generateOpenApiDoc(): any {
+    const paths: any = {};
+
+    for (const [key, doc] of this.documentation.entries()) {
+      const [method, path] = key.split(':');
+
+      if (!paths[path]) {
+        paths[path] = {};
+      }
+
+      paths[path][method.toLowerCase()] = {
+        summary: doc.summary,
+        description: doc.description,
+        tags: doc.tags || [],
+        parameters: this.formatParameters(doc.parameters || []),
+        requestBody: doc.requestBody ? {
+          required: doc.requestBody.required || false,
+          content: {
+            'application/json': {
+              schema: doc.requestBody.schema
+            }
+          }
+        } : undefined,
+        responses: this.formatResponses(doc.responses || {}),
+        security: doc.requiresAuth ? [{ bearerAuth: [] }] : undefined
+      };
+    }
+
+    return {
+      openapi: '3.0.3',
+      info: {
+        title: 'Webshop Scraper API',
+        version: '1.0.0',
+        description: 'API for managing webshop scraping operations'
+      },
+      servers: [
+        {
+          url: '/api',
+          description: 'API server'
+        }
+      ],
+      components: {
+        securitySchemes: {
+          bearerAuth: {
+            type: 'http',
+            scheme: 'bearer',
+            bearerFormat: 'API-Key'
+          }
+        }
+      },
+      paths
+    };
+  }
+
+  /**
+   * Generate simple HTML documentation
+   */
+  generateHtmlDoc(): string {
+    const docs = this.getDocumentation();
+    const groupedByTag: { [key: string]: EndpointDocumentation[] } = {};
+
+    // Group by tags
+    for (const doc of docs) {
+      const tag = doc.tags?.[0] || 'Other';
+      if (!groupedByTag[tag]) {
+        groupedByTag[tag] = [];
+      }
+      groupedByTag[tag].push(doc);
+    }
+
+    const sections = Object.entries(groupedByTag)
+      .map(([tag, endpoints]) => {
+        const endpointItems = endpoints.map(ep => `
+          <div class="endpoint">
+            <div class="endpoint-header">
+              <span class="method method-${ep.method?.toLowerCase()}">${ep.method}</span>
+              <span class="path">${ep.path}</span>
+              ${ep.requiresAuth ? '<span class="auth-required">🔒 Auth Required</span>' : ''}
+            </div>
+            <h4>${ep.summary}</h4>
+            ${ep.description ? `<p>${ep.description}</p>` : ''}
+
+            ${ep.parameters && ep.parameters.length > 0 ? `
+              <h5>Parameters</h5>
+              <ul class="parameters">
+                ${ep.parameters.map(p => `
+                  <li>
+                    <strong>${p.name}</strong>
+                    <span class="param-type">(${p.in}: ${p.type})</span>
+                    ${p.required ? '<span class="required">*</span>' : ''}
+                    ${p.description ? ` - ${p.description}` : ''}
+                  </li>
+                `).join('')}
+              </ul>
+            ` : ''}
+
+            ${ep.requestBody ? `
+              <h5>Request Body</h5>
+              <pre><code>${JSON.stringify(ep.requestBody.schema, null, 2)}</code></pre>
+            ` : ''}
+
+            ${ep.responses && Object.keys(ep.responses).length > 0 ? `
+              <h5>Responses</h5>
+              <ul class="responses">
+                ${Object.entries(ep.responses).map(([status, resp]) => `
+                  <li><strong>${status}</strong> - ${resp.description}</li>
+                `).join('')}
+              </ul>
+            ` : ''}
+          </div>
+        `).join('');
+
+        return `
+          <section>
+            <h3>${tag}</h3>
+            ${endpointItems}
+          </section>
+        `;
+      }).join('');
+
+    return `
+      <!DOCTYPE html>
+      <html>
+        <head>
+          <title>Webshop Scraper API Documentation</title>
+          <style>
+            body { font-family: Arial, sans-serif; max-width: 1200px; margin: 0 auto; padding: 20px; }
+            .endpoint { border: 1px solid #ddd; margin: 10px 0; padding: 15px; border-radius: 5px; }
+            .endpoint-header { margin-bottom: 10px; display: flex; align-items: center; gap: 10px; }
+            .method { padding: 4px 8px; border-radius: 3px; color: white; font-size: 12px; font-weight: bold; }
+            .method-get { background-color: #61affe; }
+            .method-post { background-color: #49cc90; }
+            .method-put { background-color: #fca130; }
+            .method-patch { background-color: #50e3c2; }
+            .method-delete { background-color: #f93e3e; }
+            .path { font-family: monospace; background: #f5f5f5; padding: 4px 8px; border-radius: 3px; }
+            .auth-required { background: #ff6b6b; color: white; padding: 2px 6px; border-radius: 3px; font-size: 11px; }
+            .parameters, .responses { margin-left: 20px; }
+            .param-type { color: #666; font-style: italic; }
+            .required { color: red; }
+            pre { background: #f5f5f5; padding: 10px; border-radius: 3px; overflow-x: auto; }
+            h3 { color: #333; border-bottom: 2px solid #eee; padding-bottom: 10px; }
+            h4 { margin: 0 0 10px 0; color: #555; }
+            h5 { margin: 15px 0 5px 0; color: #666; }
+          </style>
+        </head>
+        <body>
+          <h1>Webshop Scraper API Documentation</h1>
+          <p>Generated automatically from registered endpoints.</p>
+          ${sections}
+        </body>
+      </html>
+    `;
+  }
+
+  private formatParameters(parameters: any[]): any[] {
+    return parameters.map(param => ({
+      name: param.name,
+      in: param.in,
+      required: param.required || false,
+      schema: {
+        type: param.type || 'string'
+      },
+      description: param.description
+    }));
+  }
+
+  private formatResponses(responses: { [key: string]: any }): any {
+    const formatted: any = {};
+
+    for (const [status, response] of Object.entries(responses)) {
+      formatted[status] = {
+        description: response.description || '',
+        content: response.schema ? {
+          'application/json': {
+            schema: response.schema
+          }
+        } : undefined
+      };
+    }
+
+    // Default responses if none provided
+    if (Object.keys(formatted).length === 0) {
+      formatted['200'] = { description: 'Success' };
+      formatted['500'] = { description: 'Internal server error' };
+    }
+
+    return formatted;
+  }
+}

src/api/core/types.ts

@@ -0,0 +1,101 @@
+import { Request, Response, NextFunction } from 'express';
+
+export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
+
+export interface EndpointParameter {
+  name: string;
+  in: 'query' | 'path' | 'header' | 'body';
+  type: 'string' | 'number' | 'boolean' | 'object' | 'array';
+  required?: boolean;
+  description?: string;
+}
+
+export interface EndpointRequestBody {
+  required?: boolean;
+  schema: any; // JSON Schema
+  description?: string;
+}
+
+export interface EndpointResponse {
+  description: string;
+  schema?: any; // JSON Schema
+}
+
+export interface EndpointDocumentation {
+  method?: HttpMethod;
+  path?: string;
+  summary: string;
+  description?: string;
+  tags?: string[];
+  parameters?: EndpointParameter[];
+  requestBody?: EndpointRequestBody;
+  responses?: { [statusCode: string]: EndpointResponse };
+  requiresAuth?: boolean;
+}
+
+export interface EndpointHandler {
+  (req: Request, res: Response, next?: NextFunction): Promise<void> | void;
+}
+
+export type EndpointMiddleware = (req: Request, res: Response, next?: NextFunction) => Promise<void> | void;
+
+export interface EndpointComponent {
+  method: HttpMethod;
+  path: string;
+  handler: EndpointHandler;
+  middleware?: EndpointMiddleware[];
+  documentation: EndpointDocumentation;
+}
+
+export interface EndpointMethod {
+  method: HttpMethod;
+  path: string;
+  handler: EndpointHandler;
+  middleware?: EndpointMiddleware[];
+  doc: Omit<EndpointDocumentation, 'method' | 'path'>;
+}
+
+/**
+ * Base class for endpoint components
+ */
+export abstract class BaseEndpointComponent {
+  abstract getEndpoints(): EndpointMethod[];
+
+  /**
+   * Helper method to create endpoint method definitions
+   */
+  protected createEndpoint(
+    method: HttpMethod,
+    path: string,
+    handler: EndpointHandler,
+    doc: Omit<EndpointDocumentation, 'method' | 'path'>,
+    middleware?: EndpointMiddleware[]
+  ): EndpointMethod {
+    return {
+      method,
+      path,
+      handler,
+      middleware,
+      doc
+    };
+  }
+
+  /**
+   * Register all endpoints from this component
+   */
+  registerEndpoints(registry: any): void {
+    for (const endpoint of this.getEndpoints()) {
+      registry.register({
+        method: endpoint.method,
+        path: endpoint.path,
+        handler: endpoint.handler,
+        middleware: endpoint.middleware,
+        documentation: {
+          ...endpoint.doc,
+          method: endpoint.method,
+          path: endpoint.path
+        }
+      });
+    }
+  }
+}

src/api/routes.ts

@@ -1,909 +1,64 @@
-import { Router, Request, Response } from 'express';
-import { v4 as uuidv4 } from 'uuid';
+import { Router } from 'express';
 import { JobQueue } from '../queue/JobQueue';
-import { ScraperJob } from '../types';
-import { logger } from '../utils/logger';
 import { ShopDatabase } from '../database/Database';
-
-export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
-  const router = Router();
-
-  /**
-   * POST /jobs - Create a new scraping job
-   */
-  router.post('/jobs', async (req: Request, res: Response): Promise<void> => {
-    try {
-      const { url, custom_id } = req.body;
-
-      if (!url) {
-        res.status(400).json({ error: 'URL is required' });
-        return;
-      }
-
-      // Validate URL format
-      try {
-        new URL(url);
-      } catch (error) {
-        res.status(400).json({ error: 'Invalid URL format' });
-        return;
-      }
-
-      let shopId: string | undefined;
-
-      // If database is available, create or find shop
-      if (db) {
-        const { SitemapParser } = require('../scraper/SitemapParser');
-        const { sitemapUrl, webshopType } = SitemapParser.getSitemapUrl(url);
-
-        let shop = db.getShopByUrl(url);
-        if (!shop) {
-          // Validate custom_id format if provided
-          if (custom_id && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(custom_id)) {
-            res.status(400).json({ error: 'custom_id must be a valid UUID format' });
-            return;
-          }
-
-          // Check if custom_id is already in use
-          if (custom_id && db.getShopByCustomId(custom_id)) {
-            res.status(409).json({ error: 'custom_id is already in use' });
-            return;
-          }
-
-          shop = db.createShop(url, sitemapUrl, webshopType, custom_id);
-          logger.info(`Created new shop: ${shop.id}${shop.custom_id ? ` with custom ID ${shop.custom_id}` : ''}`);
-
-          // Create initial schedule - we'll get more info from sitemap during first scrape
-          const { ScrapeScheduler } = require('../scheduler/ScrapeScheduler');
-          // Note: Initial schedule will be created after first successful scrape
+import { EndpointRegistry } from './core/EndpointRegistry';
+import { authMiddleware } from './middleware/auth';
+import { BaseEndpointComponent } from './core/types';
+
+// Import all endpoint components
+import { DocumentationEndpoint } from './components/DocumentationEndpoint';
+import { JobsEndpoint } from './components/JobsEndpoint';
+import { ShopsEndpoint } from './components/ShopsEndpoint';
+import { WebhooksEndpoint } from './components/WebhooksEndpoint';
+import { CustomUrlsEndpoint } from './components/CustomUrlsEndpoint';
+import { HealthEndpoint } from './components/HealthEndpoint';
+
+export function createRouter(jobQueue: JobQueue, apiKey: string, db?: ShopDatabase): Router {
+  // Create the endpoint registry
+  const registry = new EndpointRegistry();
+
+  // Create auth middleware instance
+  const auth = authMiddleware(apiKey);
+
+  // Register all endpoint components
+  const components: BaseEndpointComponent[] = [
+    new HealthEndpoint(jobQueue),
+    new JobsEndpoint(jobQueue, db),
+  ];
+
+  // Only register database-dependent endpoints if database is available
+  if (db) {
+    components.push(
+      new ShopsEndpoint(db),
+      new WebhooksEndpoint(db),
+      new CustomUrlsEndpoint(db)
+    );
+  }
+
+  // Register documentation endpoint (needs registry reference)
+  const docEndpoint = new DocumentationEndpoint(registry);
+  components.push(docEndpoint);
+
+  // Register all endpoints with appropriate middleware
+  for (const component of components) {
+    const endpoints = component.getEndpoints();
+    for (const endpoint of endpoints) {
+      const middleware: Array<(req: any, res: any, next?: any) => void | Promise<void>> = endpoint.doc.requiresAuth ? [auth, ...(endpoint.middleware || [])] : endpoint.middleware || [];
+
+      registry.register({
+        method: endpoint.method,
+        path: endpoint.path,
+        handler: endpoint.handler,
+        middleware,
+        documentation: {
+          ...endpoint.doc,
+          method: endpoint.method,
+          path: endpoint.path
         }
-        shopId = shop.id;
-      }
-
-      // Create new job
-      const job: ScraperJob = {
-        id: uuidv4(),
-        sitemapUrl: url,
-        status: 'pending',
-        createdAt: new Date(),
-        updatedAt: new Date(),
-        shopId
-      };
-
-      jobQueue.addJob(job);
-
-      logger.info(`New job created: ${job.id} for shop: ${shopId || 'N/A'}`);
-
-      res.status(201).json({
-        job_id: job.id,
-        shop_id: shopId,
-        status: job.status,
-        message: 'Job created successfully'
-      });
-    } catch (error) {
-      logger.error('Error creating job', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /jobs/:id - Get job status and result
-   */
-  router.get('/jobs/:id', (req: Request, res: Response): void => {
-    try {
-      const { id } = req.params;
-      const job = jobQueue.getJob(id);
-
-      if (!job) {
-        res.status(404).json({ error: 'Job not found' });
-        return;
-      }
-
-      const response: any = {
-        job_id: job.id,
-        status: job.status,
-        created_at: job.createdAt,
-        updated_at: job.updatedAt
-      };
-
-      if (job.status === 'completed' && job.result) {
-        response.result = job.result;
-      }
-
-      if (job.status === 'failed' && job.error) {
-        response.error = job.error;
-      }
-
-      res.json(response);
-    } catch (error) {
-      logger.error('Error fetching job', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /jobs - List all jobs
-   */
-  router.get('/jobs', (req: Request, res: Response): void => {
-    try {
-      const jobs = jobQueue.getAllJobs();
-      const stats = jobQueue.getStats();
-
-      res.json({
-        stats,
-        jobs: jobs.map(job => ({
-          job_id: job.id,
-          status: job.status,
-          sitemap_url: job.sitemapUrl,
-          created_at: job.createdAt,
-          updated_at: job.updatedAt
-        }))
       });
-    } catch (error) {
-      logger.error('Error listing jobs', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /shops - List all shops
-   */
-  router.get('/shops', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const shops = db.getAllShops();
-      const shopsWithAnalytics = shops.map(shop => {
-        const analytics = db.getShopAnalytics(shop.id);
-        return {
-          ...shop,
-          analytics
-        };
-      });
-
-      res.json({ shops: shopsWithAnalytics });
-    } catch (error) {
-      logger.error('Error listing shops', error);
-      res.status(500).json({ error: 'Internal server error' });
     }
-  });
-
-  /**
-   * GET /shops/:id - Get detailed shop information with analytics (without full results)
-   */
-  router.get('/shops/:id', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      // Get analytics
-      const analytics = db.getShopAnalytics(shop.id);
-
-      // Get scrape history
-      const scrapeHistory = db.getScrapeHistory(shop.id, 10);
-
-      // Get latest content (without full content, just metadata)
-      const latestContent = db.getLatestContentByType(shop.id);
-
-      // Get scheduled jobs
-      const scheduledJobs = db.getScheduledJobs(shop.id);
-
-      // Build response with content change markers and URLs only (no content)
-      const contentMetadata: any = {
-        shipping_informations: latestContent.shipping.map(c => ({
-          url: c.url,
-          changed: Boolean(c.changed),
-          last_updated: c.updated_at
-        })),
-        contacts: latestContent.contacts.map(c => ({
-          url: c.url,
-          changed: Boolean(c.changed),
-          last_updated: c.updated_at
-        })),
-        terms_of_conditions: latestContent.terms.map(c => ({
-          url: c.url,
-          changed: Boolean(c.changed),
-          last_updated: c.updated_at
-        })),
-        faq: latestContent.faq.map(c => ({
-          url: c.url,
-          changed: Boolean(c.changed),
-          last_updated: c.updated_at
-        }))
-      };
-
-      res.json({
-        shop: {
-          id: shop.id,
-          custom_id: shop.custom_id,
-          url: shop.url,
-          sitemap_url: shop.sitemap_url,
-          webshop_type: shop.webshop_type,
-          created_at: shop.created_at,
-          updated_at: shop.updated_at
-        },
-        analytics: {
-          total_scrapes: analytics?.total_scrapes || 0,
-          last_scraped_at: analytics?.last_scraped_at || null,
-          next_scrape_at: analytics?.next_scrape_at || null,
-          total_urls_found: analytics?.total_urls_found || 0,
-          average_scrape_time: analytics?.average_scrape_time || null,
-          average_page_scrape_time: analytics?.average_page_scrape_time || null
-        },
-        content_metadata: contentMetadata,
-        scrape_history: scrapeHistory.map(h => ({
-          id: h.id,
-          job_id: h.job_id,
-          started_at: h.started_at,
-          completed_at: h.completed_at,
-          status: h.status,
-          error: h.error,
-          scrape_time_ms: h.scrape_time_ms,
-          urls_found: h.urls_found
-        })),
-        scheduled_jobs: scheduledJobs.map(j => ({
-          id: j.id,
-          next_run_at: j.next_run_at,
-          frequency: j.frequency,
-          last_modified: j.last_modified,
-          status: j.status,
-          enabled: j.enabled,
-          created_at: j.created_at
-        }))
-      });
-    } catch (error) {
-      logger.error('Error fetching shop details', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /shops/:id/results - Get shop results with full content (supports filters)
-   */
-  router.get('/shops/:id/results', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      // Parse query parameters
-      const limit = req.query.limit ? parseInt(req.query.limit as string) : undefined;
-      const dateFrom = req.query.date_from as string | undefined;
-      const dateTo = req.query.date_to as string | undefined;
-      const contentType = req.query.content_type as string | undefined;
-
-      // Get content with filters
-      const allContent = db.getAllContent(shop.id, {
-        limit,
-        dateFrom,
-        dateTo,
-        contentType
-      });
-
-      // Group by type and URL (latest version of each URL)
-      const grouped: { [key: string]: any[] } = {
-        shipping_informations: [],
-        contacts: [],
-        terms_of_conditions: [],
-        faq: []
-      };
-
-      const urlMap = new Map<string, any>();
-      for (const content of allContent) {
-        const key = `${content.content_type}:${content.url}`;
-        if (!urlMap.has(key)) {
-          urlMap.set(key, {
-            url: content.url,
-            content: content.content,
-            changed: Boolean(content.changed),
-            last_updated: content.updated_at
-          });
-        }
-      }
-
-      // Organize by type
-      for (const content of allContent) {
-        const key = `${content.content_type}:${content.url}`;
-        const urlKey = urlMap.get(key);
-
-        if (!urlKey) continue;
-
-        const typeKey = content.content_type === 'shipping' ? 'shipping_informations' :
-                       content.content_type === 'contacts' ? 'contacts' :
-                       content.content_type === 'terms' ? 'terms_of_conditions' :
-                       'faq';
-
-        // Only add if not already added
-        if (!grouped[typeKey].find((c: any) => c.url === urlKey.url)) {
-          grouped[typeKey].push(urlKey);
-        }
-      }
-
-      res.json({
-        shop_id: shop.id,
-        filters: {
-          limit: limit || null,
-          date_from: dateFrom || null,
-          date_to: dateTo || null,
-          content_type: contentType || null
-        },
-        results: grouped
-      });
-    } catch (error) {
-      logger.error('Error fetching shop results', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * PATCH /shops/:id/schedule - Enable or disable scheduled scraping
-   */
-  router.patch('/shops/:id/schedule', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const { enabled } = req.body;
-
-      if (typeof enabled !== 'boolean') {
-        res.status(400).json({ error: 'enabled field must be a boolean' });
-        return;
-      }
-
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      db.setScheduleEnabled(shop.id, enabled);
-
-      res.json({
-        shop_id: shop.id,
-        schedule_enabled: enabled,
-        message: `Schedule ${enabled ? 'enabled' : 'disabled'} successfully`
-      });
-    } catch (error) {
-      logger.error('Error updating schedule status', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * PATCH /shops/:id/custom-id - Set or update custom ID for a shop
-   */
-  router.patch('/shops/:id/custom-id', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const { custom_id } = req.body;
-
-      const shop = db.getShopByAnyId(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      // Validate custom_id format if provided
-      if (custom_id !== null && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(custom_id)) {
-        res.status(400).json({ error: 'custom_id must be a valid UUID format or null' });
-        return;
-      }
-
-      // Check if custom_id is already in use by another shop
-      if (custom_id) {
-        const existingShop = db.getShopByCustomId(custom_id);
-        if (existingShop && existingShop.id !== shop.id) {
-          res.status(409).json({ error: 'custom_id is already in use' });
-          return;
-        }
-      }
-
-      const success = db.setCustomId(shop.id, custom_id);
-      if (!success) {
-        res.status(500).json({ error: 'Failed to update custom_id' });
-        return;
-      }
-
-      res.json({
-        shop_id: shop.id,
-        custom_id: custom_id,
-        message: custom_id ? `Custom ID set to ${custom_id}` : 'Custom ID removed'
-      });
-    } catch (error) {
-      logger.error('Error updating custom ID', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * DELETE /shops/:id - Delete a shop and all related data
-   */
-  router.delete('/shops/:id', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      db.deleteShop(shop.id);
-
-      res.json({
-        message: 'Shop and all related data deleted successfully',
-        shop_id: shop.id
-      });
-    } catch (error) {
-      logger.error('Error deleting shop', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * POST /shops/:id/webhooks - Create or replace webhook for a shop
-   */
-  router.post('/shops/:id/webhooks', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const { url, secret } = req.body;
-
-      if (!url) {
-        res.status(400).json({ error: 'url field is required' });
-        return;
-      }
-
-      // Validate URL format
-      try {
-        new URL(url);
-      } catch (error) {
-        res.status(400).json({ error: 'Invalid URL format' });
-        return;
-      }
-
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      // Check if webhook already exists
-      const existingWebhook = db.getWebhook(shop.id);
-      if (existingWebhook) {
-        // Update existing webhook
-        db.updateWebhook(shop.id, url, secret);
-        const updated = db.getWebhook(shop.id);
-
-        res.json({
-          message: 'Webhook updated successfully',
-          webhook: {
-            id: updated!.id,
-            shop_id: updated!.shop_id,
-            url: updated!.url,
-            enabled: updated!.enabled,
-            created_at: updated!.created_at,
-            updated_at: updated!.updated_at
-          }
-        });
-      } else {
-        // Create new webhook
-        const webhookId = db.createWebhook(shop.id, url, secret);
-        const webhook = db.getWebhook(shop.id);
-
-        res.status(201).json({
-          message: 'Webhook created successfully',
-          webhook: {
-            id: webhook!.id,
-            shop_id: webhook!.shop_id,
-            url: webhook!.url,
-            enabled: webhook!.enabled,
-            created_at: webhook!.created_at,
-            updated_at: webhook!.updated_at
-          }
-        });
-      }
-    } catch (error) {
-      logger.error('Error creating/updating webhook', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /shops/:id/webhooks - Get webhook configuration for a shop
-   */
-  router.get('/shops/:id/webhooks', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const webhook = db.getWebhook(shop.id);
-
-      if (!webhook) {
-        res.status(404).json({ error: 'No webhook configured for this shop' });
-        return;
-      }
-
-      // Get recent deliveries
-      const deliveries = db.getWebhookDeliveries(shop.id, 10);
-
-      res.json({
-        webhook: {
-          id: webhook.id,
-          shop_id: webhook.shop_id,
-          url: webhook.url,
-          enabled: webhook.enabled,
-          created_at: webhook.created_at,
-          updated_at: webhook.updated_at
-        },
-        recent_deliveries: deliveries.map(d => ({
-          id: d.id,
-          event: d.event,
-          status: d.status,
-          response_code: d.response_code,
-          error: d.error,
-          attempted_at: d.attempted_at
-        }))
-      });
-    } catch (error) {
-      logger.error('Error fetching webhook', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * PATCH /shops/:id/webhooks - Enable or disable webhook
-   */
-  router.patch('/shops/:id/webhooks', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const { enabled } = req.body;
-
-      if (typeof enabled !== 'boolean') {
-        res.status(400).json({ error: 'enabled field must be a boolean' });
-        return;
-      }
-
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const webhook = db.getWebhook(shop.id);
-      if (!webhook && enabled) {
-        res.status(404).json({ error: 'No webhook configured for this shop' });
-        return;
-      }
-
-      db.setWebhookEnabled(shop.id, enabled);
-
-      res.json({
-        shop_id: shop.id,
-        webhook_enabled: enabled,
-        message: `Webhook ${enabled ? 'enabled' : 'disabled'} successfully`
-      });
-    } catch (error) {
-      logger.error('Error updating webhook status', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * DELETE /shops/:id/webhooks - Delete webhook for a shop
-   */
-  router.delete('/shops/:id/webhooks', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const webhook = db.getWebhook(shop.id);
-      if (!webhook) {
-        res.status(404).json({ error: 'No webhook configured for this shop' });
-        return;
-      }
-
-      db.deleteWebhook(shop.id);
-
-      res.json({
-        message: 'Webhook deleted successfully',
-        shop_id: shop.id
-      });
-    } catch (error) {
-      logger.error('Error deleting webhook', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * POST /shops/:id/custom-urls - Add a custom URL to scrape for a shop
-   */
-  router.post('/shops/:id/custom-urls', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const { url, content_type } = req.body;
-
-      // Validate required fields
-      if (!url || !content_type) {
-        res.status(400).json({ error: 'url and content_type fields are required' });
-        return;
-      }
-
-      // Validate content_type
-      const validContentTypes = ['shipping', 'contacts', 'terms', 'faq'];
-      if (!validContentTypes.includes(content_type)) {
-        res.status(400).json({
-          error: 'content_type must be one of: shipping, contacts, terms, faq'
-        });
-        return;
-      }
-
-      // Validate URL format
-      let parsedUrl: URL;
-      try {
-        parsedUrl = new URL(url);
-      } catch (error) {
-        res.status(400).json({ error: 'Invalid URL format' });
-        return;
-      }
-
-      // Check if shop exists
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      // Validate that the URL belongs to the same domain as the shop
-      const shopUrl = new URL(shop.url);
-      if (parsedUrl.hostname !== shopUrl.hostname) {
-        res.status(400).json({
-          error: `URL must belong to the same domain as the shop (${shopUrl.hostname})`
-        });
-        return;
-      }
-
-      // Check if URL was already scraped
-      const alreadyScraped = db.isUrlAlreadyScraped(shop.id, url);
-      if (alreadyScraped) {
-        res.status(409).json({
-          error: 'This URL has already been scraped from the sitemap',
-          message: 'URL already exists in scraped content'
-        });
-        return;
-      }
-
-      // Create custom URL
-      try {
-        const customUrl = db.createCustomUrl(shop.id, url, content_type);
-
-        res.status(201).json({
-          message: 'Custom URL added successfully',
-          custom_url: {
-            id: customUrl.id,
-            shop_id: customUrl.shop_id,
-            url: customUrl.url,
-            content_type: customUrl.content_type,
-            enabled: customUrl.enabled,
-            created_at: customUrl.created_at
-          }
-        });
-      } catch (error: any) {
-        if (error.message?.includes('already been added')) {
-          res.status(409).json({ error: error.message });
-          return;
-        }
-        throw error;
-      }
-    } catch (error) {
-      logger.error('Error creating custom URL', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /shops/:id/custom-urls - List all custom URLs for a shop
-   */
-  router.get('/shops/:id/custom-urls', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id } = req.params;
-      const shop = db.getShopByAnyId(id);
-
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const customUrls = db.getCustomUrls(shop.id);
-
-      res.json({
-        shop_id: shop.id,
-        custom_urls: customUrls.map(cu => ({
-          id: cu.id,
-          url: cu.url,
-          content_type: cu.content_type,
-          enabled: cu.enabled,
-          created_at: cu.created_at
-        }))
-      });
-    } catch (error) {
-      logger.error('Error fetching custom URLs', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * PATCH /shops/:id/custom-urls/:customUrlId - Enable or disable a custom URL
-   */
-  router.patch('/shops/:id/custom-urls/:customUrlId', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id, customUrlId } = req.params;
-      const { enabled } = req.body;
-
-      if (typeof enabled !== 'boolean') {
-        res.status(400).json({ error: 'enabled field must be a boolean' });
-        return;
-      }
-
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const customUrl = db.getCustomUrlById(customUrlId);
-      if (!customUrl) {
-        res.status(404).json({ error: 'Custom URL not found' });
-        return;
-      }
-
-      if (customUrl.shop_id !== shop.id) {
-        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
-        return;
-      }
-
-      db.setCustomUrlEnabled(customUrlId, enabled);
-
-      res.json({
-        message: `Custom URL ${enabled ? 'enabled' : 'disabled'} successfully`,
-        custom_url_id: customUrlId,
-        enabled
-      });
-    } catch (error) {
-      logger.error('Error updating custom URL', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * DELETE /shops/:id/custom-urls/:customUrlId - Delete a custom URL
-   */
-  router.delete('/shops/:id/custom-urls/:customUrlId', (req: Request, res: Response): void => {
-    try {
-      if (!db) {
-        res.status(503).json({ error: 'Database not available' });
-        return;
-      }
-
-      const { id, customUrlId } = req.params;
-
-      const shop = db.getShopById(id);
-      if (!shop) {
-        res.status(404).json({ error: 'Shop not found' });
-        return;
-      }
-
-      const customUrl = db.getCustomUrlById(customUrlId);
-      if (!customUrl) {
-        res.status(404).json({ error: 'Custom URL not found' });
-        return;
-      }
-
-      if (customUrl.shop_id !== shop.id) {
-        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
-        return;
-      }
-
-      db.deleteCustomUrl(customUrlId);
-
-      res.json({
-        message: 'Custom URL deleted successfully',
-        custom_url_id: customUrlId
-      });
-    } catch (error) {
-      logger.error('Error deleting custom URL', error);
-      res.status(500).json({ error: 'Internal server error' });
-    }
-  });
-
-  /**
-   * GET /health - Health check endpoint
-   */
-  router.get('/health', (req: Request, res: Response): void => {
-    res.json({
-      status: 'ok',
-      timestamp: new Date().toISOString(),
-      queue_stats: jobQueue.getStats()
-    });
-  });
+  }
 
-  return router;
-}
+  // Build and return the router
+  return registry.buildRouter();
+}

src/api/server.ts

@@ -2,7 +2,6 @@ import express, { Express } from 'express';
 import path from 'path';
 import { JobQueue } from '../queue/JobQueue';
 import { createRouter } from './routes';
-import { authMiddleware } from './middleware/auth';
 import { logger } from '../utils/logger';
 import { ShopDatabase } from '../database/Database';
 
@@ -22,18 +21,9 @@ export function createServer(apiKey: string, jobQueue: JobQueue, db?: ShopDataba
   const webUIPath = path.join(__dirname, '../../dist/web');
   app.use(express.static(webUIPath));
 
-  // Health check (no auth required)
-  app.get('/health', (req, res) => {
-    res.json({
-      status: 'ok',
-      timestamp: new Date().toISOString(),
-      queue_stats: jobQueue.getStats()
-    });
-  });
-
-  // Protected routes
-  const router = createRouter(jobQueue, db);
-  app.use('/api', authMiddleware(apiKey), router);
+  // API routes (authentication handled per endpoint based on documentation)
+  const router = createRouter(jobQueue, apiKey, db);
+  app.use('/', router);
 
   // Catch all handler for SPA - serve index.html for all non-API routes
   app.get('*', (req, res) => {