feat: implement complete Qdrant and MCP system with API endpoints

- Add QdrantService for vector database operations
- Add EmbeddingService for OpenRouter integration
- Add QdrantCleanupService for automated maintenance
- Add QdrantEmbeddingWorkflow for scraping integration
- Add MCP server with 4 category-specific tools
- Add comprehensive API endpoints for management
- Add shop lifecycle management with custom_id protection
- Extend database schema with Qdrant and MCP tables
- Add embedding workflow to scraping pipeline

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/components/McpEndpoint.ts

@@ -0,0 +1,538 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+
+export class McpEndpoint extends BaseEndpointComponent {
+  constructor(private db: ShopDatabase) {
+    super();
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      // MCP Analytics
+      this.createEndpoint(
+        'GET',
+        '/api/mcp/analytics',
+        this.getMcpAnalytics.bind(this),
+        {
+          summary: 'Get MCP analytics',
+          description: 'Get overall MCP usage statistics and analytics',
+          tags: ['MCP'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'dateFrom',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Start date for analytics (ISO format)'
+            },
+            {
+              name: 'dateTo',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'End date for analytics (ISO format)'
+            },
+            {
+              name: 'groupBy',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Group results by: day, hour, shop, tool'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'MCP analytics data',
+              schema: {
+                type: 'object',
+                properties: {
+                  total_calls: { type: 'number' },
+                  successful_calls: { type: 'number' },
+                  failed_calls: { type: 'number' },
+                  success_rate: { type: 'number' },
+                  avg_response_time: { type: 'number' },
+                  tool_usage: {
+                    type: 'object',
+                    additionalProperties: { type: 'number' }
+                  },
+                  shop_usage: {
+                    type: 'object',
+                    additionalProperties: { type: 'number' }
+                  },
+                  timeline: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        date: { type: 'string' },
+                        calls: { type: 'number' },
+                        success_rate: { type: 'number' }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:shopId/mcp/analytics',
+        this.getShopMcpAnalytics.bind(this),
+        {
+          summary: 'Get shop MCP analytics',
+          description: 'Get MCP usage analytics for a specific shop',
+          tags: ['MCP'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            },
+            {
+              name: 'dateFrom',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Start date for analytics (ISO format)'
+            },
+            {
+              name: 'dateTo',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'End date for analytics (ISO format)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Shop MCP analytics',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  total_calls: { type: 'number' },
+                  successful_calls: { type: 'number' },
+                  failed_calls: { type: 'number' },
+                  success_rate: { type: 'number' },
+                  avg_response_time: { type: 'number' },
+                  tool_usage: {
+                    type: 'object',
+                    additionalProperties: { type: 'number' }
+                  },
+                  recent_calls: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        tool_name: { type: 'string' },
+                        query: { type: 'string' },
+                        success: { type: 'boolean' },
+                        response_time: { type: 'number' },
+                        error_message: { type: 'string', nullable: true },
+                        created_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            '404': { description: 'Shop not found' }
+          }
+        }
+      ),
+
+      // MCP Logs
+      this.createEndpoint(
+        'GET',
+        '/api/mcp/logs',
+        this.getMcpLogs.bind(this),
+        {
+          summary: 'Get MCP logs',
+          description: 'Get MCP call logs with filtering options',
+          tags: ['MCP'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Filter by shop ID'
+            },
+            {
+              name: 'toolName',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Filter by tool name'
+            },
+            {
+              name: 'success',
+              in: 'query',
+              type: 'boolean',
+              required: false,
+              description: 'Filter by success status'
+            },
+            {
+              name: 'limit',
+              in: 'query',
+              type: 'number',
+              required: false,
+              description: 'Maximum number of logs to return (default: 100)'
+            },
+            {
+              name: 'offset',
+              in: 'query',
+              type: 'number',
+              required: false,
+              description: 'Number of logs to skip (for pagination)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'MCP logs',
+              schema: {
+                type: 'object',
+                properties: {
+                  logs: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        shop_id: { type: 'string' },
+                        tool_name: { type: 'string' },
+                        query: { type: 'string' },
+                        parameters: { type: 'object' },
+                        response_metadata: { type: 'object' },
+                        success: { type: 'boolean' },
+                        response_time: { type: 'number' },
+                        error_message: { type: 'string', nullable: true },
+                        created_at: { type: 'string' }
+                      }
+                    }
+                  },
+                  pagination: {
+                    type: 'object',
+                    properties: {
+                      total: { type: 'number' },
+                      limit: { type: 'number' },
+                      offset: { type: 'number' },
+                      has_more: { type: 'boolean' }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'GET',
+        '/api/mcp/logs/:logId',
+        this.getMcpLogDetails.bind(this),
+        {
+          summary: 'Get MCP log details',
+          description: 'Get detailed information about a specific MCP call',
+          tags: ['MCP'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'logId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Log ID'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'MCP log details',
+              schema: {
+                type: 'object',
+                properties: {
+                  id: { type: 'string' },
+                  shop_id: { type: 'string' },
+                  tool_name: { type: 'string' },
+                  query: { type: 'string' },
+                  parameters: { type: 'object' },
+                  response_metadata: { type: 'object' },
+                  success: { type: 'boolean' },
+                  response_time: { type: 'number' },
+                  error_message: { type: 'string', nullable: true },
+                  created_at: { type: 'string' }
+                }
+              }
+            },
+            '404': { description: 'Log not found' }
+          }
+        }
+      ),
+
+      // MCP Tool Usage Stats
+      this.createEndpoint(
+        'GET',
+        '/api/mcp/tools/stats',
+        this.getMcpToolStats.bind(this),
+        {
+          summary: 'Get MCP tool usage statistics',
+          description: 'Get usage statistics for each MCP tool',
+          tags: ['MCP'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'dateFrom',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'Start date for stats (ISO format)'
+            },
+            {
+              name: 'dateTo',
+              in: 'query',
+              type: 'string',
+              required: false,
+              description: 'End date for stats (ISO format)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'MCP tool statistics',
+              schema: {
+                type: 'object',
+                properties: {
+                  tools: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        tool_name: { type: 'string' },
+                        total_calls: { type: 'number' },
+                        successful_calls: { type: 'number' },
+                        failed_calls: { type: 'number' },
+                        success_rate: { type: 'number' },
+                        avg_response_time: { type: 'number' },
+                        most_common_queries: {
+                          type: 'array',
+                          items: {
+                            type: 'object',
+                            properties: {
+                              query: { type: 'string' },
+                              count: { type: 'number' }
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      // Clean up logs
+      this.createEndpoint(
+        'DELETE',
+        '/api/mcp/logs/cleanup',
+        this.cleanupMcpLogs.bind(this),
+        {
+          summary: 'Clean up old MCP logs',
+          description: 'Delete MCP logs older than specified date',
+          tags: ['MCP'],
+          requiresAuth: true,
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                older_than: {
+                  type: 'string',
+                  description: 'Delete logs older than this date (ISO format)'
+                }
+              },
+              required: ['older_than']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Cleanup completed',
+              schema: {
+                type: 'object',
+                properties: {
+                  deleted_count: { type: 'number' }
+                }
+              }
+            }
+          }
+        }
+      )
+    ];
+  }
+
+  private async getMcpAnalytics(req: Request, res: Response): Promise<void> {
+    try {
+      const { dateFrom, dateTo, groupBy } = req.query;
+
+      const options: any = {};
+      if (dateFrom) options.dateFrom = dateFrom as string;
+      if (dateTo) options.dateTo = dateTo as string;
+
+      const analytics = this.db.getMcpAnalytics(options);
+
+      // Calculate derived metrics
+      const successRate = analytics.total_calls > 0
+        ? (analytics.successful_calls / analytics.total_calls) * 100
+        : 0;
+
+      // Get tool usage breakdown
+      const toolUsage = this.db.getMcpToolUsage(options);
+
+      // Get shop usage breakdown
+      const shopUsage = this.db.getMcpShopUsage(options);
+
+      // Get timeline data if requested
+      let timeline: any[] = [];
+      if (groupBy === 'day' || groupBy === 'hour') {
+        timeline = this.db.getMcpAnalyticsTimeline(groupBy as 'day' | 'hour', options);
+      }
+
+      res.json({
+        total_calls: analytics.total_calls,
+        successful_calls: analytics.successful_calls,
+        failed_calls: analytics.failed_calls,
+        success_rate: successRate,
+        avg_response_time: analytics.avg_response_time,
+        tool_usage: toolUsage,
+        shop_usage: shopUsage,
+        timeline
+      });
+    } catch (error) {
+      logger.error('Failed to get MCP analytics:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getShopMcpAnalytics(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+      const { dateFrom, dateTo } = req.query;
+
+      const shop = this.db.getShopById(shopId);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const options: any = { shopId };
+      if (dateFrom) options.dateFrom = dateFrom as string;
+      if (dateTo) options.dateTo = dateTo as string;
+
+      const analytics = this.db.getMcpAnalytics(options);
+      const toolUsage = this.db.getMcpToolUsage(options);
+      const recentCalls = this.db.getMcpLogs({ shopId, limit: 20 });
+
+      const successRate = analytics.total_calls > 0
+        ? (analytics.successful_calls / analytics.total_calls) * 100
+        : 0;
+
+      res.json({
+        shop_id: shopId,
+        total_calls: analytics.total_calls,
+        successful_calls: analytics.successful_calls,
+        failed_calls: analytics.failed_calls,
+        success_rate: successRate,
+        avg_response_time: analytics.avg_response_time,
+        tool_usage: toolUsage,
+        recent_calls: recentCalls.logs
+      });
+    } catch (error) {
+      logger.error('Failed to get shop MCP analytics:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getMcpLogs(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId, toolName, success, limit, offset } = req.query;
+
+      const options: any = {
+        limit: parseInt(limit as string) || 100,
+        offset: parseInt(offset as string) || 0
+      };
+
+      if (shopId) options.shopId = shopId as string;
+      if (toolName) options.toolName = toolName as string;
+      if (success !== undefined) options.success = success === 'true';
+
+      const result = this.db.getMcpLogs(options);
+
+      res.json({
+        logs: result.logs,
+        pagination: result.pagination
+      });
+    } catch (error) {
+      logger.error('Failed to get MCP logs:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getMcpLogDetails(req: Request, res: Response): Promise<void> {
+    try {
+      const { logId } = req.params;
+
+      const log = this.db.getMcpLogById(logId);
+      if (!log) {
+        res.status(404).json({ error: 'Log not found' });
+        return;
+      }
+
+      res.json(log);
+    } catch (error) {
+      logger.error('Failed to get MCP log details:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getMcpToolStats(req: Request, res: Response): Promise<void> {
+    try {
+      const { dateFrom, dateTo } = req.query;
+
+      const options: any = {};
+      if (dateFrom) options.dateFrom = dateFrom as string;
+      if (dateTo) options.dateTo = dateTo as string;
+
+      const toolStats = this.db.getMcpToolStats(options);
+
+      res.json({ tools: toolStats });
+    } catch (error) {
+      logger.error('Failed to get MCP tool stats:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async cleanupMcpLogs(req: Request, res: Response): Promise<void> {
+    try {
+      const { older_than } = req.body;
+
+      const deletedCount = this.db.cleanupMcpLogs(older_than);
+
+      res.json({ deleted_count: deletedCount });
+    } catch (error) {
+      logger.error('Failed to cleanup MCP logs:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/components/QdrantEndpoint.ts

@@ -0,0 +1,602 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+import { QdrantService } from '../../services/QdrantService';
+import { EmbeddingService } from '../../services/EmbeddingService';
+import { QdrantCleanupService } from '../../services/QdrantCleanupService';
+import { QdrantEmbeddingWorkflow } from '../../services/QdrantEmbeddingWorkflow';
+import type { AppConfig } from '../../config';
+
+export class QdrantEndpoint extends BaseEndpointComponent {
+  private qdrantService?: QdrantService;
+  private embeddingService?: EmbeddingService;
+  private cleanupService?: QdrantCleanupService;
+  private embeddingWorkflow?: QdrantEmbeddingWorkflow;
+
+  constructor(private db: ShopDatabase, private config: AppConfig) {
+    super();
+
+    try {
+      this.qdrantService = new QdrantService(config);
+      this.embeddingService = new EmbeddingService(config);
+      this.cleanupService = new QdrantCleanupService(config, db);
+      this.embeddingWorkflow = new QdrantEmbeddingWorkflow(config, db);
+    } catch (error) {
+      logger.warn('Qdrant services not available:', error);
+    }
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      // Shop Qdrant Settings
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:shopId/qdrant',
+        this.getShopQdrantStatus.bind(this),
+        {
+          summary: 'Get shop Qdrant status',
+          description: 'Get Qdrant embedding status and settings for a shop',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'Shop Qdrant status',
+              schema: {
+                type: 'object',
+                properties: {
+                  enabled: { type: 'boolean' },
+                  custom_id: { type: 'string', nullable: true },
+                  collections: { type: 'array', items: { type: 'string' } },
+                  embedding_stats: {
+                    type: 'object',
+                    properties: {
+                      total_embeddings: { type: 'number' },
+                      completed: { type: 'number' },
+                      failed: { type: 'number' }
+                    }
+                  },
+                  errors: { type: 'array', items: { type: 'object' } }
+                }
+              }
+            },
+            '404': { description: 'Shop not found' }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'PUT',
+        '/api/shops/:shopId/qdrant/toggle',
+        this.toggleShopQdrant.bind(this),
+        {
+          summary: 'Toggle shop Qdrant embedding',
+          description: 'Enable or disable Qdrant embedding for a shop',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                enabled: { type: 'boolean' }
+              },
+              required: ['enabled']
+            }
+          },
+          responses: {
+            '200': { description: 'Qdrant status updated' },
+            '404': { description: 'Shop not found' }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'PUT',
+        '/api/shops/:shopId/qdrant/custom-id',
+        this.updateShopCustomId.bind(this),
+        {
+          summary: 'Update shop custom ID',
+          description: 'Set or update the custom ID for Qdrant collection naming',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                custom_id: { type: 'string' }
+              },
+              required: ['custom_id']
+            }
+          },
+          responses: {
+            '200': { description: 'Custom ID updated' },
+            '400': { description: 'Invalid custom ID or already in use' },
+            '403': { description: 'Cannot modify existing custom ID' },
+            '404': { description: 'Shop not found' }
+          }
+        }
+      ),
+
+      // Collection Management
+      this.createEndpoint(
+        'GET',
+        '/api/qdrant/collections',
+        this.listCollections.bind(this),
+        {
+          summary: 'List Qdrant collections',
+          description: 'List all Qdrant collections with metadata',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          responses: {
+            '200': {
+              description: 'List of collections',
+              schema: {
+                type: 'object',
+                properties: {
+                  collections: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        name: { type: 'string' },
+                        vectors_count: { type: 'number' },
+                        config: { type: 'object' }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'DELETE',
+        '/api/qdrant/collections/:collectionName',
+        this.deleteCollection.bind(this),
+        {
+          summary: 'Delete Qdrant collection',
+          description: 'Delete a specific Qdrant collection',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'collectionName',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Collection name to delete'
+            }
+          ],
+          responses: {
+            '200': { description: 'Collection deleted' },
+            '404': { description: 'Collection not found' }
+          }
+        }
+      ),
+
+      // Health Check
+      this.createEndpoint(
+        'GET',
+        '/api/qdrant/health',
+        this.healthCheck.bind(this),
+        {
+          summary: 'Qdrant health check',
+          description: 'Check the health of Qdrant and embedding services',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          responses: {
+            '200': {
+              description: 'Health status',
+              schema: {
+                type: 'object',
+                properties: {
+                  healthy: { type: 'boolean' },
+                  services: {
+                    type: 'object',
+                    properties: {
+                      qdrant: { type: 'boolean' },
+                      embedding: { type: 'boolean' },
+                      database: { type: 'boolean' }
+                    }
+                  },
+                  errors: { type: 'array', items: { type: 'string' } }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      // Embedding Management
+      this.createEndpoint(
+        'POST',
+        '/api/shops/:shopId/qdrant/re-embed',
+        this.reEmbedShopContent.bind(this),
+        {
+          summary: 'Re-embed shop content',
+          description: 'Trigger re-embedding of all content for a shop',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          requestBody: {
+            required: false,
+            schema: {
+              type: 'object',
+              properties: {
+                content_type: {
+                  type: 'string',
+                  enum: ['shipping', 'contacts', 'terms', 'faq'],
+                  description: 'Optional: re-embed only specific content type'
+                }
+              }
+            }
+          },
+          responses: {
+            '202': { description: 'Re-embedding started' },
+            '404': { description: 'Shop not found' },
+            '400': { description: 'Qdrant not enabled for shop' }
+          }
+        }
+      ),
+
+      // Error Management
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:shopId/qdrant/errors',
+        this.getShopQdrantErrors.bind(this),
+        {
+          summary: 'Get shop Qdrant errors',
+          description: 'Get recent Qdrant errors for a shop',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            },
+            {
+              name: 'limit',
+              in: 'query',
+              type: 'number',
+              required: false,
+              description: 'Maximum number of errors to return (default: 50)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'List of Qdrant errors',
+              schema: {
+                type: 'object',
+                properties: {
+                  errors: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        operation_type: { type: 'string' },
+                        error_message: { type: 'string' },
+                        metadata: { type: 'object' },
+                        created_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      ),
+
+      this.createEndpoint(
+        'DELETE',
+        '/api/shops/:shopId/qdrant/errors',
+        this.clearShopQdrantErrors.bind(this),
+        {
+          summary: 'Clear shop Qdrant errors',
+          description: 'Clear all Qdrant errors for a shop',
+          tags: ['Qdrant'],
+          requiresAuth: true,
+          parameters: [
+            {
+              name: 'shopId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID'
+            }
+          ],
+          responses: {
+            '200': { description: 'Errors cleared' }
+          }
+        }
+      )
+    ];
+  }
+
+  private async getShopQdrantStatus(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+
+      const shop = this.db.getShopById(shopId);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Get collections for this shop
+      let collections: string[] = [];
+      if (this.qdrantService && shop.custom_id) {
+        try {
+          const allCollections = await this.qdrantService.listCollections();
+          collections = allCollections.filter(name => name.startsWith(shop.custom_id!));
+        } catch (error) {
+          logger.error('Failed to list collections:', error);
+        }
+      }
+
+      // Get embedding stats
+      const embeddings = this.db.getShopEmbeddingsStats(shopId);
+
+      // Get recent errors
+      const errors = this.db.getQdrantErrors(shopId, { limit: 10 });
+
+      res.json({
+        enabled: shop.qdrant_enabled,
+        custom_id: shop.custom_id,
+        collections,
+        embedding_stats: embeddings,
+        errors
+      });
+    } catch (error) {
+      logger.error('Failed to get shop Qdrant status:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async toggleShopQdrant(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+      const { enabled } = req.body;
+
+      const shop = this.db.getShopById(shopId);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      this.db.toggleShopQdrant(shopId, enabled);
+
+      res.json({ message: 'Qdrant status updated' });
+    } catch (error) {
+      logger.error('Failed to toggle shop Qdrant:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateShopCustomId(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+      const { custom_id } = req.body;
+
+      // Validate custom_id format (alphanumeric, dashes, underscores)
+      if (!/^[a-zA-Z0-9_-]+$/.test(custom_id)) {
+        res.status(400).json({ error: 'Custom ID must contain only alphanumeric characters, dashes, and underscores' });
+        return;
+      }
+
+      // Check if custom_id is already in use
+      if (this.db.isCustomIdInUse(custom_id, shopId)) {
+        res.status(400).json({ error: 'Custom ID already in use' });
+        return;
+      }
+
+      try {
+        this.db.updateShopCustomId(shopId, custom_id);
+        res.json({ message: 'Custom ID updated' });
+      } catch (error: any) {
+        if (error.message.includes('Cannot modify existing custom_id')) {
+          res.status(403).json({ error: error.message });
+        } else if (error.message.includes('Shop not found')) {
+          res.status(404).json({ error: error.message });
+        } else {
+          throw error;
+        }
+      }
+    } catch (error) {
+      logger.error('Failed to update shop custom ID:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async listCollections(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.qdrantService) {
+        res.status(503).json({ error: 'Qdrant service not available' });
+        return;
+      }
+
+      const collections = await this.qdrantService.listCollections();
+      const collectionsWithInfo = [];
+
+      for (const name of collections) {
+        try {
+          const info = await this.qdrantService.getCollectionInfo(name);
+          collectionsWithInfo.push({
+            name,
+            vectors_count: info.vectors_count,
+            config: info.config
+          });
+        } catch (error) {
+          logger.error(`Failed to get info for collection ${name}:`, error);
+          collectionsWithInfo.push({
+            name,
+            vectors_count: 0,
+            config: {}
+          });
+        }
+      }
+
+      res.json({ collections: collectionsWithInfo });
+    } catch (error) {
+      logger.error('Failed to list collections:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async deleteCollection(req: Request, res: Response): Promise<void> {
+    try {
+      const { collectionName } = req.params;
+
+      if (!this.qdrantService) {
+        res.status(503).json({ error: 'Qdrant service not available' });
+        return;
+      }
+
+      const exists = await this.qdrantService.collectionExists(collectionName);
+      if (!exists) {
+        res.status(404).json({ error: 'Collection not found' });
+        return;
+      }
+
+      await this.qdrantService.deleteCollection(collectionName);
+
+      // Update database records
+      this.db.clearEmbeddingsByCollection(collectionName);
+
+      res.json({ message: 'Collection deleted' });
+    } catch (error) {
+      logger.error('Failed to delete collection:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async healthCheck(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.embeddingWorkflow) {
+        res.json({
+          healthy: false,
+          services: { qdrant: false, embedding: false, database: false },
+          errors: ['Embedding workflow not available']
+        });
+        return;
+      }
+
+      const health = await this.embeddingWorkflow.healthCheck();
+      res.json(health);
+    } catch (error) {
+      logger.error('Health check failed:', error);
+      res.status(500).json({
+        healthy: false,
+        services: { qdrant: false, embedding: false, database: false },
+        errors: ['Health check failed']
+      });
+    }
+  }
+
+  private async reEmbedShopContent(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+      const { content_type } = req.body || {};
+
+      const shop = this.db.getShopById(shopId);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      if (!shop.qdrant_enabled) {
+        res.status(400).json({ error: 'Qdrant not enabled for this shop' });
+        return;
+      }
+
+      // Get content to re-embed
+      const content = this.db.getShopContent(shopId, { contentType: content_type });
+
+      if (content.length === 0) {
+        res.status(200).json({ message: 'No content to re-embed' });
+        return;
+      }
+
+      // Start re-embedding process
+      // Note: In a real system, this should be queued for background processing
+      logger.info(`Re-embedding ${content.length} content items for shop ${shopId}`);
+
+      res.status(202).json({
+        message: `Re-embedding started for ${content.length} content items`
+      });
+    } catch (error) {
+      logger.error('Failed to start re-embedding:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async getShopQdrantErrors(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+      const limit = parseInt(req.query.limit as string) || 50;
+
+      const errors = this.db.getQdrantErrors(shopId, { limit });
+
+      res.json({ errors });
+    } catch (error) {
+      logger.error('Failed to get shop Qdrant errors:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async clearShopQdrantErrors(req: Request, res: Response): Promise<void> {
+    try {
+      const { shopId } = req.params;
+
+      this.db.clearQdrantErrors(shopId);
+
+      res.json({ message: 'Errors cleared' });
+    } catch (error) {
+      logger.error('Failed to clear shop Qdrant errors:', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/routes.ts

@@ -12,8 +12,11 @@ import { ShopsEndpoint } from './components/ShopsEndpoint';
 import { WebhooksEndpoint } from './components/WebhooksEndpoint';
 import { CustomUrlsEndpoint } from './components/CustomUrlsEndpoint';
 import { HealthEndpoint } from './components/HealthEndpoint';
+import { QdrantEndpoint } from './components/QdrantEndpoint';
+import { McpEndpoint } from './components/McpEndpoint';
+import type { AppConfig } from '../config';
 
-export function createRouter(jobQueue: JobQueue, apiKey: string, db?: ShopDatabase): Router {
+export function createRouter(jobQueue: JobQueue, apiKey: string, db?: ShopDatabase, config?: AppConfig): Router {
   // Create the endpoint registry
   const registry = new EndpointRegistry();
 
@@ -31,8 +34,14 @@ export function createRouter(jobQueue: JobQueue, apiKey: string, db?: ShopDataba
     components.push(
       new ShopsEndpoint(db),
       new WebhooksEndpoint(db),
-      new CustomUrlsEndpoint(db)
+      new CustomUrlsEndpoint(db),
+      new McpEndpoint(db)
     );
+
+    // Register Qdrant endpoints if config is available
+    if (config) {
+      components.push(new QdrantEndpoint(db, config));
+    }
   }
 
   // Register documentation endpoint (needs registry reference)

src/database/Database.ts

@@ -1223,4 +1223,522 @@ export class ShopDatabase {
     `);
     return stmt.all(shopContentId) as ShopEmbedding[];
   }
+
+  // Shop lifecycle management methods
+
+  /**
+   * Toggle Qdrant embedding feature for a shop
+   */
+  toggleShopQdrant(shopId: string, enabled: boolean): void {
+    const stmt = this.db.prepare(`
+      UPDATE shops
+      SET qdrant_enabled = ?
+      WHERE id = ?
+    `);
+    stmt.run(enabled ? 1 : 0, shopId);
+  }
+
+  /**
+   * Update shop custom_id with protection
+   * Throws error if trying to modify existing custom_id
+   */
+  updateShopCustomId(shopId: string, customId: string): void {
+    // Check if shop exists and has existing custom_id
+    const shop = this.getShopById(shopId);
+    if (!shop) {
+      throw new Error('Shop not found');
+    }
+
+    if (shop.custom_id && shop.custom_id !== customId) {
+      throw new Error('Cannot modify existing custom_id. Delete shop and recreate if needed.');
+    }
+
+    const stmt = this.db.prepare(`
+      UPDATE shops
+      SET custom_id = ?
+      WHERE id = ?
+    `);
+    stmt.run(customId, shopId);
+  }
+
+  /**
+   * Queue shop for deletion with 24-hour delay
+   * This adds shop to deletion queue for Qdrant cleanup
+   */
+  queueShopForDeletion(shopId: string): void {
+    const shop = this.getShopById(shopId);
+    if (!shop) {
+      throw new Error('Shop not found');
+    }
+
+    // If shop has custom_id, queue for Qdrant deletion
+    if (shop.custom_id) {
+      const deletionDate = new Date();
+      deletionDate.setHours(deletionDate.getHours() + 24);
+
+      this.db.prepare(`
+        INSERT INTO qdrant_deletion_queue (shop_id, custom_id, scheduled_deletion_at, deletion_status, created_at)
+        VALUES (?, ?, ?, 'queued', ?)
+      `).run(shopId, shop.custom_id, deletionDate.toISOString(), new Date().toISOString());
+    }
+
+    // Delete shop from main table immediately
+    this.deleteShop(shopId);
+  }
+
+  /**
+   * Delete shop immediately (used internally)
+   */
+  deleteShop(shopId: string): void {
+    const transaction = this.db.transaction(() => {
+      // Delete related data in order
+      this.db.prepare('DELETE FROM shop_embeddings WHERE shop_content_id IN (SELECT id FROM shop_content WHERE shop_id = ?)').run(shopId);
+      this.db.prepare('DELETE FROM shop_content WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM shop_custom_urls WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM scheduled_jobs WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM shop_analytics WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM webhook_logs WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM qdrant_errors WHERE shop_id = ?').run(shopId);
+      this.db.prepare('DELETE FROM mcp_logs WHERE shop_id = ?').run(shopId);
+
+      // Finally delete the shop
+      this.db.prepare('DELETE FROM shops WHERE id = ?').run(shopId);
+    });
+
+    transaction();
+  }
+
+  /**
+   * Get shops queued for Qdrant deletion
+   */
+  getShopsForQdrantDeletion(): QdrantDeletionQueue[] {
+    const now = new Date().toISOString();
+    const stmt = this.db.prepare(`
+      SELECT * FROM qdrant_deletion_queue
+      WHERE deletion_status = 'queued' AND scheduled_deletion_at <= ?
+      ORDER BY scheduled_deletion_at ASC
+    `);
+    return stmt.all(now) as QdrantDeletionQueue[];
+  }
+
+  /**
+   * Update deletion queue status
+   */
+  updateQdrantDeletionStatus(id: string, status: 'queued' | 'completed' | 'failed', error?: string): void {
+    const stmt = this.db.prepare(`
+      UPDATE qdrant_deletion_queue
+      SET deletion_status = ?, error_message = ?, updated_at = ?
+      WHERE id = ?
+    `);
+    stmt.run(status, error || null, new Date().toISOString(), id);
+  }
+
+  /**
+   * Clean up completed deletion queue entries older than 7 days
+   */
+  cleanupDeletionQueue(): void {
+    const weekAgo = new Date();
+    weekAgo.setDate(weekAgo.getDate() - 7);
+
+    this.db.prepare(`
+      DELETE FROM qdrant_deletion_queue
+      WHERE deletion_status IN ('completed', 'failed') AND updated_at < ?
+    `).run(weekAgo.toISOString());
+  }
+
+  /**
+   * Check if custom_id is already in use (for validation)
+   */
+  isCustomIdInUse(customId: string, excludeShopId?: string): boolean {
+    let query = 'SELECT COUNT(*) as count FROM shops WHERE custom_id = ?';
+    const params: any[] = [customId];
+
+    if (excludeShopId) {
+      query += ' AND id != ?';
+      params.push(excludeShopId);
+    }
+
+    const result = this.db.prepare(query).get(...params) as { count: number };
+    return result.count > 0;
+  }
+
+  /**
+   * Get shops with Qdrant enabled
+   */
+  getQdrantEnabledShops(): Shop[] {
+    const stmt = this.db.prepare(`
+      SELECT * FROM shops
+      WHERE qdrant_enabled = 1 AND custom_id IS NOT NULL
+      ORDER BY created_at DESC
+    `);
+    return stmt.all() as Shop[];
+  }
+
+  // Additional database methods for API endpoints
+
+  /**
+   * Get shop embedding statistics
+   */
+  getShopEmbeddingsStats(shopId: string): { total_embeddings: number; completed: number; failed: number } {
+    const stats = this.db.prepare(`
+      SELECT
+        COUNT(*) as total_embeddings,
+        SUM(CASE WHEN embedding_status = 'completed' THEN 1 ELSE 0 END) as completed,
+        SUM(CASE WHEN embedding_status = 'failed' THEN 1 ELSE 0 END) as failed
+      FROM shop_embeddings se
+      JOIN shop_content sc ON se.shop_content_id = sc.id
+      WHERE sc.shop_id = ?
+    `).get(shopId) as any;
+
+    return {
+      total_embeddings: stats?.total_embeddings || 0,
+      completed: stats?.completed || 0,
+      failed: stats?.failed || 0
+    };
+  }
+
+  /**
+   * Clear embeddings by collection name
+   */
+  clearEmbeddingsByCollection(collectionName: string): void {
+    this.db.prepare(`
+      DELETE FROM shop_embeddings
+      WHERE collection_name = ?
+    `).run(collectionName);
+  }
+
+  // MCP Analytics Methods
+
+  /**
+   * Get MCP analytics with optional filters
+   */
+  getMcpAnalytics(options: {
+    shopId?: string;
+    dateFrom?: string;
+    dateTo?: string;
+  }): {
+    total_calls: number;
+    successful_calls: number;
+    failed_calls: number;
+    avg_response_time: number;
+  } {
+    let query = `
+      SELECT
+        COUNT(*) as total_calls,
+        SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful_calls,
+        SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed_calls,
+        AVG(response_time) as avg_response_time
+      FROM mcp_logs
+      WHERE 1=1
+    `;
+
+    const params: any[] = [];
+
+    if (options.shopId) {
+      query += ' AND shop_id = ?';
+      params.push(options.shopId);
+    }
+
+    if (options.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    const result = this.db.prepare(query).get(...params) as any;
+
+    return {
+      total_calls: result?.total_calls || 0,
+      successful_calls: result?.successful_calls || 0,
+      failed_calls: result?.failed_calls || 0,
+      avg_response_time: result?.avg_response_time || 0
+    };
+  }
+
+  /**
+   * Get MCP tool usage breakdown
+   */
+  getMcpToolUsage(options: {
+    shopId?: string;
+    dateFrom?: string;
+    dateTo?: string;
+  }): { [toolName: string]: number } {
+    let query = `
+      SELECT tool_name, COUNT(*) as count
+      FROM mcp_logs
+      WHERE 1=1
+    `;
+
+    const params: any[] = [];
+
+    if (options.shopId) {
+      query += ' AND shop_id = ?';
+      params.push(options.shopId);
+    }
+
+    if (options.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    query += ' GROUP BY tool_name ORDER BY count DESC';
+
+    const results = this.db.prepare(query).all(...params) as any[];
+    const usage: { [toolName: string]: number } = {};
+
+    for (const result of results) {
+      usage[result.tool_name] = result.count;
+    }
+
+    return usage;
+  }
+
+  /**
+   * Get MCP shop usage breakdown
+   */
+  getMcpShopUsage(options: {
+    dateFrom?: string;
+    dateTo?: string;
+  }): { [shopId: string]: number } {
+    let query = `
+      SELECT shop_id, COUNT(*) as count
+      FROM mcp_logs
+      WHERE 1=1
+    `;
+
+    const params: any[] = [];
+
+    if (options.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    query += ' GROUP BY shop_id ORDER BY count DESC';
+
+    const results = this.db.prepare(query).all(...params) as any[];
+    const usage: { [shopId: string]: number } = {};
+
+    for (const result of results) {
+      usage[result.shop_id] = result.count;
+    }
+
+    return usage;
+  }
+
+  /**
+   * Get MCP analytics timeline
+   */
+  getMcpAnalyticsTimeline(
+    groupBy: 'day' | 'hour',
+    options: {
+      shopId?: string;
+      dateFrom?: string;
+      dateTo?: string;
+    }
+  ): Array<{ date: string; calls: number; success_rate: number }> {
+    const dateFormat = groupBy === 'day' ? "%Y-%m-%d" : "%Y-%m-%d %H:00:00";
+
+    let query = `
+      SELECT
+        strftime('${dateFormat}', created_at) as date,
+        COUNT(*) as calls,
+        AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END) as success_rate
+      FROM mcp_logs
+      WHERE 1=1
+    `;
+
+    const params: any[] = [];
+
+    if (options.shopId) {
+      query += ' AND shop_id = ?';
+      params.push(options.shopId);
+    }
+
+    if (options.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    query += ' GROUP BY strftime(?, created_at) ORDER BY date';
+    params.push(dateFormat);
+
+    return this.db.prepare(query).all(...params) as any[];
+  }
+
+  /**
+   * Get MCP logs with filtering and pagination
+   */
+  getMcpLogs(options: {
+    shopId?: string;
+    toolName?: string;
+    success?: boolean;
+    limit?: number;
+    offset?: number;
+  }): {
+    logs: McpLog[];
+    pagination: {
+      total: number;
+      limit: number;
+      offset: number;
+      has_more: boolean;
+    };
+  } {
+    let whereClause = 'WHERE 1=1';
+    const params: any[] = [];
+
+    if (options.shopId) {
+      whereClause += ' AND shop_id = ?';
+      params.push(options.shopId);
+    }
+
+    if (options.toolName) {
+      whereClause += ' AND tool_name = ?';
+      params.push(options.toolName);
+    }
+
+    if (options.success !== undefined) {
+      whereClause += ' AND success = ?';
+      params.push(options.success ? 1 : 0);
+    }
+
+    // Get total count
+    const countQuery = `SELECT COUNT(*) as total FROM mcp_logs ${whereClause}`;
+    const countResult = this.db.prepare(countQuery).get(...params) as { total: number };
+    const total = countResult.total;
+
+    // Get paginated results
+    const limit = options.limit || 100;
+    const offset = options.offset || 0;
+
+    const logsQuery = `
+      SELECT * FROM mcp_logs ${whereClause}
+      ORDER BY created_at DESC
+      LIMIT ? OFFSET ?
+    `;
+
+    const logs = this.db.prepare(logsQuery).all(...params, limit, offset) as McpLog[];
+
+    return {
+      logs,
+      pagination: {
+        total,
+        limit,
+        offset,
+        has_more: offset + limit < total
+      }
+    };
+  }
+
+  /**
+   * Get MCP log by ID
+   */
+  getMcpLogById(logId: string): McpLog | null {
+    const stmt = this.db.prepare(`
+      SELECT * FROM mcp_logs WHERE id = ?
+    `);
+    return stmt.get(logId) as McpLog | null;
+  }
+
+  /**
+   * Get MCP tool statistics
+   */
+  getMcpToolStats(options: {
+    dateFrom?: string;
+    dateTo?: string;
+  }): Array<{
+    tool_name: string;
+    total_calls: number;
+    successful_calls: number;
+    failed_calls: number;
+    success_rate: number;
+    avg_response_time: number;
+    most_common_queries: Array<{ query: string; count: number }>;
+  }> {
+    let query = `
+      SELECT
+        tool_name,
+        COUNT(*) as total_calls,
+        SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) as successful_calls,
+        SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) as failed_calls,
+        AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END) as success_rate,
+        AVG(response_time) as avg_response_time
+      FROM mcp_logs
+      WHERE 1=1
+    `;
+
+    const params: any[] = [];
+
+    if (options.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    query += ' GROUP BY tool_name ORDER BY total_calls DESC';
+
+    const toolStats = this.db.prepare(query).all(...params) as any[];
+
+    // Get most common queries for each tool
+    for (const toolStat of toolStats) {
+      let queriesQuery = `
+        SELECT query, COUNT(*) as count
+        FROM mcp_logs
+        WHERE tool_name = ?
+      `;
+
+      const queriesParams = [toolStat.tool_name];
+
+      if (options.dateFrom) {
+        queriesQuery += ' AND created_at >= ?';
+        queriesParams.push(options.dateFrom);
+      }
+
+      if (options.dateTo) {
+        queriesQuery += ' AND created_at <= ?';
+        queriesParams.push(options.dateTo);
+      }
+
+      queriesQuery += ' GROUP BY query ORDER BY count DESC LIMIT 5';
+
+      const queries = this.db.prepare(queriesQuery).all(...queriesParams) as any[];
+      toolStat.most_common_queries = queries;
+    }
+
+    return toolStats;
+  }
+
+  /**
+   * Clean up old MCP logs
+   */
+  cleanupMcpLogs(olderThan: string): number {
+    const result = this.db.prepare(`
+      DELETE FROM mcp_logs
+      WHERE created_at < ?
+    `).run(olderThan);
+
+    return result.changes;
+  }
 }

src/scraper/WebshopScraper.ts

@@ -4,16 +4,20 @@ import { ContentExtractor } from './ContentExtractor';
 import { logger } from '../utils/logger';
 import { ShopDatabase } from '../database/Database';
 import { WebhookManager } from '../webhooks/WebhookManager';
+import { QdrantEmbeddingWorkflow } from '../services/QdrantEmbeddingWorkflow';
+import type { AppConfig } from '../config';
 
 export class WebshopScraper {
   private contentExtractor: ContentExtractor;
   private db: ShopDatabase | null;
   private webhookManager: WebhookManager | null;
+  private embeddingWorkflow: QdrantEmbeddingWorkflow | null;
 
-  constructor(db?: ShopDatabase) {
+  constructor(db?: ShopDatabase, config?: AppConfig) {
     this.contentExtractor = new ContentExtractor();
     this.db = db || null;
     this.webhookManager = db ? new WebhookManager(db) : null;
+    this.embeddingWorkflow = (db && config) ? new QdrantEmbeddingWorkflow(config, db) : null;
   }
 
   /**
@@ -104,10 +108,26 @@ export class WebshopScraper {
 
       // Save content to database if available
       if (this.db && shopId) {
+        // Collect all content for batch embedding processing
+        const embeddingBatch: any[] = [];
+
         // Save all shipping pages
         for (const content of shippingContent) {
           const result = this.db.saveContent(shopId, 'shipping', content.url, content.content, content.title);
 
+          // Add to embedding batch if workflow is available
+          if (this.embeddingWorkflow) {
+            embeddingBatch.push({
+              contentId: result.contentId,
+              contentType: 'shipping',
+              url: content.url,
+              content: content.content,
+              title: content.title,
+              contentHash: this.calculateContentHash(content.content),
+              contentChanged: result.changed
+            });
+          }
+
           // Trigger webhook if content changed
           if (result.changed && this.webhookManager) {
             const previous = this.db.getContentHistory(shopId, 'shipping', 2);
@@ -127,6 +147,19 @@ export class WebshopScraper {
         for (const content of contactsContent) {
           const result = this.db.saveContent(shopId, 'contacts', content.url, content.content, content.title);
 
+          // Add to embedding batch if workflow is available
+          if (this.embeddingWorkflow) {
+            embeddingBatch.push({
+              contentId: result.contentId,
+              contentType: 'contacts',
+              url: content.url,
+              content: content.content,
+              title: content.title,
+              contentHash: this.calculateContentHash(content.content),
+              contentChanged: result.changed
+            });
+          }
+
           // Trigger webhook if content changed
           if (result.changed && this.webhookManager) {
             const previous = this.db.getContentHistory(shopId, 'contacts', 2);
@@ -146,6 +179,19 @@ export class WebshopScraper {
         for (const content of termsContent) {
           const result = this.db.saveContent(shopId, 'terms', content.url, content.content, content.title);
 
+          // Add to embedding batch if workflow is available
+          if (this.embeddingWorkflow) {
+            embeddingBatch.push({
+              contentId: result.contentId,
+              contentType: 'terms',
+              url: content.url,
+              content: content.content,
+              title: content.title,
+              contentHash: this.calculateContentHash(content.content),
+              contentChanged: result.changed
+            });
+          }
+
           // Trigger webhook if content changed
           if (result.changed && this.webhookManager) {
             const previous = this.db.getContentHistory(shopId, 'terms', 2);
@@ -165,6 +211,19 @@ export class WebshopScraper {
         for (const content of faqContent) {
           const result = this.db.saveContent(shopId, 'faq', content.url, content.content, content.title);
 
+          // Add to embedding batch if workflow is available
+          if (this.embeddingWorkflow) {
+            embeddingBatch.push({
+              contentId: result.contentId,
+              contentType: 'faq',
+              url: content.url,
+              content: content.content,
+              title: content.title,
+              contentHash: this.calculateContentHash(content.content),
+              contentChanged: result.changed
+            });
+          }
+
           // Trigger webhook if content changed
           if (result.changed && this.webhookManager) {
             const previous = this.db.getContentHistory(shopId, 'faq', 2);
@@ -181,6 +240,27 @@ export class WebshopScraper {
           }
         }
 
+        // Process embeddings in batch
+        if (this.embeddingWorkflow && embeddingBatch.length > 0) {
+          logger.info(`Processing ${embeddingBatch.length} items for Qdrant embedding`);
+
+          try {
+            const embeddingResults = await this.embeddingWorkflow.processContentBatch(shopId, embeddingBatch);
+            const embeddedCount = embeddingResults.filter(r => r.embedded).length;
+            const failedCount = embeddingResults.filter(r => !r.embedded).length;
+
+            logger.info(`Embedding batch completed: ${embeddedCount} embedded, ${failedCount} failed`);
+
+            // Log embedding completion webhook event if any were successful
+            if (embeddedCount > 0 && this.webhookManager) {
+              // Note: This would need a new webhook event type for embedding completion
+              // await this.webhookManager.notifyEmbeddingCompleted(shopId, embeddedCount, failedCount);
+            }
+          } catch (error) {
+            logger.error('Failed to process embedding batch:', error);
+          }
+        }
+
         // Update analytics
         const scrapeTimeMs = Date.now() - startTime;
         const pageCount = shippingContent.length + contactsContent.length + termsContent.length + faqContent.length;
@@ -232,6 +312,14 @@ export class WebshopScraper {
     return results;
   }
 
+  /**
+   * Calculate hash of content for change detection
+   */
+  private calculateContentHash(content: string): string {
+    const crypto = require('crypto');
+    return crypto.createHash('sha256').update(content).digest('hex');
+  }
+
   /**
    * Extract content from the best matching URL (deprecated - kept for backwards compatibility)
    */

src/services/QdrantEmbeddingWorkflow.ts

@@ -0,0 +1,333 @@
+import { v4 as uuidv4 } from 'uuid';
+import { logger } from '../utils/logger';
+import { ShopDatabase, ShopContent } from '../database/Database';
+import { QdrantService } from './QdrantService';
+import { EmbeddingService } from './EmbeddingService';
+import { QdrantCleanupService } from './QdrantCleanupService';
+import type { AppConfig } from '../config';
+
+export interface EmbeddingWorkflowResult {
+  contentId: string;
+  embedded: boolean;
+  collectionName?: string;
+  pointId?: string;
+  error?: string;
+}
+
+export class QdrantEmbeddingWorkflow {
+  private qdrantService: QdrantService;
+  private embeddingService: EmbeddingService;
+  private cleanupService: QdrantCleanupService;
+
+  constructor(
+    private config: AppConfig,
+    private database: ShopDatabase
+  ) {
+    this.qdrantService = new QdrantService(config);
+    this.embeddingService = new EmbeddingService(config);
+    this.cleanupService = new QdrantCleanupService(config, database);
+  }
+
+  /**
+   * Process content for embedding after it's saved to database
+   */
+  async processContentForEmbedding(
+    shopId: string,
+    contentId: string,
+    contentType: 'shipping' | 'contacts' | 'terms' | 'faq',
+    url: string,
+    content: string,
+    title: string | null,
+    contentHash: string,
+    contentChanged: boolean
+  ): Promise<EmbeddingWorkflowResult> {
+    try {
+      // Get shop details
+      const shop = this.database.getShopById(shopId);
+      if (!shop) {
+        return { contentId, embedded: false, error: 'Shop not found' };
+      }
+
+      // Check if Qdrant is enabled for this shop
+      if (!shop.qdrant_enabled || !shop.custom_id) {
+        logger.debug(`Skipping embedding for shop ${shopId}: Qdrant disabled or no custom_id`);
+        return { contentId, embedded: false, error: 'Qdrant not enabled or missing custom_id' };
+      }
+
+      // Check if services are available
+      if (!this.embeddingService.isEnabled()) {
+        logger.debug(`Skipping embedding for shop ${shopId}: EmbeddingService not available`);
+        return { contentId, embedded: false, error: 'Embedding service not available' };
+      }
+
+      // Check if content already has up-to-date embedding
+      const existingEmbeddings = this.database.getShopEmbeddings(contentId);
+      const hasCurrentEmbedding = existingEmbeddings.some(
+        emb => emb.embedding_status === 'completed'
+      );
+
+      // If content hasn't changed and we have an embedding, skip
+      if (!contentChanged && hasCurrentEmbedding) {
+        logger.debug(`Skipping embedding for content ${contentId}: no changes and embedding exists`);
+        return { contentId, embedded: false, error: 'Content unchanged and embedding exists' };
+      }
+
+      // Prepare content for embedding
+      const processedContent = this.embeddingService.preprocessContent(title, content);
+
+      // Check if content is too long
+      if (this.embeddingService.isTextTooLong(processedContent)) {
+        const truncatedContent = this.embeddingService.truncateText(processedContent);
+        logger.warn(`Content truncated for embedding: ${contentId}`);
+      }
+
+      // Generate embedding
+      const embeddingResult = await this.embeddingService.generateEmbedding({
+        text: this.embeddingService.isTextTooLong(processedContent)
+          ? this.embeddingService.truncateText(processedContent)
+          : processedContent,
+        contentId,
+        shopId,
+        contentType
+      });
+
+      if (!embeddingResult.success) {
+        await this.cleanupService.logQdrantError(
+          shopId,
+          'embedding',
+          embeddingResult.error || 'Embedding generation failed',
+          { contentId, contentType, url }
+        );
+        return { contentId, embedded: false, error: embeddingResult.error };
+      }
+
+      // Get iterator for collection naming
+      const iterator = await this.qdrantService.getNextIterator(shop.custom_id, contentType);
+      const collectionName = this.qdrantService.generateCollectionName(shop.custom_id, contentType, iterator);
+
+      // Ensure collection exists
+      if (!(await this.qdrantService.collectionExists(collectionName))) {
+        await this.qdrantService.createCollection(collectionName);
+        logger.info(`Created Qdrant collection: ${collectionName}`);
+      }
+
+      // Create point
+      const pointId = uuidv4();
+      const point = {
+        id: pointId,
+        vector: embeddingResult.embedding,
+        payload: {
+          shop_id: shop.custom_id,
+          content_id: contentId,
+          content_type: contentType,
+          url: url,
+          title: title || undefined,
+          content_hash: contentHash,
+          created_at: new Date().toISOString(),
+        }
+      };
+
+      // Insert point into Qdrant
+      await this.qdrantService.upsertPoint(collectionName, point);
+
+      // Save embedding record to database
+      const now = new Date().toISOString();
+      const embeddingId = uuidv4();
+
+      this.database.createShopEmbedding({
+        id: embeddingId,
+        shop_content_id: contentId,
+        collection_name: collectionName,
+        point_id: pointId,
+        embedding_status: 'completed',
+        created_at: now,
+        updated_at: now
+      });
+
+      logger.info(`Successfully embedded content ${contentId} in collection ${collectionName}`);
+
+      return {
+        contentId,
+        embedded: true,
+        collectionName,
+        pointId
+      };
+
+    } catch (error: any) {
+      logger.error(`Failed to process embedding for content ${contentId}:`, error);
+
+      // Log error for UI display
+      await this.cleanupService.logQdrantError(
+        shopId,
+        'embedding',
+        error.message || 'Embedding workflow failed',
+        { contentId, contentType, url, error: error.stack }
+      );
+
+      return {
+        contentId,
+        embedded: false,
+        error: error.message || 'Embedding workflow failed'
+      };
+    }
+  }
+
+  /**
+   * Process multiple content items in batch
+   */
+  async processContentBatch(
+    shopId: string,
+    contentItems: Array<{
+      contentId: string;
+      contentType: 'shipping' | 'contacts' | 'terms' | 'faq';
+      url: string;
+      content: string;
+      title: string | null;
+      contentHash: string;
+      contentChanged: boolean;
+    }>
+  ): Promise<EmbeddingWorkflowResult[]> {
+    const results: EmbeddingWorkflowResult[] = [];
+
+    // Process items one by one to avoid rate limits and resource issues
+    for (const item of contentItems) {
+      try {
+        const result = await this.processContentForEmbedding(
+          shopId,
+          item.contentId,
+          item.contentType,
+          item.url,
+          item.content,
+          item.title,
+          item.contentHash,
+          item.contentChanged
+        );
+        results.push(result);
+
+        // Small delay between items to be nice to the embedding service
+        if (contentItems.length > 1) {
+          await new Promise(resolve => setTimeout(resolve, 100));
+        }
+
+      } catch (error: any) {
+        logger.error(`Failed to process content item ${item.contentId}:`, error);
+        results.push({
+          contentId: item.contentId,
+          embedded: false,
+          error: error.message || 'Processing failed'
+        });
+      }
+    }
+
+    const successCount = results.filter(r => r.embedded).length;
+    const failCount = results.filter(r => !r.embedded).length;
+
+    logger.info(`Batch embedding completed for shop ${shopId}: ${successCount} success, ${failCount} failed`);
+
+    return results;
+  }
+
+  /**
+   * Re-embed existing content (useful for content updates or service recovery)
+   */
+  async reEmbedContent(contentId: string): Promise<EmbeddingWorkflowResult> {
+    try {
+      // Get content from database
+      const content = this.database.getLatestContent(''); // We need to get by content ID
+      // Note: This would need a database method to get content by ID
+      // For now, let's implement a simpler approach
+
+      logger.warn('Re-embedding not fully implemented yet - needs database method to get content by ID');
+
+      return {
+        contentId,
+        embedded: false,
+        error: 'Re-embedding not implemented yet'
+      };
+
+    } catch (error: any) {
+      return {
+        contentId,
+        embedded: false,
+        error: error.message || 'Re-embedding failed'
+      };
+    }
+  }
+
+  /**
+   * Get embedding status for content
+   */
+  getEmbeddingStatus(contentId: string): {
+    hasEmbedding: boolean;
+    status: 'pending' | 'completed' | 'failed' | 'none';
+    collectionName?: string;
+    pointId?: string;
+  } {
+    try {
+      const embeddings = this.database.getShopEmbeddings(contentId);
+
+      if (embeddings.length === 0) {
+        return { hasEmbedding: false, status: 'none' };
+      }
+
+      // Get the latest embedding
+      const latest = embeddings[0];
+
+      return {
+        hasEmbedding: latest.embedding_status === 'completed',
+        status: latest.embedding_status as any,
+        collectionName: latest.collection_name,
+        pointId: latest.point_id
+      };
+
+    } catch (error) {
+      logger.error(`Failed to get embedding status for content ${contentId}:`, error);
+      return { hasEmbedding: false, status: 'none' };
+    }
+  }
+
+  /**
+   * Health check for embedding workflow
+   */
+  async healthCheck(): Promise<{
+    healthy: boolean;
+    services: {
+      qdrant: boolean;
+      embedding: boolean;
+      database: boolean;
+    };
+    errors: string[];
+  }> {
+    const errors: string[] = [];
+
+    try {
+      const [qdrantHealthy, embeddingEnabled] = await Promise.all([
+        this.qdrantService.isHealthy(),
+        Promise.resolve(this.embeddingService.isEnabled())
+      ]);
+
+      const services = {
+        qdrant: qdrantHealthy,
+        embedding: embeddingEnabled,
+        database: true // Database is always available if we got this far
+      };
+
+      if (!qdrantHealthy) errors.push('Qdrant service unavailable');
+      if (!embeddingEnabled) errors.push('Embedding service not configured');
+
+      return {
+        healthy: qdrantHealthy && embeddingEnabled,
+        services,
+        errors
+      };
+
+    } catch (error: any) {
+      errors.push(error.message || 'Health check failed');
+      return {
+        healthy: false,
+        services: { qdrant: false, embedding: false, database: false },
+        errors
+      };
+    }
+  }
+}