feat: enhance Qdrant product payloads with comprehensive details #77

- WooCommerce: Add descriptions, categories, tags, attributes, variations, images, pricing details, stock management
- Shopify: Add descriptions, vendor, tags, variants, options, images, barcode, compare_at_price, timestamps
- ShopRenter: Add full Qdrant integration with descriptions, categories, tags, attributes, images, manufacturer, model
- All platforms: Store structured data for filtering and direct access without additional DB queries
- Enhanced payload indexes for better search performance
- Maintain backward compatibility

supabase/functions/shopify-sync/index.ts

@@ -176,26 +176,80 @@ async function syncProductsToQdrant(
     const embeddings = await generateEmbeddingBatch(productTexts)
     console.log(`[Qdrant] Embeddings generated successfully`)
 
-    // Convert products to Qdrant points with embeddings
+    // Convert products to Qdrant points with embeddings and comprehensive details
     const points: QdrantPoint[] = products.map((product, index) => {
       const primaryVariant = product.variants?.[0]
       return {
         id: generatePointId('shopify', storeId, product.id),
         vector: embeddings[index],
         payload: {
+          // Basic identification
           store_id: storeId,
           product_id: product.id.toString(),
           platform: 'shopify',
           title: product.title,
           handle: product.handle,
+
+          // Vendor and categorization
           vendor: product.vendor || null,
           product_type: product.product_type || null,
-          status: product.status,
+          tags: product.tags ? product.tags.split(',').map(t => t.trim()) : [],
+
+          // Descriptions
+          description: product.body_html || null,
+
+          // Pricing (from primary variant)
           price: primaryVariant ? parseFloat(primaryVariant.price) : 0,
+          compare_at_price: primaryVariant?.compare_at_price ? parseFloat(primaryVariant.compare_at_price) : null,
+          currency: primaryVariant?.currency_code || 'USD',
+
+          // Primary variant details
           sku: primaryVariant?.sku || null,
+          barcode: primaryVariant?.barcode || null,
           inventory_quantity: primaryVariant?.inventory_quantity || 0,
-          description: product.body_html || null,
-          tags: product.tags ? product.tags.split(',').map(t => t.trim()) : [],
+          inventory_policy: primaryVariant?.inventory_policy || 'deny',
+
+          // All variants (for variable products)
+          variants: product.variants?.map((v: any) => ({
+            id: v.id,
+            title: v.title,
+            price: parseFloat(v.price),
+            compare_at_price: v.compare_at_price ? parseFloat(v.compare_at_price) : null,
+            sku: v.sku,
+            barcode: v.barcode,
+            inventory_quantity: v.inventory_quantity,
+            option1: v.option1,
+            option2: v.option2,
+            option3: v.option3,
+            weight: v.weight,
+            weight_unit: v.weight_unit
+          })) || [],
+
+          // Product options (Color, Size, etc.)
+          options: product.options?.map((opt: any) => ({
+            id: opt.id,
+            name: opt.name,
+            position: opt.position,
+            values: opt.values
+          })) || [],
+
+          // Images
+          images: product.images?.map((img: any) => ({
+            id: img.id,
+            src: img.src,
+            alt: img.alt || product.title,
+            position: img.position,
+            width: img.width,
+            height: img.height
+          })) || [],
+
+          // Product status and metadata
+          status: product.status,
+          published_at: product.published_at || null,
+          created_at: product.created_at || null,
+          updated_at: product.updated_at || null,
+
+          // Metadata
           synced_at: new Date().toISOString(),
         }
       }

supabase/functions/shoprenter-sync/index.ts

@@ -3,12 +3,230 @@ import { createClient } from 'https://esm.sh/@supabase/supabase-js@2'
 import { wrapHandler, logError } from '../_shared/error-handler.ts'
 import { fetchProducts, fetchOrders, fetchCustomers } from '../_shared/shoprenter-client.ts'
 import { formatFirstValidPhone, detectCountryCode } from '../_shared/phone-formatter.ts'
+import {
+  collectionExists,
+  createCollection,
+  upsertPoints,
+  deletePointsByFilter,
+  scrollPoints,
+  getCollectionName,
+  generatePointId,
+  initializeStoreCollections,
+  generateEmbeddingBatch,
+  createProductText,
+  createOrderText,
+  createCustomerText,
+  QdrantPoint
+} from '../_shared/qdrant-client.ts'
 
 const corsHeaders = {
   'Access-Control-Allow-Origin': '*',
   'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
 }
 
+// Log Qdrant sync operation
+async function logQdrantSync(
+  supabaseAdmin: any,
+  storeId: string,
+  syncType: string,
+  collectionName: string,
+  operation: string,
+  itemsProcessed: number,
+  itemsSucceeded: number,
+  itemsFailed: number,
+  startedAt: Date,
+  errorMessage?: string
+): Promise<void> {
+  try {
+    await supabaseAdmin
+      .from('qdrant_sync_logs')
+      .insert({
+        store_id: storeId,
+        sync_type: syncType,
+        collection_name: collectionName,
+        operation,
+        items_processed: itemsProcessed,
+        items_succeeded: itemsSucceeded,
+        items_failed: itemsFailed,
+        error_message: errorMessage,
+        started_at: startedAt.toISOString(),
+        completed_at: new Date().toISOString(),
+      })
+  } catch (error) {
+    console.error('[Qdrant] Failed to log sync operation:', error)
+  }
+}
+
+// Sync products to Qdrant
+async function syncProductsToQdrant(
+  storeId: string,
+  storeName: string,
+  products: any[],
+  supabaseAdmin: any
+): Promise<{ synced: number; errors: number }> {
+  const startTime = new Date()
+  const collectionName = getCollectionName(storeName, 'products')
+
+  console.log(`[Qdrant] Syncing ${products.length} products to ${collectionName}`)
+
+  let synced = 0
+  let errors = 0
+
+  try {
+    if (!(await collectionExists(collectionName))) {
+      await createCollection(collectionName, [
+        { field: 'store_id', type: 'keyword' },
+        { field: 'product_id', type: 'keyword' },
+        { field: 'platform', type: 'keyword' },
+        { field: 'active', type: 'keyword' },
+        { field: 'price', type: 'float' },
+        { field: 'sku', type: 'keyword' },
+      ])
+    }
+
+    // Get existing product IDs from Qdrant to detect deletions
+    const existingPoints = await scrollPoints(collectionName, {
+      must: [{ key: 'store_id', match: { value: storeId } }]
+    }, 1000)
+
+    const existingProductIds = new Set(
+      existingPoints.points.map((p: any) => p.payload?.product_id).filter(Boolean)
+    )
+
+    const currentProductIds = new Set(products.map(p => p.id.toString()))
+
+    // Find deleted products
+    const deletedProductIds = Array.from(existingProductIds).filter(
+      id => !currentProductIds.has(id)
+    )
+
+    // Delete removed products from Qdrant
+    if (deletedProductIds.length > 0) {
+      console.log(`[Qdrant] Deleting ${deletedProductIds.length} removed products`)
+      await deletePointsByFilter(collectionName, {
+        must: [
+          { key: 'store_id', match: { value: storeId } },
+          { key: 'product_id', match: { any: deletedProductIds } }
+        ]
+      })
+    }
+
+    // Generate text representations for all products
+    const productTexts = products.map((product) =>
+      createProductText({
+        name: product.name,
+        description: product.description,
+        short_description: product.short_description,
+        sku: product.sku,
+        categories: product.categories || [],
+        tags: product.tags || [],
+        attributes: product.attributes || [],
+        price: product.price,
+        meta_description: product.meta_description,
+      })
+    )
+
+    // Generate embeddings in batch
+    console.log(`[Qdrant] Generating embeddings for ${productTexts.length} products...`)
+    const embeddings = await generateEmbeddingBatch(productTexts)
+    console.log(`[Qdrant] Embeddings generated successfully`)
+
+    // Convert products to Qdrant points with embeddings and comprehensive details
+    const points: QdrantPoint[] = products.map((product, index) => ({
+      id: generatePointId('shoprenter', storeId, product.id),
+      vector: embeddings[index],
+      payload: {
+        // Basic identification
+        store_id: storeId,
+        product_id: product.id.toString(),
+        platform: 'shoprenter',
+        name: product.name,
+        sku: product.sku || null,
+
+        // Pricing
+        price: parseFloat(product.price) || 0,
+        currency: product.currency || 'HUF',
+        price_gross: product.price_gross ? parseFloat(product.price_gross) : null,
+
+        // Descriptions
+        description: product.description || null,
+        short_description: product.short_description || null,
+        meta_description: product.meta_description || null,
+
+        // Categorization
+        categories: product.categories?.map((cat: any) => ({
+          id: cat.id,
+          name: cat.name,
+          slug: cat.slug || null
+        })) || [],
+        tags: product.tags || [],
+
+        // Attributes
+        attributes: product.attributes || [],
+
+        // Images
+        images: product.images?.map((img: any) => ({
+          id: img.id,
+          src: img.src || img.url,
+          alt: img.alt || product.name,
+          position: img.position || 0
+        })) || [],
+
+        // Stock information
+        stock: product.stock || 0,
+        stock_status: product.stock > 0 ? 'instock' : 'outofstock',
+
+        // Product status
+        active: product.active !== false,
+        status: product.active !== false ? 'active' : 'inactive',
+
+        // Additional fields
+        manufacturer: product.manufacturer || null,
+        model: product.model || null,
+        weight: product.weight || null,
+        weight_unit: product.weight_unit || 'kg',
+
+        // Metadata
+        synced_at: new Date().toISOString(),
+      }
+    }))
+
+    await upsertPoints(collectionName, points)
+    synced = points.length
+
+    await logQdrantSync(
+      supabaseAdmin,
+      storeId,
+      'products',
+      collectionName,
+      'upsert',
+      products.length,
+      synced,
+      errors,
+      startTime
+    )
+
+    console.log(`[Qdrant] Products sync complete: ${synced} synced, ${deletedProductIds.length} deleted`)
+  } catch (error: any) {
+    console.error('[Qdrant] Product sync error:', error)
+    errors = products.length
+    await logQdrantSync(
+      supabaseAdmin,
+      storeId,
+      'products',
+      collectionName,
+      'upsert',
+      products.length,
+      synced,
+      errors,
+      startTime,
+      error.message
+    )
+  }
+
+  return { synced, errors }
+}
+
 serve(wrapHandler('shoprenter-sync', async (req) => {
   if (req.method === 'OPTIONS') {
     return new Response('ok', { headers: corsHeaders })
@@ -53,7 +271,7 @@ serve(wrapHandler('shoprenter-sync', async (req) => {
     // Verify store belongs to user
     const { data: store, error: storeError } = await supabase
       .from('stores')
-      .select('id, store_name, platform_name, store_url')
+      .select('id, store_name, platform_name, store_url, qdrant_sync_enabled, data_access_permissions')
       .eq('id', storeId)
       .eq('user_id', user.id)
       .eq('platform_name', 'shoprenter')
@@ -72,6 +290,20 @@ serve(wrapHandler('shoprenter-sync', async (req) => {
 
     console.log(`[ShopRenter] Starting full sync for store ${storeId}`)
 
+    // Check data access permissions
+    const permissions = store.data_access_permissions || {}
+    const canSyncProducts = permissions.allow_product_access !== false
+    const canSyncOrders = permissions.allow_order_access !== false
+    const canSyncCustomers = permissions.allow_customer_access !== false
+    const qdrantEnabled = store.qdrant_sync_enabled !== false
+
+    console.log('[ShopRenter] Sync permissions:', {
+      products: canSyncProducts,
+      orders: canSyncOrders,
+      customers: canSyncCustomers,
+      qdrant: qdrantEnabled
+    })
+
     const syncStats = {
       products: { synced: 0, errors: 0 },
       orders: { synced: 0, errors: 0 },
@@ -81,59 +313,86 @@ serve(wrapHandler('shoprenter-sync', async (req) => {
     const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!
     const supabaseAdmin = createClient(supabaseUrl, supabaseServiceKey)
 
+    // Initialize Qdrant collections if enabled
+    if (qdrantEnabled) {
+      try {
+        await initializeStoreCollections(
+          store.store_name,
+          canSyncOrders,
+          canSyncCustomers
+        )
+      } catch (error) {
+        console.error('[Qdrant] Failed to initialize collections:', error)
+      }
+    }
+
     // Sync Products
+    const allProducts: any[] = []
     try {
-      console.log('[ShopRenter] Syncing products...')
-      let page = 1
-      let hasMore = true
-      const limit = 50
-
-      while (hasMore) {
-        const productsData = await fetchProducts(storeId, page, limit)
-
-        if (productsData.items && productsData.items.length > 0) {
-          const productsToCache = productsData.items.map((product: any) => ({
-            store_id: storeId,
-            shoprenter_product_id: product.id,
-            name: product.name,
-            sku: product.sku,
-            price: parseFloat(product.price) || 0,
-            currency: product.currency || 'HUF',
-            description: product.description,
-            stock: product.stock,
-            active: product.active !== false,
-            raw_data: product,
-            last_synced_at: new Date().toISOString()
-          }))
-
-          const { error: upsertError } = await supabaseAdmin
-            .from('shoprenter_products_cache')
-            .upsert(productsToCache, {
-              onConflict: 'store_id,shoprenter_product_id'
-            })
-
-          if (upsertError) {
-            console.error('[ShopRenter] Error caching products:', upsertError)
-            syncStats.products.errors += productsToCache.length
+      if (!canSyncProducts) {
+        console.log('[ShopRenter] Product sync disabled by store permissions')
+      } else {
+        console.log('[ShopRenter] Syncing products...')
+        let page = 1
+        let hasMore = true
+        const limit = 50
+
+        while (hasMore) {
+          const productsData = await fetchProducts(storeId, page, limit)
+
+          if (productsData.items && productsData.items.length > 0) {
+            // Collect all products for Qdrant sync
+            allProducts.push(...productsData.items)
+
+            const productsToCache = productsData.items.map((product: any) => ({
+              store_id: storeId,
+              shoprenter_product_id: product.id,
+              name: product.name,
+              sku: product.sku,
+              price: parseFloat(product.price) || 0,
+              currency: product.currency || 'HUF',
+              description: product.description,
+              stock: product.stock,
+              active: product.active !== false,
+              raw_data: product,
+              last_synced_at: new Date().toISOString()
+            }))
+
+            const { error: upsertError } = await supabaseAdmin
+              .from('shoprenter_products_cache')
+              .upsert(productsToCache, {
+                onConflict: 'store_id,shoprenter_product_id'
+              })
+
+            if (upsertError) {
+              console.error('[ShopRenter] Error caching products:', upsertError)
+              syncStats.products.errors += productsToCache.length
+            } else {
+              syncStats.products.synced += productsToCache.length
+            }
+
+            // Check if there are more pages
+            if (productsData.pagination && productsData.pagination.total) {
+              const totalPages = Math.ceil(productsData.pagination.total / limit)
+              hasMore = page < totalPages
+            } else {
+              hasMore = productsData.items.length === limit
+            }
+
+            page++
           } else {
-            syncStats.products.synced += productsToCache.length
+            hasMore = false
           }
+        }
 
-          // Check if there are more pages
-          if (productsData.pagination && productsData.pagination.total) {
-            const totalPages = Math.ceil(productsData.pagination.total / limit)
-            hasMore = page < totalPages
-          } else {
-            hasMore = productsData.items.length === limit
-          }
+        console.log(`[ShopRenter] Products synced: ${syncStats.products.synced}`)
 
-          page++
-        } else {
-          hasMore = false
+        // Sync to Qdrant if enabled
+        if (qdrantEnabled && allProducts.length > 0) {
+          const qdrantResult = await syncProductsToQdrant(storeId, store.store_name, allProducts, supabaseAdmin)
+          console.log(`[ShopRenter] Qdrant sync complete: ${qdrantResult.synced} synced, ${qdrantResult.errors} errors`)
         }
       }
-
-      console.log(`[ShopRenter] Products synced: ${syncStats.products.synced}`)
     } catch (error) {
       console.error('[ShopRenter] Product sync error:', error)
       syncStats.products.errors++

supabase/functions/woocommerce-sync/index.ts

@@ -196,20 +196,55 @@ async function syncProductsToQdrant(
     const embeddings = await generateEmbeddingBatch(productTexts)
     console.log(`[Qdrant] Embeddings generated successfully`)
 
-    // Convert products to Qdrant points with embeddings
+    // Convert products to Qdrant points with embeddings and comprehensive details
     const points: QdrantPoint[] = products.map((product, index) => ({
       id: generatePointId('woocommerce', storeId, product.id),
       vector: embeddings[index],
       payload: {
+        // Basic identification
         store_id: storeId,
         product_id: product.id.toString(),
         platform: 'woocommerce',
         name: product.name,
         sku: product.sku || null,
+
+        // Pricing
         price: parseFloat(product.price) || 0,
+        regular_price: product.regular_price ? parseFloat(product.regular_price) : null,
+        sale_price: product.sale_price ? parseFloat(product.sale_price) : null,
+
+        // Descriptions
+        description: product.description || null,
+        short_description: product.short_description || null,
+
+        // Categorization
+        categories: product.categories || [],
+        tags: product.tags?.map((t: any) => ({ id: t.id, name: t.name, slug: t.slug })) || [],
+
+        // Attributes and variations
+        attributes: product.attributes || [],
+        variations: product.variations || [],
+
+        // Images
+        images: product.images?.map((img: any) => ({
+          id: img.id,
+          src: img.src,
+          alt: img.alt || product.name
+        })) || [],
+
+        // Stock information
         stock_status: product.stock_status,
         stock_quantity: product.stock_quantity,
+        manage_stock: product.manage_stock || false,
+        backorders: product.backorders || 'no',
+
+        // Product type and status
         type: product.type || 'simple',
+        status: product.status || 'publish',
+        featured: product.featured || false,
+
+        // Metadata
+        permalink: product.permalink || null,
         synced_at: new Date().toISOString(),
       }
     }))