fix: correct sync config reading and implement smart embedding generation #90

- Fix sync config reading to use !== false instead of === true (defaults to true when undefined)
- Implement smart embedding generation that only processes new/changed products
- Reuse existing embeddings for unchanged products to save API calls and time
- Add detailed logging for embedding generation stats

supabase/functions/shoprenter-sync/index.ts

@@ -92,15 +92,17 @@ async function syncProductsToQdrant(
       ])
     }
 
-    // Get existing product IDs from Qdrant to detect deletions
+    // Get existing product IDs from Qdrant to detect deletions and changes
     const existingPoints = await scrollPoints(collectionName, {
       must: [{ key: 'store_id', match: { value: storeId } }]
     }, 1000)
 
-    const existingProductIds = new Set(
-      existingPoints.points.map((p: any) => p.payload?.product_id).filter(Boolean)
+    // Build a map of existing products for efficient lookup
+    const existingProductsMap = new Map(
+      existingPoints.points.map((p: any) => [p.payload?.product_id, p])
     )
 
+    const existingProductIds = new Set(existingProductsMap.keys())
     const currentProductIds = new Set(validProducts.map(p => p.innerId.toString()))
 
     // Find deleted products
@@ -119,9 +121,44 @@ async function syncProductsToQdrant(
       })
     }
 
-    // Generate text representations for all products
-    // Extract descriptions, tags, and attributes from ShopRenter's nested structure
-    const productTexts = validProducts.map((product) => {
+    // Helper function to check if product data has changed
+    const hasProductChanged = (product: any, existingPoint: any): boolean => {
+      if (!existingPoint) return true // New product
+
+      const payload = existingPoint.payload
+      const productDesc = product.productDescriptions?.[0] || {}
+
+      // Compare key fields that would affect the embedding
+      return (
+        payload.name !== (productDesc.name || product.name) ||
+        payload.sku !== product.sku ||
+        payload.price !== parseFloat(product.price || '0') ||
+        payload.description !== (productDesc.description || null) ||
+        payload.short_description !== (productDesc.shortDescription || null) ||
+        payload.stock !== parseInt(product.stock1 || '0') ||
+        payload.status !== (product.status === '1' ? 'active' : 'inactive')
+      )
+    }
+
+    // Separate products into new/changed vs unchanged
+    const productsNeedingEmbedding: any[] = []
+    const productsToReuse: { product: any; existingPoint: any }[] = []
+
+    validProducts.forEach(product => {
+      const productId = product.innerId.toString()
+      const existingPoint = existingProductsMap.get(productId)
+
+      if (hasProductChanged(product, existingPoint)) {
+        productsNeedingEmbedding.push(product)
+      } else {
+        productsToReuse.push({ product, existingPoint })
+      }
+    })
+
+    console.log(`[Qdrant] Products analysis: ${productsNeedingEmbedding.length} new/changed, ${productsToReuse.length} unchanged`)
+
+    // Generate text representations only for new/changed products
+    const productTexts = productsNeedingEmbedding.map((product) => {
       // Extract first language description from productDescriptions array
       const productDesc = product.productDescriptions?.[0] || {}
 
@@ -149,13 +186,18 @@ async function syncProductsToQdrant(
       })
     })
 
-    // Generate embeddings in batch
-    console.log(`[Qdrant] Generating embeddings for ${productTexts.length} products...`)
-    const embeddings = await generateEmbeddingBatch(productTexts)
-    console.log(`[Qdrant] Embeddings generated successfully`)
+    // Generate embeddings only for new/changed products
+    let embeddings: number[][] = []
+    if (productTexts.length > 0) {
+      console.log(`[Qdrant] Generating embeddings for ${productTexts.length} new/changed products...`)
+      embeddings = await generateEmbeddingBatch(productTexts)
+      console.log(`[Qdrant] Embeddings generated successfully`)
+    } else {
+      console.log(`[Qdrant] No new/changed products - skipping embedding generation`)
+    }
 
-    // Convert products to Qdrant points with embeddings and comprehensive details
-    const points: QdrantPoint[] = validProducts.map((product, index) => {
+    // Convert new/changed products to Qdrant points with embeddings
+    const newPoints: QdrantPoint[] = productsNeedingEmbedding.map((product, index) => {
       // Extract first language description from productDescriptions array
       const productDesc = product.productDescriptions?.[0] || {}
 
@@ -228,9 +270,39 @@ async function syncProductsToQdrant(
       }
     })
 
+    // Create points for unchanged products (reuse existing embeddings and update metadata)
+    const reusedPoints: QdrantPoint[] = productsToReuse.map(({ product, existingPoint }) => {
+      const productDesc = product.productDescriptions?.[0] || {}
+      const tags = (product.productTags || []).map((t: any) => t.name || t).filter(Boolean)
+      const attributes = (product.productAttributeExtend || []).map((attr: any) => {
+        if (attr.name && attr.value) {
+          return { name: attr.name, value: attr.value }
+        }
+        return null
+      }).filter(Boolean)
+      const categories = (product.productCategoryRelations || []).map((rel: any) => ({
+        id: rel.category?.id || null,
+      })).filter((cat: any) => cat.id)
+      const manufacturer = product.manufacturer?.name || null
+
+      return {
+        id: existingPoint.id,
+        vector: existingPoint.vector,  // Reuse existing embedding
+        payload: {
+          ...existingPoint.payload,
+          // Only update metadata fields that don't affect embeddings
+          synced_at: new Date().toISOString(),
+        }
+      }
+    })
+
+    // Combine new and reused points
+    const allPoints = [...newPoints, ...reusedPoints]
 
-    await upsertPoints(collectionName, points)
-    synced = points.length
+    if (allPoints.length > 0) {
+      await upsertPoints(collectionName, allPoints)
+      synced = allPoints.length
+    }
 
     await logQdrantSync(
       supabaseAdmin,
@@ -244,7 +316,7 @@ async function syncProductsToQdrant(
       startTime
     )
 
-    console.log(`[Qdrant] Products sync complete: ${synced} synced, ${deletedProductIds.length} deleted`)
+    console.log(`[Qdrant] Products sync complete: ${synced} total (${newPoints.length} new/changed, ${reusedPoints.length} unchanged), ${deletedProductIds.length} deleted`)
   } catch (error: any) {
     console.error('[Qdrant] Product sync error:', error)
     errors = products.length
@@ -700,8 +772,8 @@ serve(wrapHandler('shoprenter-sync', async (req) => {
 
     // Check store_sync_config for orders/customers sync flags
     const syncConfig = (store as any).store_sync_config?.[0] || {}
-    const shouldSyncOrders = syncConfig.sync_orders === true
-    const shouldSyncCustomers = syncConfig.sync_customers === true
+    const shouldSyncOrders = syncConfig.sync_orders !== false  // Default to true if not explicitly disabled
+    const shouldSyncCustomers = syncConfig.sync_customers !== false  // Default to true if not explicitly disabled
 
     console.log('[ShopRenter] Sync permissions:', {
       products: canSyncProducts,

supabase/test_query.ts

@@ -0,0 +1,31 @@
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2'
+
+const supabaseUrl = Deno.env.get('SUPABASE_URL') || ''
+const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') || ''
+const supabaseAdmin = createClient(supabaseUrl, supabaseServiceKey)
+
+const { data: store, error: storeError } = await supabaseAdmin
+  .from('stores')
+  .select(`
+    id,
+    store_name,
+    qdrant_sync_enabled,
+    store_sync_config (
+      sync_orders,
+      sync_customers
+    )
+  `)
+  .eq('id', 'd164a10f-580c-4500-9850-fed3079dd6af')
+  .single()
+
+console.log('Store data:', JSON.stringify(store, null, 2))
+console.log('store_sync_config:', store?.store_sync_config)
+console.log('Type of store_sync_config:', typeof store?.store_sync_config)
+console.log('Is array?:', Array.isArray(store?.store_sync_config))
+
+if (store) {
+  const syncConfig = (store as any).store_sync_config?.[0] || {}
+  console.log('syncConfig:', syncConfig)
+  console.log('syncConfig.sync_orders:', syncConfig.sync_orders)
+  console.log('syncConfig.sync_orders === true:', syncConfig.sync_orders === true)
+}