fix: check Qdrant before generating embeddings to prevent duplicates

- Add getPointsByUrl() to check existing points for a URL
- Add findContentByHash() to detect duplicate content across URLs
- Check Qdrant FIRST before generating any embeddings:
  - Skip if same content_hash exists at different URL (duplicate content)
  - Skip if same URL has same content_hash (unchanged)
  - Delete old points and re-embed if URL exists with different hash

This prevents unnecessary embedding API calls and duplicate points.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/services/QdrantEmbeddingWorkflow.ts

@@ -79,54 +79,40 @@ export class QdrantEmbeddingWorkflow {
         return { contentId, embedded: false, error: 'Embedding service not available' };
       }
 
-      // Check if content already has up-to-date embedding
-      const existingEmbeddings = this.database.getShopEmbeddings(contentId);
-
-      // Only skip if:
-      // 1. Content hasn't changed (contentChanged = false)
-      // 2. AND we have a completed embedding
-      // 3. AND the embedding was created for this exact content (by checking if it exists in Qdrant)
-      if (!contentChanged && existingEmbeddings.length > 0) {
-        const hasCompletedEmbedding = existingEmbeddings.some(
-          emb => emb.embedding_status === 'completed'
-        );
-
-        if (hasCompletedEmbedding) {
-          logger.debug(`Skipping embedding for content ${contentId}: no changes and embedding exists`);
-          return { contentId, embedded: false, error: 'Content unchanged and embedding exists' };
-        }
-      }
-
-      // Split content into chunks
-      const chunks = this.chunkingService.chunkContent(content, title);
-      const chunkStats = this.chunkingService.getChunkingStats(chunks);
-
-      logger.info(`Content ${contentId} split into ${chunks.length} chunks (avg: ${chunkStats.avgWordCount} words/chunk)`);
-
       // Generate collection name for the category (one collection per shop+category)
-      // All URLs in this category will be stored as points in this collection
       const collectionName = this.qdrantService.generateCollectionName(shop.custom_id, contentType);
       logger.debug(`Using collection ${collectionName} for content ${contentId} (URL: ${url})`);
 
-      // With deterministic point IDs based on URL + content_hash + chunk_index:
-      // - Different URLs: Different point IDs (unique within collection)
-      // - Same URL, unchanged content: Same point ID → Upsert = no-op
-      // - Same URL, changed content: Different point ID → New points created
-      // - Old points with different content hashes should be cleaned up manually
+      // Check Qdrant FIRST before generating any embeddings
+      // This prevents duplicate embeddings and unnecessary API calls
 
-      // Delete old points for this URL when content changes
-      if (contentChanged) {
-        try {
-          await this.qdrantService.deletePointsByContentId(collectionName, contentId);
-          logger.debug(`Deleted old points for content ${contentId} in collection ${collectionName}`);
-        } catch (error: any) {
-          // Collection might not exist yet, which is fine
-          if (error.status !== 404) {
-            logger.warn(`Failed to delete old points for content ${contentId}:`, error);
-          }
+      // 1. Check if this exact content_hash already exists in Qdrant (anywhere in this collection)
+      const existingByHash = await this.qdrantService.findContentByHash(collectionName, contentHash);
+      if (existingByHash && existingByHash.url !== url) {
+        // Same content exists under a different URL - skip (duplicate content)
+        logger.debug(`Skipping embedding for ${url}: identical content already exists at ${existingByHash.url}`);
+        return { contentId, embedded: false, error: `Duplicate content exists at ${existingByHash.url}` };
+      }
+
+      // 2. Check if points already exist for this URL
+      const existingPoints = await this.qdrantService.getPointsByUrl(collectionName, url);
+      if (existingPoints && existingPoints.length > 0) {
+        const existingHash = existingPoints[0].content_hash;
+
+        if (existingHash === contentHash) {
+          // Same URL, same content hash - nothing to do
+          logger.debug(`Skipping embedding for ${url}: content unchanged (hash: ${contentHash})`);
+          return { contentId, embedded: false, error: 'Content unchanged in Qdrant' };
+        } else {
+          // Same URL, different content hash - delete old points before creating new ones
+          logger.info(`Content changed for ${url}, deleting ${existingPoints.length} old points`);
+          await this.qdrantService.deletePointsByUrl(collectionName, url);
         }
       }
 
+      // Check database embedding records
+      const existingEmbeddings = this.database.getShopEmbeddings(contentId);
+
       // Delete old embedding records from database (we'll create new ones)
       if (existingEmbeddings.length > 0) {
         for (const emb of existingEmbeddings) {
@@ -135,6 +121,12 @@ export class QdrantEmbeddingWorkflow {
         logger.debug(`Cleared ${existingEmbeddings.length} old embedding records for content ${contentId}`);
       }
 
+      // Split content into chunks
+      const chunks = this.chunkingService.chunkContent(content, title);
+      const chunkStats = this.chunkingService.getChunkingStats(chunks);
+
+      logger.info(`Content ${contentId} split into ${chunks.length} chunks (avg: ${chunkStats.avgWordCount} words/chunk)`);
+
       // Ensure collection exists
       if (!(await this.qdrantService.collectionExists(collectionName))) {
         await this.qdrantService.createCollection(collectionName);

src/services/QdrantService.ts

@@ -277,6 +277,94 @@ export class QdrantService {
     }
   }
 
+  /**
+   * Check if points exist for a URL and get their content_hash
+   * Returns existing points info or null if none exist
+   */
+  async getPointsByUrl(collectionName: string, url: string): Promise<Array<{ id: string; content_hash: string; chunk_index?: number }> | null> {
+    if (!this.enabled) {
+      return null;
+    }
+
+    try {
+      const response = await this.client.scroll(collectionName, {
+        limit: 100,
+        filter: {
+          must: [
+            {
+              key: 'url',
+              match: {
+                value: url,
+              },
+            },
+          ],
+        },
+        with_payload: true,
+        with_vector: false,
+      });
+
+      if (!response.points || response.points.length === 0) {
+        return null;
+      }
+
+      return response.points.map(point => ({
+        id: String(point.id),
+        content_hash: (point.payload as any)?.content_hash || '',
+        chunk_index: (point.payload as any)?.chunk_index,
+      }));
+    } catch (error: any) {
+      if (error.status === 404) {
+        return null;
+      }
+      logger.error(`Failed to get points by URL ${url} in collection ${collectionName}:`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Check if content with the same hash already exists in a collection (in any category/URL)
+   * Returns the URL where this content exists, or null if not found
+   */
+  async findContentByHash(collectionName: string, contentHash: string): Promise<{ url: string; content_id: string } | null> {
+    if (!this.enabled) {
+      return null;
+    }
+
+    try {
+      const response = await this.client.scroll(collectionName, {
+        limit: 1,
+        filter: {
+          must: [
+            {
+              key: 'content_hash',
+              match: {
+                value: contentHash,
+              },
+            },
+          ],
+        },
+        with_payload: true,
+        with_vector: false,
+      });
+
+      if (!response.points || response.points.length === 0) {
+        return null;
+      }
+
+      const payload = response.points[0].payload as any;
+      return {
+        url: payload?.url || '',
+        content_id: payload?.content_id || '',
+      };
+    } catch (error: any) {
+      if (error.status === 404) {
+        return null;
+      }
+      logger.error(`Failed to find content by hash ${contentHash} in collection ${collectionName}:`, error);
+      throw error;
+    }
+  }
+
   /**
    * Find and remove duplicate points in a collection (keep only latest by content_hash)
    * Returns the number of duplicates removed