fix: prevent duplicate content entries in database and Qdrant

- Update existing content records instead of always inserting new rows
- Maintain consistent content_id for Qdrant point management
- Add cleanupDuplicateContent() to remove existing duplicates from DB
- Add deletePointsByUrl() to delete Qdrant points by URL
- Add deduplicateCollection() to find and remove duplicate Qdrant points

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/database/Database.ts

@@ -794,19 +794,33 @@ export class ShopDatabase {
     // 1. It's new content (no existing record)
     // 2. The content hash has changed from the previous version
     const changed = existing ? existing.content_hash !== contentHash : true;
-    const id = uuidv4();
 
-    // Insert new content record
-    this.db.prepare(`
-      INSERT INTO shop_content (id, shop_id, content_type, url, content, content_hash, changed, enabled, created_at, updated_at, title)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(id, shopId, contentType, url, content, contentHash, changed ? 1 : 0, 1, now, now, title || null);
+    if (existing) {
+      // Update existing record instead of creating a new one
+      // This prevents duplicate entries and maintains consistent content_id for Qdrant
+      this.db.prepare(`
+        UPDATE shop_content
+        SET content = ?, content_hash = ?, changed = ?, updated_at = ?, title = ?
+        WHERE id = ?
+      `).run(content, contentHash, changed ? 1 : 0, now, title || null, existing.id);
 
-    if (changed) {
-      logger.info(`Content changed detected for ${contentType} in shop ${shopId}`);
-    }
+      if (changed) {
+        logger.info(`Content changed detected for ${contentType} in shop ${shopId} (URL: ${url})`);
+      }
+
+      return { changed, contentId: existing.id };
+    } else {
+      // Insert new content record only if it doesn't exist
+      const id = uuidv4();
+      this.db.prepare(`
+        INSERT INTO shop_content (id, shop_id, content_type, url, content, content_hash, changed, enabled, created_at, updated_at, title)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(id, shopId, contentType, url, content, contentHash, 1, 1, now, now, title || null);
+
+      logger.info(`New content saved for ${contentType} in shop ${shopId} (URL: ${url})`);
 
-    return { changed, contentId: id };
+      return { changed: true, contentId: id };
+    }
   }
 
   getLatestContent(shopId: string): { [key: string]: ShopContent } {
@@ -1029,6 +1043,54 @@ export class ShopDatabase {
     return deletedCount;
   }
 
+  /**
+   * Clean up duplicate content entries in the database
+   * Keeps only the latest entry for each shop_id + content_type + url combination
+   * Returns the number of duplicate entries removed
+   */
+  cleanupDuplicateContent(shopId?: string): number {
+    // Find all duplicates (entries that are not the latest for their shop_id + content_type + url)
+    let query = `
+      DELETE FROM shop_content
+      WHERE id NOT IN (
+        SELECT id FROM (
+          SELECT id, ROW_NUMBER() OVER (
+            PARTITION BY shop_id, content_type, url
+            ORDER BY created_at DESC
+          ) as rn
+          FROM shop_content
+        ) WHERE rn = 1
+      )
+    `;
+
+    if (shopId) {
+      query = `
+        DELETE FROM shop_content
+        WHERE shop_id = ? AND id NOT IN (
+          SELECT id FROM (
+            SELECT id, ROW_NUMBER() OVER (
+              PARTITION BY shop_id, content_type, url
+              ORDER BY created_at DESC
+            ) as rn
+            FROM shop_content
+            WHERE shop_id = ?
+          ) WHERE rn = 1
+        )
+      `;
+      const result = this.db.prepare(query).run(shopId, shopId);
+      if (result.changes > 0) {
+        logger.info(`Cleaned up ${result.changes} duplicate content entries for shop ${shopId}`);
+      }
+      return result.changes;
+    } else {
+      const result = this.db.prepare(query).run();
+      if (result.changes > 0) {
+        logger.info(`Cleaned up ${result.changes} duplicate content entries across all shops`);
+      }
+      return result.changes;
+    }
+  }
+
   // Scheduled jobs operations
   createScheduledJob(shopId: string, nextRunAt: string, frequency: string | null, lastModified: string | null): string {
     const id = uuidv4();

src/services/QdrantService.ts

@@ -242,6 +242,131 @@ export class QdrantService {
     }
   }
 
+  /**
+   * Delete all points for a specific URL in a collection
+   */
+  async deletePointsByUrl(collectionName: string, url: string): Promise<void> {
+    if (!this.enabled) {
+      logger.warn('Attempted to delete points while Qdrant is disabled');
+      return;
+    }
+
+    try {
+      await this.client.delete(collectionName, {
+        wait: true,
+        filter: {
+          must: [
+            {
+              key: 'url',
+              match: {
+                value: url,
+              },
+            },
+          ],
+        },
+      });
+
+      logger.debug(`Deleted all points for URL ${url} in collection ${collectionName}`);
+    } catch (error: any) {
+      if (error.status === 404) {
+        logger.debug(`Collection ${collectionName} does not exist (cannot delete points)`);
+        return;
+      }
+      logger.error(`Failed to delete points for URL ${url} in collection ${collectionName}:`, error);
+      throw error;
+    }
+  }
+
+  /**
+   * Find and remove duplicate points in a collection (keep only latest by content_hash)
+   * Returns the number of duplicates removed
+   */
+  async deduplicateCollection(collectionName: string): Promise<number> {
+    if (!this.enabled) {
+      logger.warn('Attempted to deduplicate while Qdrant is disabled');
+      return 0;
+    }
+
+    try {
+      // Scroll through all points to find duplicates
+      const allPoints: Array<{ id: string; payload: any }> = [];
+      let offset: string | undefined = undefined;
+
+      // Collect all points
+      while (true) {
+        const response = await this.client.scroll(collectionName, {
+          limit: 100,
+          offset: offset,
+          with_payload: true,
+          with_vector: false,
+        });
+
+        if (!response.points || response.points.length === 0) break;
+
+        for (const point of response.points) {
+          allPoints.push({
+            id: String(point.id),
+            payload: point.payload,
+          });
+        }
+
+        offset = response.next_page_offset as string | undefined;
+        if (!offset) break;
+      }
+
+      // Group by URL + content_hash
+      const groups = new Map<string, Array<{ id: string; payload: any }>>();
+      for (const point of allPoints) {
+        const key = `${point.payload?.url}:${point.payload?.content_hash}`;
+        if (!groups.has(key)) {
+          groups.set(key, []);
+        }
+        groups.get(key)!.push(point);
+      }
+
+      // Find duplicates (groups with more than one point)
+      const pointsToDelete: string[] = [];
+      for (const [key, points] of groups) {
+        if (points.length > 1) {
+          // Sort by created_at descending, keep the newest
+          points.sort((a, b) => {
+            const dateA = a.payload?.created_at || '';
+            const dateB = b.payload?.created_at || '';
+            return dateB.localeCompare(dateA);
+          });
+
+          // Mark all but the first (newest) for deletion
+          for (let i = 1; i < points.length; i++) {
+            pointsToDelete.push(points[i].id);
+          }
+        }
+      }
+
+      if (pointsToDelete.length > 0) {
+        // Delete duplicates in batches
+        const batchSize = 100;
+        for (let i = 0; i < pointsToDelete.length; i += batchSize) {
+          const batch = pointsToDelete.slice(i, i + batchSize);
+          await this.client.delete(collectionName, {
+            wait: true,
+            points: batch,
+          });
+        }
+
+        logger.info(`Removed ${pointsToDelete.length} duplicate points from collection ${collectionName}`);
+      }
+
+      return pointsToDelete.length;
+    } catch (error: any) {
+      if (error.status === 404) {
+        logger.debug(`Collection ${collectionName} does not exist`);
+        return 0;
+      }
+      logger.error(`Failed to deduplicate collection ${collectionName}:`, error);
+      throw error;
+    }
+  }
+
   /**
    * Search in a single collection
    */