fix: prevent duplicate content by content_hash across categories

- Skip saving content if same content_hash already exists in another category
- Add skipped flag to saveContent return value
- Skip embedding processing for skipped duplicate content
- Add cleanupDuplicateContentByHash() to remove existing duplicates

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/components/ContentEndpoint.ts

@@ -267,8 +267,8 @@ export class ContentEndpoint extends BaseEndpointComponent {
             extractionResult.title
           );
 
-          // If embedding workflow is available, process embeddings
-          if (this.embeddingWorkflow) {
+          // If embedding workflow is available and content was not skipped (duplicate), process embeddings
+          if (this.embeddingWorkflow && !saveResult.skipped) {
             const contentHash = createHash('sha256').update(extractionResult.content).digest('hex');
             await this.embeddingWorkflow.processContentForEmbedding(
               shop.id,
@@ -282,7 +282,7 @@ export class ContentEndpoint extends BaseEndpointComponent {
             );
           }
 
-          actionTaken = 'rescraped_and_saved';
+          actionTaken = saveResult.skipped ? 'skipped_duplicate' : 'rescraped_and_saved';
           logger.info(`Successfully re-scraped content ${contentId} for shop ${shop.id}`);
         } catch (error) {
           logger.error(`Failed to re-scrape content ${contentId}:`, error);

src/database/Database.ts

@@ -777,11 +777,24 @@ export class ShopDatabase {
     url: string,
     content: string,
     title?: string
-  ): { changed: boolean; contentId: string } {
+  ): { changed: boolean; contentId: string; skipped?: boolean } {
     const crypto = require('crypto');
     const contentHash = crypto.createHash('md5').update(content).digest('hex');
     const now = new Date().toISOString();
 
+    // Check if this exact content_hash already exists for this shop (in ANY content type)
+    // This prevents the same page from being stored multiple times under different categories
+    const duplicateContent = this.db.prepare(`
+      SELECT id, content_type, url FROM shop_content
+      WHERE shop_id = ? AND content_hash = ? AND content_type != ?
+      LIMIT 1
+    `).get(shopId, contentHash, contentType) as { id: string; content_type: string; url: string } | undefined;
+
+    if (duplicateContent) {
+      logger.debug(`Skipping duplicate content for ${contentType} (URL: ${url}) - same content already exists in ${duplicateContent.content_type} (URL: ${duplicateContent.url})`);
+      return { changed: false, contentId: duplicateContent.id, skipped: true };
+    }
+
     // Check if content exists and get previous hash for this specific URL
     const existing = this.db.prepare(`
       SELECT id, content_hash FROM shop_content
@@ -1091,6 +1104,54 @@ export class ShopDatabase {
     }
   }
 
+  /**
+   * Clean up duplicate content entries by content_hash
+   * When the same content appears in multiple categories, keep only the first occurrence
+   * Returns the number of duplicate entries removed
+   */
+  cleanupDuplicateContentByHash(shopId?: string): number {
+    // Find entries where the same content_hash appears in multiple content_types
+    // Keep the one with the earliest created_at, delete the rest
+    let query: string;
+    let params: any[] = [];
+
+    if (shopId) {
+      query = `
+        DELETE FROM shop_content
+        WHERE shop_id = ? AND id NOT IN (
+          SELECT id FROM (
+            SELECT id, ROW_NUMBER() OVER (
+              PARTITION BY shop_id, content_hash
+              ORDER BY created_at ASC
+            ) as rn
+            FROM shop_content
+            WHERE shop_id = ?
+          ) WHERE rn = 1
+        )
+      `;
+      params = [shopId, shopId];
+    } else {
+      query = `
+        DELETE FROM shop_content
+        WHERE id NOT IN (
+          SELECT id FROM (
+            SELECT id, ROW_NUMBER() OVER (
+              PARTITION BY shop_id, content_hash
+              ORDER BY created_at ASC
+            ) as rn
+            FROM shop_content
+          ) WHERE rn = 1
+        )
+      `;
+    }
+
+    const result = this.db.prepare(query).run(...params);
+    if (result.changes > 0) {
+      logger.info(`Cleaned up ${result.changes} duplicate content entries by hash${shopId ? ` for shop ${shopId}` : ' across all shops'}`);
+    }
+    return result.changes;
+  }
+
   // Scheduled jobs operations
   createScheduledJob(shopId: string, nextRunAt: string, frequency: string | null, lastModified: string | null): string {
     const id = uuidv4();

src/scraper/WebshopScraper.ts

@@ -130,6 +130,9 @@ export class WebshopScraper {
         for (const content of shippingContent) {
           const result = this.db.saveContent(shopId, 'shipping', content.url, content.content, content.title);
 
+          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
+          if (result.skipped) continue;
+
           // Add to embedding batch if workflow is available
           if (this.embeddingWorkflow) {
             embeddingBatch.push({
@@ -162,6 +165,9 @@ export class WebshopScraper {
         for (const content of contactsContent) {
           const result = this.db.saveContent(shopId, 'contacts', content.url, content.content, content.title);
 
+          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
+          if (result.skipped) continue;
+
           // Add to embedding batch if workflow is available
           if (this.embeddingWorkflow) {
             embeddingBatch.push({
@@ -194,6 +200,9 @@ export class WebshopScraper {
         for (const content of termsContent) {
           const result = this.db.saveContent(shopId, 'terms', content.url, content.content, content.title);
 
+          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
+          if (result.skipped) continue;
+
           // Add to embedding batch if workflow is available
           if (this.embeddingWorkflow) {
             embeddingBatch.push({
@@ -226,6 +235,9 @@ export class WebshopScraper {
         for (const content of faqContent) {
           const result = this.db.saveContent(shopId, 'faq', content.url, content.content, content.title);
 
+          // Skip embedding and webhooks if content was skipped (duplicate hash in another category)
+          if (result.skipped) continue;
+
           // Add to embedding batch if workflow is available
           if (this.embeddingWorkflow) {
             embeddingBatch.push({