feat: implement URL disabling for sitemap-discovered content

This commit adds the ability to disable sitemap-discovered URLs with
comprehensive content and embedding cleanup, plus immediate re-scraping
when URLs are re-enabled.

Features:
- Add enabled column to shop_content table (defaults to true)
- New API endpoints for content management:
  * GET /api/shops/:id/content - List all content with enabled status
  * PATCH /api/shops/:id/content/:contentId - Enable/disable content
- Database methods for content enable/disable operations
- Scraper integration to skip disabled URLs during scraping
- Automatic cleanup of disabled URLs missing from sitemap
- Qdrant embedding cleanup when disabling URLs
- Immediate re-scraping with embedding generation when re-enabling URLs

When disabling:
- Deletes all Qdrant embeddings for the content
- Clears content, hash, and title from database
- Keeps URL record in disabled state

When re-enabling:
- Immediately scrapes the URL
- Saves content to database
- Generates Qdrant embeddings
- Returns success/failure status

During scraping:
- Filters out disabled URLs before processing
- Cleans up disabled URLs no longer in sitemap

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/components/ContentEndpoint.ts

@@ -0,0 +1,302 @@
+import { Request, Response } from 'express';
+import { BaseEndpointComponent, EndpointMethod } from '../core/types';
+import { logger } from '../../utils/logger';
+import { ShopDatabase } from '../../database/Database';
+import { QdrantService } from '../../services/QdrantService';
+import { ContentExtractor } from '../../scraper/ContentExtractor';
+import { QdrantEmbeddingWorkflow } from '../../services/QdrantEmbeddingWorkflow';
+import { createHash } from 'crypto';
+import type { AppConfig } from '../../config';
+
+export class ContentEndpoint extends BaseEndpointComponent {
+  private qdrantService?: QdrantService;
+  private contentExtractor: ContentExtractor;
+  private embeddingWorkflow?: QdrantEmbeddingWorkflow;
+
+  constructor(private db: ShopDatabase, private config?: AppConfig) {
+    super();
+    this.contentExtractor = new ContentExtractor();
+    if (config) {
+      this.qdrantService = new QdrantService(config);
+      this.embeddingWorkflow = new QdrantEmbeddingWorkflow(config, db);
+    }
+  }
+
+  getEndpoints(): EndpointMethod[] {
+    return [
+      this.createEndpoint(
+        'GET',
+        '/api/shops/:id/content',
+        this.getAllContent.bind(this),
+        {
+          summary: 'List all content for a shop',
+          description: 'Retrieves all content (from sitemap and custom URLs) for a shop, including disabled content',
+          tags: ['Content'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            },
+            {
+              name: 'include_disabled',
+              in: 'query',
+              type: 'boolean',
+              required: false,
+              description: 'Include disabled content in the results (default: true)'
+            }
+          ],
+          responses: {
+            '200': {
+              description: 'List of content items',
+              schema: {
+                type: 'object',
+                properties: {
+                  shop_id: { type: 'string' },
+                  content: {
+                    type: 'array',
+                    items: {
+                      type: 'object',
+                      properties: {
+                        id: { type: 'string' },
+                        url: { type: 'string' },
+                        content_type: { type: 'string' },
+                        enabled: { type: 'boolean' },
+                        title: { type: 'string' },
+                        created_at: { type: 'string' },
+                        updated_at: { type: 'string' }
+                      }
+                    }
+                  }
+                }
+              }
+            },
+            '404': {
+              description: 'Shop not found'
+            }
+          },
+          requiresAuth: true
+        }
+      ),
+      this.createEndpoint(
+        'PATCH',
+        '/api/shops/:id/content/:contentId',
+        this.updateContent.bind(this),
+        {
+          summary: 'Enable or disable content',
+          description: 'Updates the enabled status of a content item. When disabling, removes content from database and Qdrant. When re-enabling, triggers a rescrape.',
+          tags: ['Content'],
+          parameters: [
+            {
+              name: 'id',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Shop ID (internal UUID or custom UUID)'
+            },
+            {
+              name: 'contentId',
+              in: 'path',
+              type: 'string',
+              required: true,
+              description: 'Content ID'
+            }
+          ],
+          requestBody: {
+            required: true,
+            schema: {
+              type: 'object',
+              properties: {
+                enabled: {
+                  type: 'boolean',
+                  description: 'Whether to enable or disable the content'
+                }
+              },
+              required: ['enabled']
+            }
+          },
+          responses: {
+            '200': {
+              description: 'Content status updated',
+              schema: {
+                type: 'object',
+                properties: {
+                  message: { type: 'string' },
+                  content_id: { type: 'string' },
+                  enabled: { type: 'boolean' },
+                  action_taken: { type: 'string' }
+                }
+              }
+            },
+            '400': {
+              description: 'Invalid request data'
+            },
+            '404': {
+              description: 'Shop or content not found'
+            }
+          },
+          requiresAuth: true
+        }
+      )
+    ];
+  }
+
+  private async getAllContent(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const includeDisabled = req.query.include_disabled !== 'false'; // Default true
+
+      const shop = this.db.getShopByAnyId(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const content = this.db.getAllShopContent(shop.id, includeDisabled);
+
+      res.json({
+        shop_id: shop.id,
+        content: content.map(c => ({
+          id: c.id,
+          url: c.url,
+          content_type: c.content_type,
+          enabled: c.enabled,
+          title: c.title,
+          created_at: c.created_at,
+          updated_at: c.updated_at
+        }))
+      });
+    } catch (error) {
+      logger.error('Error fetching content', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+
+  private async updateContent(req: Request, res: Response): Promise<void> {
+    try {
+      if (!this.db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id, contentId } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      // Verify shop exists
+      const shop = this.db.getShopByAnyId(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Verify content exists and belongs to this shop
+      const content = this.db.getContentById(contentId);
+      if (!content) {
+        res.status(404).json({ error: 'Content not found' });
+        return;
+      }
+
+      if (content.shop_id !== shop.id) {
+        res.status(404).json({ error: 'Content does not belong to this shop' });
+        return;
+      }
+
+      // Check if status is already correct
+      if (content.enabled === enabled) {
+        res.json({
+          message: `Content is already ${enabled ? 'enabled' : 'disabled'}`,
+          content_id: contentId,
+          enabled,
+          action_taken: 'none'
+        });
+        return;
+      }
+
+      let actionTaken = '';
+
+      if (!enabled) {
+        // Disabling: clear content and Qdrant data
+        // First, delete from Qdrant if available
+        if (this.qdrantService) {
+          try {
+            const embeddings = this.db.getShopEmbeddings(contentId);
+            for (const emb of embeddings) {
+              await this.qdrantService.deletePointsByContentId(emb.collection_name, contentId);
+              logger.info(`Deleted Qdrant points for content ${contentId} from collection ${emb.collection_name}`);
+            }
+          } catch (error) {
+            logger.error(`Failed to delete Qdrant points for content ${contentId}:`, error);
+            // Continue with database cleanup even if Qdrant cleanup fails
+          }
+        }
+
+        // Then delete from database (clears content but keeps URL record)
+        this.db.deleteContentData(contentId);
+        actionTaken = 'content_and_embeddings_deleted';
+        logger.info(`Disabled content ${contentId} and cleared data for shop ${shop.id}`);
+      } else {
+        // Re-enabling: update the flag and scrape immediately
+        this.db.setContentEnabled(contentId, true);
+        logger.info(`Re-enabled content ${contentId} for shop ${shop.id}, starting immediate scrape`);
+
+        // Scrape the content immediately
+        try {
+          const extractionResult = await this.contentExtractor.extractContent(content.url);
+
+          // Save the scraped content
+          const saveResult = this.db.saveContent(
+            shop.id,
+            content.content_type as 'shipping' | 'contacts' | 'terms' | 'faq',
+            content.url,
+            extractionResult.content,
+            extractionResult.title
+          );
+
+          // If embedding workflow is available, process embeddings
+          if (this.embeddingWorkflow) {
+            const contentHash = createHash('sha256').update(extractionResult.content).digest('hex');
+            await this.embeddingWorkflow.processContentForEmbedding(
+              shop.id,
+              saveResult.contentId,
+              content.content_type as 'shipping' | 'contacts' | 'terms' | 'faq',
+              content.url,
+              extractionResult.content,
+              extractionResult.title,
+              contentHash,
+              saveResult.changed
+            );
+          }
+
+          actionTaken = 'rescraped_and_saved';
+          logger.info(`Successfully re-scraped content ${contentId} for shop ${shop.id}`);
+        } catch (error) {
+          logger.error(`Failed to re-scrape content ${contentId}:`, error);
+          actionTaken = 'enabled_but_scrape_failed';
+          // Note: Content is still enabled, it will be retried on next scheduled scrape
+        }
+      }
+
+      res.json({
+        message: `Content ${enabled ? 'enabled' : 'disabled'} successfully`,
+        content_id: contentId,
+        enabled,
+        action_taken: actionTaken
+      });
+    } catch (error) {
+      logger.error('Error updating content status', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  }
+}

src/api/routes.ts

@@ -11,6 +11,7 @@ import { JobsEndpoint } from './components/JobsEndpoint';
 import { ShopsEndpoint } from './components/ShopsEndpoint';
 import { WebhooksEndpoint } from './components/WebhooksEndpoint';
 import { CustomUrlsEndpoint } from './components/CustomUrlsEndpoint';
+import { ContentEndpoint } from './components/ContentEndpoint';
 import { HealthEndpoint } from './components/HealthEndpoint';
 import { QdrantEndpoint } from './components/QdrantEndpoint';
 import { McpEndpoint } from './components/McpEndpoint';
@@ -35,6 +36,7 @@ export function createRouter(jobQueue: JobQueue, apiKey: string, db?: ShopDataba
       new ShopsEndpoint(db),
       new WebhooksEndpoint(db),
       new CustomUrlsEndpoint(db),
+      new ContentEndpoint(db, config),
       new McpEndpoint(db)
     );
 

src/database/Database.ts

@@ -45,6 +45,7 @@ export interface ShopContent {
   content: string;
   content_hash: string; // MD5 hash of content to detect changes
   changed: boolean; // true if content changed from previous scrape
+  enabled: boolean; // true if URL should be scraped, false if disabled
   created_at: string;
   updated_at: string;
   title: string | null; // Page title from <title> tag
@@ -257,6 +258,13 @@ export class ShopDatabase {
       // Column might already exist, ignore error
     }
 
+    // Add enabled column to existing databases (migration)
+    try {
+      this.db.exec(`ALTER TABLE shop_content ADD COLUMN enabled INTEGER DEFAULT 1`);
+    } catch (error) {
+      // Column might already exist, ignore error
+    }
+
     // Create indexes on shop_content for faster lookups
     this.db.exec(`
       CREATE INDEX IF NOT EXISTS idx_shop_content_shop_type
@@ -790,9 +798,9 @@ export class ShopDatabase {
 
     // Insert new content record
     this.db.prepare(`
-      INSERT INTO shop_content (id, shop_id, content_type, url, content, content_hash, changed, created_at, updated_at, title)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-    `).run(id, shopId, contentType, url, content, contentHash, changed ? 1 : 0, now, now, title || null);
+      INSERT INTO shop_content (id, shop_id, content_type, url, content, content_hash, changed, enabled, created_at, updated_at, title)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(id, shopId, contentType, url, content, contentHash, changed ? 1 : 0, 1, now, now, title || null);
 
     if (changed) {
       logger.info(`Content changed detected for ${contentType} in shop ${shopId}`);
@@ -921,6 +929,106 @@ export class ShopDatabase {
     return stmt.all(...params) as ShopContent[];
   }
 
+  // Content enable/disable operations
+
+  /**
+   * Set enabled status for a specific content item
+   * When disabling, this will trigger cleanup of the content and Qdrant points
+   */
+  setContentEnabled(contentId: string, enabled: boolean): void {
+    const now = new Date().toISOString();
+
+    this.db.prepare(`
+      UPDATE shop_content
+      SET enabled = ?, updated_at = ?
+      WHERE id = ?
+    `).run(enabled ? 1 : 0, now, contentId);
+
+    logger.info(`Content ${contentId} ${enabled ? 'enabled' : 'disabled'}`);
+  }
+
+  /**
+   * Get content by ID
+   */
+  getContentById(contentId: string): ShopContent | null {
+    const stmt = this.db.prepare('SELECT * FROM shop_content WHERE id = ?');
+    const result = stmt.get(contentId) as any;
+    return result ? { ...result, changed: Boolean(result.changed), enabled: Boolean(result.enabled) } : null;
+  }
+
+  /**
+   * Get all content for a shop (including disabled)
+   */
+  getAllShopContent(shopId: string, includeDisabled: boolean = false): ShopContent[] {
+    let query = 'SELECT * FROM shop_content WHERE shop_id = ?';
+    const params: any[] = [shopId];
+
+    if (!includeDisabled) {
+      query += ' AND enabled = 1';
+    }
+
+    query += ' ORDER BY created_at DESC';
+
+    const stmt = this.db.prepare(query);
+    const results = stmt.all(...params) as any[];
+    return results.map(r => ({ ...r, changed: Boolean(r.changed), enabled: Boolean(r.enabled) }));
+  }
+
+  /**
+   * Get disabled content URLs for a shop
+   */
+  getDisabledContentUrls(shopId: string): string[] {
+    const stmt = this.db.prepare(`
+      SELECT DISTINCT url FROM shop_content
+      WHERE shop_id = ? AND enabled = 0
+    `);
+    const results = stmt.all(shopId) as any[];
+    return results.map(r => r.url);
+  }
+
+  /**
+   * Delete content and embeddings (used when disabling a URL)
+   */
+  deleteContentData(contentId: string): void {
+    // Delete embeddings first
+    const embeddings = this.getShopEmbeddings(contentId);
+    for (const emb of embeddings) {
+      this.deleteShopEmbedding(emb.id);
+    }
+
+    // Update content to clear the actual content but keep the URL record
+    this.db.prepare(`
+      UPDATE shop_content
+      SET content = '', content_hash = '', title = NULL, updated_at = ?
+      WHERE id = ?
+    `).run(new Date().toISOString(), contentId);
+
+    logger.info(`Deleted content data for ${contentId}, keeping URL record`);
+  }
+
+  /**
+   * Remove disabled URLs that are no longer in sitemap
+   */
+  cleanupMissingDisabledUrls(shopId: string, currentSitemapUrls: string[]): number {
+    const disabledUrls = this.getDisabledContentUrls(shopId);
+    let deletedCount = 0;
+
+    for (const url of disabledUrls) {
+      if (!currentSitemapUrls.includes(url)) {
+        // URL is disabled AND no longer in sitemap → delete completely
+        this.db.prepare(`
+          DELETE FROM shop_content
+          WHERE shop_id = ? AND url = ? AND enabled = 0
+        `).run(shopId, url);
+
+        logger.info(`Removed disabled URL no longer in sitemap: ${url}`);
+        deletedCount++;
+      }
+    }
+
+    return deletedCount;
+  }
+
   // Scheduled jobs operations
   createScheduledJob(shopId: string, nextRunAt: string, frequency: string | null, lastModified: string | null): string {
     const id = uuidv4();

src/scraper/WebshopScraper.ts

@@ -51,6 +51,21 @@ export class WebshopScraper {
       // Filter URLs by page type
       const filteredUrls = SitemapParser.filterUrlsByType(urls);
 
+      // Filter out disabled URLs if database and shopId are available
+      if (this.db && shopId) {
+        const disabledUrls = this.db.getDisabledContentUrls(shopId);
+        const disabledUrlSet = new Set(disabledUrls);
+
+        // Remove disabled URLs from each category
+        if (disabledUrlSet.size > 0) {
+          filteredUrls.shipping = filteredUrls.shipping.filter((u: any) => !disabledUrlSet.has(u.loc));
+          filteredUrls.contacts = filteredUrls.contacts.filter((u: any) => !disabledUrlSet.has(u.loc));
+          filteredUrls.terms = filteredUrls.terms.filter((u: any) => !disabledUrlSet.has(u.loc));
+          filteredUrls.faq = filteredUrls.faq.filter((u: any) => !disabledUrlSet.has(u.loc));
+          logger.info(`Filtered out ${disabledUrlSet.size} disabled URLs from scraping`);
+        }
+      }
+
       // Merge custom URLs if database and shopId are available
       if (this.db && shopId) {
         const customUrls = this.db.getEnabledCustomUrls(shopId);
@@ -270,6 +285,13 @@ export class WebshopScraper {
           urlsFound: urls.length,
           pageCount
         });
+
+        // Cleanup disabled URLs that are no longer in sitemap
+        const currentSitemapUrls = urls.map((u: any) => u.loc);
+        const deletedCount = this.db.cleanupMissingDisabledUrls(shopId, currentSitemapUrls);
+        if (deletedCount > 0) {
+          logger.info(`Cleaned up ${deletedCount} disabled URLs that are no longer in sitemap`);
+        }
       }
 
       const result: ScraperResult = {