feat: add custom URL management per shop/category #4

Implement ability to add, manage, and scrape custom URLs for each shop.

Features:
- Add custom_urls table to database with unique constraint
- API endpoints for CRUD operations on custom URLs
- Automatic merging of custom URLs with sitemap URLs during scraping
- URL validation (format, domain matching, duplicate detection)
- Enable/disable functionality for custom URLs
- Check if URL already scraped from sitemap

New endpoints:
- POST /shops/:id/custom-urls - Add custom URL
- GET /shops/:id/custom-urls - List custom URLs
- PATCH /shops/:id/custom-urls/:id - Enable/disable URL
- DELETE /shops/:id/custom-urls/:id - Remove custom URL

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/routes.ts

@@ -614,6 +614,222 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
     }
   });
 
+  /**
+   * POST /shops/:id/custom-urls - Add a custom URL to scrape for a shop
+   */
+  router.post('/shops/:id/custom-urls', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { url, content_type } = req.body;
+
+      // Validate required fields
+      if (!url || !content_type) {
+        res.status(400).json({ error: 'url and content_type fields are required' });
+        return;
+      }
+
+      // Validate content_type
+      const validContentTypes = ['shipping', 'contacts', 'terms', 'faq'];
+      if (!validContentTypes.includes(content_type)) {
+        res.status(400).json({
+          error: 'content_type must be one of: shipping, contacts, terms, faq'
+        });
+        return;
+      }
+
+      // Validate URL format
+      let parsedUrl: URL;
+      try {
+        parsedUrl = new URL(url);
+      } catch (error) {
+        res.status(400).json({ error: 'Invalid URL format' });
+        return;
+      }
+
+      // Check if shop exists
+      const shop = db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Validate that the URL belongs to the same domain as the shop
+      const shopUrl = new URL(shop.url);
+      if (parsedUrl.hostname !== shopUrl.hostname) {
+        res.status(400).json({
+          error: `URL must belong to the same domain as the shop (${shopUrl.hostname})`
+        });
+        return;
+      }
+
+      // Check if URL was already scraped
+      const alreadyScraped = db.isUrlAlreadyScraped(id, url);
+      if (alreadyScraped) {
+        res.status(409).json({
+          error: 'This URL has already been scraped from the sitemap',
+          message: 'URL already exists in scraped content'
+        });
+        return;
+      }
+
+      // Create custom URL
+      try {
+        const customUrl = db.createCustomUrl(id, url, content_type);
+
+        res.status(201).json({
+          message: 'Custom URL added successfully',
+          custom_url: {
+            id: customUrl.id,
+            shop_id: customUrl.shop_id,
+            url: customUrl.url,
+            content_type: customUrl.content_type,
+            enabled: customUrl.enabled,
+            created_at: customUrl.created_at
+          }
+        });
+      } catch (error: any) {
+        if (error.message?.includes('already been added')) {
+          res.status(409).json({ error: error.message });
+          return;
+        }
+        throw error;
+      }
+    } catch (error) {
+      logger.error('Error creating custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
+  /**
+   * GET /shops/:id/custom-urls - List all custom URLs for a shop
+   */
+  router.get('/shops/:id/custom-urls', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = db.getShopById(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrls = db.getCustomUrls(id);
+
+      res.json({
+        shop_id: id,
+        custom_urls: customUrls.map(cu => ({
+          id: cu.id,
+          url: cu.url,
+          content_type: cu.content_type,
+          enabled: cu.enabled,
+          created_at: cu.created_at
+        }))
+      });
+    } catch (error) {
+      logger.error('Error fetching custom URLs', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
+  /**
+   * PATCH /shops/:id/custom-urls/:customUrlId - Enable or disable a custom URL
+   */
+  router.patch('/shops/:id/custom-urls/:customUrlId', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id, customUrlId } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      const shop = db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrl = db.getCustomUrlById(customUrlId);
+      if (!customUrl) {
+        res.status(404).json({ error: 'Custom URL not found' });
+        return;
+      }
+
+      if (customUrl.shop_id !== id) {
+        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
+        return;
+      }
+
+      db.setCustomUrlEnabled(customUrlId, enabled);
+
+      res.json({
+        message: `Custom URL ${enabled ? 'enabled' : 'disabled'} successfully`,
+        custom_url_id: customUrlId,
+        enabled
+      });
+    } catch (error) {
+      logger.error('Error updating custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
+  /**
+   * DELETE /shops/:id/custom-urls/:customUrlId - Delete a custom URL
+   */
+  router.delete('/shops/:id/custom-urls/:customUrlId', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id, customUrlId } = req.params;
+
+      const shop = db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      const customUrl = db.getCustomUrlById(customUrlId);
+      if (!customUrl) {
+        res.status(404).json({ error: 'Custom URL not found' });
+        return;
+      }
+
+      if (customUrl.shop_id !== id) {
+        res.status(403).json({ error: 'Custom URL does not belong to this shop' });
+        return;
+      }
+
+      db.deleteCustomUrl(customUrlId);
+
+      res.json({
+        message: 'Custom URL deleted successfully',
+        custom_url_id: customUrlId
+      });
+    } catch (error) {
+      logger.error('Error deleting custom URL', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
   /**
    * GET /health - Health check endpoint
    */

src/database/Database.ts

@@ -82,6 +82,15 @@ export interface WebhookDelivery {
   attempted_at: string;
 }
 
+export interface CustomUrl {
+  id: string;
+  shop_id: string;
+  url: string;
+  content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+  enabled: boolean;
+  created_at: string;
+}
+
 export class ShopDatabase {
   private db: Database.Database;
 
@@ -207,6 +216,25 @@ export class ShopDatabase {
       )
     `);
 
+    // Custom URLs table (for manually added URLs per shop)
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS custom_urls (
+        id TEXT PRIMARY KEY,
+        shop_id TEXT NOT NULL,
+        url TEXT NOT NULL,
+        content_type TEXT NOT NULL,
+        enabled INTEGER DEFAULT 1,
+        created_at TEXT NOT NULL,
+        FOREIGN KEY (shop_id) REFERENCES shops(id) ON DELETE CASCADE
+      )
+    `);
+
+    // Create unique index to prevent duplicate URLs per shop
+    this.db.exec(`
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_custom_urls_shop_url
+      ON custom_urls(shop_id, url)
+    `);
+
     logger.info('Database tables initialized');
   }
 
@@ -667,4 +695,114 @@ export class ShopDatabase {
     this.db.close();
     logger.info('Database connection closed');
   }
+
+  // Custom URLs operations
+  createCustomUrl(
+    shopId: string,
+    url: string,
+    contentType: 'shipping' | 'contacts' | 'terms' | 'faq'
+  ): CustomUrl {
+    const id = uuidv4();
+    const now = new Date().toISOString();
+
+    try {
+      this.db.prepare(`
+        INSERT INTO custom_urls (id, shop_id, url, content_type, enabled, created_at)
+        VALUES (?, ?, ?, ?, 1, ?)
+      `).run(id, shopId, url, contentType, now);
+
+      logger.info(`Created custom URL ${id} for shop ${shopId}: ${url}`);
+
+      return {
+        id,
+        shop_id: shopId,
+        url,
+        content_type: contentType,
+        enabled: true,
+        created_at: now
+      };
+    } catch (error: any) {
+      if (error.code === 'SQLITE_CONSTRAINT_UNIQUE' || error.message?.includes('UNIQUE constraint')) {
+        throw new Error('This URL has already been added to this shop');
+      }
+      throw error;
+    }
+  }
+
+  getCustomUrls(shopId: string): CustomUrl[] {
+    const stmt = this.db.prepare(`
+      SELECT * FROM custom_urls
+      WHERE shop_id = ?
+      ORDER BY created_at DESC
+    `);
+    const results = stmt.all(shopId) as any[];
+
+    return results.map(r => ({
+      id: r.id,
+      shop_id: r.shop_id,
+      url: r.url,
+      content_type: r.content_type,
+      enabled: Boolean(r.enabled),
+      created_at: r.created_at
+    }));
+  }
+
+  getCustomUrlById(id: string): CustomUrl | null {
+    const stmt = this.db.prepare('SELECT * FROM custom_urls WHERE id = ?');
+    const result = stmt.get(id) as any;
+
+    if (!result) return null;
+
+    return {
+      id: result.id,
+      shop_id: result.shop_id,
+      url: result.url,
+      content_type: result.content_type,
+      enabled: Boolean(result.enabled),
+      created_at: result.created_at
+    };
+  }
+
+  getEnabledCustomUrls(shopId: string): CustomUrl[] {
+    const stmt = this.db.prepare(`
+      SELECT * FROM custom_urls
+      WHERE shop_id = ? AND enabled = 1
+      ORDER BY created_at DESC
+    `);
+    const results = stmt.all(shopId) as any[];
+
+    return results.map(r => ({
+      id: r.id,
+      shop_id: r.shop_id,
+      url: r.url,
+      content_type: r.content_type,
+      enabled: true,
+      created_at: r.created_at
+    }));
+  }
+
+  setCustomUrlEnabled(id: string, enabled: boolean): void {
+    this.db.prepare(`
+      UPDATE custom_urls
+      SET enabled = ?
+      WHERE id = ?
+    `).run(enabled ? 1 : 0, id);
+
+    logger.info(`${enabled ? 'Enabled' : 'Disabled'} custom URL ${id}`);
+  }
+
+  deleteCustomUrl(id: string): void {
+    this.db.prepare('DELETE FROM custom_urls WHERE id = ?').run(id);
+    logger.info(`Deleted custom URL ${id}`);
+  }
+
+  // Check if a URL exists in shop_content (already scraped)
+  isUrlAlreadyScraped(shopId: string, url: string): boolean {
+    const stmt = this.db.prepare(`
+      SELECT COUNT(*) as count FROM shop_content
+      WHERE shop_id = ? AND url = ?
+    `);
+    const result = stmt.get(shopId, url) as { count: number };
+    return result.count > 0;
+  }
 }

src/scraper/WebshopScraper.ts

@@ -47,6 +47,48 @@ export class WebshopScraper {
       // Filter URLs by page type
       const filteredUrls = SitemapParser.filterUrlsByType(urls);
 
+      // Merge custom URLs if database and shopId are available
+      if (this.db && shopId) {
+        const customUrls = this.db.getEnabledCustomUrls(shopId);
+
+        // Group custom URLs by content type
+        const customUrlsByType: { [key: string]: any[] } = {
+          shipping: [],
+          contacts: [],
+          terms: [],
+          faq: []
+        };
+
+        for (const customUrl of customUrls) {
+          const contentTypeKey = customUrl.content_type === 'shipping' ? 'shipping' :
+                                customUrl.content_type === 'contacts' ? 'contacts' :
+                                customUrl.content_type === 'terms' ? 'terms' :
+                                'faq';
+
+          // Check if this URL already exists in sitemap URLs
+          const alreadyExists = filteredUrls[contentTypeKey].some((u: any) => u.loc === customUrl.url);
+
+          if (!alreadyExists) {
+            // Add custom URL in the same format as sitemap URLs
+            customUrlsByType[contentTypeKey].push({
+              loc: customUrl.url,
+              lastmod: undefined,
+              changefreq: undefined,
+              priority: undefined
+            });
+            logger.info(`Added custom URL to ${contentTypeKey}: ${customUrl.url}`);
+          } else {
+            logger.info(`Custom URL already in sitemap, skipping: ${customUrl.url}`);
+          }
+        }
+
+        // Merge custom URLs with sitemap URLs
+        filteredUrls.shipping = [...filteredUrls.shipping, ...customUrlsByType.shipping];
+        filteredUrls.contacts = [...filteredUrls.contacts, ...customUrlsByType.contacts];
+        filteredUrls.terms = [...filteredUrls.terms, ...customUrlsByType.terms];
+        filteredUrls.faq = [...filteredUrls.faq, ...customUrlsByType.faq];
+      }
+
       logger.info('Filtered URLs', {
         shipping: filteredUrls.shipping.length,
         contacts: filteredUrls.contacts.length,