feat: add results endpoint, schedule control, and array-based content structure #2

- Create separate GET /api/shops/:id/results endpoint with limit and date filters
- Add PATCH /api/shops/:id/schedule to enable/disable scheduled scraping
- Add DELETE /api/shops/:id to delete shops with cascade deletion
- Modify results to always use arrays for all categories (supporting multiple pages)
- Update database schema to include enabled field for scheduled jobs
- Refactor content extraction to support multiple pages per category

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/api/routes.ts

@@ -159,7 +159,7 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
   });
 
   /**
-   * GET /shops/:id - Get detailed shop information with analytics
+   * GET /shops/:id - Get detailed shop information with analytics (without full results)
    */
   router.get('/shops/:id', (req: Request, res: Response): void => {
     try {
@@ -182,21 +182,35 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
       // Get scrape history
       const scrapeHistory = db.getScrapeHistory(id, 10);
 
-      // Get latest content
-      const latestContent = db.getLatestContent(id);
+      // Get latest content (without full content, just metadata)
+      const latestContent = db.getLatestContentByType(id);
 
       // Get scheduled jobs
       const scheduledJobs = db.getScheduledJobs(id);
 
-      // Build response with content change markers
-      const contentWithChanges: any = {};
-      for (const [type, content] of Object.entries(latestContent)) {
-        contentWithChanges[type] = {
-          url: content.url,
-          changed: content.changed === 1,
-          last_updated: content.updated_at
-        };
-      }
+      // Build response with content change markers and URLs only (no content)
+      const contentMetadata: any = {
+        shipping_informations: latestContent.shipping.map(c => ({
+          url: c.url,
+          changed: c.changed === 1,
+          last_updated: c.updated_at
+        })),
+        contacts: latestContent.contacts.map(c => ({
+          url: c.url,
+          changed: c.changed === 1,
+          last_updated: c.updated_at
+        })),
+        terms_of_conditions: latestContent.terms.map(c => ({
+          url: c.url,
+          changed: c.changed === 1,
+          last_updated: c.updated_at
+        })),
+        faq: latestContent.faq.map(c => ({
+          url: c.url,
+          changed: c.changed === 1,
+          last_updated: c.updated_at
+        }))
+      };
 
       res.json({
         shop: {
@@ -215,7 +229,7 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
           average_scrape_time: analytics?.average_scrape_time || null,
           average_page_scrape_time: analytics?.average_page_scrape_time || null
         },
-        content: contentWithChanges,
+        content_metadata: contentMetadata,
         scrape_history: scrapeHistory.map(h => ({
           id: h.id,
           job_id: h.job_id,
@@ -232,6 +246,7 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
           frequency: j.frequency,
           last_modified: j.last_modified,
           status: j.status,
+          enabled: j.enabled,
           created_at: j.created_at
         }))
       });
@@ -241,6 +256,160 @@ export function createRouter(jobQueue: JobQueue, db?: ShopDatabase): Router {
     }
   });
 
+  /**
+   * GET /shops/:id/results - Get shop results with full content (supports filters)
+   */
+  router.get('/shops/:id/results', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = db.getShopById(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      // Parse query parameters
+      const limit = req.query.limit ? parseInt(req.query.limit as string) : undefined;
+      const dateFrom = req.query.date_from as string | undefined;
+      const dateTo = req.query.date_to as string | undefined;
+      const contentType = req.query.content_type as string | undefined;
+
+      // Get content with filters
+      const allContent = db.getAllContent(id, {
+        limit,
+        dateFrom,
+        dateTo,
+        contentType
+      });
+
+      // Group by type and URL (latest version of each URL)
+      const grouped: { [key: string]: any[] } = {
+        shipping_informations: [],
+        contacts: [],
+        terms_of_conditions: [],
+        faq: []
+      };
+
+      const urlMap = new Map<string, any>();
+      for (const content of allContent) {
+        const key = `${content.content_type}:${content.url}`;
+        if (!urlMap.has(key)) {
+          urlMap.set(key, {
+            url: content.url,
+            content: content.content,
+            changed: content.changed === 1,
+            last_updated: content.updated_at
+          });
+        }
+      }
+
+      // Organize by type
+      for (const content of allContent) {
+        const key = `${content.content_type}:${content.url}`;
+        const urlKey = urlMap.get(key);
+
+        if (!urlKey) continue;
+
+        const typeKey = content.content_type === 'shipping' ? 'shipping_informations' :
+                       content.content_type === 'contacts' ? 'contacts' :
+                       content.content_type === 'terms' ? 'terms_of_conditions' :
+                       'faq';
+
+        // Only add if not already added
+        if (!grouped[typeKey].find((c: any) => c.url === urlKey.url)) {
+          grouped[typeKey].push(urlKey);
+        }
+      }
+
+      res.json({
+        shop_id: id,
+        filters: {
+          limit: limit || null,
+          date_from: dateFrom || null,
+          date_to: dateTo || null,
+          content_type: contentType || null
+        },
+        results: grouped
+      });
+    } catch (error) {
+      logger.error('Error fetching shop results', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
+  /**
+   * PATCH /shops/:id/schedule - Enable or disable scheduled scraping
+   */
+  router.patch('/shops/:id/schedule', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const { enabled } = req.body;
+
+      if (typeof enabled !== 'boolean') {
+        res.status(400).json({ error: 'enabled field must be a boolean' });
+        return;
+      }
+
+      const shop = db.getShopById(id);
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      db.setScheduleEnabled(id, enabled);
+
+      res.json({
+        shop_id: id,
+        schedule_enabled: enabled,
+        message: `Schedule ${enabled ? 'enabled' : 'disabled'} successfully`
+      });
+    } catch (error) {
+      logger.error('Error updating schedule status', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
+  /**
+   * DELETE /shops/:id - Delete a shop and all related data
+   */
+  router.delete('/shops/:id', (req: Request, res: Response): void => {
+    try {
+      if (!db) {
+        res.status(503).json({ error: 'Database not available' });
+        return;
+      }
+
+      const { id } = req.params;
+      const shop = db.getShopById(id);
+
+      if (!shop) {
+        res.status(404).json({ error: 'Shop not found' });
+        return;
+      }
+
+      db.deleteShop(id);
+
+      res.json({
+        message: 'Shop and all related data deleted successfully',
+        shop_id: id
+      });
+    } catch (error) {
+      logger.error('Error deleting shop', error);
+      res.status(500).json({ error: 'Internal server error' });
+    }
+  });
+
   /**
    * GET /health - Health check endpoint
    */

src/database/Database.ts

@@ -46,6 +46,10 @@ export interface ShopContent {
   updated_at: string;
 }
 
+export interface ShopContentWithResults extends ShopContent {
+  // For returning full content in results endpoint
+}
+
 export interface ScheduledJob {
   id: string;
   shop_id: string;
@@ -53,6 +57,7 @@ export interface ScheduledJob {
   frequency: string | null; // from sitemap changefreq
   last_modified: string | null; // from sitemap lastmod
   status: 'queued' | 'running' | 'completed' | 'failed';
+  enabled: boolean; // whether the scheduled job is enabled or disabled
   created_at: string;
 }
 
@@ -146,6 +151,7 @@ export class ShopDatabase {
         frequency TEXT,
         last_modified TEXT,
         status TEXT NOT NULL,
+        enabled INTEGER DEFAULT 1,
         created_at TEXT NOT NULL,
         FOREIGN KEY (shop_id) REFERENCES shops(id) ON DELETE CASCADE
       )
@@ -211,6 +217,12 @@ export class ShopDatabase {
     `).run(...values);
   }
 
+  deleteShop(id: string): void {
+    // Cascade delete will handle related records
+    this.db.prepare('DELETE FROM shops WHERE id = ?').run(id);
+    logger.info(`Deleted shop ${id} and all related data`);
+  }
+
   // Analytics operations
   getShopAnalytics(shopId: string): ShopAnalytics | null {
     const stmt = this.db.prepare('SELECT * FROM shop_analytics WHERE shop_id = ?');
@@ -352,6 +364,41 @@ export class ShopDatabase {
     return result;
   }
 
+  // Get all latest content grouped by type (for array results)
+  getLatestContentByType(shopId: string): { [key: string]: ShopContent[] } {
+    const stmt = this.db.prepare(`
+      SELECT * FROM shop_content
+      WHERE shop_id = ?
+      ORDER BY content_type, created_at DESC
+    `);
+
+    const rows = stmt.all(shopId) as ShopContent[];
+    const result: { [key: string]: ShopContent[] } = {
+      shipping: [],
+      contacts: [],
+      terms: [],
+      faq: []
+    };
+
+    // Group by content_type and URL (latest version of each URL)
+    const urlMap = new Map<string, ShopContent>();
+    for (const row of rows) {
+      const key = `${row.content_type}:${row.url}`;
+      if (!urlMap.has(key)) {
+        urlMap.set(key, row);
+      }
+    }
+
+    // Organize by type
+    for (const content of urlMap.values()) {
+      if (result[content.content_type]) {
+        result[content.content_type].push(content);
+      }
+    }
+
+    return result;
+  }
+
   getContentHistory(shopId: string, contentType: string, limit: number = 5): ShopContent[] {
     const stmt = this.db.prepare(`
       SELECT * FROM shop_content
@@ -362,14 +409,53 @@ export class ShopDatabase {
     return stmt.all(shopId, contentType, limit) as ShopContent[];
   }
 
+  // Get all content with optional filters
+  getAllContent(
+    shopId: string,
+    options?: {
+      limit?: number;
+      dateFrom?: string;
+      dateTo?: string;
+      contentType?: string;
+    }
+  ): ShopContent[] {
+    let query = 'SELECT * FROM shop_content WHERE shop_id = ?';
+    const params: any[] = [shopId];
+
+    if (options?.contentType) {
+      query += ' AND content_type = ?';
+      params.push(options.contentType);
+    }
+
+    if (options?.dateFrom) {
+      query += ' AND created_at >= ?';
+      params.push(options.dateFrom);
+    }
+
+    if (options?.dateTo) {
+      query += ' AND created_at <= ?';
+      params.push(options.dateTo);
+    }
+
+    query += ' ORDER BY created_at DESC';
+
+    if (options?.limit) {
+      query += ' LIMIT ?';
+      params.push(options.limit);
+    }
+
+    const stmt = this.db.prepare(query);
+    return stmt.all(...params) as ShopContent[];
+  }
+
   // Scheduled jobs operations
   createScheduledJob(shopId: string, nextRunAt: string, frequency: string | null, lastModified: string | null): string {
     const id = uuidv4();
     const now = new Date().toISOString();
 
     this.db.prepare(`
-      INSERT INTO scheduled_jobs (id, shop_id, next_run_at, frequency, last_modified, status, created_at)
-      VALUES (?, ?, ?, ?, ?, 'queued', ?)
+      INSERT INTO scheduled_jobs (id, shop_id, next_run_at, frequency, last_modified, status, enabled, created_at)
+      VALUES (?, ?, ?, ?, ?, 'queued', 1, ?)
     `).run(id, shopId, nextRunAt, frequency, lastModified, now);
 
     return id;
@@ -388,7 +474,7 @@ export class ShopDatabase {
     const now = new Date().toISOString();
     const stmt = this.db.prepare(`
       SELECT * FROM scheduled_jobs
-      WHERE next_run_at <= ? AND status = 'queued'
+      WHERE next_run_at <= ? AND status = 'queued' AND enabled = 1
       ORDER BY next_run_at ASC
     `);
     return stmt.all(now) as ScheduledJob[];
@@ -405,6 +491,16 @@ export class ShopDatabase {
     `).run(...values);
   }
 
+  // Enable or disable schedule for a shop
+  setScheduleEnabled(shopId: string, enabled: boolean): void {
+    this.db.prepare(`
+      UPDATE scheduled_jobs
+      SET enabled = ?
+      WHERE shop_id = ? AND status = 'queued'
+    `).run(enabled ? 1 : 0, shopId);
+    logger.info(`${enabled ? 'Enabled' : 'Disabled'} schedule for shop ${shopId}`);
+  }
+
   close(): void {
     this.db.close();
     logger.info('Database connection closed');

src/scraper/WebshopScraper.ts

@@ -51,30 +51,34 @@ export class WebshopScraper {
         faq: filteredUrls.faq.length
       });
 
-      // Extract content from each page type
-      const shippingContent = await this.extractPageContent(filteredUrls.shipping);
-      const contactsContent = await this.extractPageContent(filteredUrls.contacts);
-      const termsContent = await this.extractPageContent(filteredUrls.terms);
-      const faqContent = await this.extractPageContent(filteredUrls.faq);
+      // Extract content from each page type - now supporting multiple pages per category
+      const shippingContent = await this.extractAllPageContent(filteredUrls.shipping);
+      const contactsContent = await this.extractAllPageContent(filteredUrls.contacts);
+      const termsContent = await this.extractAllPageContent(filteredUrls.terms);
+      const faqContent = await this.extractAllPageContent(filteredUrls.faq);
 
       // Save content to database if available
       if (this.db && shopId) {
-        if (shippingContent) {
-          this.db.saveContent(shopId, 'shipping', shippingContent.url, shippingContent.content);
+        // Save all shipping pages
+        for (const content of shippingContent) {
+          this.db.saveContent(shopId, 'shipping', content.url, content.content);
         }
-        if (contactsContent) {
-          this.db.saveContent(shopId, 'contacts', contactsContent.url, contactsContent.content);
+        // Save all contact pages
+        for (const content of contactsContent) {
+          this.db.saveContent(shopId, 'contacts', content.url, content.content);
         }
-        if (termsContent) {
-          this.db.saveContent(shopId, 'terms', termsContent.url, termsContent.content);
+        // Save all terms pages
+        for (const content of termsContent) {
+          this.db.saveContent(shopId, 'terms', content.url, content.content);
         }
-        if (faqContent) {
-          this.db.saveContent(shopId, 'faq', faqContent.url, faqContent.content);
+        // Save all FAQ pages
+        for (const content of faqContent) {
+          this.db.saveContent(shopId, 'faq', content.url, content.content);
         }
 
         // Update analytics
         const scrapeTimeMs = Date.now() - startTime;
-        const pageCount = [shippingContent, contactsContent, termsContent, faqContent].filter(c => c !== null).length;
+        const pageCount = shippingContent.length + contactsContent.length + termsContent.length + faqContent.length;
 
         this.db.updateAnalytics(shopId, {
           scrapeTimeMs,
@@ -100,7 +104,27 @@ export class WebshopScraper {
   }
 
   /**
-   * Extract content from the best matching URL
+   * Extract content from all URLs in a category
+   */
+  private async extractAllPageContent(urls: any[]): Promise<PageContent[]> {
+    const results: PageContent[] = [];
+
+    for (const urlObj of urls) {
+      try {
+        const url = urlObj.loc;
+        const content = await this.contentExtractor.extractContent(url);
+        results.push({ url, content });
+      } catch (error) {
+        logger.error(`Failed to extract content from ${urlObj.loc}`, error);
+        // Continue with next URL even if one fails
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Extract content from the best matching URL (deprecated - kept for backwards compatibility)
    */
   private async extractPageContent(urls: any[]): Promise<PageContent | null> {
     if (urls.length === 0) {

src/types/index.ts

@@ -12,10 +12,10 @@ export interface ScraperJob {
 
 export interface ScraperResult {
   initial_sitemap: string;
-  shipping_informations: PageContent | null;
-  contacts: PageContent | null;
-  terms_of_conditions: PageContent | null;
-  faq: PageContent | null;
+  shipping_informations: PageContent[];
+  contacts: PageContent[];
+  terms_of_conditions: PageContent[];
+  faq: PageContent[];
 }
 
 export interface PageContent {