webshop-scraper/src/index.ts

import { loadConfig } from './config';
import { createServer } from './api/server';
import { JobQueue } from './queue/JobQueue';
import { WebshopScraper } from './scraper/WebshopScraper';
import { logger } from './utils/logger';
import { ShopDatabase } from './database/Database';
import { ScrapeScheduler } from './scheduler/ScrapeScheduler';
import { WebhookManager } from './webhooks/WebhookManager';

async function main() {
  try {
    logger.info('Starting Webshop Scraper application...');

    // Load configuration
    const config = loadConfig();

    // Initialize database
    const db = new ShopDatabase();

    // Initialize job queue
    const jobQueue = new JobQueue(config.maxConcurrentJobs);

    // Initialize scraper with database
    const scraper = new WebshopScraper(db);

    // Initialize scheduler
    const scheduler = new ScrapeScheduler(db, jobQueue);

    // Initialize webhook manager
    const webhookManager = new WebhookManager(db);

    // Handle job processing
    jobQueue.on('process-job', async (job) => {
      logger.info(`Processing job ${job.id}`);

      const startTime = Date.now();
      let scrapeHistoryId: string | undefined;

      try {
        // Create scrape history record if shopId is available
        if (job.shopId) {
          scrapeHistoryId = db.createScrapeHistory(job.shopId, job.id);
          db.updateScrapeHistory(scrapeHistoryId, { status: 'processing' });

          // Trigger webhook: scrape_started
          await webhookManager.notifyScrapeStarted(
            job.shopId,
            job.id,
            job.sitemapUrl
          );
        }

        const result = await scraper.scrape(job.sitemapUrl, job.shopId);

        jobQueue.updateJob(job.id, {
          status: 'completed',
          result
        });

        // Update scrape history
        if (scrapeHistoryId) {
          const scrapeTimeMs = Date.now() - startTime;
          db.updateScrapeHistory(scrapeHistoryId, {
            status: 'completed',
            completed_at: new Date().toISOString(),
            scrape_time_ms: scrapeTimeMs
          });
        }

        // Trigger webhook: scrape_completed
        if (job.shopId) {
          const totalUrls = (result.shipping_informations?.length || 0) +
                           (result.contacts?.length || 0) +
                           (result.terms_of_conditions?.length || 0) +
                           (result.faq?.length || 0);

          const totalPages = totalUrls;

          await webhookManager.notifyScrapeCompleted(
            job.shopId,
            job.id,
            {
              scrape_time_ms: Date.now() - startTime,
              urls_found: totalUrls,
              pages_scraped: totalPages
            }
          );
        }

        // Schedule next scrape if this is a new shop or first scrape
        if (job.shopId) {
          const existingSchedules = db.getScheduledJobs(job.shopId);
          const hasQueuedJobs = existingSchedules.some(s => s.status === 'queued');

          if (!hasQueuedJobs) {
            // Create initial schedule for this shop
            scheduler.createInitialSchedule(job.shopId, 'weekly', null);
            logger.info(`Created initial schedule for shop ${job.shopId}`);
          }
        }

        logger.info(`Job ${job.id} completed successfully`);
      } catch (error) {
        const errorMessage = error instanceof Error ? error.message : 'Unknown error';
        jobQueue.updateJob(job.id, {
          status: 'failed',
          error: errorMessage
        });

        // Update scrape history with error
        if (scrapeHistoryId) {
          db.updateScrapeHistory(scrapeHistoryId, {
            status: 'failed',
            completed_at: new Date().toISOString(),
            error: errorMessage
          });
        }

        // Trigger webhook: scrape_failed
        if (job.shopId) {
          await webhookManager.notifyScrapeFailed(
            job.shopId,
            job.id,
            errorMessage
          );
        }

        logger.error(`Job ${job.id} failed`, error);
      }
    });

    // Create and start HTTP server
    const app = createServer(config.apiKey, jobQueue, db);

    const server = app.listen(config.port, () => {
      logger.info(`Server listening on port ${config.port}`);
      logger.info('API endpoints:');
      logger.info(`  POST   /api/jobs        - Create new scraping job`);
      logger.info(`  GET    /api/jobs/:id    - Get job status and result`);
      logger.info(`  GET    /api/jobs        - List all jobs`);
      logger.info(`  GET    /api/shops       - List all shops`);
      logger.info(`  GET    /api/shops/:id   - Get shop details and analytics`);
      logger.info(`  GET    /health          - Health check`);
      logger.info('');
      logger.info('Authentication: Bearer token required for /api/* endpoints');
    });

    // Start scheduler
    scheduler.start();

    // Graceful shutdown
    const shutdown = async () => {
      logger.info('Shutting down gracefully...');

      // Stop scheduler
      scheduler.stop();

      // Close database
      db.close();

      server.close(() => {
        logger.info('Server closed');
        process.exit(0);
      });

      // Force shutdown after 10 seconds
      setTimeout(() => {
        logger.error('Forced shutdown after timeout');
        process.exit(1);
      }, 10000);
    };

    process.on('SIGTERM', shutdown);
    process.on('SIGINT', shutdown);

  } catch (error) {
    logger.error('Failed to start application', error);
    process.exit(1);
  }
}

main();