feat: implement webshop scraper integration backend

- Add scraper API URL and secret environment variables to both frontend and backend
- Create database migration for scraper columns in stores table
- Implement comprehensive scraper API client with multi-scraper support
- Create webhook handler Edge Function for scraper status updates
- Create scraper management Edge Function with full CRUD operations
- Support per-store scraper configuration with global fallback
- Include domain validation for custom URLs
- Ready for integration with store connection flows

shopcall.ai-main/.env.example

@@ -20,3 +20,9 @@ VITE_FRONTEND_URL=http://localhost:8080
 # and stores will use api_only mode by default (direct API access, no local caching)
 VITE_HIDE_ORDERS_ACCESS_SETTINGS=true
 VITE_HIDE_CUSTOMERS_ACCESS_SETTINGS=true
+
+# Webshop Scraper Configuration
+# Default scraper API URL (can be overridden per store)
+# For local development: http://127.0.0.1:3000
+# For production: https://your-scraper-api.com
+VITE_DEFAULT_SCRAPER_API_URL=http://127.0.0.1:3000

supabase/.env.example

@@ -47,6 +47,15 @@ INTERNAL_SYNC_SECRET=your_random_secure_secret_here
 # Get this from: https://resend.com/api-keys
 RESEND_API_KEY=your_resend_api_key_here
 
+# -----------------------------------------------------------------------------
+# Webshop Scraper Configuration
+# -----------------------------------------------------------------------------
+# Default scraper API configuration (can be overridden per store)
+# For local development: http://127.0.0.1:3000
+# For production: https://your-scraper-api.com
+DEFAULT_SCRAPER_API_URL=http://127.0.0.1:3000
+DEFAULT_SCRAPER_API_SECRET=your_shared_secret_key_here
+
 # -----------------------------------------------------------------------------
 # Database Configuration (for pg_cron scheduled sync)
 # -----------------------------------------------------------------------------

supabase/functions/_shared/scraper-client.ts

@@ -0,0 +1,344 @@
+/**
+ * Webshop Scraper API Client
+ *
+ * Handles communication with the webshop scraper service including:
+ * - Shop registration and management
+ * - Content retrieval and filtering
+ * - Custom URL management
+ * - Webhook configuration
+ * - Multi-scraper support with per-store configuration
+ */
+
+export interface ScraperConfig {
+  apiUrl: string;
+  apiSecret: string;
+}
+
+export interface ScraperJob {
+  id: string;
+  status: 'pending' | 'in_progress' | 'completed' | 'failed';
+  url: string;
+  custom_id?: string;
+  created_at: string;
+  updated_at: string;
+  error?: string;
+}
+
+export interface ScraperShop {
+  id: string;
+  url: string;
+  custom_id?: string;
+  status: 'active' | 'inactive';
+  last_scraped_at?: string;
+  next_scheduled_scrape?: string;
+  scheduled_enabled: boolean;
+  total_urls_found: number;
+  total_content_items: number;
+  created_at: string;
+  updated_at: string;
+}
+
+export interface ScraperContent {
+  id: string;
+  url: string;
+  content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+  title?: string;
+  content: string;
+  scraped_at: string;
+  metadata?: Record<string, any>;
+}
+
+export interface ScraperCustomUrl {
+  id: string;
+  url: string;
+  content_type: 'shipping' | 'contacts' | 'terms' | 'faq';
+  enabled: boolean;
+  created_at: string;
+  last_scraped_at?: string;
+  status?: 'pending' | 'completed' | 'failed';
+}
+
+export interface ScraperWebhook {
+  id: string;
+  url: string;
+  secret?: string;
+  enabled: boolean;
+  created_at: string;
+  last_delivery_at?: string;
+  total_deliveries: number;
+  failed_deliveries: number;
+}
+
+export interface ContentFilter {
+  content_type?: 'shipping' | 'contacts' | 'terms' | 'faq';
+  date_from?: string;
+  date_to?: string;
+  limit?: number;
+}
+
+/**
+ * Scraper API Client
+ */
+export class ScraperClient {
+  private config: ScraperConfig;
+
+  constructor(config: ScraperConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Get authentication headers for API requests
+   */
+  private getAuthHeaders(): Record<string, string> {
+    return {
+      'Authorization': `Bearer ${this.config.apiSecret}`,
+      'Content-Type': 'application/json',
+    };
+  }
+
+  /**
+   * Make authenticated API request
+   */
+  private async makeRequest<T>(
+    endpoint: string,
+    options: RequestInit = {}
+  ): Promise<T> {
+    const url = `${this.config.apiUrl}${endpoint}`;
+
+    const response = await fetch(url, {
+      ...options,
+      headers: {
+        ...this.getAuthHeaders(),
+        ...options.headers,
+      },
+    });
+
+    if (!response.ok) {
+      const errorText = await response.text();
+      throw new Error(`Scraper API error (${response.status}): ${errorText}`);
+    }
+
+    const contentType = response.headers.get('content-type');
+    if (contentType && contentType.includes('application/json')) {
+      return await response.json();
+    }
+
+    return response.text() as T;
+  }
+
+  /**
+   * Register a new shop for scraping
+   */
+  async registerShop(url: string, customId: string): Promise<ScraperJob> {
+    return await this.makeRequest<ScraperJob>('/api/jobs', {
+      method: 'POST',
+      body: JSON.stringify({
+        url,
+        custom_id: customId,
+      }),
+    });
+  }
+
+  /**
+   * Get job status by job ID
+   */
+  async getJobStatus(jobId: string): Promise<ScraperJob> {
+    return await this.makeRequest<ScraperJob>(`/api/jobs/${jobId}`);
+  }
+
+  /**
+   * List all jobs
+   */
+  async listJobs(): Promise<{ jobs: ScraperJob[], queue_stats: any }> {
+    return await this.makeRequest<{ jobs: ScraperJob[], queue_stats: any }>('/api/jobs');
+  }
+
+  /**
+   * Get shop information by shop ID (using store_id as custom_id)
+   */
+  async getShop(shopId: string): Promise<ScraperShop> {
+    return await this.makeRequest<ScraperShop>(`/api/shops/${shopId}`);
+  }
+
+  /**
+   * List all shops
+   */
+  async listShops(): Promise<{ shops: ScraperShop[] }> {
+    return await this.makeRequest<{ shops: ScraperShop[] }>('/api/shops');
+  }
+
+  /**
+   * Get scraped content for a shop with optional filtering
+   */
+  async getShopContent(shopId: string, filter?: ContentFilter): Promise<{ content: ScraperContent[] }> {
+    const params = new URLSearchParams();
+
+    if (filter?.content_type) {
+      params.append('content_type', filter.content_type);
+    }
+    if (filter?.date_from) {
+      params.append('date_from', filter.date_from);
+    }
+    if (filter?.date_to) {
+      params.append('date_to', filter.date_to);
+    }
+    if (filter?.limit) {
+      params.append('limit', filter.limit.toString());
+    }
+
+    const endpoint = `/api/shops/${shopId}/results${params.toString() ? `?${params.toString()}` : ''}`;
+    return await this.makeRequest<{ content: ScraperContent[] }>(endpoint);
+  }
+
+  /**
+   * Enable or disable scheduled scraping for a shop
+   */
+  async setScheduling(shopId: string, enabled: boolean): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/schedule`, {
+      method: 'PATCH',
+      body: JSON.stringify({ enabled }),
+    });
+  }
+
+  /**
+   * Set or update custom ID for a shop
+   */
+  async setCustomId(shopId: string, customId: string | null): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/custom-id`, {
+      method: 'PATCH',
+      body: JSON.stringify({ custom_id: customId }),
+    });
+  }
+
+  /**
+   * Delete a shop and all its data
+   */
+  async deleteShop(shopId: string): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}`, {
+      method: 'DELETE',
+    });
+  }
+
+  /**
+   * Configure webhook for a shop
+   */
+  async setWebhook(shopId: string, webhookUrl: string, secret?: string): Promise<ScraperWebhook> {
+    return await this.makeRequest<ScraperWebhook>(`/api/shops/${shopId}/webhooks`, {
+      method: 'POST',
+      body: JSON.stringify({
+        url: webhookUrl,
+        secret,
+      }),
+    });
+  }
+
+  /**
+   * Get webhook configuration for a shop
+   */
+  async getWebhook(shopId: string): Promise<ScraperWebhook> {
+    return await this.makeRequest<ScraperWebhook>(`/api/shops/${shopId}/webhooks`);
+  }
+
+  /**
+   * Enable or disable webhook for a shop
+   */
+  async setWebhookEnabled(shopId: string, enabled: boolean): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/webhooks`, {
+      method: 'PATCH',
+      body: JSON.stringify({ enabled }),
+    });
+  }
+
+  /**
+   * Delete webhook for a shop
+   */
+  async deleteWebhook(shopId: string): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/webhooks`, {
+      method: 'DELETE',
+    });
+  }
+
+  /**
+   * Add a custom URL to scrape for a shop
+   */
+  async addCustomUrl(shopId: string, url: string, contentType: ScraperCustomUrl['content_type']): Promise<ScraperCustomUrl> {
+    return await this.makeRequest<ScraperCustomUrl>(`/api/shops/${shopId}/custom-urls`, {
+      method: 'POST',
+      body: JSON.stringify({
+        url,
+        content_type: contentType,
+      }),
+    });
+  }
+
+  /**
+   * List all custom URLs for a shop
+   */
+  async listCustomUrls(shopId: string): Promise<{ custom_urls: ScraperCustomUrl[] }> {
+    return await this.makeRequest<{ custom_urls: ScraperCustomUrl[] }>(`/api/shops/${shopId}/custom-urls`);
+  }
+
+  /**
+   * Enable or disable a custom URL
+   */
+  async setCustomUrlEnabled(shopId: string, customUrlId: string, enabled: boolean): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/custom-urls/${customUrlId}`, {
+      method: 'PATCH',
+      body: JSON.stringify({ enabled }),
+    });
+  }
+
+  /**
+   * Delete a custom URL
+   */
+  async deleteCustomUrl(shopId: string, customUrlId: string): Promise<void> {
+    await this.makeRequest(`/api/shops/${shopId}/custom-urls/${customUrlId}`, {
+      method: 'DELETE',
+    });
+  }
+}
+
+/**
+ * Create a scraper client with configuration resolution
+ * Supports per-store configuration with fallback to global defaults
+ */
+export async function createScraperClient(storeConfig?: {
+  scraper_api_url?: string;
+  scraper_api_secret?: string;
+}): Promise<ScraperClient> {
+  // Use store-specific configuration if available
+  if (storeConfig?.scraper_api_url && storeConfig?.scraper_api_secret) {
+    return new ScraperClient({
+      apiUrl: storeConfig.scraper_api_url,
+      apiSecret: storeConfig.scraper_api_secret,
+    });
+  }
+
+  // Fall back to global configuration from environment
+  const apiUrl = Deno.env.get('DEFAULT_SCRAPER_API_URL');
+  const apiSecret = Deno.env.get('DEFAULT_SCRAPER_API_SECRET');
+
+  if (!apiUrl || !apiSecret) {
+    throw new Error('Scraper API configuration not found. Please set DEFAULT_SCRAPER_API_URL and DEFAULT_SCRAPER_API_SECRET environment variables or configure per-store settings.');
+  }
+
+  return new ScraperClient({
+    apiUrl,
+    apiSecret,
+  });
+}
+
+/**
+ * Utility function to validate if a URL belongs to the same domain as the store
+ */
+export function validateSameDomain(storeUrl: string, customUrl: string): boolean {
+  try {
+    const storeDomain = new URL(storeUrl).hostname.toLowerCase();
+    const customDomain = new URL(customUrl).hostname.toLowerCase();
+
+    // Allow exact matches and subdomains
+    return customDomain === storeDomain || customDomain.endsWith(`.${storeDomain}`);
+  } catch (error) {
+    return false; // Invalid URLs
+  }
+}

supabase/functions/scraper-management/index.ts

@@ -0,0 +1,420 @@
+/**
+ * Scraper Management API
+ *
+ * Provides endpoints for managing webshop scraping functionality:
+ * - Register/unregister shops with scraper
+ * - Get shop status and scraped content
+ * - Manage custom URLs
+ * - Configure webhooks and scheduling
+ * - Multi-scraper support with per-store configuration
+ */
+
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.39.3';
+import { createScraperClient, validateSameDomain } from '../_shared/scraper-client.ts';
+
+const corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+  'Access-Control-Allow-Methods': 'GET, POST, PATCH, DELETE, OPTIONS',
+};
+
+interface StoreRow {
+  id: string;
+  store_url: string;
+  scraper_api_url: string | null;
+  scraper_api_secret: string | null;
+  scraper_registered: boolean;
+  scraper_enabled: boolean;
+  user_id: string;
+}
+
+/**
+ * Get scraper configuration for a store
+ */
+async function getStoreScraperConfig(supabase: any, storeId: string, userId: string): Promise<StoreRow> {
+  const { data: store, error } = await supabase
+    .from('stores')
+    .select('id, store_url, scraper_api_url, scraper_api_secret, scraper_registered, scraper_enabled, user_id')
+    .eq('id', storeId)
+    .eq('user_id', userId)
+    .single();
+
+  if (error || !store) {
+    throw new Error('Store not found or access denied');
+  }
+
+  return store;
+}
+
+/**
+ * Update store scraper registration status
+ */
+async function updateStoreScraperStatus(
+  supabase: any,
+  storeId: string,
+  updates: { scraper_registered?: boolean; scraper_enabled?: boolean }
+) {
+  const { error } = await supabase
+    .from('stores')
+    .update(updates)
+    .eq('id', storeId);
+
+  if (error) {
+    throw new Error(`Failed to update store status: ${error.message}`);
+  }
+}
+
+Deno.serve(async (req) => {
+  // Handle CORS preflight requests
+  if (req.method === 'OPTIONS') {
+    return new Response(null, { headers: corsHeaders });
+  }
+
+  try {
+    // Initialize Supabase client
+    const supabaseUrl = Deno.env.get('SUPABASE_URL')!;
+    const supabaseKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!;
+    const supabase = createClient(supabaseUrl, supabaseKey);
+
+    // Get user from authorization header
+    const authHeader = req.headers.get('Authorization');
+    if (!authHeader) {
+      return new Response(
+        JSON.stringify({ error: 'Authorization header required' }),
+        { status: 401, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+      );
+    }
+
+    const { data: { user }, error: authError } = await supabase.auth.getUser(
+      authHeader.replace('Bearer ', '')
+    );
+
+    if (authError || !user) {
+      return new Response(
+        JSON.stringify({ error: 'Invalid authorization token' }),
+        { status: 401, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+      );
+    }
+
+    const url = new URL(req.url);
+    const pathParts = url.pathname.split('/').filter(Boolean);
+
+    // Route to different endpoints based on path
+    if (pathParts.length < 1) {
+      return new Response(
+        JSON.stringify({ error: 'Invalid endpoint' }),
+        { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+      );
+    }
+
+    const action = pathParts[0];
+
+    switch (action) {
+      case 'register-shop': {
+        // POST /register-shop
+        // Register a store with the scraper
+        if (req.method !== 'POST') {
+          return new Response(
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const { store_id } = await req.json();
+        if (!store_id) {
+          return new Response(
+            JSON.stringify({ error: 'store_id is required' }),
+            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const store = await getStoreScraperConfig(supabase, store_id, user.id);
+
+        if (store.scraper_registered) {
+          return new Response(
+            JSON.stringify({ success: true, message: 'Store already registered', shop_id: store_id }),
+            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        // Create scraper client with store configuration
+        const scraperClient = await createScraperClient({
+          scraper_api_url: store.scraper_api_url,
+          scraper_api_secret: store.scraper_api_secret,
+        });
+
+        // Register shop with scraper using store_id as custom_id
+        const job = await scraperClient.registerShop(store.store_url, store.id);
+
+        // Set up webhook
+        const frontendUrl = Deno.env.get('FRONTEND_URL');
+        const webhookUrl = `${Deno.env.get('SUPABASE_URL')}/functions/v1/scraper-webhook`;
+
+        try {
+          await scraperClient.setWebhook(store.id, webhookUrl);
+          console.log(`Webhook configured for store ${store.id}`);
+        } catch (webhookError) {
+          console.warn(`Failed to configure webhook for store ${store.id}:`, webhookError);
+          // Continue anyway - webhook is optional
+        }
+
+        // Enable scheduled scraping
+        try {
+          await scraperClient.setScheduling(store.id, true);
+          console.log(`Scheduled scraping enabled for store ${store.id}`);
+        } catch (scheduleError) {
+          console.warn(`Failed to enable scheduling for store ${store.id}:`, scheduleError);
+          // Continue anyway
+        }
+
+        // Update database
+        await updateStoreScraperStatus(supabase, store.id, { scraper_registered: true });
+
+        return new Response(
+          JSON.stringify({
+            success: true,
+            message: 'Store registered with scraper successfully',
+            job_id: job.id,
+            shop_id: store.id,
+          }),
+          { status: 201, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+        );
+      }
+
+      case 'shop-status': {
+        // GET /shop-status?store_id=xxx
+        if (req.method !== 'GET') {
+          return new Response(
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const storeId = url.searchParams.get('store_id');
+        if (!storeId) {
+          return new Response(
+            JSON.stringify({ error: 'store_id parameter is required' }),
+            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const store = await getStoreScraperConfig(supabase, storeId, user.id);
+
+        if (!store.scraper_registered) {
+          return new Response(
+            JSON.stringify({
+              registered: false,
+              enabled: store.scraper_enabled,
+              message: 'Store not registered with scraper',
+            }),
+            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        // Get shop status from scraper
+        const scraperClient = await createScraperClient({
+          scraper_api_url: store.scraper_api_url,
+          scraper_api_secret: store.scraper_api_secret,
+        });
+
+        try {
+          const shopData = await scraperClient.getShop(store.id);
+          return new Response(
+            JSON.stringify({
+              registered: true,
+              enabled: store.scraper_enabled,
+              shop_data: shopData,
+            }),
+            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        } catch (error) {
+          console.error(`Failed to get shop status for ${storeId}:`, error);
+          return new Response(
+            JSON.stringify({
+              registered: true,
+              enabled: store.scraper_enabled,
+              error: 'Failed to fetch shop data from scraper',
+              message: error.message,
+            }),
+            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+      }
+
+      case 'shop-content': {
+        // GET /shop-content?store_id=xxx&content_type=faq&limit=50
+        if (req.method !== 'GET') {
+          return new Response(
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const storeId = url.searchParams.get('store_id');
+        if (!storeId) {
+          return new Response(
+            JSON.stringify({ error: 'store_id parameter is required' }),
+            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const store = await getStoreScraperConfig(supabase, storeId, user.id);
+
+        if (!store.scraper_registered) {
+          return new Response(
+            JSON.stringify({ error: 'Store not registered with scraper' }),
+            { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const scraperClient = await createScraperClient({
+          scraper_api_url: store.scraper_api_url,
+          scraper_api_secret: store.scraper_api_secret,
+        });
+
+        // Build filter from query parameters
+        const filter: any = {};
+        const contentType = url.searchParams.get('content_type');
+        const dateFrom = url.searchParams.get('date_from');
+        const dateTo = url.searchParams.get('date_to');
+        const limit = url.searchParams.get('limit');
+
+        if (contentType) filter.content_type = contentType;
+        if (dateFrom) filter.date_from = dateFrom;
+        if (dateTo) filter.date_to = dateTo;
+        if (limit) filter.limit = parseInt(limit, 10);
+
+        const content = await scraperClient.getShopContent(store.id, filter);
+
+        return new Response(
+          JSON.stringify(content),
+          { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+        );
+      }
+
+      case 'custom-urls': {
+        const storeId = url.searchParams.get('store_id') || (await req.json())?.store_id;
+        if (!storeId) {
+          return new Response(
+            JSON.stringify({ error: 'store_id is required' }),
+            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const store = await getStoreScraperConfig(supabase, storeId, user.id);
+
+        if (!store.scraper_registered) {
+          return new Response(
+            JSON.stringify({ error: 'Store not registered with scraper' }),
+            { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const scraperClient = await createScraperClient({
+          scraper_api_url: store.scraper_api_url,
+          scraper_api_secret: store.scraper_api_secret,
+        });
+
+        if (req.method === 'GET') {
+          // List custom URLs
+          const customUrls = await scraperClient.listCustomUrls(store.id);
+          return new Response(
+            JSON.stringify(customUrls),
+            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+
+        } else if (req.method === 'POST') {
+          // Add custom URL
+          const { url: customUrl, content_type } = await req.json();
+
+          if (!customUrl || !content_type) {
+            return new Response(
+              JSON.stringify({ error: 'url and content_type are required' }),
+              { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
+
+          // Validate same domain
+          if (!validateSameDomain(store.store_url, customUrl)) {
+            return new Response(
+              JSON.stringify({ error: 'Custom URL must be from the same domain as the store' }),
+              { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
+
+          const result = await scraperClient.addCustomUrl(store.id, customUrl, content_type);
+
+          return new Response(
+            JSON.stringify(result),
+            { status: 201, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        return new Response(
+          JSON.stringify({ error: 'Method not allowed' }),
+          { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+        );
+      }
+
+      case 'scheduling': {
+        // PATCH /scheduling
+        if (req.method !== 'PATCH') {
+          return new Response(
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const { store_id, enabled } = await req.json();
+
+        if (!store_id || typeof enabled !== 'boolean') {
+          return new Response(
+            JSON.stringify({ error: 'store_id and enabled (boolean) are required' }),
+            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const store = await getStoreScraperConfig(supabase, store_id, user.id);
+
+        if (!store.scraper_registered) {
+          return new Response(
+            JSON.stringify({ error: 'Store not registered with scraper' }),
+            { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+          );
+        }
+
+        const scraperClient = await createScraperClient({
+          scraper_api_url: store.scraper_api_url,
+          scraper_api_secret: store.scraper_api_secret,
+        });
+
+        await scraperClient.setScheduling(store.id, enabled);
+
+        return new Response(
+          JSON.stringify({ success: true, message: `Scheduling ${enabled ? 'enabled' : 'disabled'}` }),
+          { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+        );
+      }
+
+      default:
+        return new Response(
+          JSON.stringify({ error: 'Unknown endpoint' }),
+          { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+        );
+    }
+
+  } catch (error) {
+    console.error('Error in scraper management API:', error);
+
+    return new Response(
+      JSON.stringify({
+        error: 'Internal server error',
+        message: error instanceof Error ? error.message : 'Unknown error',
+      }),
+      {
+        status: 500,
+        headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+      }
+    );
+  }
+});

supabase/functions/scraper-webhook/index.ts

@@ -0,0 +1,172 @@
+/**
+ * Scraper Webhook Handler
+ *
+ * Receives webhook notifications from the scraper service when:
+ * - Shop scraping status changes
+ * - New content is discovered
+ * - Scraping jobs complete or fail
+ * - Custom URLs are processed
+ *
+ * This handler primarily logs events and can trigger UI refresh notifications.
+ * All content data is retrieved on-demand from the scraper API.
+ */
+
+import { createClient } from 'https://esm.sh/@supabase/supabase-js@2.39.3';
+
+const corsHeaders = {
+  'Access-Control-Allow-Origin': '*',
+  'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type',
+  'Access-Control-Allow-Methods': 'POST, OPTIONS',
+};
+
+interface WebhookPayload {
+  shop_id: string;
+  custom_id?: string;
+  event_type: 'job_completed' | 'job_failed' | 'content_updated' | 'custom_url_processed';
+  timestamp: string;
+  data?: Record<string, any>;
+}
+
+Deno.serve(async (req) => {
+  // Handle CORS preflight requests
+  if (req.method === 'OPTIONS') {
+    return new Response(null, { headers: corsHeaders });
+  }
+
+  try {
+    // Only accept POST requests
+    if (req.method !== 'POST') {
+      return new Response(
+        JSON.stringify({ error: 'Method not allowed' }),
+        {
+          status: 405,
+          headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+        }
+      );
+    }
+
+    // Parse the webhook payload
+    const payload: WebhookPayload = await req.json();
+
+    // Validate required fields
+    if (!payload.shop_id || !payload.event_type || !payload.timestamp) {
+      return new Response(
+        JSON.stringify({ error: 'Missing required fields: shop_id, event_type, timestamp' }),
+        {
+          status: 400,
+          headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+        }
+      );
+    }
+
+    console.log(`Received scraper webhook for shop ${payload.shop_id}: ${payload.event_type}`, payload);
+
+    // Initialize Supabase client
+    const supabaseUrl = Deno.env.get('SUPABASE_URL')!;
+    const supabaseKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY')!;
+    const supabase = createClient(supabaseUrl, supabaseKey);
+
+    // Find the store by the custom_id (which should be the store.id)
+    const storeId = payload.custom_id || payload.shop_id;
+
+    const { data: store, error: storeError } = await supabase
+      .from('stores')
+      .select('id, store_name, platform_name, user_id')
+      .eq('id', storeId)
+      .single();
+
+    if (storeError || !store) {
+      console.error('Store not found for scraper webhook:', storeId, storeError);
+      return new Response(
+        JSON.stringify({ error: 'Store not found' }),
+        {
+          status: 404,
+          headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+        }
+      );
+    }
+
+    // Process different webhook events
+    switch (payload.event_type) {
+      case 'job_completed':
+        console.log(`Scraping job completed for store ${store.store_name} (${store.platform_name})`);
+        // Update last scrape timestamp or trigger UI refresh
+        // All actual content will be retrieved on-demand from scraper API
+        break;
+
+      case 'job_failed':
+        console.error(`Scraping job failed for store ${store.store_name}:`, payload.data);
+        // Could send notification to store owner about failed scraping
+        break;
+
+      case 'content_updated':
+        console.log(`New content discovered for store ${store.store_name}:`, payload.data);
+        // Could trigger real-time UI updates if needed
+        break;
+
+      case 'custom_url_processed':
+        console.log(`Custom URL processed for store ${store.store_name}:`, payload.data);
+        // Log custom URL processing results
+        break;
+
+      default:
+        console.warn(`Unknown webhook event type: ${payload.event_type}`);
+    }
+
+    // TODO: Could implement real-time notifications here using Supabase realtime
+    // For example, insert into a notifications table or publish to a channel
+
+    // Log the webhook event for debugging/monitoring
+    const { error: logError } = await supabase
+      .from('system_logs')
+      .insert({
+        log_type: 'scraper_webhook',
+        message: `${payload.event_type} for store ${storeId}`,
+        details: {
+          shop_id: payload.shop_id,
+          custom_id: payload.custom_id,
+          event_type: payload.event_type,
+          timestamp: payload.timestamp,
+          data: payload.data,
+          store_info: {
+            id: store.id,
+            name: store.store_name,
+            platform: store.platform_name,
+          },
+        },
+        created_at: new Date().toISOString(),
+      });
+
+    if (logError) {
+      console.error('Failed to log webhook event:', logError);
+      // Don't fail the webhook for logging errors
+    }
+
+    return new Response(
+      JSON.stringify({
+        success: true,
+        message: 'Webhook processed successfully',
+        event_type: payload.event_type,
+        shop_id: payload.shop_id,
+      }),
+      {
+        status: 200,
+        headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+      }
+    );
+
+  } catch (error) {
+    console.error('Error processing scraper webhook:', error);
+
+    return new Response(
+      JSON.stringify({
+        error: 'Internal server error',
+        message: error instanceof Error ? error.message : 'Unknown error',
+      }),
+      {
+        status: 500,
+        headers: { ...corsHeaders, 'Content-Type': 'application/json' },
+      }
+    );
+  }
+});

supabase/migrations/20251121_webshop_scraper_integration.sql

@@ -0,0 +1,25 @@
+-- Migration: Add webshop scraper integration support
+-- Date: 2024-11-21
+-- Description: Add scraper configuration columns to stores table
+
+-- Add scraper-related columns to the stores table
+ALTER TABLE stores ADD COLUMN IF NOT EXISTS scraper_api_url TEXT;
+ALTER TABLE stores ADD COLUMN IF NOT EXISTS scraper_api_secret TEXT; -- Will encrypt later when vault setup is complete
+ALTER TABLE stores ADD COLUMN IF NOT EXISTS scraper_registered BOOLEAN DEFAULT false;
+ALTER TABLE stores ADD COLUMN IF NOT EXISTS scraper_enabled BOOLEAN DEFAULT true;
+
+-- Add comments to document the columns
+COMMENT ON COLUMN stores.scraper_api_url IS 'Per-store scraper API URL override (null = use global default)';
+COMMENT ON COLUMN stores.scraper_api_secret IS 'Per-store scraper API secret override (null = use global default) - TODO: encrypt with vault';
+COMMENT ON COLUMN stores.scraper_registered IS 'Whether this store is registered with the scraper service';
+COMMENT ON COLUMN stores.scraper_enabled IS 'Whether scraper functionality is enabled for this store';
+
+-- Create indexes for efficient querying
+CREATE INDEX IF NOT EXISTS idx_stores_scraper_registered ON stores(scraper_registered);
+CREATE INDEX IF NOT EXISTS idx_stores_scraper_enabled ON stores(scraper_enabled);
+
+-- Update existing stores to have scraper_enabled = true by default
+-- but scraper_registered = false (they need to be registered)
+UPDATE stores
+SET scraper_enabled = true, scraper_registered = false
+WHERE scraper_enabled IS NULL OR scraper_registered IS NULL;