feat: add crawler information page with enhanced scraper UI

Add new public /crawler page explaining ShopCall.ai web crawler mechanism:
- User agent format and source hosts documentation
- Crawler features and process workflow
- Multi-language support (EN, DE, HU)
- Sitemap and robots.txt compliance details
- Content categories (FAQ, terms, shopping, contacts)

Enhance scraper management UI:
- Improved scraper status display with analytics
- Content breakdown by type
- Custom URL management with toggle and delete
- Enhanced content viewer with page grouping
- Better error display and timestamp formatting

Backend improvements:
- Custom URL toggle and delete endpoints
- Domain validation for custom URLs
- Enhanced error handling

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

shopcall.ai-main/src/App.tsx

@@ -30,6 +30,7 @@ const Settings = lazy(() => import("./pages/Settings"));
 const About = lazy(() => import("./pages/About"));
 const Privacy = lazy(() => import("./pages/Privacy"));
 const Terms = lazy(() => import("./pages/Terms"));
+const Crawler = lazy(() => import("./pages/Crawler"));
 const NotFound = lazy(() => import("./pages/NotFound"));
 const ShopRenterIntegration = lazy(() => import("./pages/ShopRenterIntegration"));
 const IntegrationsRedirect = lazy(() => import("./pages/IntegrationsRedirect"));
@@ -77,6 +78,7 @@ const App = () => (
               <Route path="/about" element={<About />} />
               <Route path="/privacy" element={<Privacy />} />
               <Route path="/terms" element={<Terms />} />
+              <Route path="/crawler" element={<Crawler />} />
               <Route path="/integrations" element={<IntegrationsRedirect />} />
               <Route path="/integrations/shoprenter" element={<ShopRenterIntegration />} />
               {/*<Route path="/contact" element={<Contact />} />*/}

shopcall.ai-main/src/components/ManageStoreDataContent.tsx

@@ -83,12 +83,22 @@ interface ScraperCustomUrl {
 interface ScraperShopStatus {
   id: string;
   url: string;
+  custom_id?: string;
   status: 'active' | 'inactive';
   last_scraped_at?: string;
   next_scheduled_scrape?: string;
   scheduled_enabled: boolean;
   total_urls_found: number;
   total_content_items: number;
+  scrape_count?: number;
+  last_error?: string;
+  last_error_at?: string;
+  analytics?: {
+    shipping_count: number;
+    contacts_count: number;
+    terms_count: number;
+    faq_count: number;
+  };
   created_at: string;
   updated_at: string;
 }
@@ -427,6 +437,30 @@ export function ManageStoreDataContent() {
   const handleAddCustomUrl = async () => {
     if (!selectedStore || !newCustomUrl) return;
 
+    // Validate same domain
+    if (selectedStore.store_url) {
+      try {
+        const storeDomain = new URL(selectedStore.store_url).hostname.toLowerCase();
+        const customDomain = new URL(newCustomUrl).hostname.toLowerCase();
+
+        if (customDomain !== storeDomain && !customDomain.endsWith(`.${storeDomain}`)) {
+          toast({
+            title: t('common.error'),
+            description: t('manageStoreData.website.toast.differentDomainError') || 'Custom URL must be from the same domain as your store',
+            variant: "destructive"
+          });
+          return;
+        }
+      } catch (urlError) {
+        toast({
+          title: t('common.error'),
+          description: t('manageStoreData.website.toast.invalidUrlError') || 'Invalid URL format',
+          variant: "destructive"
+        });
+        return;
+      }
+    }
+
     try {
       const sessionData = localStorage.getItem('session_data');
       if (!sessionData) throw new Error('No session data found');
@@ -447,8 +481,8 @@ export function ManageStoreDataContent() {
 
       if (response.ok) {
         toast({
-          title: t('manageStoreData.website.toast.urlAddSuccess'),
-          description: t('manageStoreData.website.toast.urlAddSuccess'),
+          title: t('common.success'),
+          description: t('manageStoreData.website.toast.urlAddSuccess') || 'Custom URL added successfully',
         });
         setNewCustomUrl("");
         fetchCustomUrls();
@@ -465,6 +499,85 @@ export function ManageStoreDataContent() {
     }
   };
 
+  const handleToggleCustomUrl = async (customUrlId: string, currentEnabled: boolean) => {
+    if (!selectedStore) return;
+
+    try {
+      const sessionData = localStorage.getItem('session_data');
+      if (!sessionData) throw new Error('No session data found');
+
+      const session = JSON.parse(sessionData);
+
+      // Update the scraper-management endpoint to support this
+      const response = await fetch(`${API_URL}/scraper-management/custom-urls/${customUrlId}/toggle`, {
+        method: 'PATCH',
+        headers: {
+          'Authorization': `Bearer ${session.session.access_token}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          store_id: selectedStore.id,
+          enabled: !currentEnabled,
+        })
+      });
+
+      if (response.ok) {
+        toast({
+          title: t('common.success'),
+          description: !currentEnabled ? t('manageStoreData.website.toast.urlEnabled') || 'URL enabled' : t('manageStoreData.website.toast.urlDisabled') || 'URL disabled',
+        });
+        fetchCustomUrls();
+      } else {
+        const error = await response.json();
+        throw new Error(error.error || 'Failed to update custom URL');
+      }
+    } catch (error) {
+      toast({
+        title: t('common.error'),
+        description: error.message || t('manageStoreData.website.toast.urlToggleError') || 'Failed to update URL',
+        variant: "destructive"
+      });
+    }
+  };
+
+  const handleDeleteCustomUrl = async (customUrlId: string) => {
+    if (!selectedStore) return;
+
+    try {
+      const sessionData = localStorage.getItem('session_data');
+      if (!sessionData) throw new Error('No session data found');
+
+      const session = JSON.parse(sessionData);
+      const response = await fetch(`${API_URL}/scraper-management/custom-urls/${customUrlId}`, {
+        method: 'DELETE',
+        headers: {
+          'Authorization': `Bearer ${session.session.access_token}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+          store_id: selectedStore.id,
+        })
+      });
+
+      if (response.ok) {
+        toast({
+          title: t('common.success'),
+          description: t('manageStoreData.website.toast.urlDeleted') || 'Custom URL deleted successfully',
+        });
+        fetchCustomUrls();
+      } else {
+        const error = await response.json();
+        throw new Error(error.error || 'Failed to delete custom URL');
+      }
+    } catch (error) {
+      toast({
+        title: t('common.error'),
+        description: error.message || t('manageStoreData.website.toast.urlDeleteError') || 'Failed to delete URL',
+        variant: "destructive"
+      });
+    }
+  };
+
   const handleToggleScheduling = async (enabled: boolean) => {
     if (!selectedStore) return;
 
@@ -1098,45 +1211,112 @@ export function ManageStoreDataContent() {
                       <CardHeader>
                         <CardTitle className="text-white flex items-center gap-2">
                           <Settings className="w-5 h-5" />
-                          {t('manageStoreData.website.scraperStatus.title')}
+                          {t('manageStoreData.website.scraperStatus.title') || 'Scraper Status'}
                         </CardTitle>
                       </CardHeader>
-                      <CardContent>
+                      <CardContent className="space-y-4">
+                        {/* Main Statistics Grid */}
                         <div className="grid grid-cols-2 md:grid-cols-4 gap-4">
                           <div className="bg-slate-700/50 rounded-lg p-3">
                             <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.status')}</div>
-                            <div className="text-white font-medium flex items-center gap-1">
+                            <div className="text-white font-medium flex items-center gap-1 mt-1">
                               {scraperStatus.status === 'active' ? (
-                                <CheckCircle className="w-4 h-4 text-green-500" />
+                                <><CheckCircle className="w-4 h-4 text-green-500" /> Active</>
                               ) : (
-                                <AlertCircle className="w-4 h-4 text-red-500" />
+                                <><AlertCircle className="w-4 h-4 text-yellow-500" /> Inactive</>
                               )}
-                              {scraperStatus.status}
                             </div>
                           </div>
                           <div className="bg-slate-700/50 rounded-lg p-3">
-                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.urlsFound')}</div>
-                            <div className="text-white font-medium">{scraperStatus.total_urls_found}</div>
+                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.urlsFound') || 'URLs Found'}</div>
+                            <div className="text-white font-medium text-xl mt-1">{scraperStatus.total_urls_found}</div>
                           </div>
                           <div className="bg-slate-700/50 rounded-lg p-3">
-                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.contentItems')}</div>
-                            <div className="text-white font-medium">{scraperStatus.total_content_items}</div>
+                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.contentItems') || 'Content Sections'}</div>
+                            <div className="text-white font-medium text-xl mt-1">{scraperStatus.total_content_items}</div>
                           </div>
                           <div className="bg-slate-700/50 rounded-lg p-3">
-                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.scheduled')}</div>
-                            <div className="text-white font-medium flex items-center gap-2">
-                              <Switch
-                                checked={scraperStatus.scheduled_enabled}
-                                onCheckedChange={handleToggleScheduling}
-                                className="data-[state=checked]:bg-cyan-500"
-                              />
-                              {scraperStatus.scheduled_enabled ? t('manageStoreData.website.scraperStatus.enabled') : t('manageStoreData.website.scraperStatus.disabled')}
+                            <div className="text-slate-400 text-sm">{t('manageStoreData.website.contentViewer.statusLabels.totalScrapes') || 'Total Scrapes'}</div>
+                            <div className="text-white font-medium text-xl mt-1">{scraperStatus.scrape_count || 0}</div>
+                          </div>
+                        </div>
+
+                        {/* Content Type Breakdown */}
+                        {scraperStatus.analytics && (
+                          <div className="border border-slate-700 rounded-lg p-4">
+                            <div className="text-white font-medium mb-3">{t('manageStoreData.website.contentViewer.statusLabels.contentBreakdown') || 'Content Breakdown'}</div>
+                            <div className="grid grid-cols-2 md:grid-cols-4 gap-3">
+                              <div className="flex items-center justify-between p-2 bg-slate-700/30 rounded">
+                                <span className="text-slate-400 text-sm">FAQ</span>
+                                <Badge variant="secondary">{scraperStatus.analytics.faq_count}</Badge>
+                              </div>
+                              <div className="flex items-center justify-between p-2 bg-slate-700/30 rounded">
+                                <span className="text-slate-400 text-sm">Shipping</span>
+                                <Badge variant="secondary">{scraperStatus.analytics.shipping_count}</Badge>
+                              </div>
+                              <div className="flex items-center justify-between p-2 bg-slate-700/30 rounded">
+                                <span className="text-slate-400 text-sm">Terms</span>
+                                <Badge variant="secondary">{scraperStatus.analytics.terms_count}</Badge>
+                              </div>
+                              <div className="flex items-center justify-between p-2 bg-slate-700/30 rounded">
+                                <span className="text-slate-400 text-sm">Contacts</span>
+                                <Badge variant="secondary">{scraperStatus.analytics.contacts_count}</Badge>
+                              </div>
+                            </div>
+                          </div>
+                        )}
+
+                        {/* Scheduling Control */}
+                        <div className="border border-slate-700 rounded-lg p-4">
+                          <div className="flex items-center justify-between">
+                            <div>
+                              <div className="text-white font-medium">{t('manageStoreData.website.contentViewer.statusLabels.scheduledScraping') || 'Scheduled Scraping'}</div>
+                              <div className="text-slate-400 text-sm mt-1">
+                                {scraperStatus.scheduled_enabled
+                                  ? (t('manageStoreData.website.scraperStatus.scheduledDescription') || 'Automatically scrapes your website for new content')
+                                  : (t('manageStoreData.website.scraperStatus.scheduledDisabledDescription') || 'Enable to automatically collect content')
+                                }
+                              </div>
                             </div>
+                            <Switch
+                              checked={scraperStatus.scheduled_enabled}
+                              onCheckedChange={handleToggleScheduling}
+                              className="data-[state=checked]:bg-cyan-500"
+                            />
                           </div>
                         </div>
-                        {scraperStatus.last_scraped_at && (
-                          <div className="mt-4 text-sm text-slate-400">
-                            Last scraped: {new Date(scraperStatus.last_scraped_at).toLocaleString()}
+
+                        {/* Timestamps and Error Display */}
+                        <div className="flex flex-wrap gap-4 text-sm">
+                          {scraperStatus.last_scraped_at && (
+                            <div className="text-slate-400">
+                              <Clock className="w-4 h-4 inline mr-1" />
+                              Last scraped: <span className="text-white">{new Date(scraperStatus.last_scraped_at).toLocaleString()}</span>
+                            </div>
+                          )}
+                          {scraperStatus.next_scheduled_scrape && scraperStatus.scheduled_enabled && (
+                            <div className="text-slate-400">
+                              <Clock className="w-4 h-4 inline mr-1" />
+                              Next scrape: <span className="text-white">{new Date(scraperStatus.next_scheduled_scrape).toLocaleString()}</span>
+                            </div>
+                          )}
+                        </div>
+
+                        {/* Error Display */}
+                        {scraperStatus.last_error && (
+                          <div className="border border-red-800 bg-red-900/20 rounded-lg p-4">
+                            <div className="flex items-start gap-2">
+                              <AlertCircle className="w-5 h-5 text-red-400 flex-shrink-0 mt-0.5" />
+                              <div className="flex-1">
+                                <div className="text-red-300 font-medium mb-1">{t('manageStoreData.website.scraperStatus.lastError') || 'Last Error'}</div>
+                                <div className="text-red-200 text-sm">{scraperStatus.last_error}</div>
+                                {scraperStatus.last_error_at && (
+                                  <div className="text-red-400 text-xs mt-2">
+                                    {new Date(scraperStatus.last_error_at).toLocaleString()}
+                                  </div>
+                                )}
+                              </div>
+                            </div>
                           </div>
                         )}
                       </CardContent>
@@ -1209,15 +1389,50 @@ export function ManageStoreDataContent() {
                               <div className="flex-1">
                                 <div className="text-white text-sm flex items-center gap-2">
                                   <ExternalLink className="w-4 h-4" />
-                                  {customUrl.url}
+                                  <a href={customUrl.url} target="_blank" rel="noopener noreferrer" className="hover:text-cyan-400 underline">
+                                    {customUrl.url}
+                                  </a>
                                 </div>
-                                <div className="text-slate-400 text-xs">
-                                  {t('manageStoreData.website.customUrls.contentType')}: {customUrl.content_type} | Status: {customUrl.status || 'pending'}
+                                <div className="text-slate-400 text-xs flex items-center gap-2 mt-1">
+                                  <Badge variant="outline" className="text-xs">
+                                    {customUrl.content_type}
+                                  </Badge>
+                                  {customUrl.status && (
+                                    <Badge variant={
+                                      customUrl.status === 'completed' ? 'default' :
+                                      customUrl.status === 'failed' ? 'destructive' : 'secondary'
+                                    } className="text-xs">
+                                      {customUrl.status}
+                                    </Badge>
+                                  )}
+                                  {customUrl.last_scraped_at && (
+                                    <span className="text-xs">
+                                      <Clock className="w-3 h-3 inline mr-1" />
+                                      {new Date(customUrl.last_scraped_at).toLocaleDateString()}
+                                    </span>
+                                  )}
                                 </div>
                               </div>
-                              <Badge variant={customUrl.enabled ? "default" : "secondary"}>
-                                {customUrl.enabled ? t('manageStoreData.website.scraperStatus.enabled') : t('manageStoreData.website.scraperStatus.disabled')}
-                              </Badge>
+                              <div className="flex items-center gap-2">
+                                <Switch
+                                  checked={customUrl.enabled}
+                                  onCheckedChange={() => handleToggleCustomUrl(customUrl.id, customUrl.enabled)}
+                                  className="data-[state=checked]:bg-cyan-500"
+                                />
+                                <Button
+                                  size="sm"
+                                  variant="ghost"
+                                  onClick={() => setConfirmDialog({
+                                    open: true,
+                                    title: t('manageStoreData.website.customUrls.deleteTitle') || 'Delete Custom URL',
+                                    description: t('manageStoreData.website.customUrls.deleteDescription') || 'Are you sure you want to delete this custom URL? This action cannot be undone.',
+                                    action: () => handleDeleteCustomUrl(customUrl.id)
+                                  })}
+                                  className="text-red-400 hover:text-red-300 hover:bg-red-900/20"
+                                >
+                                  <Trash2 className="w-4 h-4" />
+                                </Button>
+                              </div>
                             </div>
                           ))}
                         </div>
@@ -1228,35 +1443,115 @@ export function ManageStoreDataContent() {
                   {/* Scraped Content */}
                   <Card className="bg-slate-800 border-slate-700">
                     <CardHeader>
-                      <CardTitle className="text-white">{t('manageStoreData.website.contentViewer.title')} ({scraperContent.length})</CardTitle>
+                      <div className="flex items-center justify-between">
+                        <CardTitle className="text-white">{t('manageStoreData.website.contentViewer.title')} ({scraperContent.length})</CardTitle>
+                        {scraperContent.length > 0 && (
+                          <div className="text-sm text-slate-400">
+                            {t('manageStoreData.website.contentViewer.groupedByPage') || 'Grouped by page'}
+                          </div>
+                        )}
+                      </div>
                     </CardHeader>
                     <CardContent>
                       {scraperContent.length === 0 ? (
                         <div className="text-center py-8 text-slate-400">
-                          {t('manageStoreData.website.contentViewer.noContentDescription')}
+                          {scraperStatus?.status === 'active' ? (
+                            <div>
+                              <Loader2 className="w-8 h-8 mx-auto mb-3 animate-spin text-cyan-500" />
+                              <p>{t('manageStoreData.website.contentViewer.scrapingInProgress') || 'Scraping in progress...'}</p>
+                            </div>
+                          ) : (
+                            <div>
+                              <p>{t('manageStoreData.website.contentViewer.noContentDescription')}</p>
+                              {scraperStatus && !scraperStatus.scheduled_enabled && (
+                                <p className="mt-2 text-sm">
+                                  {t('manageStoreData.website.contentViewer.enableScheduling') || 'Enable scheduled scraping above to automatically collect content.'}
+                                </p>
+                              )}
+                            </div>
+                          )}
                         </div>
                       ) : (
                         <div className="space-y-4">
-                          {scraperContent.map((content) => (
-                            <div key={content.id} className="border border-slate-600 rounded-lg p-4 bg-slate-700/30">
-                              <div className="flex items-start justify-between mb-2">
-                                <div className="flex-1">
-                                  <div className="text-white font-medium flex items-center gap-2">
-                                    <ExternalLink className="w-4 h-4" />
-                                    {content.title || t('manageStoreData.website.contentViewer.untitled')}
+                          {/* Group content by URL (page) */}
+                          {Object.entries(
+                            scraperContent.reduce((acc, content) => {
+                              if (!acc[content.url]) {
+                                acc[content.url] = [];
+                              }
+                              acc[content.url].push(content);
+                              return acc;
+                            }, {} as Record<string, typeof scraperContent>)
+                          ).map(([url, contents]) => (
+                            <div key={url} className="border border-slate-600 rounded-lg overflow-hidden">
+                              {/* Page Header */}
+                              <div className="bg-slate-700/50 p-3 border-b border-slate-600">
+                                <div className="flex items-center justify-between">
+                                  <div className="flex-1 min-w-0">
+                                    <div className="flex items-center gap-2">
+                                      <Globe className="w-4 h-4 text-cyan-400 flex-shrink-0" />
+                                      <a
+                                        href={url}
+                                        target="_blank"
+                                        rel="noopener noreferrer"
+                                        className="text-white hover:text-cyan-400 underline truncate text-sm"
+                                      >
+                                        {url}
+                                      </a>
+                                      <ExternalLink className="w-3 h-3 text-slate-400 flex-shrink-0" />
+                                    </div>
+                                    <div className="text-slate-400 text-xs mt-1">
+                                      {contents.length} {contents.length === 1 ? 'section' : 'sections'} found
+                                    </div>
                                   </div>
-                                  <div className="text-cyan-400 text-sm break-all">{content.url}</div>
                                 </div>
-                                <Badge variant="outline" className="text-xs">
-                                  {content.content_type}
-                                </Badge>
                               </div>
-                              <div className="text-slate-300 text-sm line-clamp-3 mb-2">
-                                {content.content.substring(0, 200)}...
-                              </div>
-                              <div className="text-slate-400 text-xs flex items-center gap-1">
-                                <Clock className="w-3 h-3" />
-                                {t('manageStoreData.website.contentViewer.scraped')}: {new Date(content.scraped_at).toLocaleString()}
+
+                              {/* Content Sections */}
+                              <div className="divide-y divide-slate-600">
+                                {contents.map((content) => (
+                                  <div key={content.id} className="p-4 bg-slate-800/50 hover:bg-slate-700/30 transition-colors">
+                                    <div className="flex items-start justify-between mb-2">
+                                      <div className="flex-1">
+                                        <div className="text-white font-medium">
+                                          {content.title || t('manageStoreData.website.contentViewer.untitled')}
+                                        </div>
+                                      </div>
+                                      <Badge
+                                        variant="outline"
+                                        className="text-xs ml-2"
+                                      >
+                                        {content.content_type}
+                                      </Badge>
+                                    </div>
+                                    <div className="text-slate-300 text-sm mb-3 whitespace-pre-wrap">
+                                      {content.content.length > 500 ? (
+                                        <details className="cursor-pointer">
+                                          <summary className="hover:text-white">
+                                            {content.content.substring(0, 500)}...
+                                            <span className="text-cyan-400 ml-2">{t('manageStoreData.website.contentViewer.readMore') || 'Read more'}</span>
+                                          </summary>
+                                          <div className="mt-2 pt-2 border-t border-slate-600">
+                                            {content.content}
+                                          </div>
+                                        </details>
+                                      ) : (
+                                        content.content
+                                      )}
+                                    </div>
+                                    <div className="flex items-center gap-4 text-slate-400 text-xs">
+                                      <div className="flex items-center gap-1">
+                                        <Clock className="w-3 h-3" />
+                                        {t('manageStoreData.website.contentViewer.scraped')}: {new Date(content.scraped_at).toLocaleString()}
+                                      </div>
+                                      {content.metadata && Object.keys(content.metadata).length > 0 && (
+                                        <Badge variant="secondary" className="text-xs">
+                                          {Object.keys(content.metadata).length} metadata fields
+                                        </Badge>
+                                      )}
+                                    </div>
+                                  </div>
+                                ))}
                               </div>
                             </div>
                           ))}

shopcall.ai-main/src/i18n/locales/de.json

@@ -1107,7 +1107,10 @@
         "registering": "Wird registriert...",
         "enabling": "Wird aktiviert...",
         "disabling": "Wird deaktiviert...",
-        "error": "Fehler: {{message}}"
+        "error": "Fehler: {{message}}",
+        "lastError": "Letzter Fehler",
+        "scheduledDescription": "Scannt Ihre Website automatisch nach neuen Inhalten",
+        "scheduledDisabledDescription": "Aktivieren Sie das automatische Sammeln von Inhalten"
       },
       "customUrls": {
         "title": "Benutzerdefinierte URLs",
@@ -1128,7 +1131,9 @@
         },
         "addingUrl": "URL wird hinzugefügt...",
         "noCustomUrls": "Noch keine benutzerdefinierten URLs hinzugefügt",
-        "domainError": "Die URL muss von derselben Domain wie Ihr Shop stammen"
+        "domainError": "Die URL muss von derselben Domain wie Ihr Shop stammen",
+        "deleteTitle": "Benutzerdefinierte URL löschen",
+        "deleteDescription": "Möchten Sie diese benutzerdefinierte URL wirklich löschen? Diese Aktion kann nicht rückgängig gemacht werden."
       },
       "contentViewer": {
         "title": "Gescrapte Inhalte",
@@ -1147,7 +1152,10 @@
           "status": "Status",
           "urlsFound": "URLs gefunden",
           "contentItems": "Inhaltselemente",
-          "scheduled": "Geplant"
+          "scheduled": "Geplant",
+          "totalScrapes": "Gesamte Scrapings",
+          "contentBreakdown": "Inhaltsaufschlüsselung",
+          "scheduledScraping": "Geplantes Scraping"
         },
         "table": {
           "url": "URL",
@@ -1160,6 +1168,10 @@
         "noContentDescription": "Keine Inhalte gefunden. Der Scraper wird automatisch Inhalte entdecken.",
         "untitled": "Ohne Titel",
         "scraped": "Gescrapt",
+        "groupedByPage": "Nach Seite gruppiert",
+        "scrapingInProgress": "Scraping läuft...",
+        "enableScheduling": "Aktivieren Sie oben das geplante Scraping, um automatisch Inhalte zu sammeln.",
+        "readMore": "Mehr lesen",
         "contentDialog": {
           "title": "Inhaltsdetails",
           "close": "Schließen"
@@ -1173,7 +1185,14 @@
         "disableSuccess": "Scraping erfolgreich deaktiviert",
         "disableError": "Fehler beim Deaktivieren des Scrapings",
         "urlAddSuccess": "Benutzerdefinierte URL erfolgreich hinzugefügt",
-        "urlAddError": "Fehler beim Hinzufügen der benutzerdefinierten URL"
+        "urlAddError": "Fehler beim Hinzufügen der benutzerdefinierten URL",
+        "differentDomainError": "Benutzerdefinierte URL muss von derselben Domain wie Ihr Shop stammen",
+        "invalidUrlError": "Ungültiges URL-Format",
+        "urlEnabled": "Benutzerdefinierte URL erfolgreich aktiviert",
+        "urlDisabled": "Benutzerdefinierte URL erfolgreich deaktiviert",
+        "urlToggleError": "Fehler beim Aktualisieren der benutzerdefinierten URL",
+        "urlDeleted": "Benutzerdefinierte URL erfolgreich gelöscht",
+        "urlDeleteError": "Fehler beim Löschen der benutzerdefinierten URL"
       }
     }
   },
@@ -1183,5 +1202,95 @@
     "bankLevelEncryption": "Verschlüsselung auf Bankniveau",
     "lightningFastAuth": "Blitzschnelle Authentifizierung",
     "poweredByAI": "Powered by fortschrittliche KI-Technologie"
+  },
+  "callDetails": {
+    "title": "Anrufdetails",
+    "contactInformation": "Kontaktinformationen",
+    "customer": "Kunde",
+    "addedAt": "Hinzugefügt",
+    "totalCalls": "Anrufe gesamt",
+    "intent": "Absicht",
+    "totalDuration": "Gesamtdauer",
+    "totalCost": "Gesamtkosten",
+    "callOutcome": "Anrufergebnis",
+    "callHistory": "Anrufverlauf",
+    "duration": "Dauer",
+    "transcript": "Transkript",
+    "summary": "Zusammenfassung",
+    "assistant": "Assistent",
+    "noTranscript": "Kein Transkript verfügbar",
+    "noSummary": "Keine Zusammenfassung verfügbar",
+    "callRecording": "Anrufaufzeichnung",
+    "playMono": "Mono-Aufzeichnung abspielen",
+    "playStereo": "Stereo-Aufzeichnung abspielen",
+    "noRecording": "Keine Aufzeichnung verfügbar"
+  },
+  "crawler": {
+    "title": "ShopCall.ai Web-Crawler",
+    "backHome": "Zurück zur Startseite",
+    "hero": {
+      "description": "Unser Web-Crawler wurde entwickelt, um Ihren E-Commerce-Shop besser zu verstehen und unserer KI zu helfen, genaueren und persönlicheren Kundensupport zu bieten. Erfahren Sie, wie wir crawlen, was wir sammeln und wie Sie die volle Kontrolle behalten."
+    },
+    "userAgent": {
+      "title": "Identifizierung unseres Crawlers",
+      "formatLabel": "User-Agent-Format",
+      "hostLabel": "Quellhosts",
+      "hostDescription": "Der Crawler arbeitet von mehreren Servern aus, wobei [n] eine Servernummer darstellt (z.B. crawler-1, crawler-2, crawler-3, usw.)"
+    },
+    "features": {
+      "title": "Wie unser Crawler funktioniert",
+      "subtitle": "Entwickelt mit Respekt für Ihre Inhalte und vollständiger Einhaltung von Webstandards",
+      "respectful": {
+        "title": "Respektvolles Crawling",
+        "description": "Wir respektieren vollständig Ihre robots.txt-Datei und folgen allen Crawling-Direktiven. Unser Crawler identifiziert sich klar und arbeitet innerhalb von Ratenlimits, um die Leistung Ihrer Website nicht zu beeinträchtigen."
+      },
+      "registered": {
+        "title": "Nur registrierte Shops",
+        "description": "Wir crawlen nur Webshops, die in unserem System registriert sind. Wenn Sie Ihren Shop nicht mit ShopCall.ai verbunden haben, wird unser Crawler Ihre Website niemals besuchen."
+      },
+      "manageable": {
+        "title": "Volle Benutzerkontrolle",
+        "description": "Shop-Besitzer haben vollständige Kontrolle darüber, welche URLs gecrawlt werden können. Sie können Crawling-Berechtigungen direkt über Ihr Dashboard verwalten und jederzeit bestimmte Seiten zulassen oder blockieren."
+      },
+      "targeted": {
+        "title": "Gezielte Inhaltserkennung",
+        "description": "Wir konzentrieren uns auf spezifische Seitenkategorien, die zur Verbesserung der KI-Antworten beitragen: FAQ-Seiten, Nutzungsbedingungen, Einkaufsführer und Kontaktinformationen. Wir crawlen keine unnötigen Inhalte."
+      }
+    },
+    "howItWorks": {
+      "title": "Der Crawling-Prozess",
+      "step1": {
+        "title": "Sitemap-Erkennung",
+        "description": "Wir beginnen mit dem Lesen Ihrer sitemap.xml-Datei, um die auf Ihrem Webshop verfügbaren Seiten zu entdecken. Dies gewährleistet effizientes Crawling und respektiert Ihre Seitenstruktur."
+      },
+      "step2": {
+        "title": "robots.txt-Konformität",
+        "description": "Bevor wir eine Seite crawlen, überprüfen wir Ihre robots.txt-Datei, um sicherzustellen, dass wir darauf zugreifen dürfen. Seiten, die als nicht erlaubt markiert sind, werden niemals gecrawlt."
+      },
+      "step3": {
+        "title": "Kategoriebasierte Filterung",
+        "description": "Wir analysieren entdeckte URLs und identifizieren Seiten, die in spezifische, für den Kundensupport relevante Kategorien fallen:",
+        "categories": {
+          "faq": "FAQ- und Hilfeseiten",
+          "terms": "Nutzungsbedingungen und Richtlinien",
+          "shopping": "Einkaufsführer und Produktinformationen",
+          "contacts": "Kontaktinformationen und Support-Seiten"
+        }
+      },
+      "step4": {
+        "title": "KI-Wissensdatenbank",
+        "description": "Der gesammelte Inhalt wird verarbeitet und zu unserer KI-Wissensdatenbank hinzugefügt, wodurch genauere und kontextbewusste Antworten ermöglicht werden, wenn Kunden über Ihre Produkte, Richtlinien oder Dienstleistungen anrufen."
+      }
+    },
+    "cta": {
+      "title": "Bereit, Ihren KI-Support zu verbessern?",
+      "description": "Verbinden Sie Ihren Shop mit ShopCall.ai und lassen Sie unseren Crawler dabei helfen, eine personalisierte Wissensdatenbank für besseren Kundenservice aufzubauen.",
+      "button": "Jetzt starten"
+    },
+    "footer": {
+      "copyright": "© 2024 ShopCall.ai. Alle Rechte vorbehalten.",
+      "privacy": "Datenschutzrichtlinie",
+      "about": "Über uns"
+    }
   }
 }

shopcall.ai-main/src/i18n/locales/en.json

@@ -1263,7 +1263,10 @@
         "registering": "Registering...",
         "enabling": "Enabling...",
         "disabling": "Disabling...",
-        "error": "Error: {{message}}"
+        "error": "Error: {{message}}",
+        "lastError": "Last Error",
+        "scheduledDescription": "Automatically scrapes your website for new content",
+        "scheduledDisabledDescription": "Enable to automatically collect content"
       },
       "customUrls": {
         "title": "Custom URLs",
@@ -1284,7 +1287,9 @@
         },
         "addingUrl": "Adding URL...",
         "noCustomUrls": "No custom URLs added yet",
-        "domainError": "URL must be from the same domain as your store"
+        "domainError": "URL must be from the same domain as your store",
+        "deleteTitle": "Delete Custom URL",
+        "deleteDescription": "Are you sure you want to delete this custom URL? This action cannot be undone."
       },
       "contentViewer": {
         "title": "Scraped Content",
@@ -1303,7 +1308,10 @@
           "status": "Status",
           "urlsFound": "URLs Found",
           "contentItems": "Content Items",
-          "scheduled": "Scheduled"
+          "scheduled": "Scheduled",
+          "totalScrapes": "Total Scrapes",
+          "contentBreakdown": "Content Breakdown",
+          "scheduledScraping": "Scheduled Scraping"
         },
         "table": {
           "url": "URL",
@@ -1316,6 +1324,10 @@
         "noContentDescription": "No content found. The scraper will discover content automatically.",
         "untitled": "Untitled",
         "scraped": "Scraped",
+        "groupedByPage": "Grouped by page",
+        "scrapingInProgress": "Scraping in progress...",
+        "enableScheduling": "Enable scheduled scraping above to automatically collect content.",
+        "readMore": "Read more",
         "contentDialog": {
           "title": "Content Details",
           "close": "Close"
@@ -1329,7 +1341,14 @@
         "disableSuccess": "Scraping disabled successfully",
         "disableError": "Failed to disable scraping",
         "urlAddSuccess": "Custom URL added successfully",
-        "urlAddError": "Failed to add custom URL"
+        "urlAddError": "Failed to add custom URL",
+        "differentDomainError": "Custom URL must be from the same domain as your store",
+        "invalidUrlError": "Invalid URL format",
+        "urlEnabled": "Custom URL enabled successfully",
+        "urlDisabled": "Custom URL disabled successfully",
+        "urlToggleError": "Failed to update custom URL",
+        "urlDeleted": "Custom URL deleted successfully",
+        "urlDeleteError": "Failed to delete custom URL"
       }
     }
   },
@@ -1361,5 +1380,73 @@
     "playMono": "Play Mono Recording",
     "playStereo": "Play Stereo Recording",
     "noRecording": "No recording available"
+  },
+  "crawler": {
+    "title": "ShopCall.ai Web Crawler",
+    "backHome": "Back to Home",
+    "hero": {
+      "description": "Our web crawler is designed to understand your e-commerce store better, helping our AI provide more accurate and personalized customer support. Learn how we crawl, what we collect, and how you maintain full control."
+    },
+    "userAgent": {
+      "title": "Identifying Our Crawler",
+      "formatLabel": "User Agent Format",
+      "hostLabel": "Source Hosts",
+      "hostDescription": "The crawler operates from multiple servers, where [n] represents a server number (e.g., crawler-1, crawler-2, crawler-3, etc.)"
+    },
+    "features": {
+      "title": "How Our Crawler Works",
+      "subtitle": "Built with respect for your content and full compliance with web standards",
+      "respectful": {
+        "title": "Respectful Crawling",
+        "description": "We fully respect your robots.txt file and follow all crawling directives. Our crawler identifies itself clearly and operates within rate limits to avoid impacting your site performance."
+      },
+      "registered": {
+        "title": "Registered Stores Only",
+        "description": "We only crawl webshops that are registered in our system. If you haven't connected your store to ShopCall.ai, our crawler will never visit your site."
+      },
+      "manageable": {
+        "title": "Full User Control",
+        "description": "Store owners have complete control over which URLs can be crawled. You can manage crawling permissions directly from your dashboard, allowing or blocking specific pages at any time."
+      },
+      "targeted": {
+        "title": "Targeted Content Discovery",
+        "description": "We focus on specific page categories that help improve AI responses: FAQ pages, terms of service, shopping guides, and contact information. We don't crawl unnecessary content."
+      }
+    },
+    "howItWorks": {
+      "title": "The Crawling Process",
+      "step1": {
+        "title": "Sitemap Discovery",
+        "description": "We start by reading your sitemap.xml file to discover the pages available on your webshop. This ensures efficient crawling and respects your site structure."
+      },
+      "step2": {
+        "title": "robots.txt Compliance",
+        "description": "Before crawling any page, we check your robots.txt file to ensure we're allowed to access it. Pages marked as disallowed are never crawled."
+      },
+      "step3": {
+        "title": "Category-Based Filtering",
+        "description": "We analyze discovered URLs and identify pages that fall into specific categories relevant to customer support:",
+        "categories": {
+          "faq": "FAQ and Help Pages",
+          "terms": "Terms of Service and Policies",
+          "shopping": "Shopping Guides and Product Information",
+          "contacts": "Contact Information and Support Pages"
+        }
+      },
+      "step4": {
+        "title": "AI Knowledge Base",
+        "description": "The collected content is processed and added to our AI knowledge base, enabling more accurate and context-aware responses when customers call about your products, policies, or services."
+      }
+    },
+    "cta": {
+      "title": "Ready to Enhance Your AI Support?",
+      "description": "Connect your store to ShopCall.ai and let our crawler help build a personalized knowledge base for better customer service.",
+      "button": "Get Started"
+    },
+    "footer": {
+      "copyright": "© 2024 ShopCall.ai. All rights reserved.",
+      "privacy": "Privacy Policy",
+      "about": "About Us"
+    }
   }
 }

shopcall.ai-main/src/i18n/locales/hu.json

@@ -1253,7 +1253,10 @@
         "registering": "Regisztráció folyamatban...",
         "enabling": "Engedélyezés...",
         "disabling": "Letiltás...",
-        "error": "Hiba: {{message}}"
+        "error": "Hiba: {{message}}",
+        "lastError": "Utolsó Hiba",
+        "scheduledDescription": "Automatikusan letölti a weboldal tartalmát",
+        "scheduledDisabledDescription": "Engedélyezze az automatikus tartalom gyűjtést"
       },
       "customUrls": {
         "title": "Egyedi URL-ek",
@@ -1274,7 +1277,9 @@
         },
         "addingUrl": "URL hozzáadása...",
         "noCustomUrls": "Még nincsenek egyedi URL-ek hozzáadva",
-        "domainError": "Az URL-nek ugyanarról a domainről kell származnia, mint az áruházának"
+        "domainError": "Az URL-nek ugyanarról a domainről kell származnia, mint az áruházának",
+        "deleteTitle": "Egyedi URL Törlése",
+        "deleteDescription": "Biztosan törölni szeretné ezt az egyedi URL-t? Ez a művelet nem vonható vissza."
       },
       "contentViewer": {
         "title": "Scraped Tartalom",
@@ -1293,7 +1298,10 @@
           "status": "Állapot",
           "urlsFound": "Talált URL-ek",
           "contentItems": "Tartalom Elemek",
-          "scheduled": "Ütemezett"
+          "scheduled": "Ütemezett",
+          "totalScrapes": "Összes Letöltés",
+          "contentBreakdown": "Tartalom Bontás",
+          "scheduledScraping": "Ütemezett Scraping"
         },
         "table": {
           "url": "URL",
@@ -1306,6 +1314,10 @@
         "noContentDescription": "Nem található tartalom.",
         "untitled": "Névtelen",
         "scraped": "Letöltve",
+        "groupedByPage": "Oldalak szerint csoportosítva",
+        "scrapingInProgress": "Scraping folyamatban...",
+        "enableScheduling": "Engedélyezze fent az ütemezett scrapinget a tartalom automatikus gyűjtéséhez.",
+        "readMore": "Tovább olvas",
         "contentDialog": {
           "title": "Tartalom Részletei",
           "close": "Bezárás"
@@ -1319,7 +1331,14 @@
         "disableSuccess": "Scraping sikeresen letiltva",
         "disableError": "Hiba a scraping letiltása során",
         "urlAddSuccess": "Egyedi URL sikeresen hozzáadva",
-        "urlAddError": "Hiba az egyedi URL hozzáadása során"
+        "urlAddError": "Hiba az egyedi URL hozzáadása során",
+        "differentDomainError": "Az egyedi URL-nek ugyanarról a domainről kell származnia, mint az áruházának",
+        "invalidUrlError": "Érvénytelen URL formátum",
+        "urlEnabled": "Egyedi URL sikeresen engedélyezve",
+        "urlDisabled": "Egyedi URL sikeresen letiltva",
+        "urlToggleError": "Hiba az egyedi URL frissítése során",
+        "urlDeleted": "Egyedi URL sikeresen törölve",
+        "urlDeleteError": "Hiba az egyedi URL törlése során"
       }
     }
   },
@@ -1351,5 +1370,73 @@
     "bankLevelEncryption": "Banki szintű titkosítás",
     "lightningFastAuth": "Villámgyors hitelesítés",
     "poweredByAI": "Fejlett AI technológiával hajtva"
+  },
+  "crawler": {
+    "title": "ShopCall.ai Web Crawler",
+    "backHome": "Vissza a Főoldalra",
+    "hero": {
+      "description": "Webes crawlerünk arra lett tervezve, hogy jobban megértse e-kereskedelmi áruházát, segítve AI-nkat pontosabb és személyre szabottabb ügyfélszolgálat nyújtásában. Tudja meg, hogyan gyűjtünk adatokat, mit gyűjtünk, és hogyan tartja meg a teljes irányítást."
+    },
+    "userAgent": {
+      "title": "Crawlerünk Azonosítása",
+      "formatLabel": "User Agent Formátum",
+      "hostLabel": "Forrás Hosztok",
+      "hostDescription": "A crawler több szerverről működik, ahol [n] a szerver számát jelöli (pl. crawler-1, crawler-2, crawler-3, stb.)"
+    },
+    "features": {
+      "title": "Hogyan Működik a Crawlerünk",
+      "subtitle": "A tartalmak tiszteletével és a webes szabványok teljes betartásával készült",
+      "respectful": {
+        "title": "Tiszteletteljes Crawling",
+        "description": "Teljes mértékben tiszteletben tartjuk a robots.txt fájlját és követjük az összes crawling irányelvet. Crawlerünk egyértelműen azonosítja magát, és sebességkorlátokon belül működik, hogy ne befolyásolja weboldalának teljesítményét."
+      },
+      "registered": {
+        "title": "Csak Regisztrált Áruházak",
+        "description": "Csak olyan webáruházakat crawlolunk, amelyek regisztrálva vannak rendszerünkben. Ha nem kapcsolta össze áruházát a ShopCall.ai-jal, crawlerünk soha nem fogja meglátogatni weboldalát."
+      },
+      "manageable": {
+        "title": "Teljes Felhasználói Kontroll",
+        "description": "Az áruháztulajdonosok teljes mértékben irányíthatják, mely URL-eket lehet crawlolni. A crawling engedélyeket közvetlenül a műszerfalról kezelheti, bármikor engedélyezhet vagy letilthat bizonyos oldalakat."
+      },
+      "targeted": {
+        "title": "Célzott Tartalom Felfedezés",
+        "description": "Olyan specifikus oldalkategóriákra összpontosítunk, amelyek javítják az AI válaszokat: GYIK oldalak, felhasználási feltételek, vásárlási útmutatók és kapcsolattartási információk. Nem crawlolunk felesleges tartalmat."
+      }
+    },
+    "howItWorks": {
+      "title": "A Crawling Folyamat",
+      "step1": {
+        "title": "Sitemap Felfedezés",
+        "description": "A sitemap.xml fájl olvasásával kezdjük, hogy felfedezzük a webáruházában elérhető oldalakat. Ez hatékony crawling-ot biztosít és tiszteletben tartja webhelye szerkezetét."
+      },
+      "step2": {
+        "title": "robots.txt Megfelelés",
+        "description": "Mielőtt bármely oldalt crawlolnánk, ellenőrizzük a robots.txt fájlt, hogy biztosítsuk, hozzáférhetünk-e hozzá. A tiltott oldalakat soha nem crawloljuk."
+      },
+      "step3": {
+        "title": "Kategória Alapú Szűrés",
+        "description": "Elemezzük a felfedezett URL-eket és azonosítjuk azokat az oldalakat, amelyek az ügyfélszolgálathoz kapcsolódó specifikus kategóriákba tartoznak:",
+        "categories": {
+          "faq": "GYIK és Segítség Oldalak",
+          "terms": "Felhasználási Feltételek és Irányelvek",
+          "shopping": "Vásárlási Útmutatók és Termékinformációk",
+          "contacts": "Kapcsolattartási Információk és Támogatási Oldalak"
+        }
+      },
+      "step4": {
+        "title": "AI Tudásbázis",
+        "description": "A gyűjtött tartalmat feldolgozzuk és hozzáadjuk AI tudásbázisunkhoz, lehetővé téve pontosabb és kontextus-tudatos válaszokat, amikor ügyfelek termékeiről, irányelveikről vagy szolgáltatásaikról hívnak."
+      }
+    },
+    "cta": {
+      "title": "Készen Áll AI Támogatásának Fejlesztésére?",
+      "description": "Kapcsolja össze áruházát a ShopCall.ai-jal, és hagyja, hogy crawlerünk segítsen személyre szabott tudásbázis építésében a jobb ügyfélszolgálat érdekében.",
+      "button": "Kezdje El Most"
+    },
+    "footer": {
+      "copyright": "© 2024 ShopCall.ai. Minden jog fenntartva.",
+      "privacy": "Adatvédelmi Irányelvek",
+      "about": "Rólunk"
+    }
   }
 }

shopcall.ai-main/src/pages/Crawler.tsx

@@ -0,0 +1,223 @@
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Bot, Globe, Shield, FileSearch, Lock, Settings } from "lucide-react";
+import { useTranslation } from "react-i18next";
+
+const Crawler = () => {
+  const { t } = useTranslation();
+
+  const features = [
+    {
+      icon: Shield,
+      title: t('crawler.features.respectful.title'),
+      description: t('crawler.features.respectful.description')
+    },
+    {
+      icon: Lock,
+      title: t('crawler.features.registered.title'),
+      description: t('crawler.features.registered.description')
+    },
+    {
+      icon: Settings,
+      title: t('crawler.features.manageable.title'),
+      description: t('crawler.features.manageable.description')
+    },
+    {
+      icon: FileSearch,
+      title: t('crawler.features.targeted.title'),
+      description: t('crawler.features.targeted.description')
+    }
+  ];
+
+  return (
+    <div className="min-h-screen bg-slate-900 text-white">
+      {/* Header */}
+      <header className="border-b border-slate-800">
+        <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
+          <div className="flex items-center justify-between h-16">
+            <div className="flex items-center gap-3">
+              <img src="/uploads/e0ddbf09-622c-426a-851f-149776e300c0.png" alt="ShopCall.ai" className="w-8 h-8" />
+              <span className="text-xl font-bold text-white">ShopCall.ai</span>
+            </div>
+            <Button variant="outline" className="border-slate-600 text-slate-300 hover:bg-slate-800" asChild>
+              <a href="/">{t('crawler.backHome')}</a>
+            </Button>
+          </div>
+        </div>
+      </header>
+
+      {/* Hero Section */}
+      <section className="py-20">
+        <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 text-center">
+          <div className="w-16 h-16 bg-[#52b3d0]/20 rounded-lg flex items-center justify-center mb-6 mx-auto">
+            <Bot className="h-8 w-8 text-[#52b3d0]" />
+          </div>
+          <h1 className="text-5xl font-bold mb-6">{t('crawler.title')}</h1>
+          <p className="text-xl text-slate-300 mb-8 leading-relaxed">
+            {t('crawler.hero.description')}
+          </p>
+        </div>
+      </section>
+
+      {/* User Agent Section */}
+      <section className="py-12 bg-slate-800/50">
+        <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8">
+          <h2 className="text-3xl font-bold mb-6 text-center">{t('crawler.userAgent.title')}</h2>
+          <Card className="bg-slate-700/50 border-slate-600">
+            <CardContent className="pt-6">
+              <div className="space-y-4">
+                <div>
+                  <h3 className="text-lg font-semibold text-[#52b3d0] mb-2">{t('crawler.userAgent.formatLabel')}</h3>
+                  <code className="block bg-slate-900 p-4 rounded-lg text-slate-300 text-sm overflow-x-auto">
+                    ShopCallCrawler/1.0.0+0e76ee9 (+https://shopcall.ai/crawler)
+                  </code>
+                </div>
+                <div>
+                  <h3 className="text-lg font-semibold text-[#52b3d0] mb-2">{t('crawler.userAgent.hostLabel')}</h3>
+                  <code className="block bg-slate-900 p-4 rounded-lg text-slate-300 text-sm overflow-x-auto">
+                    crawler-[n].shop.static.shopcall.ai
+                  </code>
+                  <p className="text-slate-400 text-sm mt-2">{t('crawler.userAgent.hostDescription')}</p>
+                </div>
+              </div>
+            </CardContent>
+          </Card>
+        </div>
+      </section>
+
+      {/* Features Section */}
+      <section className="py-20">
+        <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
+          <div className="text-center mb-16">
+            <h2 className="text-4xl font-bold mb-6">{t('crawler.features.title')}</h2>
+            <p className="text-xl text-slate-300 max-w-3xl mx-auto">
+              {t('crawler.features.subtitle')}
+            </p>
+          </div>
+
+          <div className="grid md:grid-cols-2 gap-8">
+            {features.map((feature, index) => (
+              <Card key={index} className="bg-slate-700/50 border-slate-600">
+                <CardHeader>
+                  <div className="w-12 h-12 bg-[#52b3d0]/20 rounded-lg flex items-center justify-center mb-4">
+                    <feature.icon className="h-6 w-6 text-[#52b3d0]" />
+                  </div>
+                  <CardTitle className="text-white text-xl">{feature.title}</CardTitle>
+                </CardHeader>
+                <CardContent>
+                  <p className="text-slate-300">{feature.description}</p>
+                </CardContent>
+              </Card>
+            ))}
+          </div>
+        </div>
+      </section>
+
+      {/* How It Works Section */}
+      <section className="py-20 bg-slate-800/50">
+        <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8">
+          <h2 className="text-4xl font-bold mb-12 text-center">{t('crawler.howItWorks.title')}</h2>
+
+          <div className="space-y-6">
+            <Card className="bg-slate-700/50 border-slate-600">
+              <CardHeader>
+                <CardTitle className="text-white flex items-center gap-3">
+                  <span className="w-8 h-8 bg-[#52b3d0] rounded-full flex items-center justify-center text-sm font-bold">1</span>
+                  {t('crawler.howItWorks.step1.title')}
+                </CardTitle>
+              </CardHeader>
+              <CardContent>
+                <p className="text-slate-300">{t('crawler.howItWorks.step1.description')}</p>
+              </CardContent>
+            </Card>
+
+            <Card className="bg-slate-700/50 border-slate-600">
+              <CardHeader>
+                <CardTitle className="text-white flex items-center gap-3">
+                  <span className="w-8 h-8 bg-[#52b3d0] rounded-full flex items-center justify-center text-sm font-bold">2</span>
+                  {t('crawler.howItWorks.step2.title')}
+                </CardTitle>
+              </CardHeader>
+              <CardContent>
+                <p className="text-slate-300">{t('crawler.howItWorks.step2.description')}</p>
+              </CardContent>
+            </Card>
+
+            <Card className="bg-slate-700/50 border-slate-600">
+              <CardHeader>
+                <CardTitle className="text-white flex items-center gap-3">
+                  <span className="w-8 h-8 bg-[#52b3d0] rounded-full flex items-center justify-center text-sm font-bold">3</span>
+                  {t('crawler.howItWorks.step3.title')}
+                </CardTitle>
+              </CardHeader>
+              <CardContent>
+                <p className="text-slate-300">{t('crawler.howItWorks.step3.description')}</p>
+                <div className="mt-4 space-y-2">
+                  <div className="flex items-center gap-2 text-slate-400">
+                    <span className="w-2 h-2 bg-[#52b3d0] rounded-full"></span>
+                    <span>{t('crawler.howItWorks.step3.categories.faq')}</span>
+                  </div>
+                  <div className="flex items-center gap-2 text-slate-400">
+                    <span className="w-2 h-2 bg-[#52b3d0] rounded-full"></span>
+                    <span>{t('crawler.howItWorks.step3.categories.terms')}</span>
+                  </div>
+                  <div className="flex items-center gap-2 text-slate-400">
+                    <span className="w-2 h-2 bg-[#52b3d0] rounded-full"></span>
+                    <span>{t('crawler.howItWorks.step3.categories.shopping')}</span>
+                  </div>
+                  <div className="flex items-center gap-2 text-slate-400">
+                    <span className="w-2 h-2 bg-[#52b3d0] rounded-full"></span>
+                    <span>{t('crawler.howItWorks.step3.categories.contacts')}</span>
+                  </div>
+                </div>
+              </CardContent>
+            </Card>
+
+            <Card className="bg-slate-700/50 border-slate-600">
+              <CardHeader>
+                <CardTitle className="text-white flex items-center gap-3">
+                  <span className="w-8 h-8 bg-[#52b3d0] rounded-full flex items-center justify-center text-sm font-bold">4</span>
+                  {t('crawler.howItWorks.step4.title')}
+                </CardTitle>
+              </CardHeader>
+              <CardContent>
+                <p className="text-slate-300">{t('crawler.howItWorks.step4.description')}</p>
+              </CardContent>
+            </Card>
+          </div>
+        </div>
+      </section>
+
+      {/* CTA Section */}
+      <section className="py-20">
+        <div className="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 text-center">
+          <h2 className="text-4xl font-bold mb-6">{t('crawler.cta.title')}</h2>
+          <p className="text-xl text-slate-300 mb-8">
+            {t('crawler.cta.description')}
+          </p>
+          <Button size="lg" className="bg-[#52b3d0] hover:bg-[#4a9fbc] text-white px-8 py-4 text-lg" asChild>
+            <a href="/signup">
+              {t('crawler.cta.button')}
+              <Globe className="ml-2 h-5 w-5" />
+            </a>
+          </Button>
+        </div>
+      </section>
+
+      {/* Footer */}
+      <footer className="bg-slate-900 border-t border-slate-800 py-8">
+        <div className="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 text-center">
+          <div className="flex justify-center items-center gap-4 text-sm text-slate-500">
+            <span>{t('crawler.footer.copyright')}</span>
+            <span>•</span>
+            <a href="/privacy" className="hover:text-slate-300">{t('crawler.footer.privacy')}</a>
+            <span>•</span>
+            <a href="/about" className="hover:text-slate-300">{t('crawler.footer.about')}</a>
+          </div>
+        </div>
+      </footer>
+    </div>
+  );
+};
+
+export default Crawler;

supabase/functions/scraper-management/index.ts

@@ -293,67 +293,132 @@ Deno.serve(async (req) => {
       }
 
       case 'custom-urls': {
-        const storeId = url.searchParams.get('store_id') || (await req.json())?.store_id;
-        if (!storeId) {
-          return new Response(
-            JSON.stringify({ error: 'store_id is required' }),
-            { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
-          );
-        }
+        // Handle GET and POST for listing/adding custom URLs
+        if (pathParts.length === 1) {
+          const storeId = url.searchParams.get('store_id') || (await req.json())?.store_id;
+          if (!storeId) {
+            return new Response(
+              JSON.stringify({ error: 'store_id is required' }),
+              { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
 
-        const store = await getStoreScraperConfig(supabase, storeId, user.id);
+          const store = await getStoreScraperConfig(supabase, storeId, user.id);
 
-        if (!store.scraper_registered) {
-          return new Response(
-            JSON.stringify({ error: 'Store not registered with scraper' }),
-            { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
-          );
-        }
+          if (!store.scraper_registered) {
+            return new Response(
+              JSON.stringify({ error: 'Store not registered with scraper' }),
+              { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
 
-        const scraperClient = await createScraperClient({
-          scraper_api_url: store.scraper_api_url,
-          scraper_api_secret: store.scraper_api_secret,
-        });
+          const scraperClient = await createScraperClient({
+            scraper_api_url: store.scraper_api_url,
+            scraper_api_secret: store.scraper_api_secret,
+          });
+
+          if (req.method === 'GET') {
+            // List custom URLs
+            const customUrls = await scraperClient.listCustomUrls(store.id);
+            return new Response(
+              JSON.stringify(customUrls),
+              { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+
+          } else if (req.method === 'POST') {
+            // Add custom URL
+            const { url: customUrl, content_type } = await req.json();
+
+            if (!customUrl || !content_type) {
+              return new Response(
+                JSON.stringify({ error: 'url and content_type are required' }),
+                { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+              );
+            }
+
+            // Validate same domain
+            if (!validateSameDomain(store.store_url, customUrl)) {
+              return new Response(
+                JSON.stringify({ error: 'Custom URL must be from the same domain as the store' }),
+                { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+              );
+            }
+
+            const result = await scraperClient.addCustomUrl(store.id, customUrl, content_type);
+
+            return new Response(
+              JSON.stringify(result),
+              { status: 201, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
 
-        if (req.method === 'GET') {
-          // List custom URLs
-          const customUrls = await scraperClient.listCustomUrls(store.id);
           return new Response(
-            JSON.stringify(customUrls),
-            { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
           );
+        }
 
-        } else if (req.method === 'POST') {
-          // Add custom URL
-          const { url: customUrl, content_type } = await req.json();
+        // Handle PATCH /custom-urls/:customUrlId/toggle and DELETE /custom-urls/:customUrlId
+        if (pathParts.length >= 2) {
+          const customUrlId = pathParts[1];
+          const subAction = pathParts[2]; // 'toggle' or undefined
 
-          if (!customUrl || !content_type) {
+          const { store_id, enabled } = await req.json().catch(() => ({ store_id: null, enabled: null }));
+
+          if (!store_id) {
             return new Response(
-              JSON.stringify({ error: 'url and content_type are required' }),
+              JSON.stringify({ error: 'store_id is required' }),
               { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
             );
           }
 
-          // Validate same domain
-          if (!validateSameDomain(store.store_url, customUrl)) {
+          const store = await getStoreScraperConfig(supabase, store_id, user.id);
+
+          if (!store.scraper_registered) {
             return new Response(
-              JSON.stringify({ error: 'Custom URL must be from the same domain as the store' }),
-              { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+              JSON.stringify({ error: 'Store not registered with scraper' }),
+              { status: 404, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
             );
           }
 
-          const result = await scraperClient.addCustomUrl(store.id, customUrl, content_type);
+          const scraperClient = await createScraperClient({
+            scraper_api_url: store.scraper_api_url,
+            scraper_api_secret: store.scraper_api_secret,
+          });
+
+          if (req.method === 'PATCH' && subAction === 'toggle') {
+            // Toggle custom URL enabled status
+            if (typeof enabled !== 'boolean') {
+              return new Response(
+                JSON.stringify({ error: 'enabled (boolean) is required' }),
+                { status: 400, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+              );
+            }
+
+            await scraperClient.setCustomUrlEnabled(store.id, customUrlId, enabled);
+
+            return new Response(
+              JSON.stringify({ success: true, message: `Custom URL ${enabled ? 'enabled' : 'disabled'}` }),
+              { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+
+          } else if (req.method === 'DELETE') {
+            // Delete custom URL
+            await scraperClient.deleteCustomUrl(store.id, customUrlId);
+
+            return new Response(
+              JSON.stringify({ success: true, message: 'Custom URL deleted successfully' }),
+              { status: 200, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            );
+          }
 
           return new Response(
-            JSON.stringify(result),
-            { status: 201, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
+            JSON.stringify({ error: 'Method not allowed' }),
+            { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
           );
         }
 
-        return new Response(
-          JSON.stringify({ error: 'Method not allowed' }),
-          { status: 405, headers: { ...corsHeaders, 'Content-Type': 'application/json' } }
-        );
+        break;
       }
 
       case 'scheduling': {