feat: improve scraper content management UI

- Add content preview modal with raw/rendered markdown modes using react-markdown
- Group scraped content by category instead of by URL for better organization
- Remove scraping enable/disable checkbox (not allowed for shop owners)
- Add toggle functionality for individual scraped URLs (enable/disable per URL)
- Hide content in list view, show preview button instead
- Support adding auto-discovered URLs as custom URLs when toggling off
- Add Eye and Code icons for preview modes
- Improve UX with better visual organization and controls

shopcall.ai-main/package.json

@@ -57,6 +57,7 @@
     "react-dom": "^18.3.1",
     "react-hook-form": "^7.53.0",
     "react-i18next": "^16.2.4",
+    "react-markdown": "^10.1.0",
     "react-resizable-panels": "^2.1.3",
     "react-router-dom": "^6.26.2",
     "recharts": "^2.12.7",

shopcall.ai-main/src/components/ManageStoreDataContent.tsx

@@ -10,6 +10,13 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
 import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { Badge } from "@/components/ui/badge";
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+} from "@/components/ui/dialog";
 import {
   AlertDialog,
   AlertDialogAction,
@@ -20,10 +27,11 @@ import {
   AlertDialogHeader,
   AlertDialogTitle,
 } from "@/components/ui/alert-dialog";
-import { Loader2, Search, Package, ChevronLeft, ChevronRight, Tag, Globe, ExternalLink, Plus, Trash2, Clock, CheckCircle, AlertCircle, Settings } from "lucide-react";
+import { Loader2, Search, Package, ChevronLeft, ChevronRight, Tag, Globe, ExternalLink, Plus, Trash2, Clock, CheckCircle, AlertCircle, Settings, Eye, Code } from "lucide-react";
 import { API_URL } from "@/lib/config";
 import { useToast } from "@/hooks/use-toast";
 import { useTranslation } from "react-i18next";
+import ReactMarkdown from 'react-markdown';
 
 interface StoreData {
   id: string;
@@ -159,6 +167,10 @@ export function ManageStoreDataContent() {
   const [newCustomUrl, setNewCustomUrl] = useState("");
   const [newCustomUrlType, setNewCustomUrlType] = useState<'shipping' | 'contacts' | 'terms' | 'faq'>('faq');
 
+  // Content preview modal state
+  const [previewContent, setPreviewContent] = useState<ScraperContent | null>(null);
+  const [previewMode, setPreviewMode] = useState<'raw' | 'rendered'>('rendered');
+
   // Fetch stores on mount
   useEffect(() => {
     const fetchStores = async () => {
@@ -578,45 +590,6 @@ export function ManageStoreDataContent() {
     }
   };
 
-  const handleToggleScheduling = async (enabled: boolean) => {
-    if (!selectedStore) return;
-
-    try {
-      const sessionData = localStorage.getItem('session_data');
-      if (!sessionData) throw new Error('No session data found');
-
-      const session = JSON.parse(sessionData);
-      const response = await fetch(`${API_URL}/scraper-management/scheduling`, {
-        method: 'PATCH',
-        headers: {
-          'Authorization': `Bearer ${session.session.access_token}`,
-          'Content-Type': 'application/json'
-        },
-        body: JSON.stringify({
-          store_id: selectedStore.id,
-          enabled,
-        })
-      });
-
-      if (response.ok) {
-        toast({
-          title: t('common.success'),
-          description: enabled ? t('manageStoreData.website.toast.enableSuccess') : t('manageStoreData.website.toast.disableSuccess'),
-        });
-        fetchScraperStatus(); // Refresh status
-      } else {
-        const error = await response.json();
-        throw new Error(error.error || 'Failed to update scheduling');
-      }
-    } catch (error) {
-      toast({
-        title: t('common.error'),
-        description: error.message || t('manageStoreData.website.toast.enableError'),
-        variant: "destructive"
-      });
-    }
-  };
-
   const handleToggleItem = async (itemId: string, currentEnabled: boolean, product?: Product) => {
     if (!selectedStore) return;
 
@@ -1267,25 +1240,6 @@ export function ManageStoreDataContent() {
                         )}
 
                         {/* Scheduling Control */}
-                        <div className="border border-slate-700 rounded-lg p-4">
-                          <div className="flex items-center justify-between">
-                            <div>
-                              <div className="text-white font-medium">{t('manageStoreData.website.contentViewer.statusLabels.scheduledScraping') || 'Scheduled Scraping'}</div>
-                              <div className="text-slate-400 text-sm mt-1">
-                                {scraperStatus.scheduled_enabled
-                                  ? (t('manageStoreData.website.scraperStatus.scheduledDescription') || 'Automatically scrapes your website for new content')
-                                  : (t('manageStoreData.website.scraperStatus.scheduledDisabledDescription') || 'Enable to automatically collect content')
-                                }
-                              </div>
-                            </div>
-                            <Switch
-                              checked={scraperStatus.scheduled_enabled}
-                              onCheckedChange={handleToggleScheduling}
-                              className="data-[state=checked]:bg-cyan-500"
-                            />
-                          </div>
-                        </div>
-
                         {/* Timestamps and Error Display */}
                         <div className="flex flex-wrap gap-4 text-sm">
                           {scraperStatus.last_scraped_at && (
@@ -1447,7 +1401,7 @@ export function ManageStoreDataContent() {
                         <CardTitle className="text-white">{t('manageStoreData.website.contentViewer.title')} ({scraperContent.length})</CardTitle>
                         {scraperContent.length > 0 && (
                           <div className="text-sm text-slate-400">
-                            {t('manageStoreData.website.contentViewer.groupedByPage') || 'Grouped by page'}
+                            {t('manageStoreData.website.contentViewer.groupedByCategory') || 'Grouped by category'}
                           </div>
                         )}
                       </div>
@@ -1463,92 +1417,162 @@ export function ManageStoreDataContent() {
                           ) : (
                             <div>
                               <p>{t('manageStoreData.website.contentViewer.noContentDescription')}</p>
-                              {scraperStatus && !scraperStatus.scheduled_enabled && (
-                                <p className="mt-2 text-sm">
-                                  {t('manageStoreData.website.contentViewer.enableScheduling') || 'Enable scheduled scraping above to automatically collect content.'}
-                                </p>
-                              )}
                             </div>
                           )}
                         </div>
                       ) : (
                         <div className="space-y-4">
-                          {/* Group content by URL (page) */}
+                          {/* Group content by category */}
                           {Object.entries(
                             scraperContent.reduce((acc, content) => {
-                              if (!acc[content.url]) {
-                                acc[content.url] = [];
+                              if (!acc[content.content_type]) {
+                                acc[content.content_type] = [];
                               }
-                              acc[content.url].push(content);
+                              acc[content.content_type].push(content);
                               return acc;
                             }, {} as Record<string, typeof scraperContent>)
-                          ).map(([url, contents]) => (
-                            <div key={url} className="border border-slate-600 rounded-lg overflow-hidden">
-                              {/* Page Header */}
+                          ).map(([category, contents]) => (
+                            <div key={category} className="border border-slate-600 rounded-lg overflow-hidden">
+                              {/* Category Header */}
                               <div className="bg-slate-700/50 p-3 border-b border-slate-600">
                                 <div className="flex items-center justify-between">
                                   <div className="flex-1 min-w-0">
                                     <div className="flex items-center gap-2">
-                                      <Globe className="w-4 h-4 text-cyan-400 flex-shrink-0" />
-                                      <a
-                                        href={url}
-                                        target="_blank"
-                                        rel="noopener noreferrer"
-                                        className="text-white hover:text-cyan-400 underline truncate text-sm"
-                                      >
-                                        {url}
-                                      </a>
-                                      <ExternalLink className="w-3 h-3 text-slate-400 flex-shrink-0" />
+                                      <Tag className="w-4 h-4 text-cyan-400 flex-shrink-0" />
+                                      <span className="text-white font-medium capitalize">{category}</span>
                                     </div>
                                     <div className="text-slate-400 text-xs mt-1">
-                                      {contents.length} {contents.length === 1 ? 'section' : 'sections'} found
+                                      {contents.length} {contents.length === 1 ? 'item' : 'items'}
                                     </div>
                                   </div>
                                 </div>
                               </div>
 
-                              {/* Content Sections */}
+                              {/* Content Items */}
                               <div className="divide-y divide-slate-600">
                                 {contents.map((content) => (
                                   <div key={content.id} className="p-4 bg-slate-800/50 hover:bg-slate-700/30 transition-colors">
                                     <div className="flex items-start justify-between mb-2">
-                                      <div className="flex-1">
-                                        <div className="text-white font-medium">
-                                          {content.title || t('manageStoreData.website.contentViewer.untitled')}
+                                      <div className="flex-1 min-w-0">
+                                        <div className="flex items-center gap-2 mb-1">
+                                          <Globe className="w-3 h-3 text-slate-400 flex-shrink-0" />
+                                          <a
+                                            href={content.url}
+                                            target="_blank"
+                                            rel="noopener noreferrer"
+                                            className="text-white hover:text-cyan-400 underline truncate text-sm"
+                                          >
+                                            {content.title || content.url.split('/').pop() || 'Untitled'}
+                                          </a>
+                                          <ExternalLink className="w-3 h-3 text-slate-400 flex-shrink-0" />
+                                        </div>
+                                        <div className="text-slate-400 text-xs">
+                                          {new Date(content.scraped_at).toLocaleString()}
                                         </div>
                                       </div>
-                                      <Badge
-                                        variant="outline"
-                                        className="text-xs ml-2"
-                                      >
-                                        {content.content_type}
-                                      </Badge>
-                                    </div>
-                                    <div className="text-slate-300 text-sm mb-3 whitespace-pre-wrap">
-                                      {content.content.length > 500 ? (
-                                        <details className="cursor-pointer">
-                                          <summary className="hover:text-white">
-                                            {content.content.substring(0, 500)}...
-                                            <span className="text-cyan-400 ml-2">{t('manageStoreData.website.contentViewer.readMore') || 'Read more'}</span>
-                                          </summary>
-                                          <div className="mt-2 pt-2 border-t border-slate-600">
-                                            {content.content}
-                                          </div>
-                                        </details>
-                                      ) : (
-                                        content.content
-                                      )}
-                                    </div>
-                                    <div className="flex items-center gap-4 text-slate-400 text-xs">
-                                      <div className="flex items-center gap-1">
-                                        <Clock className="w-3 h-3" />
-                                        {t('manageStoreData.website.contentViewer.scraped')}: {new Date(content.scraped_at).toLocaleString()}
+                                      <div className="flex items-center gap-2 ml-2">
+                                        <Button
+                                          variant="outline"
+                                          size="sm"
+                                          onClick={() => {
+                                            setPreviewContent(content);
+                                            setPreviewMode('rendered');
+                                          }}
+                                          className="border-slate-600 hover:bg-slate-700"
+                                        >
+                                          <Eye className="w-3 h-3 mr-1" />
+                                          Preview
+                                        </Button>
+                                        <Switch
+                                          checked={(() => {
+                                            // Check if this URL exists in customUrls as disabled
+                                            const customUrl = customUrls.find(cu => cu.url === content.url);
+                                            return customUrl ? customUrl.enabled : true;
+                                          })()}
+                                          onCheckedChange={async (checked) => {
+                                            // Toggle URL enabled/disabled
+                                            try {
+                                              const sessionData = localStorage.getItem('session_data');
+                                              if (!sessionData) throw new Error('No session data found');
+
+                                              const session = JSON.parse(sessionData);
+
+                                              // Check if URL already exists in custom URLs
+                                              const existingCustomUrl = customUrls.find(cu => cu.url === content.url);
+
+                                              if (existingCustomUrl) {
+                                                // Update existing custom URL
+                                                const response = await fetch(`${API_URL}/scraper-management/custom-urls/${existingCustomUrl.id}/toggle`, {
+                                                  method: 'PATCH',
+                                                  headers: {
+                                                    'Authorization': `Bearer ${session.session.access_token}`,
+                                                    'Content-Type': 'application/json'
+                                                  },
+                                                  body: JSON.stringify({
+                                                    store_id: selectedStore?.id,
+                                                    enabled: checked
+                                                  })
+                                                });
+
+                                                if (!response.ok) {
+                                                  throw new Error('Failed to toggle URL');
+                                                }
+                                              } else {
+                                                // Add as new custom URL with specified enabled status
+                                                const response = await fetch(`${API_URL}/scraper-management/custom-urls`, {
+                                                  method: 'POST',
+                                                  headers: {
+                                                    'Authorization': `Bearer ${session.session.access_token}`,
+                                                    'Content-Type': 'application/json'
+                                                  },
+                                                  body: JSON.stringify({
+                                                    store_id: selectedStore?.id,
+                                                    url: content.url,
+                                                    content_type: content.content_type,
+                                                  })
+                                                });
+
+                                                if (!response.ok) {
+                                                  throw new Error('Failed to add custom URL');
+                                                }
+
+                                                // If we want to disable it, toggle it right after adding
+                                                if (!checked) {
+                                                  const newCustomUrl = await response.json();
+                                                  await fetch(`${API_URL}/scraper-management/custom-urls/${newCustomUrl.id}/toggle`, {
+                                                    method: 'PATCH',
+                                                    headers: {
+                                                      'Authorization': `Bearer ${session.session.access_token}`,
+                                                      'Content-Type': 'application/json'
+                                                    },
+                                                    body: JSON.stringify({
+                                                      store_id: selectedStore?.id,
+                                                      enabled: false
+                                                    })
+                                                  });
+                                                }
+                                              }
+
+                                              toast({
+                                                title: checked ? "URL enabled" : "URL disabled",
+                                                description: checked ? "This URL will be included in future scrapes" : "This URL will be skipped in future scrapes",
+                                              });
+
+                                              // Refresh both content and custom URLs
+                                              fetchScraperContent();
+                                              fetchCustomUrls();
+                                            } catch (error) {
+                                              console.error('Error toggling URL:', error);
+                                              toast({
+                                                title: "Error",
+                                                description: "Failed to toggle URL status",
+                                                variant: "destructive",
+                                              });
+                                            }
+                                          }}
+                                          className="data-[state=checked]:bg-cyan-500"
+                                        />
                                       </div>
-                                      {content.metadata && Object.keys(content.metadata).length > 0 && (
-                                        <Badge variant="secondary" className="text-xs">
-                                          {Object.keys(content.metadata).length} metadata fields
-                                        </Badge>
-                                      )}
                                     </div>
                                   </div>
                                 ))}
@@ -1612,6 +1636,73 @@ export function ManageStoreDataContent() {
         </CardContent>
       </Card>
 
+      {/* Content Preview Dialog */}
+      <Dialog open={!!previewContent} onOpenChange={(open) => !open && setPreviewContent(null)}>
+        <DialogContent className="bg-slate-800 border-slate-700 max-w-4xl max-h-[80vh] overflow-hidden flex flex-col">
+          <DialogHeader>
+            <DialogTitle className="text-white flex items-center justify-between">
+              <span className="truncate">
+                {previewContent?.title || previewContent?.url.split('/').pop() || 'Content Preview'}
+              </span>
+              <div className="flex items-center gap-2 ml-4">
+                <Button
+                  variant={previewMode === 'rendered' ? 'default' : 'outline'}
+                  size="sm"
+                  onClick={() => setPreviewMode('rendered')}
+                  className={previewMode === 'rendered' ? 'bg-cyan-500 hover:bg-cyan-600' : 'border-slate-600'}
+                >
+                  <Eye className="w-3 h-3 mr-1" />
+                  Rendered
+                </Button>
+                <Button
+                  variant={previewMode === 'raw' ? 'default' : 'outline'}
+                  size="sm"
+                  onClick={() => setPreviewMode('raw')}
+                  className={previewMode === 'raw' ? 'bg-cyan-500 hover:bg-cyan-600' : 'border-slate-600'}
+                >
+                  <Code className="w-3 h-3 mr-1" />
+                  Raw
+                </Button>
+              </div>
+            </DialogTitle>
+            <DialogDescription className="text-slate-400 flex items-center gap-4">
+              <Badge variant="outline" className="text-xs">
+                {previewContent?.content_type}
+              </Badge>
+              <a
+                href={previewContent?.url}
+                target="_blank"
+                rel="noopener noreferrer"
+                className="text-cyan-400 hover:text-cyan-300 underline text-xs flex items-center gap-1"
+              >
+                <Globe className="w-3 h-3" />
+                Open source URL
+                <ExternalLink className="w-3 h-3" />
+              </a>
+            </DialogDescription>
+          </DialogHeader>
+          <div className="flex-1 overflow-y-auto mt-4">
+            {previewMode === 'rendered' ? (
+              <div className="prose prose-invert prose-cyan max-w-none">
+                <ReactMarkdown className="text-slate-200">
+                  {previewContent?.content || ''}
+                </ReactMarkdown>
+              </div>
+            ) : (
+              <pre className="bg-slate-900 p-4 rounded-lg text-slate-300 text-sm overflow-x-auto whitespace-pre-wrap">
+                {previewContent?.content || ''}
+              </pre>
+            )}
+          </div>
+          {previewContent?.scraped_at && (
+            <div className="mt-4 pt-4 border-t border-slate-700 text-slate-400 text-xs">
+              <Clock className="w-3 h-3 inline mr-1" />
+              Scraped: {new Date(previewContent.scraped_at).toLocaleString()}
+            </div>
+          )}
+        </DialogContent>
+      </Dialog>
+
       {/* Confirmation Dialog */}
       <AlertDialog open={confirmDialog.open} onOpenChange={(open) => setConfirmDialog({ ...confirmDialog, open })}>
         <AlertDialogContent className="bg-slate-800 border-slate-700">