feat: comprehensive content improvements with markdown viewer and title extraction

Backend improvements:
- Enhanced ContentExtractor with comprehensive content cleanup (ContentExtractor.ts:103-126)
  - Removes excessive newlines (3+ → 2)
  - Strips trailing spaces/tabs
  - Cleans up markdown formatting
  - Normalizes headers and list spacing
- Added title extraction from <title> tags (ContentExtractor.ts:34)
- Updated PageContent interface to include optional title field (types/index.ts:24)
- Modified WebshopScraper to handle new extraction result structure

Frontend improvements:
- Added markdown viewer with Raw/Formatted toggle in content modal (ContentPage.ts:170-177)
- Implemented simple markdown renderer supporting:
  - Headers (H1, H2, H3)
  - Bold/italic text
  - Links with proper styling
  - Code blocks and inline code
  - Lists with proper formatting
- Enhanced modal title to display actual page titles instead of "undefined Content" (ContentPage.ts:679)
- Added view toggle functionality with proper button state management
- Updated ShopContent interface to include title field (web/src/types/index.ts:42)

User experience:
- Content now displays with proper page titles
- Users can switch between raw markdown and formatted view
- Cleaner content with improved readability
- Better formatted output with consistent spacing

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/scraper/ContentExtractor.ts

@@ -16,7 +16,7 @@ export class ContentExtractor {
   /**
    * Fetch and extract main content from a URL
    */
-  async extractContent(url: string): Promise<string> {
+  async extractContent(url: string): Promise<{ content: string; title: string }> {
     try {
       logger.info(`Extracting content from: ${url}`);
 
@@ -30,6 +30,9 @@ export class ContentExtractor {
       const html = response.data;
       const $ = cheerio.load(html);
 
+      // Extract page title
+      const pageTitle = $('title').text().trim() || 'Untitled Page';
+
       // Remove unwanted elements
       $('script').remove();
       $('style').remove();
@@ -79,20 +82,49 @@ export class ContentExtractor {
       // Convert HTML to Markdown
       const markdown = this.turndownService.turndown(mainContent);
 
-      // Clean up markdown (remove excessive newlines)
-      const cleanedMarkdown = markdown
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
+      // Clean up markdown with comprehensive cleanup
+      const cleanedMarkdown = this.cleanupContent(markdown);
 
       logger.info(`Extracted ${cleanedMarkdown.length} characters of content from ${url}`);
 
-      return cleanedMarkdown;
+      return {
+        content: cleanedMarkdown,
+        title: pageTitle
+      };
     } catch (error) {
       logger.error(`Failed to extract content from ${url}`, error);
       throw new Error(`Failed to extract content: ${error instanceof Error ? error.message : 'Unknown error'}`);
     }
   }
 
+  /**
+   * Comprehensive content cleanup to remove duplicate newlines and clean formatting
+   */
+  private cleanupContent(content: string): string {
+    return content
+      // Remove excessive newlines (3 or more becomes 2)
+      .replace(/\n{3,}/g, '\n\n')
+      // Remove trailing spaces and tabs from lines
+      .replace(/[ \t]+$/gm, '')
+      // Remove leading spaces/tabs from lines (but preserve markdown indentation)
+      .replace(/^[ \t]+/gm, '')
+      // Remove multiple spaces within text (but preserve intentional spacing)
+      .replace(/[ \t]{2,}/g, ' ')
+      // Clean up markdown list formatting
+      .replace(/\n\s*\n\s*[-\*\+]/g, '\n\n- ')
+      // Clean up excessive spaces around punctuation
+      .replace(/\s+([,.;:!?])/g, '$1')
+      // Normalize line breaks around headers
+      .replace(/\n+(#{1,6}\s)/g, '\n\n$1')
+      .replace(/(#{1,6}[^\n]*)\n+/g, '$1\n\n')
+      // Remove empty markdown links and formatting
+      .replace(/\[\]\([^)]*\)/g, '')
+      .replace(/\*\*\s*\*\*/g, '')
+      .replace(/__\s*__/g, '')
+      // Final trim
+      .trim();
+  }
+
   /**
    * Extract contact information from content
    */

src/scraper/WebshopScraper.ts

@@ -217,8 +217,12 @@ export class WebshopScraper {
     for (const urlObj of urls) {
       try {
         const url = urlObj.loc;
-        const content = await this.contentExtractor.extractContent(url);
-        results.push({ url, content });
+        const extractionResult = await this.contentExtractor.extractContent(url);
+        results.push({
+          url,
+          content: extractionResult.content,
+          title: extractionResult.title
+        });
       } catch (error) {
         logger.error(`Failed to extract content from ${urlObj.loc}`, error);
         // Continue with next URL even if one fails
@@ -239,8 +243,12 @@ export class WebshopScraper {
     // Try to extract from the first URL (usually the most relevant)
     try {
       const url = urls[0].loc;
-      const content = await this.contentExtractor.extractContent(url);
-      return { url, content };
+      const extractionResult = await this.contentExtractor.extractContent(url);
+      return {
+        url,
+        content: extractionResult.content,
+        title: extractionResult.title
+      };
     } catch (error) {
       logger.error('Failed to extract page content', error);
 
@@ -248,8 +256,12 @@ export class WebshopScraper {
       if (urls.length > 1) {
         try {
           const url = urls[1].loc;
-          const content = await this.contentExtractor.extractContent(url);
-          return { url, content };
+          const extractionResult = await this.contentExtractor.extractContent(url);
+          return {
+            url,
+            content: extractionResult.content,
+            title: extractionResult.title
+          };
         } catch (error2) {
           logger.error('Failed to extract from fallback URL', error2);
           return null;

src/types/index.ts

@@ -21,6 +21,7 @@ export interface ScraperResult {
 export interface PageContent {
   url: string;
   content: string;
+  title?: string;
 }
 
 export type WebshopType = 'shoprenter' | 'woocommerce' | 'shopify';

web/src/components/pages/ContentPage.ts

@@ -167,6 +167,14 @@ export class ContentPage {
                       <span class="badge" id="modal-change-badge">Status</span>
                     </div>
                     <div class="flex space-x-2">
+                      <div class="flex rounded-md shadow-sm" role="group">
+                        <button type="button" class="btn btn-secondary btn-sm rounded-r-none border-r-0" id="view-raw-btn">
+                          Raw
+                        </button>
+                        <button type="button" class="btn btn-primary btn-sm rounded-l-none" id="view-formatted-btn">
+                          Formatted
+                        </button>
+                      </div>
                       <button type="button" class="btn btn-secondary btn-sm" id="copy-content-btn">
                         <div class="w-4 h-4 mr-1">${getIcon('copy')}</div>
                         Copy
@@ -178,9 +186,12 @@ export class ContentPage {
                     </div>
                   </div>
                   <div class="bg-gray-50 dark:bg-gray-700 rounded-lg p-4 max-h-96 overflow-y-auto">
-                    <pre class="whitespace-pre-wrap text-sm text-gray-800 dark:text-gray-200" id="modal-content">
+                    <pre class="whitespace-pre-wrap text-sm text-gray-800 dark:text-gray-200 hidden" id="modal-content-raw">
                       Content will appear here...
                     </pre>
+                    <div class="text-sm text-gray-800 dark:text-gray-200 prose dark:prose-invert max-w-none" id="modal-content-formatted">
+                      Formatted content will appear here...
+                    </div>
                   </div>
                   <div class="mt-4 text-xs text-gray-500 dark:text-gray-400" id="modal-metadata">
                     Last updated: --
@@ -260,6 +271,18 @@ export class ContentPage {
       }
     });
 
+    // View toggle buttons
+    const viewRawBtn = this.element.querySelector('#view-raw-btn');
+    const viewFormattedBtn = this.element.querySelector('#view-formatted-btn');
+
+    viewRawBtn?.addEventListener('click', () => {
+      this.showRawView();
+    });
+
+    viewFormattedBtn?.addEventListener('click', () => {
+      this.showFormattedView();
+    });
+
     // ESC key to close modal
     document.addEventListener('keydown', (e) => {
       if (e.key === 'Escape') {
@@ -652,9 +675,18 @@ export class ContentPage {
     const modalTypeBadge = this.element?.querySelector('#modal-type-badge');
     const modalChangeBadge = this.element?.querySelector('#modal-change-badge');
 
-    if (modalTitle) modalTitle.textContent = `${getContentTypeLabel(foundItem.content_type)} Content`;
+    // Use page title if available, otherwise fallback to content type
+    const pageTitle = foundItem.title || `${getContentTypeLabel(foundItem.content_type)} Content`;
+    if (modalTitle) modalTitle.textContent = pageTitle;
     if (modalUrl) modalUrl.textContent = foundItem.url;
-    if (modalContent) modalContent.textContent = foundItem.content || '';
+
+    // Update both content views
+    const content = foundItem.content || '';
+    const modalContentRaw = this.element?.querySelector('#modal-content-raw');
+    const modalContentFormatted = this.element?.querySelector('#modal-content-formatted');
+
+    if (modalContentRaw) modalContentRaw.textContent = content;
+    if (modalContentFormatted) modalContentFormatted.innerHTML = this.renderMarkdown(content);
     if (modalMetadata) {
       const content = foundItem.content || '';
       const contentHash = foundItem.content_hash || '';
@@ -684,6 +716,74 @@ export class ContentPage {
     modal?.classList.remove('hidden');
   }
 
+  private showRawView(): void {
+    const rawElement = this.element?.querySelector('#modal-content-raw');
+    const formattedElement = this.element?.querySelector('#modal-content-formatted');
+    const rawBtn = this.element?.querySelector('#view-raw-btn');
+    const formattedBtn = this.element?.querySelector('#view-formatted-btn');
+
+    // Toggle visibility
+    rawElement?.classList.remove('hidden');
+    formattedElement?.classList.add('hidden');
+
+    // Update button styles
+    rawBtn?.classList.remove('btn-secondary');
+    rawBtn?.classList.add('btn-primary');
+    formattedBtn?.classList.remove('btn-primary');
+    formattedBtn?.classList.add('btn-secondary');
+  }
+
+  private showFormattedView(): void {
+    const rawElement = this.element?.querySelector('#modal-content-raw');
+    const formattedElement = this.element?.querySelector('#modal-content-formatted');
+    const rawBtn = this.element?.querySelector('#view-raw-btn');
+    const formattedBtn = this.element?.querySelector('#view-formatted-btn');
+
+    // Toggle visibility
+    rawElement?.classList.add('hidden');
+    formattedElement?.classList.remove('hidden');
+
+    // Update button styles
+    rawBtn?.classList.remove('btn-primary');
+    rawBtn?.classList.add('btn-secondary');
+    formattedBtn?.classList.remove('btn-secondary');
+    formattedBtn?.classList.add('btn-primary');
+  }
+
+  private renderMarkdown(markdown: string): string {
+    // Simple markdown renderer - handles basic formatting
+    return markdown
+      // Headers
+      .replace(/^### (.*$)/gim, '<h3 class="text-lg font-semibold text-gray-900 dark:text-white mt-4 mb-2">$1</h3>')
+      .replace(/^## (.*$)/gim, '<h2 class="text-xl font-semibold text-gray-900 dark:text-white mt-6 mb-3">$1</h2>')
+      .replace(/^# (.*$)/gim, '<h1 class="text-2xl font-bold text-gray-900 dark:text-white mt-8 mb-4">$1</h1>')
+
+      // Bold and italic
+      .replace(/\*\*(.*?)\*\*/g, '<strong class="font-semibold">$1</strong>')
+      .replace(/\*(.*?)\*/g, '<em class="italic">$1</em>')
+
+      // Links
+      .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<a href="$2" class="text-blue-600 dark:text-blue-400 hover:underline" target="_blank" rel="noopener noreferrer">$1</a>')
+
+      // Lists
+      .replace(/^\- (.*$)/gim, '<li class="ml-4">• $1</li>')
+      .replace(/^(\d+)\. (.*$)/gim, '<li class="ml-4">$1. $2</li>')
+
+      // Code blocks (basic)
+      .replace(/```([\s\S]*?)```/g, '<pre class="bg-gray-100 dark:bg-gray-800 p-3 rounded text-sm overflow-x-auto mt-2 mb-2"><code>$1</code></pre>')
+      .replace(/`([^`]+)`/g, '<code class="bg-gray-100 dark:bg-gray-800 px-1 rounded text-sm">$1</code>')
+
+      // Line breaks
+      .replace(/\n\n/g, '</p><p class="mb-2">')
+      .replace(/\n/g, '<br>')
+
+      // Wrap in paragraphs
+      .replace(/^(.*)$/gm, '<p class="mb-2">$1</p>')
+
+      // Clean up empty paragraphs
+      .replace(/<p class="mb-2"><\/p>/g, '');
+  }
+
   destroy(): void {
     this.eventsbound = false;
     // Clean up event listeners

web/src/types/index.ts

@@ -39,6 +39,7 @@ export interface ShopContent {
   changed: boolean;
   created_at: string;
   updated_at: string;
+  title?: string;
 }
 
 export interface ScrapeHistory {