feat: implement content chunking for Qdrant embeddings

Major improvements to vector search and LLM context management:

- Add ChunkingService to split content into 600-word chunks with 100-word overlap
- Split by paragraphs while respecting sentence boundaries
- Fix EmbeddingService to preserve paragraph structure (was destroying \n\n)
- Update QdrantEmbeddingWorkflow to create one vector per chunk instead of per page
- Store chunk_text in Qdrant payload for direct MCP access
- Update MCP BaseTool to group chunks by page and show relevant sections
- Add chunk metadata: chunk_id, chunk_index, total_chunks

Benefits:
- Better search precision (match specific paragraphs, not whole pages)
- Optimal LLM context (600 words per chunk vs 5000+ words per page)
- No content truncation (all content preserved across chunks)
- Controlled context size for MCP responses
- Better semantic search quality

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/mcp/tools/BaseTool.ts

@@ -129,37 +129,61 @@ export abstract class BaseTool {
       };
     }
 
-    const formattedResults = searchResults.map((result, index) => {
-      // Create snippet from content
-      const snippet = this.createSnippet(result.payload?.content || '', query, 150);
+    // Group results by content_id to combine chunks from the same page
+    const groupedResults = new Map<string, any[]>();
 
-      return {
-        id: result.id,
-        score: result.score,
-        title: result.payload?.title || 'Untitled',
-        url: result.payload?.url || '',
-        content: result.payload?.content || '',
-        contentType: this.contentType,
-        snippet,
-      };
+    searchResults.forEach(result => {
+      const contentId = result.payload?.content_id || result.id;
+      if (!groupedResults.has(contentId)) {
+        groupedResults.set(contentId, []);
+      }
+      groupedResults.get(contentId)!.push(result);
     });
 
-    // Create text response
-    let responseText = `Found ${searchResults.length} result${searchResults.length === 1 ? '' : 's'} for "${query}" in ${this.contentType}:\n\n`;
-
-    formattedResults.forEach((result, index) => {
-      responseText += `${index + 1}. **${result.title}**\n`;
-      responseText += `   URL: ${result.url}\n`;
-      responseText += `   Relevance: ${(result.score * 100).toFixed(1)}%\n`;
-      responseText += `   Preview: ${result.snippet}\n`;
+    // Format grouped results
+    let responseText = `Found ${groupedResults.size} page${groupedResults.size === 1 ? '' : 's'} with ${searchResults.length} relevant section${searchResults.length === 1 ? '' : 's'} for "${query}" in ${this.contentType}:\n\n`;
+
+    let pageIndex = 0;
+    for (const [contentId, chunks] of groupedResults) {
+      pageIndex++;
+
+      // Sort chunks by index
+      chunks.sort((a, b) => (a.payload?.chunk_index || 0) - (b.payload?.chunk_index || 0));
+
+      const firstChunk = chunks[0];
+      const title = firstChunk.payload?.title || 'Untitled';
+      const url = firstChunk.payload?.url || '';
+      const avgScore = chunks.reduce((sum, c) => sum + c.score, 0) / chunks.length;
+
+      responseText += `${pageIndex}. **${title}**\n`;
+      responseText += `   URL: ${url}\n`;
+      responseText += `   Relevance: ${(avgScore * 100).toFixed(1)}%\n`;
+
+      if (chunks.length === 1) {
+        // Single chunk - show full chunk text
+        const chunkText = firstChunk.payload?.chunk_text || '';
+        responseText += `   Content: ${chunkText}\n`;
+      } else {
+        // Multiple chunks - show each relevant section
+        responseText += `   ${chunks.length} relevant sections found:\n`;
+
+        chunks.forEach((chunk, idx) => {
+          const chunkText = chunk.payload?.chunk_text || '';
+          const chunkIndex = chunk.payload?.chunk_index ?? idx;
+          const totalChunks = chunk.payload?.total_chunks || chunks.length;
+
+          responseText += `\n   Section ${chunkIndex + 1} of ${totalChunks} (${(chunk.score * 100).toFixed(1)}% match):\n`;
+          responseText += `   ${chunkText}\n`;
+        });
+      }
 
-      if (index < formattedResults.length - 1) {
+      if (pageIndex < groupedResults.size) {
         responseText += '\n';
       }
-    });
+    }
 
     if (searchResults.length === limit) {
-      responseText += '\n\n*Results limited to top ' + limit + ' matches. Use a smaller limit or refine your query for more specific results.*';
+      responseText += '\n\n*Results limited to top ' + limit + ' matches. Refine your query for more specific results.*';
     }
 
     return {

src/services/ChunkingService.ts

@@ -0,0 +1,254 @@
+import { logger } from '../utils/logger';
+
+export interface ContentChunk {
+  index: number;
+  text: string;
+  wordCount: number;
+  charCount: number;
+}
+
+export interface ChunkingOptions {
+  targetChunkSize?: number;      // Target words per chunk (default: 600)
+  chunkOverlap?: number;          // Overlap in words (default: 100)
+  minChunkSize?: number;          // Minimum chunk size in words (default: 200)
+  maxChunkSize?: number;          // Maximum chunk size in words (default: 1000)
+  respectParagraphs?: boolean;    // Split on paragraph boundaries (default: true)
+  respectSentences?: boolean;     // Don't break mid-sentence (default: true)
+}
+
+export class ChunkingService {
+  private readonly defaultOptions: Required<ChunkingOptions> = {
+    targetChunkSize: 600,
+    chunkOverlap: 100,
+    minChunkSize: 200,
+    maxChunkSize: 1000,
+    respectParagraphs: true,
+    respectSentences: true,
+  };
+
+  /**
+   * Split content into chunks for embedding
+   */
+  chunkContent(content: string, title: string | null, options?: ChunkingOptions): ContentChunk[] {
+    const opts = { ...this.defaultOptions, ...options };
+
+    // Combine title with content if provided
+    let fullContent = content;
+    if (title && title.trim()) {
+      fullContent = `# ${title.trim()}\n\n${content}`;
+    }
+
+    // If content is small enough, return as single chunk
+    const wordCount = this.countWords(fullContent);
+    if (wordCount <= opts.targetChunkSize) {
+      return [{
+        index: 0,
+        text: fullContent.trim(),
+        wordCount,
+        charCount: fullContent.length,
+      }];
+    }
+
+    // Split into chunks
+    const chunks = opts.respectParagraphs
+      ? this.chunkByParagraphs(fullContent, opts)
+      : this.chunkByWords(fullContent, opts);
+
+    logger.debug(`Chunked content into ${chunks.length} chunks (avg: ${Math.round(wordCount / chunks.length)} words/chunk)`);
+
+    return chunks;
+  }
+
+  /**
+   * Split content by paragraphs (respecting \n\n boundaries)
+   */
+  private chunkByParagraphs(content: string, opts: Required<ChunkingOptions>): ContentChunk[] {
+    const chunks: ContentChunk[] = [];
+
+    // Split by double newlines (paragraphs)
+    const paragraphs = content.split(/\n\n+/).filter(p => p.trim().length > 0);
+
+    let currentChunk = '';
+    let currentWordCount = 0;
+
+    for (let i = 0; i < paragraphs.length; i++) {
+      const paragraph = paragraphs[i].trim();
+      const paragraphWords = this.countWords(paragraph);
+
+      // If single paragraph is too large, split it further
+      if (paragraphWords > opts.maxChunkSize) {
+        // Save current chunk if it has content
+        if (currentChunk) {
+          chunks.push(this.createChunk(chunks.length, currentChunk.trim()));
+          currentChunk = '';
+          currentWordCount = 0;
+        }
+
+        // Split large paragraph by sentences
+        const sentenceChunks = this.chunkBySentences(paragraph, opts);
+        chunks.push(...sentenceChunks.map((chunk, idx) =>
+          this.createChunk(chunks.length + idx, chunk)
+        ));
+        continue;
+      }
+
+      // Check if adding this paragraph exceeds target size
+      if (currentWordCount > 0 && currentWordCount + paragraphWords > opts.targetChunkSize) {
+        // Save current chunk
+        chunks.push(this.createChunk(chunks.length, currentChunk.trim()));
+
+        // Start new chunk with overlap from previous chunk
+        const overlapText = this.getOverlapText(currentChunk, opts.chunkOverlap);
+        currentChunk = overlapText ? overlapText + '\n\n' + paragraph : paragraph;
+        currentWordCount = this.countWords(currentChunk);
+      } else {
+        // Add paragraph to current chunk
+        currentChunk += (currentChunk ? '\n\n' : '') + paragraph;
+        currentWordCount += paragraphWords;
+      }
+    }
+
+    // Add final chunk if there's content
+    if (currentChunk.trim()) {
+      chunks.push(this.createChunk(chunks.length, currentChunk.trim()));
+    }
+
+    // If no chunks were created, return original content as single chunk
+    if (chunks.length === 0) {
+      chunks.push(this.createChunk(0, content.trim()));
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Split content by sentences (for large paragraphs)
+   */
+  private chunkBySentences(text: string, opts: Required<ChunkingOptions>): string[] {
+    const chunks: string[] = [];
+
+    // Split by sentence boundaries (., !, ?)
+    const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
+
+    let currentChunk = '';
+    let currentWordCount = 0;
+
+    for (const sentence of sentences) {
+      const sentenceWords = this.countWords(sentence);
+
+      if (currentWordCount > 0 && currentWordCount + sentenceWords > opts.targetChunkSize) {
+        chunks.push(currentChunk.trim());
+
+        // Add overlap
+        const overlapText = this.getOverlapText(currentChunk, opts.chunkOverlap);
+        currentChunk = overlapText ? overlapText + ' ' + sentence : sentence;
+        currentWordCount = this.countWords(currentChunk);
+      } else {
+        currentChunk += (currentChunk ? ' ' : '') + sentence;
+        currentWordCount += sentenceWords;
+      }
+    }
+
+    if (currentChunk.trim()) {
+      chunks.push(currentChunk.trim());
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Split content by word count (fallback method)
+   */
+  private chunkByWords(content: string, opts: Required<ChunkingOptions>): ContentChunk[] {
+    const chunks: ContentChunk[] = [];
+    const words = content.split(/\s+/);
+
+    let i = 0;
+    while (i < words.length) {
+      const chunkWords = words.slice(i, i + opts.targetChunkSize);
+      const chunkText = chunkWords.join(' ');
+
+      chunks.push(this.createChunk(chunks.length, chunkText));
+
+      // Move forward with overlap
+      i += opts.targetChunkSize - opts.chunkOverlap;
+
+      // Ensure we make progress
+      if (i <= chunks.length * opts.chunkOverlap) {
+        i = chunks.length * opts.targetChunkSize;
+      }
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Get overlap text from end of chunk
+   */
+  private getOverlapText(text: string, overlapWords: number): string {
+    const words = text.split(/\s+/);
+    if (words.length <= overlapWords) {
+      return text;
+    }
+
+    return words.slice(-overlapWords).join(' ');
+  }
+
+  /**
+   * Create chunk object
+   */
+  private createChunk(index: number, text: string): ContentChunk {
+    return {
+      index,
+      text,
+      wordCount: this.countWords(text),
+      charCount: text.length,
+    };
+  }
+
+  /**
+   * Count words in text
+   */
+  private countWords(text: string): number {
+    return text.trim().split(/\s+/).filter(word => word.length > 0).length;
+  }
+
+  /**
+   * Estimate if content needs chunking
+   */
+  shouldChunk(content: string, targetSize: number = 600): boolean {
+    return this.countWords(content) > targetSize;
+  }
+
+  /**
+   * Get chunking statistics
+   */
+  getChunkingStats(chunks: ContentChunk[]): {
+    totalChunks: number;
+    avgWordCount: number;
+    avgCharCount: number;
+    minWordCount: number;
+    maxWordCount: number;
+  } {
+    if (chunks.length === 0) {
+      return {
+        totalChunks: 0,
+        avgWordCount: 0,
+        avgCharCount: 0,
+        minWordCount: 0,
+        maxWordCount: 0,
+      };
+    }
+
+    const wordCounts = chunks.map(c => c.wordCount);
+    const charCounts = chunks.map(c => c.charCount);
+
+    return {
+      totalChunks: chunks.length,
+      avgWordCount: Math.round(wordCounts.reduce((a, b) => a + b, 0) / chunks.length),
+      avgCharCount: Math.round(charCounts.reduce((a, b) => a + b, 0) / chunks.length),
+      minWordCount: Math.min(...wordCounts),
+      maxWordCount: Math.max(...wordCounts),
+    };
+  }
+}

src/services/EmbeddingService.ts

@@ -45,13 +45,18 @@ export class EmbeddingService {
 
   /**
    * Preprocess content for embedding
-   * Combine title and content with proper formatting
+   * Normalize whitespace while preserving paragraph structure
    */
   preprocessContent(title: string | null, content: string): string {
-    // Remove excessive whitespace and normalize
+    // Normalize whitespace while preserving paragraph breaks
     const cleanContent = content
-      .replace(/\s+/g, ' ')
-      .replace(/\n\s+/g, '\n')
+      // Remove excessive spaces and tabs (but NOT newlines)
+      .replace(/[ \t]+/g, ' ')
+      // Normalize multiple newlines to double newlines (paragraph breaks)
+      .replace(/\n{3,}/g, '\n\n')
+      // Remove trailing/leading whitespace from each line
+      .replace(/[ \t]+$/gm, '')
+      .replace(/^[ \t]+/gm, '')
       .trim();
 
     // Combine title and content

src/services/QdrantEmbeddingWorkflow.ts

@@ -4,6 +4,7 @@ import { ShopDatabase, ShopContent } from '../database/Database';
 import { QdrantService } from './QdrantService';
 import { EmbeddingService } from './EmbeddingService';
 import { QdrantCleanupService } from './QdrantCleanupService';
+import { ChunkingService } from './ChunkingService';
 import { WebhookManager } from '../webhooks/WebhookManager';
 import type { AppConfig } from '../config';
 
@@ -19,6 +20,7 @@ export class QdrantEmbeddingWorkflow {
   private qdrantService: QdrantService;
   private embeddingService: EmbeddingService;
   private cleanupService: QdrantCleanupService;
+  private chunkingService: ChunkingService;
   private webhookManager: WebhookManager;
 
   constructor(
@@ -28,6 +30,7 @@ export class QdrantEmbeddingWorkflow {
     this.qdrantService = new QdrantService(config);
     this.embeddingService = new EmbeddingService(config);
     this.cleanupService = new QdrantCleanupService(config, database);
+    this.chunkingService = new ChunkingService();
     this.webhookManager = new WebhookManager(database);
   }
 
@@ -75,34 +78,11 @@ export class QdrantEmbeddingWorkflow {
         return { contentId, embedded: false, error: 'Content unchanged and embedding exists' };
       }
 
-      // Prepare content for embedding
-      const processedContent = this.embeddingService.preprocessContent(title, content);
+      // Split content into chunks
+      const chunks = this.chunkingService.chunkContent(content, title);
+      const chunkStats = this.chunkingService.getChunkingStats(chunks);
 
-      // Check if content is too long
-      if (this.embeddingService.isTextTooLong(processedContent)) {
-        const truncatedContent = this.embeddingService.truncateText(processedContent);
-        logger.warn(`Content truncated for embedding: ${contentId}`);
-      }
-
-      // Generate embedding
-      const embeddingResult = await this.embeddingService.generateEmbedding({
-        text: this.embeddingService.isTextTooLong(processedContent)
-          ? this.embeddingService.truncateText(processedContent)
-          : processedContent,
-        contentId,
-        shopId,
-        contentType
-      });
-
-      if (!embeddingResult.success) {
-        await this.cleanupService.logQdrantError(
-          shopId,
-          'embedding',
-          embeddingResult.error || 'Embedding generation failed',
-          { contentId, contentType, url }
-        );
-        return { contentId, embedded: false, error: embeddingResult.error };
-      }
+      logger.info(`Content ${contentId} split into ${chunks.length} chunks (avg: ${chunkStats.avgWordCount} words/chunk)`);
 
       // Get iterator for collection naming
       const iterator = await this.qdrantService.getNextIterator(shop.custom_id, contentType);
@@ -114,58 +94,112 @@ export class QdrantEmbeddingWorkflow {
         logger.info(`Created Qdrant collection: ${collectionName}`);
       }
 
-      // Create point
-      const pointId = uuidv4();
-      const point = {
-        id: pointId,
-        vector: embeddingResult.embedding,
-        payload: {
-          shop_id: shop.custom_id,
-          content_id: contentId,
-          content_type: contentType,
-          url: url,
-          title: title || undefined,
-          content_hash: contentHash,
-          created_at: new Date().toISOString(),
+      // Process each chunk
+      let successfulChunks = 0;
+      const pointIds: string[] = [];
+
+      for (const chunk of chunks) {
+        try {
+          // Preprocess chunk content
+          const processedChunk = this.embeddingService.preprocessContent(null, chunk.text);
+
+          // Generate embedding for this chunk
+          const embeddingResult = await this.embeddingService.generateEmbedding({
+            text: processedChunk,
+            contentId: `${contentId}-chunk-${chunk.index}`,
+            shopId,
+            contentType
+          });
+
+          if (!embeddingResult.success) {
+            logger.error(`Failed to embed chunk ${chunk.index} of content ${contentId}:`, embeddingResult.error);
+            continue;
+          }
+
+          // Create point with chunk data
+          const pointId = uuidv4();
+          const point = {
+            id: pointId,
+            vector: embeddingResult.embedding,
+            payload: {
+              shop_id: shop.custom_id,
+              content_id: contentId,
+              chunk_id: `${contentId}-chunk-${chunk.index}`,
+              chunk_index: chunk.index,
+              total_chunks: chunks.length,
+              chunk_text: chunk.text,
+              content_type: contentType,
+              url: url,
+              title: title || undefined,
+              content_hash: contentHash,
+              created_at: new Date().toISOString(),
+            }
+          };
+
+          // Insert point into Qdrant
+          await this.qdrantService.upsertPoint(collectionName, point);
+          pointIds.push(pointId);
+          successfulChunks++;
+
+          // Small delay between chunks to avoid rate limiting
+          if (chunks.length > 1) {
+            await new Promise(resolve => setTimeout(resolve, 100));
+          }
+
+        } catch (error: any) {
+          logger.error(`Error embedding chunk ${chunk.index} of content ${contentId}:`, error);
         }
-      };
+      }
+
+      // Save embedding record to database (for the first/main point)
+      if (pointIds.length > 0) {
+        const now = new Date().toISOString();
+        const embeddingId = uuidv4();
 
-      // Insert point into Qdrant
-      await this.qdrantService.upsertPoint(collectionName, point);
+        this.database.createShopEmbedding({
+          id: embeddingId,
+          shop_content_id: contentId,
+          collection_name: collectionName,
+          point_id: pointIds[0], // Store first chunk's point ID
+          embedding_status: successfulChunks === chunks.length ? 'completed' : 'failed',
+          created_at: now,
+          updated_at: now
+        });
+      }
 
-      // Save embedding record to database
-      const now = new Date().toISOString();
-      const embeddingId = uuidv4();
+      if (successfulChunks < chunks.length) {
+        const errorMsg = `Only ${successfulChunks}/${chunks.length} chunks embedded successfully`;
+        logger.warn(errorMsg);
 
-      this.database.createShopEmbedding({
-        id: embeddingId,
-        shop_content_id: contentId,
-        collection_name: collectionName,
-        point_id: pointId,
-        embedding_status: 'completed',
-        created_at: now,
-        updated_at: now
-      });
+        await this.cleanupService.logQdrantError(
+          shopId,
+          'embedding',
+          errorMsg,
+          { contentId, contentType, url, chunks: chunks.length, successful: successfulChunks }
+        );
+      }
 
-      logger.info(`Successfully embedded content ${contentId} in collection ${collectionName}`);
+      logger.info(`Successfully embedded ${successfulChunks}/${chunks.length} chunks of content ${contentId} in collection ${collectionName}`);
 
       // Trigger webhook for successful embedding
-      try {
-        await this.webhookManager.notifyQdrantEmbeddingCompleted(shopId, {
-          content_id: contentId,
-          content_type: contentType,
-          collection_name: collectionName,
-          point_id: pointId
-        });
-      } catch (error) {
-        logger.error('Failed to send embedding completed webhook:', error);
+      if (successfulChunks > 0) {
+        try {
+          await this.webhookManager.notifyQdrantEmbeddingCompleted(shopId, {
+            content_id: contentId,
+            content_type: contentType,
+            collection_name: collectionName,
+            point_id: pointIds[0] // First chunk's point ID
+          });
+        } catch (error) {
+          logger.error('Failed to send embedding completed webhook:', error);
+        }
       }
 
       return {
         contentId,
-        embedded: true,
+        embedded: successfulChunks > 0,
         collectionName,
-        pointId
+        pointId: pointIds[0]
       };
 
     } catch (error: any) {