Add content-based deduplication to IMAP Extract Attachments

- Add utils.hash() function for SHA-256 hashing in script engine
- Add deduplicateByHash config option (default: true)
- Use content hash in file path for deduplication
- Skip writing file if same content already exists on disk
- Include contentHash and deduplicated flag in metadata storage
- Prevents duplicate attachments when processing same email multiple times

nodes/imap/imap-extract-attachments.js

@@ -0,0 +1,480 @@
+/**
+ * @node imap-extract-attachments
+ * @name IMAP Extract Attachments
+ * @category email
+ * @version 1.4.0
+ * @description Extract attachments from email as binary files. Compatible with OCR Upload and other binary-accepting nodes.
+ * @icon paperclip
+ */
+
+const configSchema = {
+    type: 'object',
+    properties: {
+        emailSource: {
+            type: 'string',
+            title: 'Email Source',
+            description: 'Path to email data containing the raw body (e.g., {{item.body}} for loop iterations, or {{data.body}} for direct input)'
+        },
+        deduplicateByHash: {
+            type: 'boolean',
+            title: 'Deduplicate by Content',
+            description: 'Skip storing attachments that already exist (based on content hash). Prevents duplicates when processing the same email multiple times.',
+            default: true
+        },
+        filterMimeTypes: {
+            type: 'string',
+            title: 'Filter by MIME Types',
+            description: 'Comma-separated list of MIME types to include (e.g. application/pdf, image/*). Leave empty for all attachments.'
+        },
+        filterExtensions: {
+            type: 'string',
+            title: 'Filter by Extensions',
+            description: 'Comma-separated list of file extensions to include (e.g. pdf, jpg, png). Leave empty for all.'
+        },
+        maxSize: {
+            type: 'integer',
+            title: 'Max Size (bytes)',
+            description: 'Maximum attachment size in bytes (0 = no limit)',
+            default: 0
+        },
+        storeInDatabase: {
+            type: 'boolean',
+            title: 'Store in Database',
+            description: 'Store extracted attachments in database for later retrieval',
+            default: false
+        },
+        storageCollection: {
+            type: 'string',
+            title: 'Storage Collection',
+            description: 'Select collection to store attachments',
+            dynamicOptions: {
+                source: 'storage.collections',
+                labelField: 'name',
+                valueField: 'name',
+                filter: { access: 'read-write' }
+            }
+        },
+        storageTtlHours: {
+            type: 'number',
+            title: 'TTL (hours)',
+            default: 24,
+            description: 'Auto-delete stored files after hours (0 = never)'
+        },
+        binaryStoragePath: {
+            type: 'string',
+            title: 'Binary Storage Path',
+            default: './data/attachments',
+            description: 'Base directory for storing binary files on disk'
+        }
+    }
+};
+
+const inputSchema = {
+    type: 'object',
+    properties: {
+        raw: {
+            type: 'string',
+            description: 'Raw email content (from IMAP Fetch or IMAP Trigger with fetchContent enabled)'
+        },
+        email: {
+            type: 'object',
+            description: 'Email object containing raw property'
+        }
+    }
+};
+
+const outputSchema = {
+    type: 'object',
+    properties: {
+        attachments: {
+            type: 'array',
+            description: 'Array of extracted attachments as binary file objects',
+            items: {
+                type: 'object',
+                properties: {
+                    type: { type: 'string', const: 'binary' },
+                    data: { type: 'string', description: 'Base64-encoded content' },
+                    mimeType: { type: 'string' },
+                    filename: { type: 'string' },
+                    size: { type: 'number' },
+                    contentId: { type: 'string', description: 'Content-ID for inline attachments' },
+                    storage: {
+                        type: 'object',
+                        description: 'Storage reference (if storeInDatabase enabled)',
+                        properties: {
+                            collection: { type: 'string' },
+                            id: { type: 'string' }
+                        }
+                    }
+                }
+            }
+        },
+        count: { type: 'integer', description: 'Number of attachments extracted' },
+        totalSize: { type: 'integer', description: 'Total size of all attachments in bytes' }
+    }
+};
+
+const outputs = [
+    { name: 'main', displayName: 'Attachments', type: 'object', color: '#f59e0b' }
+];
+
+/**
+ * Parse MIME multipart content and extract attachments
+ */
+function parseMimeContent(rawEmail) {
+    const attachments = [];
+
+    // Find boundary from Content-Type header
+    const boundaryMatch = rawEmail.match(/boundary=["']?([^"'\s;]+)["']?/i);
+    if (!boundaryMatch) {
+        // Not a multipart email, no attachments
+        return attachments;
+    }
+
+    const boundary = boundaryMatch[1];
+    const parts = rawEmail.split('--' + boundary);
+
+    for (let i = 1; i < parts.length; i++) {
+        const part = parts[i];
+
+        // Skip the closing boundary marker
+        if (part.trim() === '--' || part.trim().startsWith('--')) {
+            continue;
+        }
+
+        // Parse headers and body from this part
+        const headerEndIndex = part.indexOf('\r\n\r\n');
+        const headerEndIndex2 = part.indexOf('\n\n');
+        const actualHeaderEnd = headerEndIndex !== -1 ? headerEndIndex : headerEndIndex2;
+
+        if (actualHeaderEnd === -1) continue;
+
+        const headers = part.substring(0, actualHeaderEnd);
+        const body = part.substring(actualHeaderEnd + (headerEndIndex !== -1 ? 4 : 2));
+
+        // Check if this is an attachment
+        const contentDisposition = headers.match(/Content-Disposition:\s*([^\r\n]+)/i);
+        const contentType = headers.match(/Content-Type:\s*([^\r\n;]+)/i);
+        const contentTransferEncoding = headers.match(/Content-Transfer-Encoding:\s*([^\r\n]+)/i);
+        const contentId = headers.match(/Content-ID:\s*<?([^>\r\n]+)>?/i);
+
+        // Extract filename from Content-Disposition or Content-Type
+        let filename = null;
+        const filenameMatch = headers.match(/filename=["']?([^"'\r\n;]+)["']?/i);
+        const filenameStarMatch = headers.match(/filename\*=(?:UTF-8''|utf-8'')([^\r\n;]+)/i);
+        const nameMatch = headers.match(/name=["']?([^"'\r\n;]+)["']?/i);
+
+        if (filenameStarMatch) {
+            try {
+                filename = decodeURIComponent(filenameStarMatch[1]);
+            } catch (e) {
+                filename = filenameStarMatch[1];
+            }
+        } else if (filenameMatch) {
+            filename = filenameMatch[1];
+        } else if (nameMatch) {
+            filename = nameMatch[1];
+        }
+
+        // Determine if this is an attachment (has filename or Content-Disposition: attachment)
+        const isAttachment = filename ||
+            (contentDisposition && contentDisposition[1].toLowerCase().includes('attachment'));
+
+        // Skip if not an attachment and no filename
+        if (!isAttachment) continue;
+
+        // Get MIME type
+        const mimeType = contentType ? contentType[1].trim() : 'application/octet-stream';
+
+        // Skip text/plain and text/html parts (these are usually the email body)
+        if (mimeType.startsWith('text/plain') || mimeType.startsWith('text/html')) {
+            if (!filename) continue;  // Only skip if no filename (body part)
+        }
+
+        // Decode content based on transfer encoding
+        let data = body.trim();
+        const encoding = contentTransferEncoding ? contentTransferEncoding[1].trim().toLowerCase() : '';
+
+        if (encoding === 'base64') {
+            // Already base64 encoded, just clean it up
+            data = data.replace(/[\r\n\s]/g, '');
+        } else if (encoding === 'quoted-printable') {
+            // Decode quoted-printable and then base64 encode
+            const decoded = decodeQuotedPrintable(data);
+            data = smartbotic.utils.base64Encode(decoded);
+        } else {
+            // Raw or 7bit/8bit - base64 encode it
+            data = smartbotic.utils.base64Encode(data);
+        }
+
+        // Generate filename if not present
+        if (!filename) {
+            const ext = getExtensionForMimeType(mimeType);
+            filename = 'attachment_' + (attachments.length + 1) + ext;
+        }
+
+        // Calculate size (approximate from base64)
+        const size = Math.floor(data.length * 3 / 4);
+
+        attachments.push({
+            type: 'binary',
+            data: data,
+            mimeType: mimeType,
+            filename: filename,
+            size: size,
+            contentId: contentId ? contentId[1] : null
+        });
+    }
+
+    return attachments;
+}
+
+/**
+ * Decode quoted-printable encoding
+ */
+function decodeQuotedPrintable(str) {
+    return str
+        .replace(/=\r?\n/g, '')  // Remove soft line breaks
+        .replace(/=([0-9A-Fa-f]{2})/g, (match, hex) => {
+            return String.fromCharCode(parseInt(hex, 16));
+        });
+}
+
+/**
+ * Get file extension for common MIME types
+ */
+function getExtensionForMimeType(mimeType) {
+    const mimeMap = {
+        'application/pdf': '.pdf',
+        'application/msword': '.doc',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.ms-excel': '.xls',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/zip': '.zip',
+        'application/x-rar-compressed': '.rar',
+        'application/x-7z-compressed': '.7z',
+        'application/gzip': '.gz',
+        'image/jpeg': '.jpg',
+        'image/png': '.png',
+        'image/gif': '.gif',
+        'image/bmp': '.bmp',
+        'image/webp': '.webp',
+        'image/tiff': '.tiff',
+        'text/plain': '.txt',
+        'text/csv': '.csv',
+        'text/html': '.html',
+        'application/json': '.json',
+        'application/xml': '.xml'
+    };
+
+    return mimeMap[mimeType.toLowerCase()] || '.bin';
+}
+
+/**
+ * Check if MIME type matches filter pattern (supports wildcards like "image/*")
+ */
+function matchesMimeType(mimeType, patterns) {
+    if (!patterns || patterns.length === 0) return true;
+
+    const lowerMime = mimeType.toLowerCase();
+
+    for (const pattern of patterns) {
+        const lowerPattern = pattern.trim().toLowerCase();
+        if (lowerPattern.endsWith('/*')) {
+            const prefix = lowerPattern.slice(0, -1);
+            if (lowerMime.startsWith(prefix)) return true;
+        } else if (lowerMime === lowerPattern) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+/**
+ * Check if filename matches extension filter
+ */
+function matchesExtension(filename, extensions) {
+    if (!extensions || extensions.length === 0) return true;
+
+    const lowerFilename = filename.toLowerCase();
+
+    for (const ext of extensions) {
+        const lowerExt = ext.trim().toLowerCase();
+        const dotExt = lowerExt.startsWith('.') ? lowerExt : '.' + lowerExt;
+        if (lowerFilename.endsWith(dotExt)) return true;
+    }
+
+    return false;
+}
+
+module.exports = {
+    configSchema,
+    inputSchema,
+    outputSchema,
+    outputs,
+
+    async execute(config, input, context) {
+        // Get raw email content from input
+        let rawEmail = null;
+
+        // First priority: use configured emailSource if provided
+        // The expression (e.g., {{item.body}}) is evaluated by the workflow engine,
+        // so config.emailSource contains the actual email body string
+        if (config.emailSource && typeof config.emailSource === 'string' && config.emailSource.length > 0) {
+            rawEmail = config.emailSource;
+            smartbotic.log.info('IMAP Extract Attachments: Using configured email source');
+        }
+
+        // Fallback: try to auto-detect from various input structures
+        if (!rawEmail) {
+            rawEmail = input.raw || input.body;
+
+            if (!rawEmail && input.email) {
+                rawEmail = input.email.raw || input.email.body;
+            }
+
+            if (!rawEmail && input.data) {
+                rawEmail = input.data.raw || input.data.body;
+            }
+
+            // Support loop iteration input (item/currentItem from Loop node)
+            if (!rawEmail && input.item) {
+                rawEmail = input.item.raw || input.item.body;
+            }
+
+            if (!rawEmail && input.currentItem) {
+                rawEmail = input.currentItem.raw || input.currentItem.body;
+            }
+        }
+
+        if (!rawEmail) {
+            // Log available keys for debugging
+            const availableKeys = Object.keys(input || {}).join(', ');
+            smartbotic.log.warn('IMAP Extract Attachments: No raw email found. Input keys: ' + availableKeys);
+            throw new Error('No raw email content provided. Configure the "Email Source" field with an expression like {{item.body}} for loop iterations, or ensure the email data is properly connected.');
+        }
+
+        // Parse MIME content
+        let attachments = parseMimeContent(rawEmail);
+
+        // Apply filters
+        const mimeFilters = config.filterMimeTypes
+            ? config.filterMimeTypes.split(',').map(s => s.trim()).filter(s => s)
+            : [];
+        const extFilters = config.filterExtensions
+            ? config.filterExtensions.split(',').map(s => s.trim()).filter(s => s)
+            : [];
+        const maxSize = config.maxSize || 0;
+
+        attachments = attachments.filter(att => {
+            // Filter by MIME type
+            if (!matchesMimeType(att.mimeType, mimeFilters)) return false;
+
+            // Filter by extension
+            if (!matchesExtension(att.filename, extFilters)) return false;
+
+            // Filter by size
+            if (maxSize > 0 && att.size > maxSize) return false;
+
+            return true;
+        });
+
+        // Store attachments - always write to disk, optionally store metadata in database
+        const storagePath = config.binaryStoragePath || './data/attachments';
+        const timestamp = Date.now();
+        const deduplicateByHash = config.deduplicateByHash !== false;  // Default to true
+
+        for (const att of attachments) {
+            // Generate file path - use content hash for deduplication if enabled
+            const date = new Date(timestamp);
+            const yearMonth = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`;
+            const safeFilename = att.filename.replace(/[^a-zA-Z0-9._-]/g, '_');
+
+            let filePath;
+            let skipWrite = false;
+
+            if (deduplicateByHash) {
+                // Calculate content hash for deduplication
+                const contentHash = smartbotic.utils.hash(att.data);
+                att.contentHash = contentHash;
+
+                // Use hash-based path: basePath/YYYY-MM/hash_filename
+                // Using first 16 chars of SHA-256 hash (64 bits) for reasonable uniqueness
+                const shortHash = contentHash.substring(0, 16);
+                filePath = `${storagePath}/${yearMonth}/${shortHash}_${safeFilename}`;
+
+                // Check if file already exists
+                if (smartbotic.fs.exists(filePath)) {
+                    smartbotic.log.info(`Attachment ${att.filename} already exists (hash: ${shortHash}), reusing: ${filePath}`);
+                    att.filePath = filePath;
+                    att.deduplicated = true;
+                    delete att.data;  // Remove base64 data from memory
+                    skipWrite = true;
+                }
+            } else {
+                // Use UUID-based path for unique storage
+                const uuid = smartbotic.utils.uuid();
+                filePath = `${storagePath}/${yearMonth}/${uuid}_${safeFilename}`;
+            }
+
+            if (!skipWrite) {
+                // Write binary data to disk
+                const writeResult = smartbotic.fs.writeFile(filePath, att.data, 'base64');
+                if (writeResult.success) {
+                    // Replace base64 data with file path
+                    att.filePath = filePath;
+                    att.deduplicated = false;
+                    delete att.data;  // Remove base64 data from memory
+                    smartbotic.log.info(`Stored attachment ${att.filename} to disk: ${filePath}`);
+                } else {
+                    smartbotic.log.warn(`Failed to write attachment to disk: ${writeResult.error}`);
+                    // Keep base64 data as fallback
+                }
+            }
+        }
+
+        // Store metadata in database if configured (without binary data)
+        if (config.storeInDatabase && attachments.length > 0) {
+            const collection = config.storageCollection || 'email_attachments';
+            const ttlHours = config.storageTtlHours || 0;
+            const ttlMs = ttlHours > 0 ? ttlHours * 60 * 60 * 1000 : 0;
+
+            for (const att of attachments) {
+                // Store only metadata, not the binary data
+                const storageDoc = {
+                    filename: att.filename,
+                    mimeType: att.mimeType,
+                    size: att.size,
+                    contentId: att.contentId,
+                    contentHash: att.contentHash,  // For deduplication lookup
+                    deduplicated: att.deduplicated,
+                    extractedAt: timestamp,
+                    filePath: att.filePath  // Reference to disk location
+                };
+
+                const insertResult = smartbotic.storage.insert(collection, storageDoc, null, ttlMs);
+                if (insertResult.success) {
+                    att.storage = {
+                        collection: collection,
+                        id: insertResult.id
+                    };
+                    smartbotic.log.info(`Stored attachment metadata ${att.filename} in ${collection}/${insertResult.id}`);
+                }
+            }
+        }
+
+        // Calculate total size
+        const totalSize = attachments.reduce((sum, att) => sum + att.size, 0);
+
+        smartbotic.log.info(`Extracted ${attachments.length} attachment(s), total size: ${totalSize} bytes`);
+
+        return {
+            attachments,
+            count: attachments.length,
+            totalSize
+        };
+    }
+};

src/runner/engine/script_engine.cpp

@@ -569,6 +569,34 @@ void ScriptEngine::setupBuiltinAPIs() {
         return JS_NewString(ctx, uuid.c_str());
     }, "uuid", 0));
 
+    // utils.hash(data) - Generate SHA-256 hash of a string
+    JS_SetPropertyStr(ctx, utils, "hash", JS_NewCFunction(ctx, [](JSContext* ctx, JSValue this_val, int argc, JSValue* argv) -> JSValue {
+        if (argc < 1) {
+            return JS_ThrowTypeError(ctx, "utils.hash requires a string");
+        }
+
+        const char* data_str = JS_ToCString(ctx, argv[0]);
+        if (!data_str) {
+            return JS_ThrowTypeError(ctx, "utils.hash requires a valid string");
+        }
+
+        // Calculate SHA-256 hash
+        unsigned char hash[SHA256_DIGEST_LENGTH];
+        SHA256(reinterpret_cast<const unsigned char*>(data_str), strlen(data_str), hash);
+        JS_FreeCString(ctx, data_str);
+
+        // Convert to hex string
+        std::string hex_hash;
+        hex_hash.reserve(SHA256_DIGEST_LENGTH * 2);
+        const char* hex_chars = "0123456789abcdef";
+        for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) {
+            hex_hash += hex_chars[(hash[i] >> 4) & 0xF];
+            hex_hash += hex_chars[hash[i] & 0xF];
+        }
+
+        return JS_NewString(ctx, hex_hash.c_str());
+    }, "hash", 1));
+
     // utils.sleep(ms) - Pause execution for specified milliseconds
     JS_SetPropertyStr(ctx, utils, "sleep", JS_NewCFunction(ctx, [](JSContext* ctx, JSValue this_val, int argc, JSValue* argv) -> JSValue {
         if (argc < 1) {