/** * @node rss-reader * @name RSS Reader * @category data * @version 1.1.4 * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection. * @icon rss */ const configSchema = { type: 'object', properties: { url: { type: 'string', title: 'Feed URL', description: 'URL of the RSS or Atom feed' }, filterKeyword: { type: 'string', title: 'Keyword Filter', description: 'Filter items by keyword (matches title and description)' }, filterNewerThan: { type: 'string', title: 'Newer Than', description: 'Only include items newer than this date (ISO 8601 format, e.g., 2024-01-15T00:00:00Z)' }, maxItems: { type: 'number', title: 'Max Items', description: 'Maximum number of items to return (0 = unlimited)', default: 0 }, timeout: { type: 'number', title: 'Timeout (ms)', default: 30000 }, detectNewItems: { type: 'boolean', title: 'Detect New Items', description: 'When enabled, only output items that have not been seen in previous executions', default: false }, stateCollection: { type: 'string', title: 'State Collection', description: 'Select a read-write collection to store seen item identifiers', dynamicOptions: { source: 'storage.collections', labelField: 'name', valueField: 'name', filter: { access: 'read-write' } }, showWhen: { field: 'detectNewItems', value: true } } }, required: ['url'] }; const inputSchema = { type: 'object', properties: { url: { type: 'string', description: 'Override feed URL from input' }, filterKeyword: { type: 'string', description: 'Override keyword filter from input' }, filterNewerThan: { type: 'string', description: 'Override date filter from input' }, maxItems: { type: 'number', description: 'Override max items from input' } } }; const outputSchema = { type: 'object', properties: { feedTitle: { type: 'string', description: 'Title of the feed' }, feedLink: { type: 'string', description: 'Link to the feed homepage' }, feedDescription: { type: 'string', description: 'Description of the feed' }, feedLanguage: { type: 'string', description: 'Language of the feed' }, feedPubDate: { type: 'string', description: 'Publication date of the feed' }, feedGenerator: { type: 'string', description: 'Generator of the feed' }, feedImage: { type: 'object', description: 'Feed image/logo', properties: { url: { type: 'string' }, title: { type: 'string' }, link: { type: 'string' } } }, itemCount: { type: 'number', description: 'Number of items returned' }, newItemCount: { type: 'number', description: 'Number of new items (when detectNewItems is enabled)' }, totalFeedItemCount: { type: 'number', description: 'Total items in feed before filtering (when detectNewItems is enabled)' }, items: { type: 'array', description: 'Array of feed items', items: { type: 'object', properties: { title: { type: 'string', description: 'Item title' }, link: { type: 'string', description: 'Item link/URL' }, description: { type: 'string', description: 'Item summary/description' }, content: { type: 'string', description: 'Full content (content:encoded)' }, pubDate: { type: 'string', description: 'Publication date' }, author: { type: 'string', description: 'Author name' }, categories: { type: 'array', items: { type: 'string' }, description: 'Categories/tags' }, guid: { type: 'string', description: 'Unique identifier' }, comments: { type: 'string', description: 'Comments URL' }, source: { type: 'string', description: 'Source attribution' }, enclosure: { type: 'object', description: 'Media enclosure (attachment)', properties: { url: { type: 'string', description: 'Media URL' }, type: { type: 'string', description: 'MIME type' }, length: { type: 'number', description: 'File size in bytes' } } }, media: { type: 'object', description: 'Media content (media:content)', properties: { url: { type: 'string' }, type: { type: 'string' }, width: { type: 'number' }, height: { type: 'number' } } }, thumbnail: { type: 'object', description: 'Thumbnail image (media:thumbnail)', properties: { url: { type: 'string' }, width: { type: 'number' }, height: { type: 'number' } } } } } } } }; /** * Simple XML parser utilities for RSS/Atom feeds * Handles basic XML structure without external dependencies */ function xmlGetTagContent(xml, tagName) { var pattern = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'i'); var match = xml.match(pattern); if (match) { return match[1].trim(); } return null; } function xmlGetAllTagContents(xml, tagName) { var results = []; var regex = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'gi'); var match; while ((match = regex.exec(xml)) !== null) { results.push(match[1].trim()); } return results; } function xmlGetTagAttributes(xml, tagName) { // Match self-closing tag or opening tag var regex = new RegExp('<' + tagName + '([^>]*?)\\/?>', 'i'); var match = xml.match(regex); if (!match) return null; var attrString = match[1]; var attrs = {}; // Parse attributes: name="value" or name='value' var attrRegex = /(\w+)=["']([^"']*)["']/g; var attrMatch; while ((attrMatch = attrRegex.exec(attrString)) !== null) { attrs[attrMatch[1]] = attrMatch[2]; } return attrs; } function xmlDecodeHtmlEntities(text) { if (!text) return ''; return text .replace(/</g, '<') .replace(/>/g, '>') .replace(/&/g, '&') .replace(/"/g, '"') .replace(/'/g, "'") .replace(/&#(\d+);/g, function(_, dec) { return String.fromCharCode(dec); }) .replace(/&#x([0-9a-f]+);/gi, function(_, hex) { return String.fromCharCode(parseInt(hex, 16)); }); } function xmlStripCdata(text) { if (!text) return ''; return text.replace(//g, '$1'); } function xmlCleanText(text) { if (!text) return ''; return xmlDecodeHtmlEntities(xmlStripCdata(text)).trim(); } /** * Parse RSS 2.0 feed */ function parseRss(xml) { var channel = xmlGetTagContent(xml, 'channel'); if (!channel) { throw new Error('Invalid RSS feed: no channel element found'); } var feedTitle = xmlGetTagContent(channel, 'title'); var feedLink = xmlGetTagContent(channel, 'link'); var feedDesc = xmlGetTagContent(channel, 'description'); var feedLanguage = xmlGetTagContent(channel, 'language'); var feedPubDate = xmlGetTagContent(channel, 'pubDate'); var feedLastBuildDate = xmlGetTagContent(channel, 'lastBuildDate'); var feedGenerator = xmlGetTagContent(channel, 'generator'); var feedImage = xmlGetTagContent(channel, 'image'); var feed = { title: xmlCleanText(feedTitle) || '', link: xmlCleanText(feedLink) || '', description: xmlCleanText(feedDesc) || '', language: xmlCleanText(feedLanguage) || '', pubDate: xmlCleanText(feedPubDate) || '', lastBuildDate: xmlCleanText(feedLastBuildDate) || '', generator: xmlCleanText(feedGenerator) || '', items: [] }; // Parse feed image if present if (feedImage) { feed.image = { url: xmlCleanText(xmlGetTagContent(feedImage, 'url')) || '', title: xmlCleanText(xmlGetTagContent(feedImage, 'title')) || '', link: xmlCleanText(xmlGetTagContent(feedImage, 'link')) || '' }; } var itemRegex = /]*>([\s\S]*?)<\/item>/gi; var match; var itemCount = 0; while ((match = itemRegex.exec(channel)) !== null) { var itemXml = match[1]; itemCount++; var catArray = xmlGetAllTagContents(itemXml, 'category'); var categories = []; for (var ci = 0; ci < catArray.length; ci++) { categories.push(xmlCleanText(catArray[ci])); } var item = { title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '', link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '', description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '', content: xmlCleanText(xmlGetTagContent(itemXml, 'content:encoded')) || '', pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '', author: xmlCleanText(xmlGetTagContent(itemXml, 'author') || xmlGetTagContent(itemXml, 'dc:creator')) || '', categories: categories, guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || '', comments: xmlCleanText(xmlGetTagContent(itemXml, 'comments')) || '', source: xmlCleanText(xmlGetTagContent(itemXml, 'source')) || '' }; // Parse enclosure (media attachment) var enclosureAttrs = xmlGetTagAttributes(itemXml, 'enclosure'); if (enclosureAttrs) { item.enclosure = { url: enclosureAttrs.url || '', type: enclosureAttrs.type || '', length: enclosureAttrs.length ? parseInt(enclosureAttrs.length, 10) : 0 }; } // Parse media:content (common in media RSS) var mediaAttrs = xmlGetTagAttributes(itemXml, 'media:content'); if (mediaAttrs) { item.media = { url: mediaAttrs.url || '', type: mediaAttrs.type || mediaAttrs.medium || '', width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0, height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0 }; } // Parse media:thumbnail var thumbAttrs = xmlGetTagAttributes(itemXml, 'media:thumbnail'); if (thumbAttrs) { item.thumbnail = { url: thumbAttrs.url || '', width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0, height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0 }; } feed.items.push(item); } return feed; } /** * Parse Atom feed */ function parseAtom(xml) { var feedContent = xmlGetTagContent(xml, 'feed'); if (!feedContent) { throw new Error('Invalid Atom feed: no feed element found'); } function getAtomLink(content, rel) { var linkRegex = /]*)>/gi; var match; while ((match = linkRegex.exec(content)) !== null) { var attrs = match[1]; var relMatch = attrs.match(/rel=["']([^"']*)["']/); var hrefMatch = attrs.match(/href=["']([^"']*)["']/); if (hrefMatch) { var linkRel = relMatch ? relMatch[1] : 'alternate'; if (linkRel === rel || (!rel && linkRel === 'alternate')) { return hrefMatch[1]; } } } return ''; } function getAtomLinkByType(content, type) { var linkRegex = /]*)>/gi; var match; while ((match = linkRegex.exec(content)) !== null) { var attrs = match[1]; var typeMatch = attrs.match(/type=["']([^"']*)["']/); var hrefMatch = attrs.match(/href=["']([^"']*)["']/); if (hrefMatch && typeMatch && typeMatch[1].indexOf(type) !== -1) { return { url: hrefMatch[1], type: typeMatch[1] }; } } return null; } var feed = { title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '', link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '', description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '', language: '', pubDate: xmlCleanText(xmlGetTagContent(feedContent, 'updated')) || '', lastBuildDate: '', generator: xmlCleanText(xmlGetTagContent(feedContent, 'generator')) || '', items: [] }; // Parse feed icon/logo var feedIcon = xmlGetTagContent(feedContent, 'icon'); var feedLogo = xmlGetTagContent(feedContent, 'logo'); if (feedIcon || feedLogo) { feed.image = { url: xmlCleanText(feedLogo || feedIcon) || '', title: feed.title, link: feed.link }; } var entryRegex = /]*>([\s\S]*?)<\/entry>/gi; var match; while ((match = entryRegex.exec(feedContent)) !== null) { var entryXml = match[1]; var categories = []; var catRegex = /]*)>/gi; var catMatch; while ((catMatch = catRegex.exec(entryXml)) !== null) { var termMatch = catMatch[1].match(/term=["']([^"']*)["']/); if (termMatch) { categories.push(xmlCleanText(termMatch[1])); } } var author = ''; var authorContent = xmlGetTagContent(entryXml, 'author'); if (authorContent) { author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || ''; } var pubDate = xmlCleanText( xmlGetTagContent(entryXml, 'published') || xmlGetTagContent(entryXml, 'updated') ) || ''; var summary = xmlCleanText(xmlGetTagContent(entryXml, 'summary')) || ''; var content = xmlCleanText(xmlGetTagContent(entryXml, 'content')) || ''; var item = { title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '', link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '', description: summary || content, content: content, pubDate: pubDate, author: author, categories: categories, guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || '', comments: '', source: '' }; // Check for enclosure link var enclosureLink = getAtomLinkByType(entryXml, 'image'); if (!enclosureLink) { enclosureLink = getAtomLinkByType(entryXml, 'audio'); } if (!enclosureLink) { enclosureLink = getAtomLinkByType(entryXml, 'video'); } if (enclosureLink) { item.enclosure = { url: enclosureLink.url, type: enclosureLink.type, length: 0 }; } // Parse media:content var mediaAttrs = xmlGetTagAttributes(entryXml, 'media:content'); if (mediaAttrs) { item.media = { url: mediaAttrs.url || '', type: mediaAttrs.type || mediaAttrs.medium || '', width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0, height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0 }; } // Parse media:thumbnail var thumbAttrs = xmlGetTagAttributes(entryXml, 'media:thumbnail'); if (thumbAttrs) { item.thumbnail = { url: thumbAttrs.url || '', width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0, height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0 }; } feed.items.push(item); } return feed; } /** * Parse date string to timestamp */ function parseDate(dateStr) { if (!dateStr) return 0; var timestamp = Date.parse(dateStr); if (!isNaN(timestamp)) { return timestamp; } var rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/; var match = dateStr.match(rfc822Regex); if (match) { var months = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 }; var day = parseInt(match[1], 10); var month = months[match[2]]; var year = parseInt(match[3], 10); var hour = parseInt(match[4], 10); var minute = parseInt(match[5], 10); var second = parseInt(match[6], 10); if (month !== undefined) { return new Date(year, month, day, hour, minute, second).getTime(); } } return 0; } /** * Filter items by keyword */ function filterByKeyword(items, keyword) { if (!keyword) return items; var lowerKeyword = keyword.toLowerCase(); return items.filter(function(item) { var title = (item.title || '').toLowerCase(); var description = (item.description || '').toLowerCase(); return title.indexOf(lowerKeyword) !== -1 || description.indexOf(lowerKeyword) !== -1; }); } /** * Filter items by date */ function filterByDate(items, newerThanStr) { if (!newerThanStr) return items; var threshold = parseDate(newerThanStr); if (threshold === 0) { smartbotic.log.warn('Could not parse date filter: ' + newerThanStr); return items; } return items.filter(function(item) { var itemDate = parseDate(item.pubDate); return itemDate > threshold; }); } /** * Limit number of items */ function limitItems(items, maxItems) { if (!maxItems || maxItems <= 0) return items; return items.slice(0, maxItems); } /** * Get unique identifier for an RSS/Atom item * Prefers GUID, falls back to link */ function getItemIdentifier(item) { return item.guid || item.link || ''; } /** * Generate a stable document ID from feed URL for state storage */ function generateStateDocId(feedUrl) { // Create a simple hash from the feed URL for a stable doc ID var hash = 0; for (var i = 0; i < feedUrl.length; i++) { var char = feedUrl.charCodeAt(i); hash = ((hash << 5) - hash) + char; hash = hash & hash; // Convert to 32-bit integer } return 'rss-state-' + Math.abs(hash).toString(36); } /** * Load seen item identifiers from storage */ function loadSeenItems(collection, feedUrl) { var docId = generateStateDocId(feedUrl); try { var result = smartbotic.storage.get(collection, docId); if (result.found && result.document && result.document.seenIds) { smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage'); // Use plain object as a set (for QuickJS compatibility) var seenObj = {}; for (var i = 0; i < result.document.seenIds.length; i++) { seenObj[result.document.seenIds[i]] = true; } return { seenIds: seenObj, docId: docId, exists: true, version: result.document._version || 1 }; } } catch (error) { smartbotic.log.warn('Error loading seen items from storage: ' + error.message); } return { seenIds: {}, docId: docId, exists: false, version: 0 }; } /** * Save seen item identifiers to storage */ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) { // Convert object keys to array var seenArray = Object.keys(seenIds); var documentData = { feedUrl: feedUrl, seenIds: seenArray, lastUpdated: new Date().toISOString(), itemCount: seenArray.length }; try { if (exists) { // Update existing document var updateResult = smartbotic.storage.update(collection, docId, documentData, version, false); if (!updateResult.success) { smartbotic.log.error('Failed to update seen items: ' + updateResult.error); return false; } smartbotic.log.debug('Updated ' + seenArray.length + ' seen item IDs in storage'); } else { // Insert new document var insertResult = smartbotic.storage.insert(collection, documentData, docId, 0); if (!insertResult.success) { smartbotic.log.error('Failed to save seen items: ' + insertResult.error); return false; } smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage'); } return true; } catch (error) { smartbotic.log.error('Error saving seen items to storage: ' + error.message); return false; } } /** * Filter items to only include new (unseen) items */ function filterNewItems(items, seenIds) { return items.filter(function(item) { var id = getItemIdentifier(item); return id && !seenIds[id]; }); } async function execute(config, input) { var url = input.url || config.url; var filterKeyword = input.filterKeyword || config.filterKeyword; var filterNewerThan = input.filterNewerThan || config.filterNewerThan; var maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems; var timeout = config.timeout || 30000; var detectNewItems = config.detectNewItems || false; var stateCollection = config.stateCollection; if (!url) { throw new Error('Feed URL is required'); } // Validate stateful mode configuration if (detectNewItems && !stateCollection) { throw new Error('State collection is required when "Detect New Items" is enabled. Please select a read-write collection in the workflow storage settings.'); } smartbotic.log.info('Fetching RSS/Atom feed from ' + url); var response = smartbotic.http.request({ method: 'GET', url: url, timeout: timeout, headers: { 'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*' } }); if (response.status < 200 || response.status >= 300) { throw new Error('Failed to fetch feed: HTTP ' + response.status); } var xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data); if (!xml || xml.trim().length === 0) { throw new Error('Empty response from feed URL'); } var feed; if (xml.indexOf(' 0) { smartbotic.log.debug('Limiting to ' + maxItems + ' items'); items = limitItems(items, maxItems); } smartbotic.log.info('Returning ' + items.length + ' items from feed'); // Persist seen items state after successful processing if (detectNewItems && stateInfo) { var saved = saveSeenItems( stateCollection, stateInfo.docId, stateInfo.seenIds, url, stateInfo.exists, stateInfo.version ); if (!saved) { smartbotic.log.warn('Failed to persist seen items state, but continuing with output'); } } var result = { feedTitle: feed.title, feedLink: feed.link, feedDescription: feed.description, itemCount: items.length, items: items }; // Add stateful mode info if enabled if (detectNewItems) { result.newItemCount = newItemCount; result.totalFeedItemCount = totalFeedItemCount; } return result; } module.exports = { configSchema, inputSchema, outputSchema, execute };