Selaa lähdekoodia

fix: RSS Reader node - add enclosure/media support and QuickJS compatibility

- Add enclosure parsing for RSS `<enclosure>` tags
- Add media:content and media:thumbnail parsing
- Add Atom feed enclosure support via link types
- Replace all ES6 features (const/let, includes, for...of) with ES5
- Remove debug logging
- Bump version to 1.1.4
fszontagh 6 kuukautta sitten
vanhempi
sitoutus
d159857f20
1 muutettua tiedostoa jossa 264 lisäystä ja 64 poistoa
  1. 264 64
      nodes/rss/rss-reader.js

+ 264 - 64
nodes/rss/rss-reader.js

@@ -2,7 +2,7 @@
  * @node rss-reader
  * @name RSS Reader
  * @category data
- * @version 1.1.2
+ * @version 1.1.4
  * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
  * @icon rss
  */
@@ -86,6 +86,18 @@ const outputSchema = {
         feedTitle: { type: 'string', description: 'Title of the feed' },
         feedLink: { type: 'string', description: 'Link to the feed homepage' },
         feedDescription: { type: 'string', description: 'Description of the feed' },
+        feedLanguage: { type: 'string', description: 'Language of the feed' },
+        feedPubDate: { type: 'string', description: 'Publication date of the feed' },
+        feedGenerator: { type: 'string', description: 'Generator of the feed' },
+        feedImage: {
+            type: 'object',
+            description: 'Feed image/logo',
+            properties: {
+                url: { type: 'string' },
+                title: { type: 'string' },
+                link: { type: 'string' }
+            }
+        },
         itemCount: { type: 'number', description: 'Number of items returned' },
         newItemCount: { type: 'number', description: 'Number of new items (when detectNewItems is enabled)' },
         totalFeedItemCount: { type: 'number', description: 'Total items in feed before filtering (when detectNewItems is enabled)' },
@@ -95,13 +107,44 @@ const outputSchema = {
             items: {
                 type: 'object',
                 properties: {
-                    title: { type: 'string' },
-                    link: { type: 'string' },
-                    description: { type: 'string' },
-                    pubDate: { type: 'string' },
-                    author: { type: 'string' },
-                    categories: { type: 'array', items: { type: 'string' } },
-                    guid: { type: 'string' }
+                    title: { type: 'string', description: 'Item title' },
+                    link: { type: 'string', description: 'Item link/URL' },
+                    description: { type: 'string', description: 'Item summary/description' },
+                    content: { type: 'string', description: 'Full content (content:encoded)' },
+                    pubDate: { type: 'string', description: 'Publication date' },
+                    author: { type: 'string', description: 'Author name' },
+                    categories: { type: 'array', items: { type: 'string' }, description: 'Categories/tags' },
+                    guid: { type: 'string', description: 'Unique identifier' },
+                    comments: { type: 'string', description: 'Comments URL' },
+                    source: { type: 'string', description: 'Source attribution' },
+                    enclosure: {
+                        type: 'object',
+                        description: 'Media enclosure (attachment)',
+                        properties: {
+                            url: { type: 'string', description: 'Media URL' },
+                            type: { type: 'string', description: 'MIME type' },
+                            length: { type: 'number', description: 'File size in bytes' }
+                        }
+                    },
+                    media: {
+                        type: 'object',
+                        description: 'Media content (media:content)',
+                        properties: {
+                            url: { type: 'string' },
+                            type: { type: 'string' },
+                            width: { type: 'number' },
+                            height: { type: 'number' }
+                        }
+                    },
+                    thumbnail: {
+                        type: 'object',
+                        description: 'Thumbnail image (media:thumbnail)',
+                        properties: {
+                            url: { type: 'string' },
+                            width: { type: 'number' },
+                            height: { type: 'number' }
+                        }
+                    }
                 }
             }
         }
@@ -131,6 +174,25 @@ function xmlGetAllTagContents(xml, tagName) {
     return results;
 }
 
+function xmlGetTagAttributes(xml, tagName) {
+    // Match self-closing tag or opening tag
+    var regex = new RegExp('<' + tagName + '([^>]*?)\\/?>', 'i');
+    var match = xml.match(regex);
+    if (!match) return null;
+
+    var attrString = match[1];
+    var attrs = {};
+
+    // Parse attributes: name="value" or name='value'
+    var attrRegex = /(\w+)=["']([^"']*)["']/g;
+    var attrMatch;
+    while ((attrMatch = attrRegex.exec(attrString)) !== null) {
+        attrs[attrMatch[1]] = attrMatch[2];
+    }
+
+    return attrs;
+}
+
 function xmlDecodeHtmlEntities(text) {
     if (!text) return '';
     return text
@@ -157,37 +219,98 @@ function xmlCleanText(text) {
  * Parse RSS 2.0 feed
  */
 function parseRss(xml) {
-    const channel = xmlGetTagContent(xml, 'channel');
+    var channel = xmlGetTagContent(xml, 'channel');
     if (!channel) {
         throw new Error('Invalid RSS feed: no channel element found');
     }
 
-    const feed = {
-        title: xmlCleanText(xmlGetTagContent(channel, 'title')) || '',
-        link: xmlCleanText(xmlGetTagContent(channel, 'link')) || '',
-        description: xmlCleanText(xmlGetTagContent(channel, 'description')) || '',
+    var feedTitle = xmlGetTagContent(channel, 'title');
+    var feedLink = xmlGetTagContent(channel, 'link');
+    var feedDesc = xmlGetTagContent(channel, 'description');
+    var feedLanguage = xmlGetTagContent(channel, 'language');
+    var feedPubDate = xmlGetTagContent(channel, 'pubDate');
+    var feedLastBuildDate = xmlGetTagContent(channel, 'lastBuildDate');
+    var feedGenerator = xmlGetTagContent(channel, 'generator');
+    var feedImage = xmlGetTagContent(channel, 'image');
+
+    var feed = {
+        title: xmlCleanText(feedTitle) || '',
+        link: xmlCleanText(feedLink) || '',
+        description: xmlCleanText(feedDesc) || '',
+        language: xmlCleanText(feedLanguage) || '',
+        pubDate: xmlCleanText(feedPubDate) || '',
+        lastBuildDate: xmlCleanText(feedLastBuildDate) || '',
+        generator: xmlCleanText(feedGenerator) || '',
         items: []
     };
 
-    const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
-    let match;
+    // Parse feed image if present
+    if (feedImage) {
+        feed.image = {
+            url: xmlCleanText(xmlGetTagContent(feedImage, 'url')) || '',
+            title: xmlCleanText(xmlGetTagContent(feedImage, 'title')) || '',
+            link: xmlCleanText(xmlGetTagContent(feedImage, 'link')) || ''
+        };
+    }
+
+    var itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
+    var match;
+    var itemCount = 0;
     while ((match = itemRegex.exec(channel)) !== null) {
-        const itemXml = match[1];
+        var itemXml = match[1];
+        itemCount++;
 
-        const categories = xmlGetAllTagContents(itemXml, 'category')
-            .map(function(c) { return xmlCleanText(c); });
+        var catArray = xmlGetAllTagContents(itemXml, 'category');
+        var categories = [];
+        for (var ci = 0; ci < catArray.length; ci++) {
+            categories.push(xmlCleanText(catArray[ci]));
+        }
 
-        const item = {
+        var item = {
             title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '',
             link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '',
             description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '',
+            content: xmlCleanText(xmlGetTagContent(itemXml, 'content:encoded')) || '',
             pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '',
             author: xmlCleanText(xmlGetTagContent(itemXml, 'author') ||
                     xmlGetTagContent(itemXml, 'dc:creator')) || '',
             categories: categories,
-            guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || ''
+            guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || '',
+            comments: xmlCleanText(xmlGetTagContent(itemXml, 'comments')) || '',
+            source: xmlCleanText(xmlGetTagContent(itemXml, 'source')) || ''
         };
 
+        // Parse enclosure (media attachment)
+        var enclosureAttrs = xmlGetTagAttributes(itemXml, 'enclosure');
+        if (enclosureAttrs) {
+            item.enclosure = {
+                url: enclosureAttrs.url || '',
+                type: enclosureAttrs.type || '',
+                length: enclosureAttrs.length ? parseInt(enclosureAttrs.length, 10) : 0
+            };
+        }
+
+        // Parse media:content (common in media RSS)
+        var mediaAttrs = xmlGetTagAttributes(itemXml, 'media:content');
+        if (mediaAttrs) {
+            item.media = {
+                url: mediaAttrs.url || '',
+                type: mediaAttrs.type || mediaAttrs.medium || '',
+                width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
+                height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
+            };
+        }
+
+        // Parse media:thumbnail
+        var thumbAttrs = xmlGetTagAttributes(itemXml, 'media:thumbnail');
+        if (thumbAttrs) {
+            item.thumbnail = {
+                url: thumbAttrs.url || '',
+                width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
+                height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
+            };
+        }
+
         feed.items.push(item);
     }
 
@@ -198,21 +321,21 @@ function parseRss(xml) {
  * Parse Atom feed
  */
 function parseAtom(xml) {
-    const feedContent = xmlGetTagContent(xml, 'feed');
+    var feedContent = xmlGetTagContent(xml, 'feed');
     if (!feedContent) {
         throw new Error('Invalid Atom feed: no feed element found');
     }
 
     function getAtomLink(content, rel) {
-        const linkRegex = /<link([^>]*)>/gi;
-        let match;
+        var linkRegex = /<link([^>]*)>/gi;
+        var match;
         while ((match = linkRegex.exec(content)) !== null) {
-            const attrs = match[1];
-            const relMatch = attrs.match(/rel=["']([^"']*)["']/);
-            const hrefMatch = attrs.match(/href=["']([^"']*)["']/);
+            var attrs = match[1];
+            var relMatch = attrs.match(/rel=["']([^"']*)["']/);
+            var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
 
             if (hrefMatch) {
-                const linkRel = relMatch ? relMatch[1] : 'alternate';
+                var linkRel = relMatch ? relMatch[1] : 'alternate';
                 if (linkRel === rel || (!rel && linkRel === 'alternate')) {
                     return hrefMatch[1];
                 }
@@ -221,54 +344,125 @@ function parseAtom(xml) {
         return '';
     }
 
-    const feed = {
+    function getAtomLinkByType(content, type) {
+        var linkRegex = /<link([^>]*)>/gi;
+        var match;
+        while ((match = linkRegex.exec(content)) !== null) {
+            var attrs = match[1];
+            var typeMatch = attrs.match(/type=["']([^"']*)["']/);
+            var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
+
+            if (hrefMatch && typeMatch && typeMatch[1].indexOf(type) !== -1) {
+                return {
+                    url: hrefMatch[1],
+                    type: typeMatch[1]
+                };
+            }
+        }
+        return null;
+    }
+
+    var feed = {
         title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '',
         link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
         description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '',
+        language: '',
+        pubDate: xmlCleanText(xmlGetTagContent(feedContent, 'updated')) || '',
+        lastBuildDate: '',
+        generator: xmlCleanText(xmlGetTagContent(feedContent, 'generator')) || '',
         items: []
     };
 
-    const entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
-    let match;
+    // Parse feed icon/logo
+    var feedIcon = xmlGetTagContent(feedContent, 'icon');
+    var feedLogo = xmlGetTagContent(feedContent, 'logo');
+    if (feedIcon || feedLogo) {
+        feed.image = {
+            url: xmlCleanText(feedLogo || feedIcon) || '',
+            title: feed.title,
+            link: feed.link
+        };
+    }
+
+    var entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
+    var match;
     while ((match = entryRegex.exec(feedContent)) !== null) {
-        const entryXml = match[1];
+        var entryXml = match[1];
 
-        const categories = [];
-        const catRegex = /<category([^>]*)>/gi;
-        let catMatch;
+        var categories = [];
+        var catRegex = /<category([^>]*)>/gi;
+        var catMatch;
         while ((catMatch = catRegex.exec(entryXml)) !== null) {
-            const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
+            var termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
             if (termMatch) {
                 categories.push(xmlCleanText(termMatch[1]));
             }
         }
 
         var author = '';
-        const authorContent = xmlGetTagContent(entryXml, 'author');
+        var authorContent = xmlGetTagContent(entryXml, 'author');
         if (authorContent) {
             author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || '';
         }
 
-        const pubDate = xmlCleanText(
+        var pubDate = xmlCleanText(
             xmlGetTagContent(entryXml, 'published') ||
             xmlGetTagContent(entryXml, 'updated')
         ) || '';
 
-        const description = xmlCleanText(
-            xmlGetTagContent(entryXml, 'summary') ||
-            xmlGetTagContent(entryXml, 'content')
-        ) || '';
+        var summary = xmlCleanText(xmlGetTagContent(entryXml, 'summary')) || '';
+        var content = xmlCleanText(xmlGetTagContent(entryXml, 'content')) || '';
 
-        const item = {
+        var item = {
             title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '',
             link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
-            description: description,
+            description: summary || content,
+            content: content,
             pubDate: pubDate,
             author: author,
             categories: categories,
-            guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || ''
+            guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || '',
+            comments: '',
+            source: ''
         };
 
+        // Check for enclosure link
+        var enclosureLink = getAtomLinkByType(entryXml, 'image');
+        if (!enclosureLink) {
+            enclosureLink = getAtomLinkByType(entryXml, 'audio');
+        }
+        if (!enclosureLink) {
+            enclosureLink = getAtomLinkByType(entryXml, 'video');
+        }
+        if (enclosureLink) {
+            item.enclosure = {
+                url: enclosureLink.url,
+                type: enclosureLink.type,
+                length: 0
+            };
+        }
+
+        // Parse media:content
+        var mediaAttrs = xmlGetTagAttributes(entryXml, 'media:content');
+        if (mediaAttrs) {
+            item.media = {
+                url: mediaAttrs.url || '',
+                type: mediaAttrs.type || mediaAttrs.medium || '',
+                width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
+                height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
+            };
+        }
+
+        // Parse media:thumbnail
+        var thumbAttrs = xmlGetTagAttributes(entryXml, 'media:thumbnail');
+        if (thumbAttrs) {
+            item.thumbnail = {
+                url: thumbAttrs.url || '',
+                width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
+                height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
+            };
+        }
+
         feed.items.push(item);
     }
 
@@ -281,24 +475,24 @@ function parseAtom(xml) {
 function parseDate(dateStr) {
     if (!dateStr) return 0;
 
-    const timestamp = Date.parse(dateStr);
+    var timestamp = Date.parse(dateStr);
     if (!isNaN(timestamp)) {
         return timestamp;
     }
 
-    const rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
-    const match = dateStr.match(rfc822Regex);
+    var rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
+    var match = dateStr.match(rfc822Regex);
     if (match) {
-        const months = {
+        var months = {
             Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
             Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
         };
-        const day = parseInt(match[1], 10);
-        const month = months[match[2]];
-        const year = parseInt(match[3], 10);
-        const hour = parseInt(match[4], 10);
-        const minute = parseInt(match[5], 10);
-        const second = parseInt(match[6], 10);
+        var day = parseInt(match[1], 10);
+        var month = months[match[2]];
+        var year = parseInt(match[3], 10);
+        var hour = parseInt(match[4], 10);
+        var minute = parseInt(match[5], 10);
+        var second = parseInt(match[6], 10);
 
         if (month !== undefined) {
             return new Date(year, month, day, hour, minute, second).getTime();
@@ -318,7 +512,7 @@ function filterByKeyword(items, keyword) {
     return items.filter(function(item) {
         var title = (item.title || '').toLowerCase();
         var description = (item.description || '').toLowerCase();
-        return title.includes(lowerKeyword) || description.includes(lowerKeyword);
+        return title.indexOf(lowerKeyword) !== -1 || description.indexOf(lowerKeyword) !== -1;
     });
 }
 
@@ -361,9 +555,9 @@ function getItemIdentifier(item) {
  */
 function generateStateDocId(feedUrl) {
     // Create a simple hash from the feed URL for a stable doc ID
-    let hash = 0;
-    for (let i = 0; i < feedUrl.length; i++) {
-        const char = feedUrl.charCodeAt(i);
+    var hash = 0;
+    for (var i = 0; i < feedUrl.length; i++) {
+        var char = feedUrl.charCodeAt(i);
         hash = ((hash << 5) - hash) + char;
         hash = hash & hash; // Convert to 32-bit integer
     }
@@ -381,8 +575,13 @@ function loadSeenItems(collection, feedUrl) {
 
         if (result.found && result.document && result.document.seenIds) {
             smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
+            // Use plain object as a set (for QuickJS compatibility)
+            var seenObj = {};
+            for (var i = 0; i < result.document.seenIds.length; i++) {
+                seenObj[result.document.seenIds[i]] = true;
+            }
             return {
-                seenIds: new Set(result.document.seenIds),
+                seenIds: seenObj,
                 docId: docId,
                 exists: true,
                 version: result.document._version || 1
@@ -393,7 +592,7 @@ function loadSeenItems(collection, feedUrl) {
     }
 
     return {
-        seenIds: new Set(),
+        seenIds: {},
         docId: docId,
         exists: false,
         version: 0
@@ -404,7 +603,8 @@ function loadSeenItems(collection, feedUrl) {
  * Save seen item identifiers to storage
  */
 function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
-    var seenArray = Array.from(seenIds);
+    // Convert object keys to array
+    var seenArray = Object.keys(seenIds);
     var documentData = {
         feedUrl: feedUrl,
         seenIds: seenArray,
@@ -443,11 +643,11 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
 function filterNewItems(items, seenIds) {
     return items.filter(function(item) {
         var id = getItemIdentifier(item);
-        return id && !seenIds.has(id);
+        return id && !seenIds[id];
     });
 }
 
-function execute(config, input) {
+async function execute(config, input) {
     var url = input.url || config.url;
     var filterKeyword = input.filterKeyword || config.filterKeyword;
     var filterNewerThan = input.filterNewerThan || config.filterNewerThan;
@@ -532,7 +732,7 @@ function execute(config, input) {
         for (var i = 0; i < items.length; i++) {
             var id = getItemIdentifier(items[i]);
             if (id) {
-                stateInfo.seenIds.add(id);
+                stateInfo.seenIds[id] = true;
             }
         }