|
|
@@ -2,7 +2,7 @@
|
|
|
* @node rss-reader
|
|
|
* @name RSS Reader
|
|
|
* @category data
|
|
|
- * @version 1.1.2
|
|
|
+ * @version 1.1.4
|
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
|
* @icon rss
|
|
|
*/
|
|
|
@@ -86,6 +86,18 @@ const outputSchema = {
|
|
|
feedTitle: { type: 'string', description: 'Title of the feed' },
|
|
|
feedLink: { type: 'string', description: 'Link to the feed homepage' },
|
|
|
feedDescription: { type: 'string', description: 'Description of the feed' },
|
|
|
+ feedLanguage: { type: 'string', description: 'Language of the feed' },
|
|
|
+ feedPubDate: { type: 'string', description: 'Publication date of the feed' },
|
|
|
+ feedGenerator: { type: 'string', description: 'Generator of the feed' },
|
|
|
+ feedImage: {
|
|
|
+ type: 'object',
|
|
|
+ description: 'Feed image/logo',
|
|
|
+ properties: {
|
|
|
+ url: { type: 'string' },
|
|
|
+ title: { type: 'string' },
|
|
|
+ link: { type: 'string' }
|
|
|
+ }
|
|
|
+ },
|
|
|
itemCount: { type: 'number', description: 'Number of items returned' },
|
|
|
newItemCount: { type: 'number', description: 'Number of new items (when detectNewItems is enabled)' },
|
|
|
totalFeedItemCount: { type: 'number', description: 'Total items in feed before filtering (when detectNewItems is enabled)' },
|
|
|
@@ -95,13 +107,44 @@ const outputSchema = {
|
|
|
items: {
|
|
|
type: 'object',
|
|
|
properties: {
|
|
|
- title: { type: 'string' },
|
|
|
- link: { type: 'string' },
|
|
|
- description: { type: 'string' },
|
|
|
- pubDate: { type: 'string' },
|
|
|
- author: { type: 'string' },
|
|
|
- categories: { type: 'array', items: { type: 'string' } },
|
|
|
- guid: { type: 'string' }
|
|
|
+ title: { type: 'string', description: 'Item title' },
|
|
|
+ link: { type: 'string', description: 'Item link/URL' },
|
|
|
+ description: { type: 'string', description: 'Item summary/description' },
|
|
|
+ content: { type: 'string', description: 'Full content (content:encoded)' },
|
|
|
+ pubDate: { type: 'string', description: 'Publication date' },
|
|
|
+ author: { type: 'string', description: 'Author name' },
|
|
|
+ categories: { type: 'array', items: { type: 'string' }, description: 'Categories/tags' },
|
|
|
+ guid: { type: 'string', description: 'Unique identifier' },
|
|
|
+ comments: { type: 'string', description: 'Comments URL' },
|
|
|
+ source: { type: 'string', description: 'Source attribution' },
|
|
|
+ enclosure: {
|
|
|
+ type: 'object',
|
|
|
+ description: 'Media enclosure (attachment)',
|
|
|
+ properties: {
|
|
|
+ url: { type: 'string', description: 'Media URL' },
|
|
|
+ type: { type: 'string', description: 'MIME type' },
|
|
|
+ length: { type: 'number', description: 'File size in bytes' }
|
|
|
+ }
|
|
|
+ },
|
|
|
+ media: {
|
|
|
+ type: 'object',
|
|
|
+ description: 'Media content (media:content)',
|
|
|
+ properties: {
|
|
|
+ url: { type: 'string' },
|
|
|
+ type: { type: 'string' },
|
|
|
+ width: { type: 'number' },
|
|
|
+ height: { type: 'number' }
|
|
|
+ }
|
|
|
+ },
|
|
|
+ thumbnail: {
|
|
|
+ type: 'object',
|
|
|
+ description: 'Thumbnail image (media:thumbnail)',
|
|
|
+ properties: {
|
|
|
+ url: { type: 'string' },
|
|
|
+ width: { type: 'number' },
|
|
|
+ height: { type: 'number' }
|
|
|
+ }
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
@@ -131,6 +174,25 @@ function xmlGetAllTagContents(xml, tagName) {
|
|
|
return results;
|
|
|
}
|
|
|
|
|
|
+function xmlGetTagAttributes(xml, tagName) {
|
|
|
+ // Match self-closing tag or opening tag
|
|
|
+ var regex = new RegExp('<' + tagName + '([^>]*?)\\/?>', 'i');
|
|
|
+ var match = xml.match(regex);
|
|
|
+ if (!match) return null;
|
|
|
+
|
|
|
+ var attrString = match[1];
|
|
|
+ var attrs = {};
|
|
|
+
|
|
|
+ // Parse attributes: name="value" or name='value'
|
|
|
+ var attrRegex = /(\w+)=["']([^"']*)["']/g;
|
|
|
+ var attrMatch;
|
|
|
+ while ((attrMatch = attrRegex.exec(attrString)) !== null) {
|
|
|
+ attrs[attrMatch[1]] = attrMatch[2];
|
|
|
+ }
|
|
|
+
|
|
|
+ return attrs;
|
|
|
+}
|
|
|
+
|
|
|
function xmlDecodeHtmlEntities(text) {
|
|
|
if (!text) return '';
|
|
|
return text
|
|
|
@@ -157,37 +219,98 @@ function xmlCleanText(text) {
|
|
|
* Parse RSS 2.0 feed
|
|
|
*/
|
|
|
function parseRss(xml) {
|
|
|
- const channel = xmlGetTagContent(xml, 'channel');
|
|
|
+ var channel = xmlGetTagContent(xml, 'channel');
|
|
|
if (!channel) {
|
|
|
throw new Error('Invalid RSS feed: no channel element found');
|
|
|
}
|
|
|
|
|
|
- const feed = {
|
|
|
- title: xmlCleanText(xmlGetTagContent(channel, 'title')) || '',
|
|
|
- link: xmlCleanText(xmlGetTagContent(channel, 'link')) || '',
|
|
|
- description: xmlCleanText(xmlGetTagContent(channel, 'description')) || '',
|
|
|
+ var feedTitle = xmlGetTagContent(channel, 'title');
|
|
|
+ var feedLink = xmlGetTagContent(channel, 'link');
|
|
|
+ var feedDesc = xmlGetTagContent(channel, 'description');
|
|
|
+ var feedLanguage = xmlGetTagContent(channel, 'language');
|
|
|
+ var feedPubDate = xmlGetTagContent(channel, 'pubDate');
|
|
|
+ var feedLastBuildDate = xmlGetTagContent(channel, 'lastBuildDate');
|
|
|
+ var feedGenerator = xmlGetTagContent(channel, 'generator');
|
|
|
+ var feedImage = xmlGetTagContent(channel, 'image');
|
|
|
+
|
|
|
+ var feed = {
|
|
|
+ title: xmlCleanText(feedTitle) || '',
|
|
|
+ link: xmlCleanText(feedLink) || '',
|
|
|
+ description: xmlCleanText(feedDesc) || '',
|
|
|
+ language: xmlCleanText(feedLanguage) || '',
|
|
|
+ pubDate: xmlCleanText(feedPubDate) || '',
|
|
|
+ lastBuildDate: xmlCleanText(feedLastBuildDate) || '',
|
|
|
+ generator: xmlCleanText(feedGenerator) || '',
|
|
|
items: []
|
|
|
};
|
|
|
|
|
|
- const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
|
|
|
- let match;
|
|
|
+ // Parse feed image if present
|
|
|
+ if (feedImage) {
|
|
|
+ feed.image = {
|
|
|
+ url: xmlCleanText(xmlGetTagContent(feedImage, 'url')) || '',
|
|
|
+ title: xmlCleanText(xmlGetTagContent(feedImage, 'title')) || '',
|
|
|
+ link: xmlCleanText(xmlGetTagContent(feedImage, 'link')) || ''
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ var itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
|
|
|
+ var match;
|
|
|
+ var itemCount = 0;
|
|
|
while ((match = itemRegex.exec(channel)) !== null) {
|
|
|
- const itemXml = match[1];
|
|
|
+ var itemXml = match[1];
|
|
|
+ itemCount++;
|
|
|
|
|
|
- const categories = xmlGetAllTagContents(itemXml, 'category')
|
|
|
- .map(function(c) { return xmlCleanText(c); });
|
|
|
+ var catArray = xmlGetAllTagContents(itemXml, 'category');
|
|
|
+ var categories = [];
|
|
|
+ for (var ci = 0; ci < catArray.length; ci++) {
|
|
|
+ categories.push(xmlCleanText(catArray[ci]));
|
|
|
+ }
|
|
|
|
|
|
- const item = {
|
|
|
+ var item = {
|
|
|
title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '',
|
|
|
link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '',
|
|
|
description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '',
|
|
|
+ content: xmlCleanText(xmlGetTagContent(itemXml, 'content:encoded')) || '',
|
|
|
pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '',
|
|
|
author: xmlCleanText(xmlGetTagContent(itemXml, 'author') ||
|
|
|
xmlGetTagContent(itemXml, 'dc:creator')) || '',
|
|
|
categories: categories,
|
|
|
- guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || ''
|
|
|
+ guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || '',
|
|
|
+ comments: xmlCleanText(xmlGetTagContent(itemXml, 'comments')) || '',
|
|
|
+ source: xmlCleanText(xmlGetTagContent(itemXml, 'source')) || ''
|
|
|
};
|
|
|
|
|
|
+ // Parse enclosure (media attachment)
|
|
|
+ var enclosureAttrs = xmlGetTagAttributes(itemXml, 'enclosure');
|
|
|
+ if (enclosureAttrs) {
|
|
|
+ item.enclosure = {
|
|
|
+ url: enclosureAttrs.url || '',
|
|
|
+ type: enclosureAttrs.type || '',
|
|
|
+ length: enclosureAttrs.length ? parseInt(enclosureAttrs.length, 10) : 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ // Parse media:content (common in media RSS)
|
|
|
+ var mediaAttrs = xmlGetTagAttributes(itemXml, 'media:content');
|
|
|
+ if (mediaAttrs) {
|
|
|
+ item.media = {
|
|
|
+ url: mediaAttrs.url || '',
|
|
|
+ type: mediaAttrs.type || mediaAttrs.medium || '',
|
|
|
+ width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
|
|
|
+ height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ // Parse media:thumbnail
|
|
|
+ var thumbAttrs = xmlGetTagAttributes(itemXml, 'media:thumbnail');
|
|
|
+ if (thumbAttrs) {
|
|
|
+ item.thumbnail = {
|
|
|
+ url: thumbAttrs.url || '',
|
|
|
+ width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
|
|
|
+ height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
feed.items.push(item);
|
|
|
}
|
|
|
|
|
|
@@ -198,21 +321,21 @@ function parseRss(xml) {
|
|
|
* Parse Atom feed
|
|
|
*/
|
|
|
function parseAtom(xml) {
|
|
|
- const feedContent = xmlGetTagContent(xml, 'feed');
|
|
|
+ var feedContent = xmlGetTagContent(xml, 'feed');
|
|
|
if (!feedContent) {
|
|
|
throw new Error('Invalid Atom feed: no feed element found');
|
|
|
}
|
|
|
|
|
|
function getAtomLink(content, rel) {
|
|
|
- const linkRegex = /<link([^>]*)>/gi;
|
|
|
- let match;
|
|
|
+ var linkRegex = /<link([^>]*)>/gi;
|
|
|
+ var match;
|
|
|
while ((match = linkRegex.exec(content)) !== null) {
|
|
|
- const attrs = match[1];
|
|
|
- const relMatch = attrs.match(/rel=["']([^"']*)["']/);
|
|
|
- const hrefMatch = attrs.match(/href=["']([^"']*)["']/);
|
|
|
+ var attrs = match[1];
|
|
|
+ var relMatch = attrs.match(/rel=["']([^"']*)["']/);
|
|
|
+ var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
|
|
|
|
|
|
if (hrefMatch) {
|
|
|
- const linkRel = relMatch ? relMatch[1] : 'alternate';
|
|
|
+ var linkRel = relMatch ? relMatch[1] : 'alternate';
|
|
|
if (linkRel === rel || (!rel && linkRel === 'alternate')) {
|
|
|
return hrefMatch[1];
|
|
|
}
|
|
|
@@ -221,54 +344,125 @@ function parseAtom(xml) {
|
|
|
return '';
|
|
|
}
|
|
|
|
|
|
- const feed = {
|
|
|
+ function getAtomLinkByType(content, type) {
|
|
|
+ var linkRegex = /<link([^>]*)>/gi;
|
|
|
+ var match;
|
|
|
+ while ((match = linkRegex.exec(content)) !== null) {
|
|
|
+ var attrs = match[1];
|
|
|
+ var typeMatch = attrs.match(/type=["']([^"']*)["']/);
|
|
|
+ var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
|
|
|
+
|
|
|
+ if (hrefMatch && typeMatch && typeMatch[1].indexOf(type) !== -1) {
|
|
|
+ return {
|
|
|
+ url: hrefMatch[1],
|
|
|
+ type: typeMatch[1]
|
|
|
+ };
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ var feed = {
|
|
|
title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '',
|
|
|
link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
|
|
|
description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '',
|
|
|
+ language: '',
|
|
|
+ pubDate: xmlCleanText(xmlGetTagContent(feedContent, 'updated')) || '',
|
|
|
+ lastBuildDate: '',
|
|
|
+ generator: xmlCleanText(xmlGetTagContent(feedContent, 'generator')) || '',
|
|
|
items: []
|
|
|
};
|
|
|
|
|
|
- const entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
|
|
|
- let match;
|
|
|
+ // Parse feed icon/logo
|
|
|
+ var feedIcon = xmlGetTagContent(feedContent, 'icon');
|
|
|
+ var feedLogo = xmlGetTagContent(feedContent, 'logo');
|
|
|
+ if (feedIcon || feedLogo) {
|
|
|
+ feed.image = {
|
|
|
+ url: xmlCleanText(feedLogo || feedIcon) || '',
|
|
|
+ title: feed.title,
|
|
|
+ link: feed.link
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ var entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
|
|
|
+ var match;
|
|
|
while ((match = entryRegex.exec(feedContent)) !== null) {
|
|
|
- const entryXml = match[1];
|
|
|
+ var entryXml = match[1];
|
|
|
|
|
|
- const categories = [];
|
|
|
- const catRegex = /<category([^>]*)>/gi;
|
|
|
- let catMatch;
|
|
|
+ var categories = [];
|
|
|
+ var catRegex = /<category([^>]*)>/gi;
|
|
|
+ var catMatch;
|
|
|
while ((catMatch = catRegex.exec(entryXml)) !== null) {
|
|
|
- const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
|
|
|
+ var termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
|
|
|
if (termMatch) {
|
|
|
categories.push(xmlCleanText(termMatch[1]));
|
|
|
}
|
|
|
}
|
|
|
|
|
|
var author = '';
|
|
|
- const authorContent = xmlGetTagContent(entryXml, 'author');
|
|
|
+ var authorContent = xmlGetTagContent(entryXml, 'author');
|
|
|
if (authorContent) {
|
|
|
author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || '';
|
|
|
}
|
|
|
|
|
|
- const pubDate = xmlCleanText(
|
|
|
+ var pubDate = xmlCleanText(
|
|
|
xmlGetTagContent(entryXml, 'published') ||
|
|
|
xmlGetTagContent(entryXml, 'updated')
|
|
|
) || '';
|
|
|
|
|
|
- const description = xmlCleanText(
|
|
|
- xmlGetTagContent(entryXml, 'summary') ||
|
|
|
- xmlGetTagContent(entryXml, 'content')
|
|
|
- ) || '';
|
|
|
+ var summary = xmlCleanText(xmlGetTagContent(entryXml, 'summary')) || '';
|
|
|
+ var content = xmlCleanText(xmlGetTagContent(entryXml, 'content')) || '';
|
|
|
|
|
|
- const item = {
|
|
|
+ var item = {
|
|
|
title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '',
|
|
|
link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
|
|
|
- description: description,
|
|
|
+ description: summary || content,
|
|
|
+ content: content,
|
|
|
pubDate: pubDate,
|
|
|
author: author,
|
|
|
categories: categories,
|
|
|
- guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || ''
|
|
|
+ guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || '',
|
|
|
+ comments: '',
|
|
|
+ source: ''
|
|
|
};
|
|
|
|
|
|
+ // Check for enclosure link
|
|
|
+ var enclosureLink = getAtomLinkByType(entryXml, 'image');
|
|
|
+ if (!enclosureLink) {
|
|
|
+ enclosureLink = getAtomLinkByType(entryXml, 'audio');
|
|
|
+ }
|
|
|
+ if (!enclosureLink) {
|
|
|
+ enclosureLink = getAtomLinkByType(entryXml, 'video');
|
|
|
+ }
|
|
|
+ if (enclosureLink) {
|
|
|
+ item.enclosure = {
|
|
|
+ url: enclosureLink.url,
|
|
|
+ type: enclosureLink.type,
|
|
|
+ length: 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ // Parse media:content
|
|
|
+ var mediaAttrs = xmlGetTagAttributes(entryXml, 'media:content');
|
|
|
+ if (mediaAttrs) {
|
|
|
+ item.media = {
|
|
|
+ url: mediaAttrs.url || '',
|
|
|
+ type: mediaAttrs.type || mediaAttrs.medium || '',
|
|
|
+ width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
|
|
|
+ height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
+ // Parse media:thumbnail
|
|
|
+ var thumbAttrs = xmlGetTagAttributes(entryXml, 'media:thumbnail');
|
|
|
+ if (thumbAttrs) {
|
|
|
+ item.thumbnail = {
|
|
|
+ url: thumbAttrs.url || '',
|
|
|
+ width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
|
|
|
+ height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
|
|
|
+ };
|
|
|
+ }
|
|
|
+
|
|
|
feed.items.push(item);
|
|
|
}
|
|
|
|
|
|
@@ -281,24 +475,24 @@ function parseAtom(xml) {
|
|
|
function parseDate(dateStr) {
|
|
|
if (!dateStr) return 0;
|
|
|
|
|
|
- const timestamp = Date.parse(dateStr);
|
|
|
+ var timestamp = Date.parse(dateStr);
|
|
|
if (!isNaN(timestamp)) {
|
|
|
return timestamp;
|
|
|
}
|
|
|
|
|
|
- const rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
|
|
|
- const match = dateStr.match(rfc822Regex);
|
|
|
+ var rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
|
|
|
+ var match = dateStr.match(rfc822Regex);
|
|
|
if (match) {
|
|
|
- const months = {
|
|
|
+ var months = {
|
|
|
Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
|
|
|
Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
|
|
|
};
|
|
|
- const day = parseInt(match[1], 10);
|
|
|
- const month = months[match[2]];
|
|
|
- const year = parseInt(match[3], 10);
|
|
|
- const hour = parseInt(match[4], 10);
|
|
|
- const minute = parseInt(match[5], 10);
|
|
|
- const second = parseInt(match[6], 10);
|
|
|
+ var day = parseInt(match[1], 10);
|
|
|
+ var month = months[match[2]];
|
|
|
+ var year = parseInt(match[3], 10);
|
|
|
+ var hour = parseInt(match[4], 10);
|
|
|
+ var minute = parseInt(match[5], 10);
|
|
|
+ var second = parseInt(match[6], 10);
|
|
|
|
|
|
if (month !== undefined) {
|
|
|
return new Date(year, month, day, hour, minute, second).getTime();
|
|
|
@@ -318,7 +512,7 @@ function filterByKeyword(items, keyword) {
|
|
|
return items.filter(function(item) {
|
|
|
var title = (item.title || '').toLowerCase();
|
|
|
var description = (item.description || '').toLowerCase();
|
|
|
- return title.includes(lowerKeyword) || description.includes(lowerKeyword);
|
|
|
+ return title.indexOf(lowerKeyword) !== -1 || description.indexOf(lowerKeyword) !== -1;
|
|
|
});
|
|
|
}
|
|
|
|
|
|
@@ -361,9 +555,9 @@ function getItemIdentifier(item) {
|
|
|
*/
|
|
|
function generateStateDocId(feedUrl) {
|
|
|
// Create a simple hash from the feed URL for a stable doc ID
|
|
|
- let hash = 0;
|
|
|
- for (let i = 0; i < feedUrl.length; i++) {
|
|
|
- const char = feedUrl.charCodeAt(i);
|
|
|
+ var hash = 0;
|
|
|
+ for (var i = 0; i < feedUrl.length; i++) {
|
|
|
+ var char = feedUrl.charCodeAt(i);
|
|
|
hash = ((hash << 5) - hash) + char;
|
|
|
hash = hash & hash; // Convert to 32-bit integer
|
|
|
}
|
|
|
@@ -381,8 +575,13 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
|
|
|
if (result.found && result.document && result.document.seenIds) {
|
|
|
smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
|
|
|
+ // Use plain object as a set (for QuickJS compatibility)
|
|
|
+ var seenObj = {};
|
|
|
+ for (var i = 0; i < result.document.seenIds.length; i++) {
|
|
|
+ seenObj[result.document.seenIds[i]] = true;
|
|
|
+ }
|
|
|
return {
|
|
|
- seenIds: new Set(result.document.seenIds),
|
|
|
+ seenIds: seenObj,
|
|
|
docId: docId,
|
|
|
exists: true,
|
|
|
version: result.document._version || 1
|
|
|
@@ -393,7 +592,7 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
}
|
|
|
|
|
|
return {
|
|
|
- seenIds: new Set(),
|
|
|
+ seenIds: {},
|
|
|
docId: docId,
|
|
|
exists: false,
|
|
|
version: 0
|
|
|
@@ -404,7 +603,8 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
* Save seen item identifiers to storage
|
|
|
*/
|
|
|
function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
- var seenArray = Array.from(seenIds);
|
|
|
+ // Convert object keys to array
|
|
|
+ var seenArray = Object.keys(seenIds);
|
|
|
var documentData = {
|
|
|
feedUrl: feedUrl,
|
|
|
seenIds: seenArray,
|
|
|
@@ -443,11 +643,11 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
function filterNewItems(items, seenIds) {
|
|
|
return items.filter(function(item) {
|
|
|
var id = getItemIdentifier(item);
|
|
|
- return id && !seenIds.has(id);
|
|
|
+ return id && !seenIds[id];
|
|
|
});
|
|
|
}
|
|
|
|
|
|
-function execute(config, input) {
|
|
|
+async function execute(config, input) {
|
|
|
var url = input.url || config.url;
|
|
|
var filterKeyword = input.filterKeyword || config.filterKeyword;
|
|
|
var filterNewerThan = input.filterNewerThan || config.filterNewerThan;
|
|
|
@@ -532,7 +732,7 @@ function execute(config, input) {
|
|
|
for (var i = 0; i < items.length; i++) {
|
|
|
var id = getItemIdentifier(items[i]);
|
|
|
if (id) {
|
|
|
- stateInfo.seenIds.add(id);
|
|
|
+ stateInfo.seenIds[id] = true;
|
|
|
}
|
|
|
}
|
|
|
|