|
@@ -2,7 +2,7 @@
|
|
|
* @node rss-reader
|
|
* @node rss-reader
|
|
|
* @name RSS Reader
|
|
* @name RSS Reader
|
|
|
* @category data
|
|
* @category data
|
|
|
- * @version 1.1.1
|
|
|
|
|
|
|
+ * @version 1.1.2
|
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
|
* @icon rss
|
|
* @icon rss
|
|
|
*/
|
|
*/
|
|
@@ -113,8 +113,8 @@ const outputSchema = {
|
|
|
* Handles basic XML structure without external dependencies
|
|
* Handles basic XML structure without external dependencies
|
|
|
*/
|
|
*/
|
|
|
function xmlGetTagContent(xml, tagName) {
|
|
function xmlGetTagContent(xml, tagName) {
|
|
|
- const pattern = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i');
|
|
|
|
|
- const match = xml.match(pattern);
|
|
|
|
|
|
|
+ var pattern = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'i');
|
|
|
|
|
+ var match = xml.match(pattern);
|
|
|
if (match) {
|
|
if (match) {
|
|
|
return match[1].trim();
|
|
return match[1].trim();
|
|
|
}
|
|
}
|
|
@@ -122,9 +122,9 @@ function xmlGetTagContent(xml, tagName) {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
function xmlGetAllTagContents(xml, tagName) {
|
|
function xmlGetAllTagContents(xml, tagName) {
|
|
|
- const results = [];
|
|
|
|
|
- const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'gi');
|
|
|
|
|
- let match;
|
|
|
|
|
|
|
+ var results = [];
|
|
|
|
|
+ var regex = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'gi');
|
|
|
|
|
+ var match;
|
|
|
while ((match = regex.exec(xml)) !== null) {
|
|
while ((match = regex.exec(xml)) !== null) {
|
|
|
results.push(match[1].trim());
|
|
results.push(match[1].trim());
|
|
|
}
|
|
}
|
|
@@ -314,10 +314,10 @@ function parseDate(dateStr) {
|
|
|
function filterByKeyword(items, keyword) {
|
|
function filterByKeyword(items, keyword) {
|
|
|
if (!keyword) return items;
|
|
if (!keyword) return items;
|
|
|
|
|
|
|
|
- const lowerKeyword = keyword.toLowerCase();
|
|
|
|
|
- return items.filter(item => {
|
|
|
|
|
- const title = (item.title || '').toLowerCase();
|
|
|
|
|
- const description = (item.description || '').toLowerCase();
|
|
|
|
|
|
|
+ var lowerKeyword = keyword.toLowerCase();
|
|
|
|
|
+ return items.filter(function(item) {
|
|
|
|
|
+ var title = (item.title || '').toLowerCase();
|
|
|
|
|
+ var description = (item.description || '').toLowerCase();
|
|
|
return title.includes(lowerKeyword) || description.includes(lowerKeyword);
|
|
return title.includes(lowerKeyword) || description.includes(lowerKeyword);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
@@ -328,14 +328,14 @@ function filterByKeyword(items, keyword) {
|
|
|
function filterByDate(items, newerThanStr) {
|
|
function filterByDate(items, newerThanStr) {
|
|
|
if (!newerThanStr) return items;
|
|
if (!newerThanStr) return items;
|
|
|
|
|
|
|
|
- const threshold = parseDate(newerThanStr);
|
|
|
|
|
|
|
+ var threshold = parseDate(newerThanStr);
|
|
|
if (threshold === 0) {
|
|
if (threshold === 0) {
|
|
|
- smartbotic.log.warn(`Could not parse date filter: ${newerThanStr}`);
|
|
|
|
|
|
|
+ smartbotic.log.warn('Could not parse date filter: ' + newerThanStr);
|
|
|
return items;
|
|
return items;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- return items.filter(item => {
|
|
|
|
|
- const itemDate = parseDate(item.pubDate);
|
|
|
|
|
|
|
+ return items.filter(function(item) {
|
|
|
|
|
+ var itemDate = parseDate(item.pubDate);
|
|
|
return itemDate > threshold;
|
|
return itemDate > threshold;
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
@@ -374,13 +374,13 @@ function generateStateDocId(feedUrl) {
|
|
|
* Load seen item identifiers from storage
|
|
* Load seen item identifiers from storage
|
|
|
*/
|
|
*/
|
|
|
function loadSeenItems(collection, feedUrl) {
|
|
function loadSeenItems(collection, feedUrl) {
|
|
|
- const docId = generateStateDocId(feedUrl);
|
|
|
|
|
|
|
+ var docId = generateStateDocId(feedUrl);
|
|
|
|
|
|
|
|
try {
|
|
try {
|
|
|
- const result = smartbotic.storage.get(collection, docId);
|
|
|
|
|
|
|
+ var result = smartbotic.storage.get(collection, docId);
|
|
|
|
|
|
|
|
if (result.found && result.document && result.document.seenIds) {
|
|
if (result.found && result.document && result.document.seenIds) {
|
|
|
- smartbotic.log.debug(`Loaded ${result.document.seenIds.length} seen item IDs from storage`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
|
|
|
return {
|
|
return {
|
|
|
seenIds: new Set(result.document.seenIds),
|
|
seenIds: new Set(result.document.seenIds),
|
|
|
docId: docId,
|
|
docId: docId,
|
|
@@ -389,7 +389,7 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
};
|
|
};
|
|
|
}
|
|
}
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
- smartbotic.log.warn(`Error loading seen items from storage: ${error.message}`);
|
|
|
|
|
|
|
+ smartbotic.log.warn('Error loading seen items from storage: ' + error.message);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
return {
|
|
return {
|
|
@@ -404,8 +404,8 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
* Save seen item identifiers to storage
|
|
* Save seen item identifiers to storage
|
|
|
*/
|
|
*/
|
|
|
function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
- const seenArray = Array.from(seenIds);
|
|
|
|
|
- const documentData = {
|
|
|
|
|
|
|
+ var seenArray = Array.from(seenIds);
|
|
|
|
|
+ var documentData = {
|
|
|
feedUrl: feedUrl,
|
|
feedUrl: feedUrl,
|
|
|
seenIds: seenArray,
|
|
seenIds: seenArray,
|
|
|
lastUpdated: new Date().toISOString(),
|
|
lastUpdated: new Date().toISOString(),
|
|
@@ -415,24 +415,24 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
try {
|
|
try {
|
|
|
if (exists) {
|
|
if (exists) {
|
|
|
// Update existing document
|
|
// Update existing document
|
|
|
- const result = smartbotic.storage.update(collection, docId, documentData, version, false);
|
|
|
|
|
- if (!result.success) {
|
|
|
|
|
- smartbotic.log.error(`Failed to update seen items: ${result.error}`);
|
|
|
|
|
|
|
+ var updateResult = smartbotic.storage.update(collection, docId, documentData, version, false);
|
|
|
|
|
+ if (!updateResult.success) {
|
|
|
|
|
+ smartbotic.log.error('Failed to update seen items: ' + updateResult.error);
|
|
|
return false;
|
|
return false;
|
|
|
}
|
|
}
|
|
|
- smartbotic.log.debug(`Updated ${seenArray.length} seen item IDs in storage`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Updated ' + seenArray.length + ' seen item IDs in storage');
|
|
|
} else {
|
|
} else {
|
|
|
// Insert new document
|
|
// Insert new document
|
|
|
- const result = smartbotic.storage.insert(collection, documentData, docId, 0);
|
|
|
|
|
- if (!result.success) {
|
|
|
|
|
- smartbotic.log.error(`Failed to save seen items: ${result.error}`);
|
|
|
|
|
|
|
+ var insertResult = smartbotic.storage.insert(collection, documentData, docId, 0);
|
|
|
|
|
+ if (!insertResult.success) {
|
|
|
|
|
+ smartbotic.log.error('Failed to save seen items: ' + insertResult.error);
|
|
|
return false;
|
|
return false;
|
|
|
}
|
|
}
|
|
|
- smartbotic.log.debug(`Saved ${seenArray.length} seen item IDs to storage`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage');
|
|
|
}
|
|
}
|
|
|
return true;
|
|
return true;
|
|
|
} catch (error) {
|
|
} catch (error) {
|
|
|
- smartbotic.log.error(`Error saving seen items to storage: ${error.message}`);
|
|
|
|
|
|
|
+ smartbotic.log.error('Error saving seen items to storage: ' + error.message);
|
|
|
return false;
|
|
return false;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
@@ -441,20 +441,20 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
* Filter items to only include new (unseen) items
|
|
* Filter items to only include new (unseen) items
|
|
|
*/
|
|
*/
|
|
|
function filterNewItems(items, seenIds) {
|
|
function filterNewItems(items, seenIds) {
|
|
|
- return items.filter(item => {
|
|
|
|
|
- const id = getItemIdentifier(item);
|
|
|
|
|
|
|
+ return items.filter(function(item) {
|
|
|
|
|
+ var id = getItemIdentifier(item);
|
|
|
return id && !seenIds.has(id);
|
|
return id && !seenIds.has(id);
|
|
|
});
|
|
});
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-async function execute(config, input) {
|
|
|
|
|
- const url = input.url || config.url;
|
|
|
|
|
- const filterKeyword = input.filterKeyword || config.filterKeyword;
|
|
|
|
|
- const filterNewerThan = input.filterNewerThan || config.filterNewerThan;
|
|
|
|
|
- const maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
|
|
|
|
|
- const timeout = config.timeout || 30000;
|
|
|
|
|
- const detectNewItems = config.detectNewItems || false;
|
|
|
|
|
- const stateCollection = config.stateCollection;
|
|
|
|
|
|
|
+function execute(config, input) {
|
|
|
|
|
+ var url = input.url || config.url;
|
|
|
|
|
+ var filterKeyword = input.filterKeyword || config.filterKeyword;
|
|
|
|
|
+ var filterNewerThan = input.filterNewerThan || config.filterNewerThan;
|
|
|
|
|
+ var maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
|
|
|
|
|
+ var timeout = config.timeout || 30000;
|
|
|
|
|
+ var detectNewItems = config.detectNewItems || false;
|
|
|
|
|
+ var stateCollection = config.stateCollection;
|
|
|
|
|
|
|
|
if (!url) {
|
|
if (!url) {
|
|
|
throw new Error('Feed URL is required');
|
|
throw new Error('Feed URL is required');
|
|
@@ -465,9 +465,9 @@ async function execute(config, input) {
|
|
|
throw new Error('State collection is required when "Detect New Items" is enabled. Please select a read-write collection in the workflow storage settings.');
|
|
throw new Error('State collection is required when "Detect New Items" is enabled. Please select a read-write collection in the workflow storage settings.');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- smartbotic.log.info(`Fetching RSS/Atom feed from ${url}`);
|
|
|
|
|
|
|
+ smartbotic.log.info('Fetching RSS/Atom feed from ' + url);
|
|
|
|
|
|
|
|
- const response = smartbotic.http.request({
|
|
|
|
|
|
|
+ var response = smartbotic.http.request({
|
|
|
method: 'GET',
|
|
method: 'GET',
|
|
|
url: url,
|
|
url: url,
|
|
|
timeout: timeout,
|
|
timeout: timeout,
|
|
@@ -477,60 +477,60 @@ async function execute(config, input) {
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
if (response.status < 200 || response.status >= 300) {
|
|
if (response.status < 200 || response.status >= 300) {
|
|
|
- throw new Error(`Failed to fetch feed: HTTP ${response.status}`);
|
|
|
|
|
|
|
+ throw new Error('Failed to fetch feed: HTTP ' + response.status);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
|
|
|
|
|
|
|
+ var xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
|
|
|
|
|
|
|
|
if (!xml || xml.trim().length === 0) {
|
|
if (!xml || xml.trim().length === 0) {
|
|
|
throw new Error('Empty response from feed URL');
|
|
throw new Error('Empty response from feed URL');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
var feed;
|
|
var feed;
|
|
|
- if (xml.includes('<feed') && xml.includes('xmlns') && xml.includes('atom')) {
|
|
|
|
|
|
|
+ if (xml.indexOf('<feed') !== -1 && xml.indexOf('xmlns') !== -1 && xml.indexOf('atom') !== -1) {
|
|
|
smartbotic.log.debug('Detected Atom feed format');
|
|
smartbotic.log.debug('Detected Atom feed format');
|
|
|
feed = parseAtom(xml);
|
|
feed = parseAtom(xml);
|
|
|
- } else if (xml.includes('<rss') || xml.includes('<channel')) {
|
|
|
|
|
|
|
+ } else if (xml.indexOf('<rss') !== -1 || xml.indexOf('<channel') !== -1) {
|
|
|
smartbotic.log.debug('Detected RSS feed format');
|
|
smartbotic.log.debug('Detected RSS feed format');
|
|
|
feed = parseRss(xml);
|
|
feed = parseRss(xml);
|
|
|
} else {
|
|
} else {
|
|
|
throw new Error('Unknown feed format: neither RSS nor Atom detected');
|
|
throw new Error('Unknown feed format: neither RSS nor Atom detected');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- let items = feed.items;
|
|
|
|
|
- const totalFeedItemCount = items.length;
|
|
|
|
|
|
|
+ var items = feed.items;
|
|
|
|
|
+ var totalFeedItemCount = items.length;
|
|
|
|
|
|
|
|
// Apply standard filters first (keyword, date)
|
|
// Apply standard filters first (keyword, date)
|
|
|
if (filterKeyword) {
|
|
if (filterKeyword) {
|
|
|
- smartbotic.log.debug(`Filtering by keyword: ${filterKeyword}`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Filtering by keyword: ' + filterKeyword);
|
|
|
items = filterByKeyword(items, filterKeyword);
|
|
items = filterByKeyword(items, filterKeyword);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if (filterNewerThan) {
|
|
if (filterNewerThan) {
|
|
|
- smartbotic.log.debug(`Filtering items newer than: ${filterNewerThan}`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Filtering items newer than: ' + filterNewerThan);
|
|
|
items = filterByDate(items, filterNewerThan);
|
|
items = filterByDate(items, filterNewerThan);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// Stateful new item detection
|
|
// Stateful new item detection
|
|
|
- let newItemCount = items.length;
|
|
|
|
|
- let stateInfo = null;
|
|
|
|
|
|
|
+ var newItemCount = items.length;
|
|
|
|
|
+ var stateInfo = null;
|
|
|
|
|
|
|
|
if (detectNewItems) {
|
|
if (detectNewItems) {
|
|
|
- smartbotic.log.info(`Detecting new items using collection: ${stateCollection}`);
|
|
|
|
|
|
|
+ smartbotic.log.info('Detecting new items using collection: ' + stateCollection);
|
|
|
|
|
|
|
|
// Load previously seen items
|
|
// Load previously seen items
|
|
|
stateInfo = loadSeenItems(stateCollection, url);
|
|
stateInfo = loadSeenItems(stateCollection, url);
|
|
|
|
|
|
|
|
// Filter to only new items
|
|
// Filter to only new items
|
|
|
- const newItems = filterNewItems(items, stateInfo.seenIds);
|
|
|
|
|
|
|
+ var newItems = filterNewItems(items, stateInfo.seenIds);
|
|
|
newItemCount = newItems.length;
|
|
newItemCount = newItems.length;
|
|
|
|
|
|
|
|
- smartbotic.log.info(`Found ${newItemCount} new items out of ${items.length} filtered items`);
|
|
|
|
|
|
|
+ smartbotic.log.info('Found ' + newItemCount + ' new items out of ' + items.length + ' filtered items');
|
|
|
|
|
|
|
|
// Update seen IDs with all current items (both new and previously seen)
|
|
// Update seen IDs with all current items (both new and previously seen)
|
|
|
// This ensures we track all items from the current feed
|
|
// This ensures we track all items from the current feed
|
|
|
- for (const item of items) {
|
|
|
|
|
- const id = getItemIdentifier(item);
|
|
|
|
|
|
|
+ for (var i = 0; i < items.length; i++) {
|
|
|
|
|
+ var id = getItemIdentifier(items[i]);
|
|
|
if (id) {
|
|
if (id) {
|
|
|
stateInfo.seenIds.add(id);
|
|
stateInfo.seenIds.add(id);
|
|
|
}
|
|
}
|
|
@@ -542,15 +542,15 @@ async function execute(config, input) {
|
|
|
|
|
|
|
|
// Apply max items limit last
|
|
// Apply max items limit last
|
|
|
if (maxItems && maxItems > 0) {
|
|
if (maxItems && maxItems > 0) {
|
|
|
- smartbotic.log.debug(`Limiting to ${maxItems} items`);
|
|
|
|
|
|
|
+ smartbotic.log.debug('Limiting to ' + maxItems + ' items');
|
|
|
items = limitItems(items, maxItems);
|
|
items = limitItems(items, maxItems);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- smartbotic.log.info(`Returning ${items.length} items from feed`);
|
|
|
|
|
|
|
+ smartbotic.log.info('Returning ' + items.length + ' items from feed');
|
|
|
|
|
|
|
|
// Persist seen items state after successful processing
|
|
// Persist seen items state after successful processing
|
|
|
if (detectNewItems && stateInfo) {
|
|
if (detectNewItems && stateInfo) {
|
|
|
- const saved = saveSeenItems(
|
|
|
|
|
|
|
+ var saved = saveSeenItems(
|
|
|
stateCollection,
|
|
stateCollection,
|
|
|
stateInfo.docId,
|
|
stateInfo.docId,
|
|
|
stateInfo.seenIds,
|
|
stateInfo.seenIds,
|
|
@@ -563,7 +563,7 @@ async function execute(config, input) {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const result = {
|
|
|
|
|
|
|
+ var result = {
|
|
|
feedTitle: feed.title,
|
|
feedTitle: feed.title,
|
|
|
feedLink: feed.link,
|
|
feedLink: feed.link,
|
|
|
feedDescription: feed.description,
|
|
feedDescription: feed.description,
|