|
|
@@ -2,7 +2,7 @@
|
|
|
* @node rss-reader
|
|
|
* @name RSS Reader
|
|
|
* @category data
|
|
|
- * @version 1.1.4
|
|
|
+ * @version 1.1.5
|
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
|
* @icon rss
|
|
|
*/
|
|
|
@@ -223,7 +223,6 @@ function parseRss(xml) {
|
|
|
if (!channel) {
|
|
|
throw new Error('Invalid RSS feed: no channel element found');
|
|
|
}
|
|
|
-
|
|
|
var feedTitle = xmlGetTagContent(channel, 'title');
|
|
|
var feedLink = xmlGetTagContent(channel, 'link');
|
|
|
var feedDesc = xmlGetTagContent(channel, 'description');
|
|
|
@@ -572,11 +571,22 @@ function loadSeenItems(collection, feedUrl) {
|
|
|
var result = smartbotic.storage.get(collection, docId);
|
|
|
|
|
|
if (result.found && result.document && result.document.seenIds) {
|
|
|
- smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
|
|
|
// Use plain object as a set (for QuickJS compatibility)
|
|
|
var seenObj = {};
|
|
|
- for (var i = 0; i < result.document.seenIds.length; i++) {
|
|
|
- seenObj[result.document.seenIds[i]] = true;
|
|
|
+ var storedIds = result.document.seenIds;
|
|
|
+ // Handle both array format and object format (in case of legacy data)
|
|
|
+ if (Array.isArray(storedIds)) {
|
|
|
+ smartbotic.log.debug('Loaded ' + storedIds.length + ' seen item IDs from storage');
|
|
|
+ for (var i = 0; i < storedIds.length; i++) {
|
|
|
+ seenObj[storedIds[i]] = true;
|
|
|
+ }
|
|
|
+ } else if (typeof storedIds === 'object' && storedIds !== null) {
|
|
|
+ // Object format - keys are the IDs
|
|
|
+ var keys = Object.keys(storedIds);
|
|
|
+ smartbotic.log.debug('Loaded ' + keys.length + ' seen item IDs from storage (object format)');
|
|
|
+ for (var j = 0; j < keys.length; j++) {
|
|
|
+ seenObj[keys[j]] = true;
|
|
|
+ }
|
|
|
}
|
|
|
return {
|
|
|
seenIds: seenObj,
|
|
|
@@ -623,10 +633,22 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
// Insert new document
|
|
|
var insertResult = smartbotic.storage.insert(collection, documentData, docId, 0);
|
|
|
if (!insertResult.success) {
|
|
|
- smartbotic.log.error('Failed to save seen items: ' + insertResult.error);
|
|
|
- return false;
|
|
|
+ // If insert fails because document exists, try update instead
|
|
|
+ if (insertResult.error && insertResult.error.indexOf('already exists') !== -1) {
|
|
|
+ smartbotic.log.debug('Document exists, trying update instead');
|
|
|
+ var retryResult = smartbotic.storage.update(collection, docId, documentData, 0, false);
|
|
|
+ if (!retryResult.success) {
|
|
|
+ smartbotic.log.error('Failed to update seen items on retry: ' + retryResult.error);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ smartbotic.log.debug('Updated ' + seenArray.length + ' seen item IDs in storage (retry)');
|
|
|
+ } else {
|
|
|
+ smartbotic.log.error('Failed to save seen items: ' + insertResult.error);
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ } else {
|
|
|
+ smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage');
|
|
|
}
|
|
|
- smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage');
|
|
|
}
|
|
|
return true;
|
|
|
} catch (error) {
|
|
|
@@ -639,10 +661,14 @@ function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
|
|
|
* Filter items to only include new (unseen) items
|
|
|
*/
|
|
|
function filterNewItems(items, seenIds) {
|
|
|
- return items.filter(function(item) {
|
|
|
- var id = getItemIdentifier(item);
|
|
|
- return id && !seenIds[id];
|
|
|
- });
|
|
|
+ var newItems = [];
|
|
|
+ for (var i = 0; i < items.length; i++) {
|
|
|
+ var id = getItemIdentifier(items[i]);
|
|
|
+ if (id && !seenIds[id]) {
|
|
|
+ newItems.push(items[i]);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return newItems;
|
|
|
}
|
|
|
|
|
|
async function execute(config, input) {
|