rss-reader.js 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781
  1. /**
  2. * @node rss-reader
  3. * @name RSS Reader
  4. * @category data
  5. * @version 1.1.4
  6. * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
  7. * @icon rss
  8. */
  9. const configSchema = {
  10. type: 'object',
  11. properties: {
  12. url: {
  13. type: 'string',
  14. title: 'Feed URL',
  15. description: 'URL of the RSS or Atom feed'
  16. },
  17. filterKeyword: {
  18. type: 'string',
  19. title: 'Keyword Filter',
  20. description: 'Filter items by keyword (matches title and description)'
  21. },
  22. filterNewerThan: {
  23. type: 'string',
  24. title: 'Newer Than',
  25. description: 'Only include items newer than this date (ISO 8601 format, e.g., 2024-01-15T00:00:00Z)'
  26. },
  27. maxItems: {
  28. type: 'number',
  29. title: 'Max Items',
  30. description: 'Maximum number of items to return (0 = unlimited)',
  31. default: 0
  32. },
  33. timeout: {
  34. type: 'number',
  35. title: 'Timeout (ms)',
  36. default: 30000
  37. },
  38. detectNewItems: {
  39. type: 'boolean',
  40. title: 'Detect New Items',
  41. description: 'When enabled, only output items that have not been seen in previous executions',
  42. default: false
  43. },
  44. stateCollection: {
  45. type: 'string',
  46. title: 'State Collection',
  47. description: 'Select a read-write collection to store seen item identifiers',
  48. dynamicOptions: {
  49. source: 'storage.collections',
  50. labelField: 'name',
  51. valueField: 'name',
  52. filter: { access: 'read-write' }
  53. },
  54. showWhen: { field: 'detectNewItems', value: true }
  55. }
  56. },
  57. required: ['url']
  58. };
  59. const inputSchema = {
  60. type: 'object',
  61. properties: {
  62. url: {
  63. type: 'string',
  64. description: 'Override feed URL from input'
  65. },
  66. filterKeyword: {
  67. type: 'string',
  68. description: 'Override keyword filter from input'
  69. },
  70. filterNewerThan: {
  71. type: 'string',
  72. description: 'Override date filter from input'
  73. },
  74. maxItems: {
  75. type: 'number',
  76. description: 'Override max items from input'
  77. }
  78. }
  79. };
  80. const outputSchema = {
  81. type: 'object',
  82. properties: {
  83. feedTitle: { type: 'string', description: 'Title of the feed' },
  84. feedLink: { type: 'string', description: 'Link to the feed homepage' },
  85. feedDescription: { type: 'string', description: 'Description of the feed' },
  86. feedLanguage: { type: 'string', description: 'Language of the feed' },
  87. feedPubDate: { type: 'string', description: 'Publication date of the feed' },
  88. feedGenerator: { type: 'string', description: 'Generator of the feed' },
  89. feedImage: {
  90. type: 'object',
  91. description: 'Feed image/logo',
  92. properties: {
  93. url: { type: 'string' },
  94. title: { type: 'string' },
  95. link: { type: 'string' }
  96. }
  97. },
  98. itemCount: { type: 'number', description: 'Number of items returned' },
  99. newItemCount: { type: 'number', description: 'Number of new items (when detectNewItems is enabled)' },
  100. totalFeedItemCount: { type: 'number', description: 'Total items in feed before filtering (when detectNewItems is enabled)' },
  101. items: {
  102. type: 'array',
  103. description: 'Array of feed items',
  104. items: {
  105. type: 'object',
  106. properties: {
  107. title: { type: 'string', description: 'Item title' },
  108. link: { type: 'string', description: 'Item link/URL' },
  109. description: { type: 'string', description: 'Item summary/description' },
  110. content: { type: 'string', description: 'Full content (content:encoded)' },
  111. pubDate: { type: 'string', description: 'Publication date' },
  112. author: { type: 'string', description: 'Author name' },
  113. categories: { type: 'array', items: { type: 'string' }, description: 'Categories/tags' },
  114. guid: { type: 'string', description: 'Unique identifier' },
  115. comments: { type: 'string', description: 'Comments URL' },
  116. source: { type: 'string', description: 'Source attribution' },
  117. enclosure: {
  118. type: 'object',
  119. description: 'Media enclosure (attachment)',
  120. properties: {
  121. url: { type: 'string', description: 'Media URL' },
  122. type: { type: 'string', description: 'MIME type' },
  123. length: { type: 'number', description: 'File size in bytes' }
  124. }
  125. },
  126. media: {
  127. type: 'object',
  128. description: 'Media content (media:content)',
  129. properties: {
  130. url: { type: 'string' },
  131. type: { type: 'string' },
  132. width: { type: 'number' },
  133. height: { type: 'number' }
  134. }
  135. },
  136. thumbnail: {
  137. type: 'object',
  138. description: 'Thumbnail image (media:thumbnail)',
  139. properties: {
  140. url: { type: 'string' },
  141. width: { type: 'number' },
  142. height: { type: 'number' }
  143. }
  144. }
  145. }
  146. }
  147. }
  148. }
  149. };
  150. /**
  151. * Simple XML parser utilities for RSS/Atom feeds
  152. * Handles basic XML structure without external dependencies
  153. */
  154. function xmlGetTagContent(xml, tagName) {
  155. var pattern = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'i');
  156. var match = xml.match(pattern);
  157. if (match) {
  158. return match[1].trim();
  159. }
  160. return null;
  161. }
  162. function xmlGetAllTagContents(xml, tagName) {
  163. var results = [];
  164. var regex = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'gi');
  165. var match;
  166. while ((match = regex.exec(xml)) !== null) {
  167. results.push(match[1].trim());
  168. }
  169. return results;
  170. }
  171. function xmlGetTagAttributes(xml, tagName) {
  172. // Match self-closing tag or opening tag
  173. var regex = new RegExp('<' + tagName + '([^>]*?)\\/?>', 'i');
  174. var match = xml.match(regex);
  175. if (!match) return null;
  176. var attrString = match[1];
  177. var attrs = {};
  178. // Parse attributes: name="value" or name='value'
  179. var attrRegex = /(\w+)=["']([^"']*)["']/g;
  180. var attrMatch;
  181. while ((attrMatch = attrRegex.exec(attrString)) !== null) {
  182. attrs[attrMatch[1]] = attrMatch[2];
  183. }
  184. return attrs;
  185. }
  186. function xmlDecodeHtmlEntities(text) {
  187. if (!text) return '';
  188. return text
  189. .replace(/&lt;/g, '<')
  190. .replace(/&gt;/g, '>')
  191. .replace(/&amp;/g, '&')
  192. .replace(/&quot;/g, '"')
  193. .replace(/&apos;/g, "'")
  194. .replace(/&#(\d+);/g, function(_, dec) { return String.fromCharCode(dec); })
  195. .replace(/&#x([0-9a-f]+);/gi, function(_, hex) { return String.fromCharCode(parseInt(hex, 16)); });
  196. }
  197. function xmlStripCdata(text) {
  198. if (!text) return '';
  199. return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
  200. }
  201. function xmlCleanText(text) {
  202. if (!text) return '';
  203. return xmlDecodeHtmlEntities(xmlStripCdata(text)).trim();
  204. }
  205. /**
  206. * Parse RSS 2.0 feed
  207. */
  208. function parseRss(xml) {
  209. var channel = xmlGetTagContent(xml, 'channel');
  210. if (!channel) {
  211. throw new Error('Invalid RSS feed: no channel element found');
  212. }
  213. var feedTitle = xmlGetTagContent(channel, 'title');
  214. var feedLink = xmlGetTagContent(channel, 'link');
  215. var feedDesc = xmlGetTagContent(channel, 'description');
  216. var feedLanguage = xmlGetTagContent(channel, 'language');
  217. var feedPubDate = xmlGetTagContent(channel, 'pubDate');
  218. var feedLastBuildDate = xmlGetTagContent(channel, 'lastBuildDate');
  219. var feedGenerator = xmlGetTagContent(channel, 'generator');
  220. var feedImage = xmlGetTagContent(channel, 'image');
  221. var feed = {
  222. title: xmlCleanText(feedTitle) || '',
  223. link: xmlCleanText(feedLink) || '',
  224. description: xmlCleanText(feedDesc) || '',
  225. language: xmlCleanText(feedLanguage) || '',
  226. pubDate: xmlCleanText(feedPubDate) || '',
  227. lastBuildDate: xmlCleanText(feedLastBuildDate) || '',
  228. generator: xmlCleanText(feedGenerator) || '',
  229. items: []
  230. };
  231. // Parse feed image if present
  232. if (feedImage) {
  233. feed.image = {
  234. url: xmlCleanText(xmlGetTagContent(feedImage, 'url')) || '',
  235. title: xmlCleanText(xmlGetTagContent(feedImage, 'title')) || '',
  236. link: xmlCleanText(xmlGetTagContent(feedImage, 'link')) || ''
  237. };
  238. }
  239. var itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
  240. var match;
  241. while ((match = itemRegex.exec(channel)) !== null) {
  242. var itemXml = match[1];
  243. var catArray = xmlGetAllTagContents(itemXml, 'category');
  244. var categories = [];
  245. for (var ci = 0; ci < catArray.length; ci++) {
  246. categories.push(xmlCleanText(catArray[ci]));
  247. }
  248. var item = {
  249. title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '',
  250. link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '',
  251. description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '',
  252. content: xmlCleanText(xmlGetTagContent(itemXml, 'content:encoded')) || '',
  253. pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '',
  254. author: xmlCleanText(xmlGetTagContent(itemXml, 'author') ||
  255. xmlGetTagContent(itemXml, 'dc:creator')) || '',
  256. categories: categories,
  257. guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || '',
  258. comments: xmlCleanText(xmlGetTagContent(itemXml, 'comments')) || '',
  259. source: xmlCleanText(xmlGetTagContent(itemXml, 'source')) || ''
  260. };
  261. // Parse enclosure (media attachment)
  262. var enclosureAttrs = xmlGetTagAttributes(itemXml, 'enclosure');
  263. if (enclosureAttrs) {
  264. item.enclosure = {
  265. url: enclosureAttrs.url || '',
  266. type: enclosureAttrs.type || '',
  267. length: enclosureAttrs.length ? parseInt(enclosureAttrs.length, 10) : 0
  268. };
  269. }
  270. // Parse media:content (common in media RSS)
  271. var mediaAttrs = xmlGetTagAttributes(itemXml, 'media:content');
  272. if (mediaAttrs) {
  273. item.media = {
  274. url: mediaAttrs.url || '',
  275. type: mediaAttrs.type || mediaAttrs.medium || '',
  276. width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
  277. height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
  278. };
  279. }
  280. // Parse media:thumbnail
  281. var thumbAttrs = xmlGetTagAttributes(itemXml, 'media:thumbnail');
  282. if (thumbAttrs) {
  283. item.thumbnail = {
  284. url: thumbAttrs.url || '',
  285. width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
  286. height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
  287. };
  288. }
  289. feed.items.push(item);
  290. }
  291. return feed;
  292. }
  293. /**
  294. * Parse Atom feed
  295. */
  296. function parseAtom(xml) {
  297. var feedContent = xmlGetTagContent(xml, 'feed');
  298. if (!feedContent) {
  299. throw new Error('Invalid Atom feed: no feed element found');
  300. }
  301. function getAtomLink(content, rel) {
  302. var linkRegex = /<link([^>]*)>/gi;
  303. var match;
  304. while ((match = linkRegex.exec(content)) !== null) {
  305. var attrs = match[1];
  306. var relMatch = attrs.match(/rel=["']([^"']*)["']/);
  307. var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
  308. if (hrefMatch) {
  309. var linkRel = relMatch ? relMatch[1] : 'alternate';
  310. if (linkRel === rel || (!rel && linkRel === 'alternate')) {
  311. return hrefMatch[1];
  312. }
  313. }
  314. }
  315. return '';
  316. }
  317. function getAtomLinkByType(content, type) {
  318. var linkRegex = /<link([^>]*)>/gi;
  319. var match;
  320. while ((match = linkRegex.exec(content)) !== null) {
  321. var attrs = match[1];
  322. var typeMatch = attrs.match(/type=["']([^"']*)["']/);
  323. var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
  324. if (hrefMatch && typeMatch && typeMatch[1].indexOf(type) !== -1) {
  325. return {
  326. url: hrefMatch[1],
  327. type: typeMatch[1]
  328. };
  329. }
  330. }
  331. return null;
  332. }
  333. var feed = {
  334. title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '',
  335. link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
  336. description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '',
  337. language: '',
  338. pubDate: xmlCleanText(xmlGetTagContent(feedContent, 'updated')) || '',
  339. lastBuildDate: '',
  340. generator: xmlCleanText(xmlGetTagContent(feedContent, 'generator')) || '',
  341. items: []
  342. };
  343. // Parse feed icon/logo
  344. var feedIcon = xmlGetTagContent(feedContent, 'icon');
  345. var feedLogo = xmlGetTagContent(feedContent, 'logo');
  346. if (feedIcon || feedLogo) {
  347. feed.image = {
  348. url: xmlCleanText(feedLogo || feedIcon) || '',
  349. title: feed.title,
  350. link: feed.link
  351. };
  352. }
  353. var entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
  354. var match;
  355. while ((match = entryRegex.exec(feedContent)) !== null) {
  356. var entryXml = match[1];
  357. var categories = [];
  358. var catRegex = /<category([^>]*)>/gi;
  359. var catMatch;
  360. while ((catMatch = catRegex.exec(entryXml)) !== null) {
  361. var termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
  362. if (termMatch) {
  363. categories.push(xmlCleanText(termMatch[1]));
  364. }
  365. }
  366. var author = '';
  367. var authorContent = xmlGetTagContent(entryXml, 'author');
  368. if (authorContent) {
  369. author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || '';
  370. }
  371. var pubDate = xmlCleanText(
  372. xmlGetTagContent(entryXml, 'published') ||
  373. xmlGetTagContent(entryXml, 'updated')
  374. ) || '';
  375. var summary = xmlCleanText(xmlGetTagContent(entryXml, 'summary')) || '';
  376. var content = xmlCleanText(xmlGetTagContent(entryXml, 'content')) || '';
  377. var item = {
  378. title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '',
  379. link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
  380. description: summary || content,
  381. content: content,
  382. pubDate: pubDate,
  383. author: author,
  384. categories: categories,
  385. guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || '',
  386. comments: '',
  387. source: ''
  388. };
  389. // Check for enclosure link
  390. var enclosureLink = getAtomLinkByType(entryXml, 'image');
  391. if (!enclosureLink) {
  392. enclosureLink = getAtomLinkByType(entryXml, 'audio');
  393. }
  394. if (!enclosureLink) {
  395. enclosureLink = getAtomLinkByType(entryXml, 'video');
  396. }
  397. if (enclosureLink) {
  398. item.enclosure = {
  399. url: enclosureLink.url,
  400. type: enclosureLink.type,
  401. length: 0
  402. };
  403. }
  404. // Parse media:content
  405. var mediaAttrs = xmlGetTagAttributes(entryXml, 'media:content');
  406. if (mediaAttrs) {
  407. item.media = {
  408. url: mediaAttrs.url || '',
  409. type: mediaAttrs.type || mediaAttrs.medium || '',
  410. width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
  411. height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
  412. };
  413. }
  414. // Parse media:thumbnail
  415. var thumbAttrs = xmlGetTagAttributes(entryXml, 'media:thumbnail');
  416. if (thumbAttrs) {
  417. item.thumbnail = {
  418. url: thumbAttrs.url || '',
  419. width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
  420. height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
  421. };
  422. }
  423. feed.items.push(item);
  424. }
  425. return feed;
  426. }
  427. /**
  428. * Parse date string to timestamp
  429. */
  430. function parseDate(dateStr) {
  431. if (!dateStr) return 0;
  432. var timestamp = Date.parse(dateStr);
  433. if (!isNaN(timestamp)) {
  434. return timestamp;
  435. }
  436. var rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
  437. var match = dateStr.match(rfc822Regex);
  438. if (match) {
  439. var months = {
  440. Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
  441. Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
  442. };
  443. var day = parseInt(match[1], 10);
  444. var month = months[match[2]];
  445. var year = parseInt(match[3], 10);
  446. var hour = parseInt(match[4], 10);
  447. var minute = parseInt(match[5], 10);
  448. var second = parseInt(match[6], 10);
  449. if (month !== undefined) {
  450. return new Date(year, month, day, hour, minute, second).getTime();
  451. }
  452. }
  453. return 0;
  454. }
  455. /**
  456. * Filter items by keyword
  457. */
  458. function filterByKeyword(items, keyword) {
  459. if (!keyword) return items;
  460. var lowerKeyword = keyword.toLowerCase();
  461. return items.filter(function(item) {
  462. var title = (item.title || '').toLowerCase();
  463. var description = (item.description || '').toLowerCase();
  464. return title.indexOf(lowerKeyword) !== -1 || description.indexOf(lowerKeyword) !== -1;
  465. });
  466. }
  467. /**
  468. * Filter items by date
  469. */
  470. function filterByDate(items, newerThanStr) {
  471. if (!newerThanStr) return items;
  472. var threshold = parseDate(newerThanStr);
  473. if (threshold === 0) {
  474. smartbotic.log.warn('Could not parse date filter: ' + newerThanStr);
  475. return items;
  476. }
  477. return items.filter(function(item) {
  478. var itemDate = parseDate(item.pubDate);
  479. return itemDate > threshold;
  480. });
  481. }
  482. /**
  483. * Limit number of items
  484. */
  485. function limitItems(items, maxItems) {
  486. if (!maxItems || maxItems <= 0) return items;
  487. return items.slice(0, maxItems);
  488. }
  489. /**
  490. * Get unique identifier for an RSS/Atom item
  491. * Prefers GUID, falls back to link
  492. */
  493. function getItemIdentifier(item) {
  494. return item.guid || item.link || '';
  495. }
  496. /**
  497. * Generate a stable document ID from feed URL for state storage
  498. */
  499. function generateStateDocId(feedUrl) {
  500. // Create a simple hash from the feed URL for a stable doc ID
  501. var hash = 0;
  502. for (var i = 0; i < feedUrl.length; i++) {
  503. var char = feedUrl.charCodeAt(i);
  504. hash = ((hash << 5) - hash) + char;
  505. hash = hash & hash; // Convert to 32-bit integer
  506. }
  507. return 'rss-state-' + Math.abs(hash).toString(36);
  508. }
  509. /**
  510. * Load seen item identifiers from storage
  511. */
  512. function loadSeenItems(collection, feedUrl) {
  513. var docId = generateStateDocId(feedUrl);
  514. try {
  515. var result = smartbotic.storage.get(collection, docId);
  516. if (result.found && result.document && result.document.seenIds) {
  517. smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
  518. // Use plain object as a set (for QuickJS compatibility)
  519. var seenObj = {};
  520. for (var i = 0; i < result.document.seenIds.length; i++) {
  521. seenObj[result.document.seenIds[i]] = true;
  522. }
  523. return {
  524. seenIds: seenObj,
  525. docId: docId,
  526. exists: true,
  527. version: result.document._version || 1
  528. };
  529. }
  530. } catch (error) {
  531. smartbotic.log.warn('Error loading seen items from storage: ' + error.message);
  532. }
  533. return {
  534. seenIds: {},
  535. docId: docId,
  536. exists: false,
  537. version: 0
  538. };
  539. }
  540. /**
  541. * Save seen item identifiers to storage
  542. */
  543. function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
  544. // Convert object keys to array
  545. var seenArray = Object.keys(seenIds);
  546. var documentData = {
  547. feedUrl: feedUrl,
  548. seenIds: seenArray,
  549. lastUpdated: new Date().toISOString(),
  550. itemCount: seenArray.length
  551. };
  552. try {
  553. if (exists) {
  554. // Update existing document
  555. var updateResult = smartbotic.storage.update(collection, docId, documentData, version, false);
  556. if (!updateResult.success) {
  557. smartbotic.log.error('Failed to update seen items: ' + updateResult.error);
  558. return false;
  559. }
  560. smartbotic.log.debug('Updated ' + seenArray.length + ' seen item IDs in storage');
  561. } else {
  562. // Insert new document
  563. var insertResult = smartbotic.storage.insert(collection, documentData, docId, 0);
  564. if (!insertResult.success) {
  565. smartbotic.log.error('Failed to save seen items: ' + insertResult.error);
  566. return false;
  567. }
  568. smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage');
  569. }
  570. return true;
  571. } catch (error) {
  572. smartbotic.log.error('Error saving seen items to storage: ' + error.message);
  573. return false;
  574. }
  575. }
  576. /**
  577. * Filter items to only include new (unseen) items
  578. */
  579. function filterNewItems(items, seenIds) {
  580. return items.filter(function(item) {
  581. var id = getItemIdentifier(item);
  582. return id && !seenIds[id];
  583. });
  584. }
  585. async function execute(config, input) {
  586. var url = input.url || config.url;
  587. var filterKeyword = input.filterKeyword || config.filterKeyword;
  588. var filterNewerThan = input.filterNewerThan || config.filterNewerThan;
  589. var maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
  590. var timeout = config.timeout || 30000;
  591. var detectNewItems = config.detectNewItems || false;
  592. var stateCollection = config.stateCollection;
  593. if (!url) {
  594. throw new Error('Feed URL is required');
  595. }
  596. // Validate stateful mode configuration
  597. if (detectNewItems && !stateCollection) {
  598. throw new Error('State collection is required when "Detect New Items" is enabled. Please select a read-write collection in the workflow storage settings.');
  599. }
  600. smartbotic.log.info('Fetching RSS/Atom feed from ' + url);
  601. var response = smartbotic.http.request({
  602. method: 'GET',
  603. url: url,
  604. timeout: timeout,
  605. headers: {
  606. 'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*'
  607. }
  608. });
  609. if (response.status < 200 || response.status >= 300) {
  610. throw new Error('Failed to fetch feed: HTTP ' + response.status);
  611. }
  612. var xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
  613. if (!xml || xml.trim().length === 0) {
  614. throw new Error('Empty response from feed URL');
  615. }
  616. var feed;
  617. if (xml.indexOf('<feed') !== -1 && xml.indexOf('xmlns') !== -1 && xml.indexOf('atom') !== -1) {
  618. smartbotic.log.debug('Detected Atom feed format');
  619. feed = parseAtom(xml);
  620. } else if (xml.indexOf('<rss') !== -1 || xml.indexOf('<channel') !== -1) {
  621. smartbotic.log.debug('Detected RSS feed format');
  622. feed = parseRss(xml);
  623. } else {
  624. throw new Error('Unknown feed format: neither RSS nor Atom detected');
  625. }
  626. var items = feed.items;
  627. var totalFeedItemCount = items.length;
  628. // Apply standard filters first (keyword, date)
  629. if (filterKeyword) {
  630. smartbotic.log.debug('Filtering by keyword: ' + filterKeyword);
  631. items = filterByKeyword(items, filterKeyword);
  632. }
  633. if (filterNewerThan) {
  634. smartbotic.log.debug('Filtering items newer than: ' + filterNewerThan);
  635. items = filterByDate(items, filterNewerThan);
  636. }
  637. // Stateful new item detection
  638. var newItemCount = items.length;
  639. var stateInfo = null;
  640. if (detectNewItems) {
  641. smartbotic.log.info('Detecting new items using collection: ' + stateCollection);
  642. // Load previously seen items
  643. stateInfo = loadSeenItems(stateCollection, url);
  644. // Filter to only new items
  645. var newItems = filterNewItems(items, stateInfo.seenIds);
  646. newItemCount = newItems.length;
  647. smartbotic.log.info('Found ' + newItemCount + ' new items out of ' + items.length + ' filtered items');
  648. // Update seen IDs with all current items (both new and previously seen)
  649. // This ensures we track all items from the current feed
  650. for (var i = 0; i < items.length; i++) {
  651. var id = getItemIdentifier(items[i]);
  652. if (id) {
  653. stateInfo.seenIds[id] = true;
  654. }
  655. }
  656. // Use only new items for output
  657. items = newItems;
  658. }
  659. // Apply max items limit last
  660. if (maxItems && maxItems > 0) {
  661. smartbotic.log.debug('Limiting to ' + maxItems + ' items');
  662. items = limitItems(items, maxItems);
  663. }
  664. smartbotic.log.info('Returning ' + items.length + ' items from feed');
  665. // Persist seen items state after successful processing
  666. if (detectNewItems && stateInfo) {
  667. var saved = saveSeenItems(
  668. stateCollection,
  669. stateInfo.docId,
  670. stateInfo.seenIds,
  671. url,
  672. stateInfo.exists,
  673. stateInfo.version
  674. );
  675. if (!saved) {
  676. smartbotic.log.warn('Failed to persist seen items state, but continuing with output');
  677. }
  678. }
  679. var result = {
  680. feedTitle: feed.title,
  681. feedLink: feed.link,
  682. feedDescription: feed.description,
  683. itemCount: items.length,
  684. items: items
  685. };
  686. // Add stateful mode info if enabled
  687. if (detectNewItems) {
  688. result.newItemCount = newItemCount;
  689. result.totalFeedItemCount = totalFeedItemCount;
  690. }
  691. return result;
  692. }
  693. module.exports = { configSchema, inputSchema, outputSchema, execute };