rss-reader.js 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783
  1. /**
  2. * @node rss-reader
  3. * @name RSS Reader
  4. * @category data
  5. * @version 1.1.4
  6. * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
  7. * @icon rss
  8. */
  9. const configSchema = {
  10. type: 'object',
  11. properties: {
  12. url: {
  13. type: 'string',
  14. title: 'Feed URL',
  15. description: 'URL of the RSS or Atom feed'
  16. },
  17. filterKeyword: {
  18. type: 'string',
  19. title: 'Keyword Filter',
  20. description: 'Filter items by keyword (matches title and description)'
  21. },
  22. filterNewerThan: {
  23. type: 'string',
  24. title: 'Newer Than',
  25. description: 'Only include items newer than this date (ISO 8601 format, e.g., 2024-01-15T00:00:00Z)'
  26. },
  27. maxItems: {
  28. type: 'number',
  29. title: 'Max Items',
  30. description: 'Maximum number of items to return (0 = unlimited)',
  31. default: 0
  32. },
  33. timeout: {
  34. type: 'number',
  35. title: 'Timeout (ms)',
  36. default: 30000
  37. },
  38. detectNewItems: {
  39. type: 'boolean',
  40. title: 'Detect New Items',
  41. description: 'When enabled, only output items that have not been seen in previous executions',
  42. default: false
  43. },
  44. stateCollection: {
  45. type: 'string',
  46. title: 'State Collection',
  47. description: 'Select a read-write collection to store seen item identifiers',
  48. dynamicOptions: {
  49. source: 'storage.collections',
  50. labelField: 'name',
  51. valueField: 'name',
  52. filter: { access: 'read-write' }
  53. },
  54. showWhen: { field: 'detectNewItems', value: true }
  55. }
  56. },
  57. required: ['url']
  58. };
  59. const inputSchema = {
  60. type: 'object',
  61. properties: {
  62. url: {
  63. type: 'string',
  64. description: 'Override feed URL from input'
  65. },
  66. filterKeyword: {
  67. type: 'string',
  68. description: 'Override keyword filter from input'
  69. },
  70. filterNewerThan: {
  71. type: 'string',
  72. description: 'Override date filter from input'
  73. },
  74. maxItems: {
  75. type: 'number',
  76. description: 'Override max items from input'
  77. }
  78. }
  79. };
  80. const outputSchema = {
  81. type: 'object',
  82. properties: {
  83. feedTitle: { type: 'string', description: 'Title of the feed' },
  84. feedLink: { type: 'string', description: 'Link to the feed homepage' },
  85. feedDescription: { type: 'string', description: 'Description of the feed' },
  86. feedLanguage: { type: 'string', description: 'Language of the feed' },
  87. feedPubDate: { type: 'string', description: 'Publication date of the feed' },
  88. feedGenerator: { type: 'string', description: 'Generator of the feed' },
  89. feedImage: {
  90. type: 'object',
  91. description: 'Feed image/logo',
  92. properties: {
  93. url: { type: 'string' },
  94. title: { type: 'string' },
  95. link: { type: 'string' }
  96. }
  97. },
  98. itemCount: { type: 'number', description: 'Number of items returned' },
  99. newItemCount: { type: 'number', description: 'Number of new items (when detectNewItems is enabled)' },
  100. totalFeedItemCount: { type: 'number', description: 'Total items in feed before filtering (when detectNewItems is enabled)' },
  101. items: {
  102. type: 'array',
  103. description: 'Array of feed items',
  104. items: {
  105. type: 'object',
  106. properties: {
  107. title: { type: 'string', description: 'Item title' },
  108. link: { type: 'string', description: 'Item link/URL' },
  109. description: { type: 'string', description: 'Item summary/description' },
  110. content: { type: 'string', description: 'Full content (content:encoded)' },
  111. pubDate: { type: 'string', description: 'Publication date' },
  112. author: { type: 'string', description: 'Author name' },
  113. categories: { type: 'array', items: { type: 'string' }, description: 'Categories/tags' },
  114. guid: { type: 'string', description: 'Unique identifier' },
  115. comments: { type: 'string', description: 'Comments URL' },
  116. source: { type: 'string', description: 'Source attribution' },
  117. enclosure: {
  118. type: 'object',
  119. description: 'Media enclosure (attachment)',
  120. properties: {
  121. url: { type: 'string', description: 'Media URL' },
  122. type: { type: 'string', description: 'MIME type' },
  123. length: { type: 'number', description: 'File size in bytes' }
  124. }
  125. },
  126. media: {
  127. type: 'object',
  128. description: 'Media content (media:content)',
  129. properties: {
  130. url: { type: 'string' },
  131. type: { type: 'string' },
  132. width: { type: 'number' },
  133. height: { type: 'number' }
  134. }
  135. },
  136. thumbnail: {
  137. type: 'object',
  138. description: 'Thumbnail image (media:thumbnail)',
  139. properties: {
  140. url: { type: 'string' },
  141. width: { type: 'number' },
  142. height: { type: 'number' }
  143. }
  144. }
  145. }
  146. }
  147. }
  148. }
  149. };
  150. /**
  151. * Simple XML parser utilities for RSS/Atom feeds
  152. * Handles basic XML structure without external dependencies
  153. */
  154. function xmlGetTagContent(xml, tagName) {
  155. var pattern = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'i');
  156. var match = xml.match(pattern);
  157. if (match) {
  158. return match[1].trim();
  159. }
  160. return null;
  161. }
  162. function xmlGetAllTagContents(xml, tagName) {
  163. var results = [];
  164. var regex = new RegExp('<' + tagName + '[^>]*>([\\s\\S]*?)<\\/' + tagName + '>', 'gi');
  165. var match;
  166. while ((match = regex.exec(xml)) !== null) {
  167. results.push(match[1].trim());
  168. }
  169. return results;
  170. }
  171. function xmlGetTagAttributes(xml, tagName) {
  172. // Match self-closing tag or opening tag
  173. var regex = new RegExp('<' + tagName + '([^>]*?)\\/?>', 'i');
  174. var match = xml.match(regex);
  175. if (!match) return null;
  176. var attrString = match[1];
  177. var attrs = {};
  178. // Parse attributes: name="value" or name='value'
  179. var attrRegex = /(\w+)=["']([^"']*)["']/g;
  180. var attrMatch;
  181. while ((attrMatch = attrRegex.exec(attrString)) !== null) {
  182. attrs[attrMatch[1]] = attrMatch[2];
  183. }
  184. return attrs;
  185. }
  186. function xmlDecodeHtmlEntities(text) {
  187. if (!text) return '';
  188. return text
  189. .replace(/&lt;/g, '<')
  190. .replace(/&gt;/g, '>')
  191. .replace(/&amp;/g, '&')
  192. .replace(/&quot;/g, '"')
  193. .replace(/&apos;/g, "'")
  194. .replace(/&#(\d+);/g, function(_, dec) { return String.fromCharCode(dec); })
  195. .replace(/&#x([0-9a-f]+);/gi, function(_, hex) { return String.fromCharCode(parseInt(hex, 16)); });
  196. }
  197. function xmlStripCdata(text) {
  198. if (!text) return '';
  199. return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
  200. }
  201. function xmlCleanText(text) {
  202. if (!text) return '';
  203. return xmlDecodeHtmlEntities(xmlStripCdata(text)).trim();
  204. }
  205. /**
  206. * Parse RSS 2.0 feed
  207. */
  208. function parseRss(xml) {
  209. var channel = xmlGetTagContent(xml, 'channel');
  210. if (!channel) {
  211. throw new Error('Invalid RSS feed: no channel element found');
  212. }
  213. var feedTitle = xmlGetTagContent(channel, 'title');
  214. var feedLink = xmlGetTagContent(channel, 'link');
  215. var feedDesc = xmlGetTagContent(channel, 'description');
  216. var feedLanguage = xmlGetTagContent(channel, 'language');
  217. var feedPubDate = xmlGetTagContent(channel, 'pubDate');
  218. var feedLastBuildDate = xmlGetTagContent(channel, 'lastBuildDate');
  219. var feedGenerator = xmlGetTagContent(channel, 'generator');
  220. var feedImage = xmlGetTagContent(channel, 'image');
  221. var feed = {
  222. title: xmlCleanText(feedTitle) || '',
  223. link: xmlCleanText(feedLink) || '',
  224. description: xmlCleanText(feedDesc) || '',
  225. language: xmlCleanText(feedLanguage) || '',
  226. pubDate: xmlCleanText(feedPubDate) || '',
  227. lastBuildDate: xmlCleanText(feedLastBuildDate) || '',
  228. generator: xmlCleanText(feedGenerator) || '',
  229. items: []
  230. };
  231. // Parse feed image if present
  232. if (feedImage) {
  233. feed.image = {
  234. url: xmlCleanText(xmlGetTagContent(feedImage, 'url')) || '',
  235. title: xmlCleanText(xmlGetTagContent(feedImage, 'title')) || '',
  236. link: xmlCleanText(xmlGetTagContent(feedImage, 'link')) || ''
  237. };
  238. }
  239. var itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
  240. var match;
  241. var itemCount = 0;
  242. while ((match = itemRegex.exec(channel)) !== null) {
  243. var itemXml = match[1];
  244. itemCount++;
  245. var catArray = xmlGetAllTagContents(itemXml, 'category');
  246. var categories = [];
  247. for (var ci = 0; ci < catArray.length; ci++) {
  248. categories.push(xmlCleanText(catArray[ci]));
  249. }
  250. var item = {
  251. title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '',
  252. link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '',
  253. description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '',
  254. content: xmlCleanText(xmlGetTagContent(itemXml, 'content:encoded')) || '',
  255. pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '',
  256. author: xmlCleanText(xmlGetTagContent(itemXml, 'author') ||
  257. xmlGetTagContent(itemXml, 'dc:creator')) || '',
  258. categories: categories,
  259. guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || '',
  260. comments: xmlCleanText(xmlGetTagContent(itemXml, 'comments')) || '',
  261. source: xmlCleanText(xmlGetTagContent(itemXml, 'source')) || ''
  262. };
  263. // Parse enclosure (media attachment)
  264. var enclosureAttrs = xmlGetTagAttributes(itemXml, 'enclosure');
  265. if (enclosureAttrs) {
  266. item.enclosure = {
  267. url: enclosureAttrs.url || '',
  268. type: enclosureAttrs.type || '',
  269. length: enclosureAttrs.length ? parseInt(enclosureAttrs.length, 10) : 0
  270. };
  271. }
  272. // Parse media:content (common in media RSS)
  273. var mediaAttrs = xmlGetTagAttributes(itemXml, 'media:content');
  274. if (mediaAttrs) {
  275. item.media = {
  276. url: mediaAttrs.url || '',
  277. type: mediaAttrs.type || mediaAttrs.medium || '',
  278. width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
  279. height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
  280. };
  281. }
  282. // Parse media:thumbnail
  283. var thumbAttrs = xmlGetTagAttributes(itemXml, 'media:thumbnail');
  284. if (thumbAttrs) {
  285. item.thumbnail = {
  286. url: thumbAttrs.url || '',
  287. width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
  288. height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
  289. };
  290. }
  291. feed.items.push(item);
  292. }
  293. return feed;
  294. }
  295. /**
  296. * Parse Atom feed
  297. */
  298. function parseAtom(xml) {
  299. var feedContent = xmlGetTagContent(xml, 'feed');
  300. if (!feedContent) {
  301. throw new Error('Invalid Atom feed: no feed element found');
  302. }
  303. function getAtomLink(content, rel) {
  304. var linkRegex = /<link([^>]*)>/gi;
  305. var match;
  306. while ((match = linkRegex.exec(content)) !== null) {
  307. var attrs = match[1];
  308. var relMatch = attrs.match(/rel=["']([^"']*)["']/);
  309. var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
  310. if (hrefMatch) {
  311. var linkRel = relMatch ? relMatch[1] : 'alternate';
  312. if (linkRel === rel || (!rel && linkRel === 'alternate')) {
  313. return hrefMatch[1];
  314. }
  315. }
  316. }
  317. return '';
  318. }
  319. function getAtomLinkByType(content, type) {
  320. var linkRegex = /<link([^>]*)>/gi;
  321. var match;
  322. while ((match = linkRegex.exec(content)) !== null) {
  323. var attrs = match[1];
  324. var typeMatch = attrs.match(/type=["']([^"']*)["']/);
  325. var hrefMatch = attrs.match(/href=["']([^"']*)["']/);
  326. if (hrefMatch && typeMatch && typeMatch[1].indexOf(type) !== -1) {
  327. return {
  328. url: hrefMatch[1],
  329. type: typeMatch[1]
  330. };
  331. }
  332. }
  333. return null;
  334. }
  335. var feed = {
  336. title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '',
  337. link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
  338. description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '',
  339. language: '',
  340. pubDate: xmlCleanText(xmlGetTagContent(feedContent, 'updated')) || '',
  341. lastBuildDate: '',
  342. generator: xmlCleanText(xmlGetTagContent(feedContent, 'generator')) || '',
  343. items: []
  344. };
  345. // Parse feed icon/logo
  346. var feedIcon = xmlGetTagContent(feedContent, 'icon');
  347. var feedLogo = xmlGetTagContent(feedContent, 'logo');
  348. if (feedIcon || feedLogo) {
  349. feed.image = {
  350. url: xmlCleanText(feedLogo || feedIcon) || '',
  351. title: feed.title,
  352. link: feed.link
  353. };
  354. }
  355. var entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
  356. var match;
  357. while ((match = entryRegex.exec(feedContent)) !== null) {
  358. var entryXml = match[1];
  359. var categories = [];
  360. var catRegex = /<category([^>]*)>/gi;
  361. var catMatch;
  362. while ((catMatch = catRegex.exec(entryXml)) !== null) {
  363. var termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
  364. if (termMatch) {
  365. categories.push(xmlCleanText(termMatch[1]));
  366. }
  367. }
  368. var author = '';
  369. var authorContent = xmlGetTagContent(entryXml, 'author');
  370. if (authorContent) {
  371. author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || '';
  372. }
  373. var pubDate = xmlCleanText(
  374. xmlGetTagContent(entryXml, 'published') ||
  375. xmlGetTagContent(entryXml, 'updated')
  376. ) || '';
  377. var summary = xmlCleanText(xmlGetTagContent(entryXml, 'summary')) || '';
  378. var content = xmlCleanText(xmlGetTagContent(entryXml, 'content')) || '';
  379. var item = {
  380. title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '',
  381. link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
  382. description: summary || content,
  383. content: content,
  384. pubDate: pubDate,
  385. author: author,
  386. categories: categories,
  387. guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || '',
  388. comments: '',
  389. source: ''
  390. };
  391. // Check for enclosure link
  392. var enclosureLink = getAtomLinkByType(entryXml, 'image');
  393. if (!enclosureLink) {
  394. enclosureLink = getAtomLinkByType(entryXml, 'audio');
  395. }
  396. if (!enclosureLink) {
  397. enclosureLink = getAtomLinkByType(entryXml, 'video');
  398. }
  399. if (enclosureLink) {
  400. item.enclosure = {
  401. url: enclosureLink.url,
  402. type: enclosureLink.type,
  403. length: 0
  404. };
  405. }
  406. // Parse media:content
  407. var mediaAttrs = xmlGetTagAttributes(entryXml, 'media:content');
  408. if (mediaAttrs) {
  409. item.media = {
  410. url: mediaAttrs.url || '',
  411. type: mediaAttrs.type || mediaAttrs.medium || '',
  412. width: mediaAttrs.width ? parseInt(mediaAttrs.width, 10) : 0,
  413. height: mediaAttrs.height ? parseInt(mediaAttrs.height, 10) : 0
  414. };
  415. }
  416. // Parse media:thumbnail
  417. var thumbAttrs = xmlGetTagAttributes(entryXml, 'media:thumbnail');
  418. if (thumbAttrs) {
  419. item.thumbnail = {
  420. url: thumbAttrs.url || '',
  421. width: thumbAttrs.width ? parseInt(thumbAttrs.width, 10) : 0,
  422. height: thumbAttrs.height ? parseInt(thumbAttrs.height, 10) : 0
  423. };
  424. }
  425. feed.items.push(item);
  426. }
  427. return feed;
  428. }
  429. /**
  430. * Parse date string to timestamp
  431. */
  432. function parseDate(dateStr) {
  433. if (!dateStr) return 0;
  434. var timestamp = Date.parse(dateStr);
  435. if (!isNaN(timestamp)) {
  436. return timestamp;
  437. }
  438. var rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
  439. var match = dateStr.match(rfc822Regex);
  440. if (match) {
  441. var months = {
  442. Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
  443. Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
  444. };
  445. var day = parseInt(match[1], 10);
  446. var month = months[match[2]];
  447. var year = parseInt(match[3], 10);
  448. var hour = parseInt(match[4], 10);
  449. var minute = parseInt(match[5], 10);
  450. var second = parseInt(match[6], 10);
  451. if (month !== undefined) {
  452. return new Date(year, month, day, hour, minute, second).getTime();
  453. }
  454. }
  455. return 0;
  456. }
  457. /**
  458. * Filter items by keyword
  459. */
  460. function filterByKeyword(items, keyword) {
  461. if (!keyword) return items;
  462. var lowerKeyword = keyword.toLowerCase();
  463. return items.filter(function(item) {
  464. var title = (item.title || '').toLowerCase();
  465. var description = (item.description || '').toLowerCase();
  466. return title.indexOf(lowerKeyword) !== -1 || description.indexOf(lowerKeyword) !== -1;
  467. });
  468. }
  469. /**
  470. * Filter items by date
  471. */
  472. function filterByDate(items, newerThanStr) {
  473. if (!newerThanStr) return items;
  474. var threshold = parseDate(newerThanStr);
  475. if (threshold === 0) {
  476. smartbotic.log.warn('Could not parse date filter: ' + newerThanStr);
  477. return items;
  478. }
  479. return items.filter(function(item) {
  480. var itemDate = parseDate(item.pubDate);
  481. return itemDate > threshold;
  482. });
  483. }
  484. /**
  485. * Limit number of items
  486. */
  487. function limitItems(items, maxItems) {
  488. if (!maxItems || maxItems <= 0) return items;
  489. return items.slice(0, maxItems);
  490. }
  491. /**
  492. * Get unique identifier for an RSS/Atom item
  493. * Prefers GUID, falls back to link
  494. */
  495. function getItemIdentifier(item) {
  496. return item.guid || item.link || '';
  497. }
  498. /**
  499. * Generate a stable document ID from feed URL for state storage
  500. */
  501. function generateStateDocId(feedUrl) {
  502. // Create a simple hash from the feed URL for a stable doc ID
  503. var hash = 0;
  504. for (var i = 0; i < feedUrl.length; i++) {
  505. var char = feedUrl.charCodeAt(i);
  506. hash = ((hash << 5) - hash) + char;
  507. hash = hash & hash; // Convert to 32-bit integer
  508. }
  509. return 'rss-state-' + Math.abs(hash).toString(36);
  510. }
  511. /**
  512. * Load seen item identifiers from storage
  513. */
  514. function loadSeenItems(collection, feedUrl) {
  515. var docId = generateStateDocId(feedUrl);
  516. try {
  517. var result = smartbotic.storage.get(collection, docId);
  518. if (result.found && result.document && result.document.seenIds) {
  519. smartbotic.log.debug('Loaded ' + result.document.seenIds.length + ' seen item IDs from storage');
  520. // Use plain object as a set (for QuickJS compatibility)
  521. var seenObj = {};
  522. for (var i = 0; i < result.document.seenIds.length; i++) {
  523. seenObj[result.document.seenIds[i]] = true;
  524. }
  525. return {
  526. seenIds: seenObj,
  527. docId: docId,
  528. exists: true,
  529. version: result.document._version || 1
  530. };
  531. }
  532. } catch (error) {
  533. smartbotic.log.warn('Error loading seen items from storage: ' + error.message);
  534. }
  535. return {
  536. seenIds: {},
  537. docId: docId,
  538. exists: false,
  539. version: 0
  540. };
  541. }
  542. /**
  543. * Save seen item identifiers to storage
  544. */
  545. function saveSeenItems(collection, docId, seenIds, feedUrl, exists, version) {
  546. // Convert object keys to array
  547. var seenArray = Object.keys(seenIds);
  548. var documentData = {
  549. feedUrl: feedUrl,
  550. seenIds: seenArray,
  551. lastUpdated: new Date().toISOString(),
  552. itemCount: seenArray.length
  553. };
  554. try {
  555. if (exists) {
  556. // Update existing document
  557. var updateResult = smartbotic.storage.update(collection, docId, documentData, version, false);
  558. if (!updateResult.success) {
  559. smartbotic.log.error('Failed to update seen items: ' + updateResult.error);
  560. return false;
  561. }
  562. smartbotic.log.debug('Updated ' + seenArray.length + ' seen item IDs in storage');
  563. } else {
  564. // Insert new document
  565. var insertResult = smartbotic.storage.insert(collection, documentData, docId, 0);
  566. if (!insertResult.success) {
  567. smartbotic.log.error('Failed to save seen items: ' + insertResult.error);
  568. return false;
  569. }
  570. smartbotic.log.debug('Saved ' + seenArray.length + ' seen item IDs to storage');
  571. }
  572. return true;
  573. } catch (error) {
  574. smartbotic.log.error('Error saving seen items to storage: ' + error.message);
  575. return false;
  576. }
  577. }
  578. /**
  579. * Filter items to only include new (unseen) items
  580. */
  581. function filterNewItems(items, seenIds) {
  582. return items.filter(function(item) {
  583. var id = getItemIdentifier(item);
  584. return id && !seenIds[id];
  585. });
  586. }
  587. async function execute(config, input) {
  588. var url = input.url || config.url;
  589. var filterKeyword = input.filterKeyword || config.filterKeyword;
  590. var filterNewerThan = input.filterNewerThan || config.filterNewerThan;
  591. var maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
  592. var timeout = config.timeout || 30000;
  593. var detectNewItems = config.detectNewItems || false;
  594. var stateCollection = config.stateCollection;
  595. if (!url) {
  596. throw new Error('Feed URL is required');
  597. }
  598. // Validate stateful mode configuration
  599. if (detectNewItems && !stateCollection) {
  600. throw new Error('State collection is required when "Detect New Items" is enabled. Please select a read-write collection in the workflow storage settings.');
  601. }
  602. smartbotic.log.info('Fetching RSS/Atom feed from ' + url);
  603. var response = smartbotic.http.request({
  604. method: 'GET',
  605. url: url,
  606. timeout: timeout,
  607. headers: {
  608. 'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*'
  609. }
  610. });
  611. if (response.status < 200 || response.status >= 300) {
  612. throw new Error('Failed to fetch feed: HTTP ' + response.status);
  613. }
  614. var xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
  615. if (!xml || xml.trim().length === 0) {
  616. throw new Error('Empty response from feed URL');
  617. }
  618. var feed;
  619. if (xml.indexOf('<feed') !== -1 && xml.indexOf('xmlns') !== -1 && xml.indexOf('atom') !== -1) {
  620. smartbotic.log.debug('Detected Atom feed format');
  621. feed = parseAtom(xml);
  622. } else if (xml.indexOf('<rss') !== -1 || xml.indexOf('<channel') !== -1) {
  623. smartbotic.log.debug('Detected RSS feed format');
  624. feed = parseRss(xml);
  625. } else {
  626. throw new Error('Unknown feed format: neither RSS nor Atom detected');
  627. }
  628. var items = feed.items;
  629. var totalFeedItemCount = items.length;
  630. // Apply standard filters first (keyword, date)
  631. if (filterKeyword) {
  632. smartbotic.log.debug('Filtering by keyword: ' + filterKeyword);
  633. items = filterByKeyword(items, filterKeyword);
  634. }
  635. if (filterNewerThan) {
  636. smartbotic.log.debug('Filtering items newer than: ' + filterNewerThan);
  637. items = filterByDate(items, filterNewerThan);
  638. }
  639. // Stateful new item detection
  640. var newItemCount = items.length;
  641. var stateInfo = null;
  642. if (detectNewItems) {
  643. smartbotic.log.info('Detecting new items using collection: ' + stateCollection);
  644. // Load previously seen items
  645. stateInfo = loadSeenItems(stateCollection, url);
  646. // Filter to only new items
  647. var newItems = filterNewItems(items, stateInfo.seenIds);
  648. newItemCount = newItems.length;
  649. smartbotic.log.info('Found ' + newItemCount + ' new items out of ' + items.length + ' filtered items');
  650. // Update seen IDs with all current items (both new and previously seen)
  651. // This ensures we track all items from the current feed
  652. for (var i = 0; i < items.length; i++) {
  653. var id = getItemIdentifier(items[i]);
  654. if (id) {
  655. stateInfo.seenIds[id] = true;
  656. }
  657. }
  658. // Use only new items for output
  659. items = newItems;
  660. }
  661. // Apply max items limit last
  662. if (maxItems && maxItems > 0) {
  663. smartbotic.log.debug('Limiting to ' + maxItems + ' items');
  664. items = limitItems(items, maxItems);
  665. }
  666. smartbotic.log.info('Returning ' + items.length + ' items from feed');
  667. // Persist seen items state after successful processing
  668. if (detectNewItems && stateInfo) {
  669. var saved = saveSeenItems(
  670. stateCollection,
  671. stateInfo.docId,
  672. stateInfo.seenIds,
  673. url,
  674. stateInfo.exists,
  675. stateInfo.version
  676. );
  677. if (!saved) {
  678. smartbotic.log.warn('Failed to persist seen items state, but continuing with output');
  679. }
  680. }
  681. var result = {
  682. feedTitle: feed.title,
  683. feedLink: feed.link,
  684. feedDescription: feed.description,
  685. itemCount: items.length,
  686. items: items
  687. };
  688. // Add stateful mode info if enabled
  689. if (detectNewItems) {
  690. result.newItemCount = newItemCount;
  691. result.totalFeedItemCount = totalFeedItemCount;
  692. }
  693. return result;
  694. }
  695. module.exports = { configSchema, inputSchema, outputSchema, execute };