|
@@ -0,0 +1,437 @@
|
|
|
|
|
+/**
|
|
|
|
|
+ * @node rss-reader
|
|
|
|
|
+ * @name RSS Reader
|
|
|
|
|
+ * @category rss
|
|
|
|
|
+ * @version 1.0.0
|
|
|
|
|
+ * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count
|
|
|
|
|
+ * @icon rss
|
|
|
|
|
+ */
|
|
|
|
|
+
|
|
|
|
|
+const configSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ url: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ title: 'Feed URL',
|
|
|
|
|
+ description: 'URL of the RSS or Atom feed'
|
|
|
|
|
+ },
|
|
|
|
|
+ filterKeyword: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ title: 'Keyword Filter',
|
|
|
|
|
+ description: 'Filter items by keyword (matches title and description)'
|
|
|
|
|
+ },
|
|
|
|
|
+ filterNewerThan: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ title: 'Newer Than',
|
|
|
|
|
+ description: 'Only include items newer than this date (ISO 8601 format, e.g., 2024-01-15T00:00:00Z)'
|
|
|
|
|
+ },
|
|
|
|
|
+ maxItems: {
|
|
|
|
|
+ type: 'number',
|
|
|
|
|
+ title: 'Max Items',
|
|
|
|
|
+ description: 'Maximum number of items to return (0 = unlimited)',
|
|
|
|
|
+ default: 0
|
|
|
|
|
+ },
|
|
|
|
|
+ timeout: {
|
|
|
|
|
+ type: 'number',
|
|
|
|
|
+ title: 'Timeout (ms)',
|
|
|
|
|
+ default: 30000
|
|
|
|
|
+ }
|
|
|
|
|
+ },
|
|
|
|
|
+ required: ['url']
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+const inputSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ url: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ description: 'Override feed URL from input'
|
|
|
|
|
+ },
|
|
|
|
|
+ filterKeyword: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ description: 'Override keyword filter from input'
|
|
|
|
|
+ },
|
|
|
|
|
+ filterNewerThan: {
|
|
|
|
|
+ type: 'string',
|
|
|
|
|
+ description: 'Override date filter from input'
|
|
|
|
|
+ },
|
|
|
|
|
+ maxItems: {
|
|
|
|
|
+ type: 'number',
|
|
|
|
|
+ description: 'Override max items from input'
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+const outputSchema = {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ feedTitle: { type: 'string', description: 'Title of the feed' },
|
|
|
|
|
+ feedLink: { type: 'string', description: 'Link to the feed homepage' },
|
|
|
|
|
+ feedDescription: { type: 'string', description: 'Description of the feed' },
|
|
|
|
|
+ itemCount: { type: 'number', description: 'Number of items returned' },
|
|
|
|
|
+ items: {
|
|
|
|
|
+ type: 'array',
|
|
|
|
|
+ description: 'Array of feed items',
|
|
|
|
|
+ items: {
|
|
|
|
|
+ type: 'object',
|
|
|
|
|
+ properties: {
|
|
|
|
|
+ title: { type: 'string' },
|
|
|
|
|
+ link: { type: 'string' },
|
|
|
|
|
+ description: { type: 'string' },
|
|
|
|
|
+ pubDate: { type: 'string' },
|
|
|
|
|
+ author: { type: 'string' },
|
|
|
|
|
+ categories: { type: 'array', items: { type: 'string' } },
|
|
|
|
|
+ guid: { type: 'string' }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Simple XML parser for RSS/Atom feeds
|
|
|
|
|
+ * Handles basic XML structure without external dependencies
|
|
|
|
|
+ */
|
|
|
|
|
+function parseXml() {
|
|
|
|
|
+ function getTagContent(xml, tagName) {
|
|
|
|
|
+ const patterns = [
|
|
|
|
|
+ new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'),
|
|
|
|
|
+ new RegExp(`<${tagName}[^>]*\\/>`, 'i')
|
|
|
|
|
+ ];
|
|
|
|
|
+
|
|
|
|
|
+ const match = xml.match(patterns[0]);
|
|
|
|
|
+ if (match) {
|
|
|
|
|
+ return match[1].trim();
|
|
|
|
|
+ }
|
|
|
|
|
+ return null;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function getAllTagContents(xml, tagName) {
|
|
|
|
|
+ const results = [];
|
|
|
|
|
+ const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'gi');
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = regex.exec(xml)) !== null) {
|
|
|
|
|
+ results.push(match[1].trim());
|
|
|
|
|
+ }
|
|
|
|
|
+ return results;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function getTagAttribute(xml, tagName, attrName) {
|
|
|
|
|
+ const regex = new RegExp(`<${tagName}[^>]*\\s${attrName}=["']([^"']*)["'][^>]*>`, 'i');
|
|
|
|
|
+ const match = xml.match(regex);
|
|
|
|
|
+ return match ? match[1] : null;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function getAllTags(xml, tagName) {
|
|
|
|
|
+ const results = [];
|
|
|
|
|
+ const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>|<${tagName}([^>]*)\\/>`, 'gi');
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = regex.exec(xml)) !== null) {
|
|
|
|
|
+ results.push({
|
|
|
|
|
+ content: match[1] ? match[1].trim() : '',
|
|
|
|
|
+ fullMatch: match[0]
|
|
|
|
|
+ });
|
|
|
|
|
+ }
|
|
|
|
|
+ return results;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function decodeHtmlEntities(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return text
|
|
|
|
|
+ .replace(/</g, '<')
|
|
|
|
|
+ .replace(/>/g, '>')
|
|
|
|
|
+ .replace(/&/g, '&')
|
|
|
|
|
+ .replace(/"/g, '"')
|
|
|
|
|
+ .replace(/'/g, "'")
|
|
|
|
|
+ .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(dec))
|
|
|
|
|
+ .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function stripCdata(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function cleanText(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return decodeHtmlEntities(stripCdata(text)).trim();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return {
|
|
|
|
|
+ getTagContent,
|
|
|
|
|
+ getAllTagContents,
|
|
|
|
|
+ getTagAttribute,
|
|
|
|
|
+ getAllTags,
|
|
|
|
|
+ cleanText
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Parse RSS 2.0 feed
|
|
|
|
|
+ */
|
|
|
|
|
+function parseRss(xml, parser) {
|
|
|
|
|
+ const channel = parser.getTagContent(xml, 'channel');
|
|
|
|
|
+ if (!channel) {
|
|
|
|
|
+ throw new Error('Invalid RSS feed: no channel element found');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const feed = {
|
|
|
|
|
+ title: parser.cleanText(parser.getTagContent(channel, 'title')) || '',
|
|
|
|
|
+ link: parser.cleanText(parser.getTagContent(channel, 'link')) || '',
|
|
|
|
|
+ description: parser.cleanText(parser.getTagContent(channel, 'description')) || '',
|
|
|
|
|
+ items: []
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = itemRegex.exec(channel)) !== null) {
|
|
|
|
|
+ const itemXml = match[1];
|
|
|
|
|
+
|
|
|
|
|
+ const categories = parser.getAllTagContents(itemXml, 'category')
|
|
|
|
|
+ .map(c => parser.cleanText(c));
|
|
|
|
|
+
|
|
|
|
|
+ const item = {
|
|
|
|
|
+ title: parser.cleanText(parser.getTagContent(itemXml, 'title')) || '',
|
|
|
|
|
+ link: parser.cleanText(parser.getTagContent(itemXml, 'link')) || '',
|
|
|
|
|
+ description: parser.cleanText(parser.getTagContent(itemXml, 'description')) || '',
|
|
|
|
|
+ pubDate: parser.cleanText(parser.getTagContent(itemXml, 'pubDate')) || '',
|
|
|
|
|
+ author: parser.cleanText(parser.getTagContent(itemXml, 'author') ||
|
|
|
|
|
+ parser.getTagContent(itemXml, 'dc:creator')) || '',
|
|
|
|
|
+ categories: categories,
|
|
|
|
|
+ guid: parser.cleanText(parser.getTagContent(itemXml, 'guid')) || ''
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ feed.items.push(item);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return feed;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Parse Atom feed
|
|
|
|
|
+ */
|
|
|
|
|
+function parseAtom(xml, parser) {
|
|
|
|
|
+ const feedContent = parser.getTagContent(xml, 'feed');
|
|
|
|
|
+ if (!feedContent) {
|
|
|
|
|
+ throw new Error('Invalid Atom feed: no feed element found');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ function getAtomLink(content, rel) {
|
|
|
|
|
+ const linkRegex = /<link([^>]*)>/gi;
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = linkRegex.exec(content)) !== null) {
|
|
|
|
|
+ const attrs = match[1];
|
|
|
|
|
+ const relMatch = attrs.match(/rel=["']([^"']*)["']/);
|
|
|
|
|
+ const hrefMatch = attrs.match(/href=["']([^"']*)["']/);
|
|
|
|
|
+
|
|
|
|
|
+ if (hrefMatch) {
|
|
|
|
|
+ const linkRel = relMatch ? relMatch[1] : 'alternate';
|
|
|
|
|
+ if (linkRel === rel || (!rel && linkRel === 'alternate')) {
|
|
|
|
|
+ return hrefMatch[1];
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return '';
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const feed = {
|
|
|
|
|
+ title: parser.cleanText(parser.getTagContent(feedContent, 'title')) || '',
|
|
|
|
|
+ link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
|
|
|
|
|
+ description: parser.cleanText(parser.getTagContent(feedContent, 'subtitle')) || '',
|
|
|
|
|
+ items: []
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ const entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = entryRegex.exec(feedContent)) !== null) {
|
|
|
|
|
+ const entryXml = match[1];
|
|
|
|
|
+
|
|
|
|
|
+ const categories = [];
|
|
|
|
|
+ const catRegex = /<category([^>]*)>/gi;
|
|
|
|
|
+ let catMatch;
|
|
|
|
|
+ while ((catMatch = catRegex.exec(entryXml)) !== null) {
|
|
|
|
|
+ const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
|
|
|
|
|
+ if (termMatch) {
|
|
|
|
|
+ categories.push(parser.cleanText(termMatch[1]));
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ let author = '';
|
|
|
|
|
+ const authorContent = parser.getTagContent(entryXml, 'author');
|
|
|
|
|
+ if (authorContent) {
|
|
|
|
|
+ author = parser.cleanText(parser.getTagContent(authorContent, 'name')) || '';
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const pubDate = parser.cleanText(
|
|
|
|
|
+ parser.getTagContent(entryXml, 'published') ||
|
|
|
|
|
+ parser.getTagContent(entryXml, 'updated')
|
|
|
|
|
+ ) || '';
|
|
|
|
|
+
|
|
|
|
|
+ const description = parser.cleanText(
|
|
|
|
|
+ parser.getTagContent(entryXml, 'summary') ||
|
|
|
|
|
+ parser.getTagContent(entryXml, 'content')
|
|
|
|
|
+ ) || '';
|
|
|
|
|
+
|
|
|
|
|
+ const item = {
|
|
|
|
|
+ title: parser.cleanText(parser.getTagContent(entryXml, 'title')) || '',
|
|
|
|
|
+ link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
|
|
|
|
|
+ description: description,
|
|
|
|
|
+ pubDate: pubDate,
|
|
|
|
|
+ author: author,
|
|
|
|
|
+ categories: categories,
|
|
|
|
|
+ guid: parser.cleanText(parser.getTagContent(entryXml, 'id')) || ''
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ feed.items.push(item);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return feed;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Parse date string to timestamp
|
|
|
|
|
+ */
|
|
|
|
|
+function parseDate(dateStr) {
|
|
|
|
|
+ if (!dateStr) return 0;
|
|
|
|
|
+
|
|
|
|
|
+ const timestamp = Date.parse(dateStr);
|
|
|
|
|
+ if (!isNaN(timestamp)) {
|
|
|
|
|
+ return timestamp;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
|
|
|
|
|
+ const match = dateStr.match(rfc822Regex);
|
|
|
|
|
+ if (match) {
|
|
|
|
|
+ const months = {
|
|
|
|
|
+ Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
|
|
|
|
|
+ Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
|
|
|
|
|
+ };
|
|
|
|
|
+ const day = parseInt(match[1], 10);
|
|
|
|
|
+ const month = months[match[2]];
|
|
|
|
|
+ const year = parseInt(match[3], 10);
|
|
|
|
|
+ const hour = parseInt(match[4], 10);
|
|
|
|
|
+ const minute = parseInt(match[5], 10);
|
|
|
|
|
+ const second = parseInt(match[6], 10);
|
|
|
|
|
+
|
|
|
|
|
+ if (month !== undefined) {
|
|
|
|
|
+ return new Date(year, month, day, hour, minute, second).getTime();
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return 0;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Filter items by keyword
|
|
|
|
|
+ */
|
|
|
|
|
+function filterByKeyword(items, keyword) {
|
|
|
|
|
+ if (!keyword) return items;
|
|
|
|
|
+
|
|
|
|
|
+ const lowerKeyword = keyword.toLowerCase();
|
|
|
|
|
+ return items.filter(item => {
|
|
|
|
|
+ const title = (item.title || '').toLowerCase();
|
|
|
|
|
+ const description = (item.description || '').toLowerCase();
|
|
|
|
|
+ return title.includes(lowerKeyword) || description.includes(lowerKeyword);
|
|
|
|
|
+ });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Filter items by date
|
|
|
|
|
+ */
|
|
|
|
|
+function filterByDate(items, newerThanStr) {
|
|
|
|
|
+ if (!newerThanStr) return items;
|
|
|
|
|
+
|
|
|
|
|
+ const threshold = parseDate(newerThanStr);
|
|
|
|
|
+ if (threshold === 0) {
|
|
|
|
|
+ smartbotic.log.warn(`Could not parse date filter: ${newerThanStr}`);
|
|
|
|
|
+ return items;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ return items.filter(item => {
|
|
|
|
|
+ const itemDate = parseDate(item.pubDate);
|
|
|
|
|
+ return itemDate > threshold;
|
|
|
|
|
+ });
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+/**
|
|
|
|
|
+ * Limit number of items
|
|
|
|
|
+ */
|
|
|
|
|
+function limitItems(items, maxItems) {
|
|
|
|
|
+ if (!maxItems || maxItems <= 0) return items;
|
|
|
|
|
+ return items.slice(0, maxItems);
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+async function execute(config, input, context) {
|
|
|
|
|
+ const url = input.url || config.url;
|
|
|
|
|
+ const filterKeyword = input.filterKeyword || config.filterKeyword;
|
|
|
|
|
+ const filterNewerThan = input.filterNewerThan || config.filterNewerThan;
|
|
|
|
|
+ const maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
|
|
|
|
|
+ const timeout = config.timeout || 30000;
|
|
|
|
|
+
|
|
|
|
|
+ if (!url) {
|
|
|
|
|
+ throw new Error('Feed URL is required');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ smartbotic.log.info(`Fetching RSS/Atom feed from ${url}`);
|
|
|
|
|
+
|
|
|
|
|
+ const response = smartbotic.http.request({
|
|
|
|
|
+ method: 'GET',
|
|
|
|
|
+ url: url,
|
|
|
|
|
+ timeout: timeout,
|
|
|
|
|
+ headers: {
|
|
|
|
|
+ 'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*'
|
|
|
|
|
+ }
|
|
|
|
|
+ });
|
|
|
|
|
+
|
|
|
|
|
+ if (response.status < 200 || response.status >= 300) {
|
|
|
|
|
+ throw new Error(`Failed to fetch feed: HTTP ${response.status}`);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);
|
|
|
|
|
+
|
|
|
|
|
+ if (!xml || xml.trim().length === 0) {
|
|
|
|
|
+ throw new Error('Empty response from feed URL');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ const parser = parseXml(xml);
|
|
|
|
|
+
|
|
|
|
|
+ let feed;
|
|
|
|
|
+ if (xml.includes('<feed') && xml.includes('xmlns') && xml.includes('atom')) {
|
|
|
|
|
+ smartbotic.log.debug('Detected Atom feed format');
|
|
|
|
|
+ feed = parseAtom(xml, parser);
|
|
|
|
|
+ } else if (xml.includes('<rss') || xml.includes('<channel')) {
|
|
|
|
|
+ smartbotic.log.debug('Detected RSS feed format');
|
|
|
|
|
+ feed = parseRss(xml, parser);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ throw new Error('Unknown feed format: neither RSS nor Atom detected');
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ let items = feed.items;
|
|
|
|
|
+
|
|
|
|
|
+ if (filterKeyword) {
|
|
|
|
|
+ smartbotic.log.debug(`Filtering by keyword: ${filterKeyword}`);
|
|
|
|
|
+ items = filterByKeyword(items, filterKeyword);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (filterNewerThan) {
|
|
|
|
|
+ smartbotic.log.debug(`Filtering items newer than: ${filterNewerThan}`);
|
|
|
|
|
+ items = filterByDate(items, filterNewerThan);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (maxItems && maxItems > 0) {
|
|
|
|
|
+ smartbotic.log.debug(`Limiting to ${maxItems} items`);
|
|
|
|
|
+ items = limitItems(items, maxItems);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ smartbotic.log.info(`Parsed ${items.length} items from feed`);
|
|
|
|
|
+
|
|
|
|
|
+ return {
|
|
|
|
|
+ feedTitle: feed.title,
|
|
|
|
|
+ feedLink: feed.link,
|
|
|
|
|
+ feedDescription: feed.description,
|
|
|
|
|
+ itemCount: items.length,
|
|
|
|
|
+ items: items
|
|
|
|
|
+ };
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+module.exports = { configSchema, inputSchema, outputSchema, execute };
|