|
@@ -2,7 +2,7 @@
|
|
|
* @node rss-reader
|
|
* @node rss-reader
|
|
|
* @name RSS Reader
|
|
* @name RSS Reader
|
|
|
* @category data
|
|
* @category data
|
|
|
- * @version 1.1.0
|
|
|
|
|
|
|
+ * @version 1.1.1
|
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
* @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count. Supports stateful new item detection.
|
|
|
* @icon rss
|
|
* @icon rss
|
|
|
*/
|
|
*/
|
|
@@ -109,96 +109,63 @@ const outputSchema = {
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
- * Simple XML parser for RSS/Atom feeds
|
|
|
|
|
|
|
+ * Simple XML parser utilities for RSS/Atom feeds
|
|
|
* Handles basic XML structure without external dependencies
|
|
* Handles basic XML structure without external dependencies
|
|
|
*/
|
|
*/
|
|
|
-function parseXml() {
|
|
|
|
|
- function getTagContent(xml, tagName) {
|
|
|
|
|
- const patterns = [
|
|
|
|
|
- new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'),
|
|
|
|
|
- new RegExp(`<${tagName}[^>]*\\/>`, 'i')
|
|
|
|
|
- ];
|
|
|
|
|
-
|
|
|
|
|
- const match = xml.match(patterns[0]);
|
|
|
|
|
- if (match) {
|
|
|
|
|
- return match[1].trim();
|
|
|
|
|
- }
|
|
|
|
|
- return null;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- function getAllTagContents(xml, tagName) {
|
|
|
|
|
- const results = [];
|
|
|
|
|
- const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'gi');
|
|
|
|
|
- let match;
|
|
|
|
|
- while ((match = regex.exec(xml)) !== null) {
|
|
|
|
|
- results.push(match[1].trim());
|
|
|
|
|
- }
|
|
|
|
|
- return results;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- function getTagAttribute(xml, tagName, attrName) {
|
|
|
|
|
- const regex = new RegExp(`<${tagName}[^>]*\\s${attrName}=["']([^"']*)["'][^>]*>`, 'i');
|
|
|
|
|
- const match = xml.match(regex);
|
|
|
|
|
- return match ? match[1] : null;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- function getAllTags(xml, tagName) {
|
|
|
|
|
- const results = [];
|
|
|
|
|
- const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>|<${tagName}([^>]*)\\/>`, 'gi');
|
|
|
|
|
- let match;
|
|
|
|
|
- while ((match = regex.exec(xml)) !== null) {
|
|
|
|
|
- results.push({
|
|
|
|
|
- content: match[1] ? match[1].trim() : '',
|
|
|
|
|
- fullMatch: match[0]
|
|
|
|
|
- });
|
|
|
|
|
- }
|
|
|
|
|
- return results;
|
|
|
|
|
|
|
+function xmlGetTagContent(xml, tagName) {
|
|
|
|
|
+ const pattern = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i');
|
|
|
|
|
+ const match = xml.match(pattern);
|
|
|
|
|
+ if (match) {
|
|
|
|
|
+ return match[1].trim();
|
|
|
}
|
|
}
|
|
|
|
|
+ return null;
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
- function decodeHtmlEntities(text) {
|
|
|
|
|
- if (!text) return '';
|
|
|
|
|
- return text
|
|
|
|
|
- .replace(/</g, '<')
|
|
|
|
|
- .replace(/>/g, '>')
|
|
|
|
|
- .replace(/&/g, '&')
|
|
|
|
|
- .replace(/"/g, '"')
|
|
|
|
|
- .replace(/'/g, "'")
|
|
|
|
|
- .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(dec))
|
|
|
|
|
- .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
|
|
|
|
|
|
+function xmlGetAllTagContents(xml, tagName) {
|
|
|
|
|
+ const results = [];
|
|
|
|
|
+ const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'gi');
|
|
|
|
|
+ let match;
|
|
|
|
|
+ while ((match = regex.exec(xml)) !== null) {
|
|
|
|
|
+ results.push(match[1].trim());
|
|
|
}
|
|
}
|
|
|
|
|
+ return results;
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
- function stripCdata(text) {
|
|
|
|
|
- if (!text) return '';
|
|
|
|
|
- return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
|
|
|
|
|
- }
|
|
|
|
|
|
|
+function xmlDecodeHtmlEntities(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return text
|
|
|
|
|
+ .replace(/</g, '<')
|
|
|
|
|
+ .replace(/>/g, '>')
|
|
|
|
|
+ .replace(/&/g, '&')
|
|
|
|
|
+ .replace(/"/g, '"')
|
|
|
|
|
+ .replace(/'/g, "'")
|
|
|
|
|
+ .replace(/&#(\d+);/g, function(_, dec) { return String.fromCharCode(dec); })
|
|
|
|
|
+ .replace(/&#x([0-9a-f]+);/gi, function(_, hex) { return String.fromCharCode(parseInt(hex, 16)); });
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
- function cleanText(text) {
|
|
|
|
|
- if (!text) return '';
|
|
|
|
|
- return decodeHtmlEntities(stripCdata(text)).trim();
|
|
|
|
|
- }
|
|
|
|
|
|
|
+function xmlStripCdata(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
- return {
|
|
|
|
|
- getTagContent,
|
|
|
|
|
- getAllTagContents,
|
|
|
|
|
- getTagAttribute,
|
|
|
|
|
- getAllTags,
|
|
|
|
|
- cleanText
|
|
|
|
|
- };
|
|
|
|
|
|
|
+function xmlCleanText(text) {
|
|
|
|
|
+ if (!text) return '';
|
|
|
|
|
+ return xmlDecodeHtmlEntities(xmlStripCdata(text)).trim();
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
|
* Parse RSS 2.0 feed
|
|
* Parse RSS 2.0 feed
|
|
|
*/
|
|
*/
|
|
|
-function parseRss(xml, parser) {
|
|
|
|
|
- const channel = parser.getTagContent(xml, 'channel');
|
|
|
|
|
|
|
+function parseRss(xml) {
|
|
|
|
|
+ const channel = xmlGetTagContent(xml, 'channel');
|
|
|
if (!channel) {
|
|
if (!channel) {
|
|
|
throw new Error('Invalid RSS feed: no channel element found');
|
|
throw new Error('Invalid RSS feed: no channel element found');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const feed = {
|
|
const feed = {
|
|
|
- title: parser.cleanText(parser.getTagContent(channel, 'title')) || '',
|
|
|
|
|
- link: parser.cleanText(parser.getTagContent(channel, 'link')) || '',
|
|
|
|
|
- description: parser.cleanText(parser.getTagContent(channel, 'description')) || '',
|
|
|
|
|
|
|
+ title: xmlCleanText(xmlGetTagContent(channel, 'title')) || '',
|
|
|
|
|
+ link: xmlCleanText(xmlGetTagContent(channel, 'link')) || '',
|
|
|
|
|
+ description: xmlCleanText(xmlGetTagContent(channel, 'description')) || '',
|
|
|
items: []
|
|
items: []
|
|
|
};
|
|
};
|
|
|
|
|
|
|
@@ -207,18 +174,18 @@ function parseRss(xml, parser) {
|
|
|
while ((match = itemRegex.exec(channel)) !== null) {
|
|
while ((match = itemRegex.exec(channel)) !== null) {
|
|
|
const itemXml = match[1];
|
|
const itemXml = match[1];
|
|
|
|
|
|
|
|
- const categories = parser.getAllTagContents(itemXml, 'category')
|
|
|
|
|
- .map(c => parser.cleanText(c));
|
|
|
|
|
|
|
+ const categories = xmlGetAllTagContents(itemXml, 'category')
|
|
|
|
|
+ .map(function(c) { return xmlCleanText(c); });
|
|
|
|
|
|
|
|
const item = {
|
|
const item = {
|
|
|
- title: parser.cleanText(parser.getTagContent(itemXml, 'title')) || '',
|
|
|
|
|
- link: parser.cleanText(parser.getTagContent(itemXml, 'link')) || '',
|
|
|
|
|
- description: parser.cleanText(parser.getTagContent(itemXml, 'description')) || '',
|
|
|
|
|
- pubDate: parser.cleanText(parser.getTagContent(itemXml, 'pubDate')) || '',
|
|
|
|
|
- author: parser.cleanText(parser.getTagContent(itemXml, 'author') ||
|
|
|
|
|
- parser.getTagContent(itemXml, 'dc:creator')) || '',
|
|
|
|
|
|
|
+ title: xmlCleanText(xmlGetTagContent(itemXml, 'title')) || '',
|
|
|
|
|
+ link: xmlCleanText(xmlGetTagContent(itemXml, 'link')) || '',
|
|
|
|
|
+ description: xmlCleanText(xmlGetTagContent(itemXml, 'description')) || '',
|
|
|
|
|
+ pubDate: xmlCleanText(xmlGetTagContent(itemXml, 'pubDate')) || '',
|
|
|
|
|
+ author: xmlCleanText(xmlGetTagContent(itemXml, 'author') ||
|
|
|
|
|
+ xmlGetTagContent(itemXml, 'dc:creator')) || '',
|
|
|
categories: categories,
|
|
categories: categories,
|
|
|
- guid: parser.cleanText(parser.getTagContent(itemXml, 'guid')) || ''
|
|
|
|
|
|
|
+ guid: xmlCleanText(xmlGetTagContent(itemXml, 'guid')) || ''
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
feed.items.push(item);
|
|
feed.items.push(item);
|
|
@@ -230,8 +197,8 @@ function parseRss(xml, parser) {
|
|
|
/**
|
|
/**
|
|
|
* Parse Atom feed
|
|
* Parse Atom feed
|
|
|
*/
|
|
*/
|
|
|
-function parseAtom(xml, parser) {
|
|
|
|
|
- const feedContent = parser.getTagContent(xml, 'feed');
|
|
|
|
|
|
|
+function parseAtom(xml) {
|
|
|
|
|
+ const feedContent = xmlGetTagContent(xml, 'feed');
|
|
|
if (!feedContent) {
|
|
if (!feedContent) {
|
|
|
throw new Error('Invalid Atom feed: no feed element found');
|
|
throw new Error('Invalid Atom feed: no feed element found');
|
|
|
}
|
|
}
|
|
@@ -255,9 +222,9 @@ function parseAtom(xml, parser) {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const feed = {
|
|
const feed = {
|
|
|
- title: parser.cleanText(parser.getTagContent(feedContent, 'title')) || '',
|
|
|
|
|
|
|
+ title: xmlCleanText(xmlGetTagContent(feedContent, 'title')) || '',
|
|
|
link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
|
|
link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
|
|
|
- description: parser.cleanText(parser.getTagContent(feedContent, 'subtitle')) || '',
|
|
|
|
|
|
|
+ description: xmlCleanText(xmlGetTagContent(feedContent, 'subtitle')) || '',
|
|
|
items: []
|
|
items: []
|
|
|
};
|
|
};
|
|
|
|
|
|
|
@@ -272,34 +239,34 @@ function parseAtom(xml, parser) {
|
|
|
while ((catMatch = catRegex.exec(entryXml)) !== null) {
|
|
while ((catMatch = catRegex.exec(entryXml)) !== null) {
|
|
|
const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
|
|
const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
|
|
|
if (termMatch) {
|
|
if (termMatch) {
|
|
|
- categories.push(parser.cleanText(termMatch[1]));
|
|
|
|
|
|
|
+ categories.push(xmlCleanText(termMatch[1]));
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- let author = '';
|
|
|
|
|
- const authorContent = parser.getTagContent(entryXml, 'author');
|
|
|
|
|
|
|
+ var author = '';
|
|
|
|
|
+ const authorContent = xmlGetTagContent(entryXml, 'author');
|
|
|
if (authorContent) {
|
|
if (authorContent) {
|
|
|
- author = parser.cleanText(parser.getTagContent(authorContent, 'name')) || '';
|
|
|
|
|
|
|
+ author = xmlCleanText(xmlGetTagContent(authorContent, 'name')) || '';
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const pubDate = parser.cleanText(
|
|
|
|
|
- parser.getTagContent(entryXml, 'published') ||
|
|
|
|
|
- parser.getTagContent(entryXml, 'updated')
|
|
|
|
|
|
|
+ const pubDate = xmlCleanText(
|
|
|
|
|
+ xmlGetTagContent(entryXml, 'published') ||
|
|
|
|
|
+ xmlGetTagContent(entryXml, 'updated')
|
|
|
) || '';
|
|
) || '';
|
|
|
|
|
|
|
|
- const description = parser.cleanText(
|
|
|
|
|
- parser.getTagContent(entryXml, 'summary') ||
|
|
|
|
|
- parser.getTagContent(entryXml, 'content')
|
|
|
|
|
|
|
+ const description = xmlCleanText(
|
|
|
|
|
+ xmlGetTagContent(entryXml, 'summary') ||
|
|
|
|
|
+ xmlGetTagContent(entryXml, 'content')
|
|
|
) || '';
|
|
) || '';
|
|
|
|
|
|
|
|
const item = {
|
|
const item = {
|
|
|
- title: parser.cleanText(parser.getTagContent(entryXml, 'title')) || '',
|
|
|
|
|
|
|
+ title: xmlCleanText(xmlGetTagContent(entryXml, 'title')) || '',
|
|
|
link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
|
|
link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
|
|
|
description: description,
|
|
description: description,
|
|
|
pubDate: pubDate,
|
|
pubDate: pubDate,
|
|
|
author: author,
|
|
author: author,
|
|
|
categories: categories,
|
|
categories: categories,
|
|
|
- guid: parser.cleanText(parser.getTagContent(entryXml, 'id')) || ''
|
|
|
|
|
|
|
+ guid: xmlCleanText(xmlGetTagContent(entryXml, 'id')) || ''
|
|
|
};
|
|
};
|
|
|
|
|
|
|
|
feed.items.push(item);
|
|
feed.items.push(item);
|
|
@@ -519,15 +486,13 @@ async function execute(config, input) {
|
|
|
throw new Error('Empty response from feed URL');
|
|
throw new Error('Empty response from feed URL');
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const parser = parseXml(xml);
|
|
|
|
|
-
|
|
|
|
|
- let feed;
|
|
|
|
|
|
|
+ var feed;
|
|
|
if (xml.includes('<feed') && xml.includes('xmlns') && xml.includes('atom')) {
|
|
if (xml.includes('<feed') && xml.includes('xmlns') && xml.includes('atom')) {
|
|
|
smartbotic.log.debug('Detected Atom feed format');
|
|
smartbotic.log.debug('Detected Atom feed format');
|
|
|
- feed = parseAtom(xml, parser);
|
|
|
|
|
|
|
+ feed = parseAtom(xml);
|
|
|
} else if (xml.includes('<rss') || xml.includes('<channel')) {
|
|
} else if (xml.includes('<rss') || xml.includes('<channel')) {
|
|
|
smartbotic.log.debug('Detected RSS feed format');
|
|
smartbotic.log.debug('Detected RSS feed format');
|
|
|
- feed = parseRss(xml, parser);
|
|
|
|
|
|
|
+ feed = parseRss(xml);
|
|
|
} else {
|
|
} else {
|
|
|
throw new Error('Unknown feed format: neither RSS nor Atom detected');
|
|
throw new Error('Unknown feed format: neither RSS nor Atom detected');
|
|
|
}
|
|
}
|