smartbotic/nodes/rss/rss-reader.js

/**
 * @node rss-reader
 * @name RSS Reader
 * @category rss
 * @version 1.0.0
 * @description Read and parse RSS/Atom feeds with optional filtering by keyword, date, or count
 * @icon rss
 */

const configSchema = {
    type: 'object',
    properties: {
        url: {
            type: 'string',
            title: 'Feed URL',
            description: 'URL of the RSS or Atom feed'
        },
        filterKeyword: {
            type: 'string',
            title: 'Keyword Filter',
            description: 'Filter items by keyword (matches title and description)'
        },
        filterNewerThan: {
            type: 'string',
            title: 'Newer Than',
            description: 'Only include items newer than this date (ISO 8601 format, e.g., 2024-01-15T00:00:00Z)'
        },
        maxItems: {
            type: 'number',
            title: 'Max Items',
            description: 'Maximum number of items to return (0 = unlimited)',
            default: 0
        },
        timeout: {
            type: 'number',
            title: 'Timeout (ms)',
            default: 30000
        }
    },
    required: ['url']
};

const inputSchema = {
    type: 'object',
    properties: {
        url: {
            type: 'string',
            description: 'Override feed URL from input'
        },
        filterKeyword: {
            type: 'string',
            description: 'Override keyword filter from input'
        },
        filterNewerThan: {
            type: 'string',
            description: 'Override date filter from input'
        },
        maxItems: {
            type: 'number',
            description: 'Override max items from input'
        }
    }
};

const outputSchema = {
    type: 'object',
    properties: {
        feedTitle: { type: 'string', description: 'Title of the feed' },
        feedLink: { type: 'string', description: 'Link to the feed homepage' },
        feedDescription: { type: 'string', description: 'Description of the feed' },
        itemCount: { type: 'number', description: 'Number of items returned' },
        items: {
            type: 'array',
            description: 'Array of feed items',
            items: {
                type: 'object',
                properties: {
                    title: { type: 'string' },
                    link: { type: 'string' },
                    description: { type: 'string' },
                    pubDate: { type: 'string' },
                    author: { type: 'string' },
                    categories: { type: 'array', items: { type: 'string' } },
                    guid: { type: 'string' }
                }
            }
        }
    }
};

/**
 * Simple XML parser for RSS/Atom feeds
 * Handles basic XML structure without external dependencies
 */
function parseXml() {
    function getTagContent(xml, tagName) {
        const patterns = [
            new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'i'),
            new RegExp(`<${tagName}[^>]*\\/>`, 'i')
        ];

        const match = xml.match(patterns[0]);
        if (match) {
            return match[1].trim();
        }
        return null;
    }

    function getAllTagContents(xml, tagName) {
        const results = [];
        const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>`, 'gi');
        let match;
        while ((match = regex.exec(xml)) !== null) {
            results.push(match[1].trim());
        }
        return results;
    }

    function getTagAttribute(xml, tagName, attrName) {
        const regex = new RegExp(`<${tagName}[^>]*\\s${attrName}=["']([^"']*)["'][^>]*>`, 'i');
        const match = xml.match(regex);
        return match ? match[1] : null;
    }

    function getAllTags(xml, tagName) {
        const results = [];
        const regex = new RegExp(`<${tagName}[^>]*>([\\s\\S]*?)<\\/${tagName}>|<${tagName}([^>]*)\\/>`, 'gi');
        let match;
        while ((match = regex.exec(xml)) !== null) {
            results.push({
                content: match[1] ? match[1].trim() : '',
                fullMatch: match[0]
            });
        }
        return results;
    }

    function decodeHtmlEntities(text) {
        if (!text) return '';
        return text
            .replace(/&lt;/g, '<')
            .replace(/&gt;/g, '>')
            .replace(/&amp;/g, '&')
            .replace(/&quot;/g, '"')
            .replace(/&apos;/g, "'")
            .replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(dec))
            .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
    }

    function stripCdata(text) {
        if (!text) return '';
        return text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1');
    }

    function cleanText(text) {
        if (!text) return '';
        return decodeHtmlEntities(stripCdata(text)).trim();
    }

    return {
        getTagContent,
        getAllTagContents,
        getTagAttribute,
        getAllTags,
        cleanText
    };
}

/**
 * Parse RSS 2.0 feed
 */
function parseRss(xml, parser) {
    const channel = parser.getTagContent(xml, 'channel');
    if (!channel) {
        throw new Error('Invalid RSS feed: no channel element found');
    }

    const feed = {
        title: parser.cleanText(parser.getTagContent(channel, 'title')) || '',
        link: parser.cleanText(parser.getTagContent(channel, 'link')) || '',
        description: parser.cleanText(parser.getTagContent(channel, 'description')) || '',
        items: []
    };

    const itemRegex = /<item[^>]*>([\s\S]*?)<\/item>/gi;
    let match;
    while ((match = itemRegex.exec(channel)) !== null) {
        const itemXml = match[1];

        const categories = parser.getAllTagContents(itemXml, 'category')
            .map(c => parser.cleanText(c));

        const item = {
            title: parser.cleanText(parser.getTagContent(itemXml, 'title')) || '',
            link: parser.cleanText(parser.getTagContent(itemXml, 'link')) || '',
            description: parser.cleanText(parser.getTagContent(itemXml, 'description')) || '',
            pubDate: parser.cleanText(parser.getTagContent(itemXml, 'pubDate')) || '',
            author: parser.cleanText(parser.getTagContent(itemXml, 'author') ||
                    parser.getTagContent(itemXml, 'dc:creator')) || '',
            categories: categories,
            guid: parser.cleanText(parser.getTagContent(itemXml, 'guid')) || ''
        };

        feed.items.push(item);
    }

    return feed;
}

/**
 * Parse Atom feed
 */
function parseAtom(xml, parser) {
    const feedContent = parser.getTagContent(xml, 'feed');
    if (!feedContent) {
        throw new Error('Invalid Atom feed: no feed element found');
    }

    function getAtomLink(content, rel) {
        const linkRegex = /<link([^>]*)>/gi;
        let match;
        while ((match = linkRegex.exec(content)) !== null) {
            const attrs = match[1];
            const relMatch = attrs.match(/rel=["']([^"']*)["']/);
            const hrefMatch = attrs.match(/href=["']([^"']*)["']/);

            if (hrefMatch) {
                const linkRel = relMatch ? relMatch[1] : 'alternate';
                if (linkRel === rel || (!rel && linkRel === 'alternate')) {
                    return hrefMatch[1];
                }
            }
        }
        return '';
    }

    const feed = {
        title: parser.cleanText(parser.getTagContent(feedContent, 'title')) || '',
        link: getAtomLink(feedContent, 'alternate') || getAtomLink(feedContent, null) || '',
        description: parser.cleanText(parser.getTagContent(feedContent, 'subtitle')) || '',
        items: []
    };

    const entryRegex = /<entry[^>]*>([\s\S]*?)<\/entry>/gi;
    let match;
    while ((match = entryRegex.exec(feedContent)) !== null) {
        const entryXml = match[1];

        const categories = [];
        const catRegex = /<category([^>]*)>/gi;
        let catMatch;
        while ((catMatch = catRegex.exec(entryXml)) !== null) {
            const termMatch = catMatch[1].match(/term=["']([^"']*)["']/);
            if (termMatch) {
                categories.push(parser.cleanText(termMatch[1]));
            }
        }

        let author = '';
        const authorContent = parser.getTagContent(entryXml, 'author');
        if (authorContent) {
            author = parser.cleanText(parser.getTagContent(authorContent, 'name')) || '';
        }

        const pubDate = parser.cleanText(
            parser.getTagContent(entryXml, 'published') ||
            parser.getTagContent(entryXml, 'updated')
        ) || '';

        const description = parser.cleanText(
            parser.getTagContent(entryXml, 'summary') ||
            parser.getTagContent(entryXml, 'content')
        ) || '';

        const item = {
            title: parser.cleanText(parser.getTagContent(entryXml, 'title')) || '',
            link: getAtomLink(entryXml, 'alternate') || getAtomLink(entryXml, null) || '',
            description: description,
            pubDate: pubDate,
            author: author,
            categories: categories,
            guid: parser.cleanText(parser.getTagContent(entryXml, 'id')) || ''
        };

        feed.items.push(item);
    }

    return feed;
}

/**
 * Parse date string to timestamp
 */
function parseDate(dateStr) {
    if (!dateStr) return 0;

    const timestamp = Date.parse(dateStr);
    if (!isNaN(timestamp)) {
        return timestamp;
    }

    const rfc822Regex = /(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})/;
    const match = dateStr.match(rfc822Regex);
    if (match) {
        const months = {
            Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5,
            Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11
        };
        const day = parseInt(match[1], 10);
        const month = months[match[2]];
        const year = parseInt(match[3], 10);
        const hour = parseInt(match[4], 10);
        const minute = parseInt(match[5], 10);
        const second = parseInt(match[6], 10);

        if (month !== undefined) {
            return new Date(year, month, day, hour, minute, second).getTime();
        }
    }

    return 0;
}

/**
 * Filter items by keyword
 */
function filterByKeyword(items, keyword) {
    if (!keyword) return items;

    const lowerKeyword = keyword.toLowerCase();
    return items.filter(item => {
        const title = (item.title || '').toLowerCase();
        const description = (item.description || '').toLowerCase();
        return title.includes(lowerKeyword) || description.includes(lowerKeyword);
    });
}

/**
 * Filter items by date
 */
function filterByDate(items, newerThanStr) {
    if (!newerThanStr) return items;

    const threshold = parseDate(newerThanStr);
    if (threshold === 0) {
        smartbotic.log.warn(`Could not parse date filter: ${newerThanStr}`);
        return items;
    }

    return items.filter(item => {
        const itemDate = parseDate(item.pubDate);
        return itemDate > threshold;
    });
}

/**
 * Limit number of items
 */
function limitItems(items, maxItems) {
    if (!maxItems || maxItems <= 0) return items;
    return items.slice(0, maxItems);
}

async function execute(config, input, context) {
    const url = input.url || config.url;
    const filterKeyword = input.filterKeyword || config.filterKeyword;
    const filterNewerThan = input.filterNewerThan || config.filterNewerThan;
    const maxItems = input.maxItems !== undefined ? input.maxItems : config.maxItems;
    const timeout = config.timeout || 30000;

    if (!url) {
        throw new Error('Feed URL is required');
    }

    smartbotic.log.info(`Fetching RSS/Atom feed from ${url}`);

    const response = smartbotic.http.request({
        method: 'GET',
        url: url,
        timeout: timeout,
        headers: {
            'Accept': 'application/rss+xml, application/atom+xml, application/xml, text/xml, */*'
        }
    });

    if (response.status < 200 || response.status >= 300) {
        throw new Error(`Failed to fetch feed: HTTP ${response.status}`);
    }

    const xml = typeof response.data === 'string' ? response.data : JSON.stringify(response.data);

    if (!xml || xml.trim().length === 0) {
        throw new Error('Empty response from feed URL');
    }

    const parser = parseXml(xml);

    let feed;
    if (xml.includes('<feed') && xml.includes('xmlns') && xml.includes('atom')) {
        smartbotic.log.debug('Detected Atom feed format');
        feed = parseAtom(xml, parser);
    } else if (xml.includes('<rss') || xml.includes('<channel')) {
        smartbotic.log.debug('Detected RSS feed format');
        feed = parseRss(xml, parser);
    } else {
        throw new Error('Unknown feed format: neither RSS nor Atom detected');
    }

    let items = feed.items;

    if (filterKeyword) {
        smartbotic.log.debug(`Filtering by keyword: ${filterKeyword}`);
        items = filterByKeyword(items, filterKeyword);
    }

    if (filterNewerThan) {
        smartbotic.log.debug(`Filtering items newer than: ${filterNewerThan}`);
        items = filterByDate(items, filterNewerThan);
    }

    if (maxItems && maxItems > 0) {
        smartbotic.log.debug(`Limiting to ${maxItems} items`);
        items = limitItems(items, maxItems);
    }

    smartbotic.log.info(`Parsed ${items.length} items from feed`);

    return {
        feedTitle: feed.title,
        feedLink: feed.link,
        feedDescription: feed.description,
        itemCount: items.length,
        items: items
    };
}

module.exports = { configSchema, inputSchema, outputSchema, execute };