← back to Norma

agents/datasource-agent/lib/frontpage-parser.js

155 lines

/**
 * Frontpage Parser — Simple RSS/Atom XML parser using regex.
 *
 * Extracts headline items (title, link, pubDate, description) from RSS and Atom feeds
 * without requiring any external XML parsing library.
 */

/**
 * Decode common HTML entities to plain text.
 *
 * @param {string} str - HTML-encoded string
 * @returns {string} Decoded string
 */
function decodeEntities(str) {
  if (!str) return '';
  return str
    .replace(/&/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#039;/g, "'")
    .replace(/&#39;/g, "'")
    .replace(/&apos;/g, "'")
    .replace(/<!\[CDATA\[(.*?)\]\]>/gs, '$1');
}

/**
 * Strip HTML tags from a string.
 *
 * @param {string} str - String potentially containing HTML tags
 * @returns {string} Plain text
 */
function stripTags(str) {
  if (!str) return '';
  return str.replace(/<[^>]*>/g, '').trim();
}

/**
 * Extract the text content of an XML element.
 *
 * @param {string} xml - XML source
 * @param {string} tag - Tag name to extract
 * @returns {string|null} Text content or null
 */
function extractTag(xml, tag) {
  const regex = new RegExp('<' + tag + '[^>]*>([\\s\\S]*?)</' + tag + '>', 'i');
  const match = xml.match(regex);
  if (!match) return null;
  return decodeEntities(match[1].trim());
}

/**
 * Extract href from an Atom link element.
 *
 * @param {string} xml - XML source (single entry block)
 * @returns {string|null} URL or null
 */
function extractAtomLink(xml) {
  // Atom links: <link href="..." rel="alternate" />
  const altMatch = xml.match(/<link[^>]*rel=["']alternate["'][^>]*href=["']([^"']+)["']/i);
  if (altMatch) return decodeEntities(altMatch[1]);

  // Fallback: first link with href
  const match = xml.match(/<link[^>]*href=["']([^"']+)["']/i);
  if (match) return decodeEntities(match[1]);

  // Fallback: <link>URL</link>
  const textLink = extractTag(xml, 'link');
  return textLink || null;
}

/**
 * Parse RSS 2.0 XML into an array of headline items.
 *
 * @param {string} xml - Raw RSS 2.0 XML string
 * @returns {Array<{ title: string, url: string, published: string, description: string }>}
 */
function parseRSS20(xml) {
  const items = [];
  const itemRegex = /<item[\s>]([\s\S]*?)<\/item>/gi;
  let match;

  while ((match = itemRegex.exec(xml)) !== null) {
    const block = match[1];
    const title = stripTags(extractTag(block, 'title') || '');
    const link = extractTag(block, 'link') || '';
    const pubDate = extractTag(block, 'pubDate') || '';
    const description = stripTags(extractTag(block, 'description') || '');

    if (title) {
      items.push({
        title: title.substring(0, 500),
        url: link,
        published: pubDate,
        description: description.substring(0, 1000),
      });
    }
  }

  return items;
}

/**
 * Parse Atom XML into an array of headline items.
 *
 * @param {string} xml - Raw Atom XML string
 * @returns {Array<{ title: string, url: string, published: string, description: string }>}
 */
function parseAtom(xml) {
  const items = [];
  const entryRegex = /<entry[\s>]([\s\S]*?)<\/entry>/gi;
  let match;

  while ((match = entryRegex.exec(xml)) !== null) {
    const block = match[1];
    const title = stripTags(extractTag(block, 'title') || '');
    const link = extractAtomLink(block) || '';
    const published = extractTag(block, 'published') || extractTag(block, 'updated') || '';
    const summary = stripTags(extractTag(block, 'summary') || extractTag(block, 'content') || '');

    if (title) {
      items.push({
        title: title.substring(0, 500),
        url: link,
        published: published,
        description: summary.substring(0, 1000),
      });
    }
  }

  return items;
}

/**
 * Auto-detect feed format and parse into headline items.
 * Supports RSS 2.0 and Atom feeds.
 *
 * @param {string} xml - Raw XML string (RSS or Atom)
 * @returns {Array<{ title: string, url: string, published: string, description: string }>}
 */
function parseRSS(xml) {
  if (!xml || typeof xml !== 'string') return [];

  // Detect format: Atom uses <feed> and <entry>, RSS uses <rss> and <item>
  const isAtom = /<feed[\s>]/i.test(xml) && /<entry[\s>]/i.test(xml);

  if (isAtom) {
    return parseAtom(xml);
  }

  return parseRSS20(xml);
}

module.exports = { parseRSS, parseRSS20, parseAtom, extractTag, stripTags, decodeEntities };