`scrapers/sources.js`

309 lines
'use strict';

// Category constants
const CAT = {
  DESIGN_CENTER: 'design-centers',
  TRADE: 'trade-pubs',
  CONSUMER: 'consumer-pubs',
  WALLPAPER: 'wallpaper-watch',
};

// RSS-based sources
const RSS_SOURCES = [
  // --- Trade pubs ---
  {
    key: 'business-of-home',
    name: 'Business of Home',
    category: CAT.TRADE,
    rssUrl: 'https://businessofhome.com/articles.rss',
  },
  {
    key: 'editor-at-large',
    name: 'Editor at Large',
    category: CAT.TRADE,
    rssUrl: 'https://editoratlarge.com/feed/',
  },
  {
    key: 'furniture-today',
    name: 'Furniture Today',
    category: CAT.TRADE,
    rssUrl: 'https://www.furnituretoday.com/category/news/feed/',
  },
  {
    key: 'hospitality-design',
    name: 'Hospitality Design',
    category: CAT.TRADE,
    rssUrl: 'https://hospitalitydesign.com/feed/',
  },
  {
    key: 'interior-design-mag',
    name: 'Interior Design Magazine',
    category: CAT.TRADE,
    rssUrl: 'https://interiordesign.net/feed/',
  },
  // --- Consumer pubs ---
  {
    key: 'architectural-digest',
    name: 'Architectural Digest',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.architecturaldigest.com/feed/rss',
  },
  {
    key: 'elle-decor',
    name: 'Elle Decor',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.elledecor.com/rss/all.xml/',
  },
  {
    key: 'house-beautiful',
    name: 'House Beautiful',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.housebeautiful.com/rss/all.xml/',
  },
  {
    key: 'veranda',
    name: 'Veranda',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.veranda.com/rss/all.xml/',
  },
  {
    key: 'domino',
    name: 'Domino',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.domino.com/feed',
  },
  {
    key: 'dezeen',
    name: 'Dezeen',
    category: CAT.CONSUMER,
    rssUrl: 'https://www.dezeen.com/feed/',
  },
  {
    key: 'design-milk',
    name: 'Design Milk',
    category: CAT.CONSUMER,
    rssUrl: 'https://design-milk.com/feed/',
  },
];

// HTML-scraped sources (design centers + wallpaper houses)
// These are scraped for linked headlines only — no excerpts
const SCRAPE_SOURCES = [
  // --- Design Centers ---
  {
    key: 'pacific-design-center',
    name: 'Pacific Design Center',
    category: CAT.DESIGN_CENTER,
    url: 'https://pacificdesigncenter.com/news',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://pacificdesigncenter.com',
  },
  {
    key: 'dd-building',
    name: 'D&D Building NYC',
    category: CAT.DESIGN_CENTER,
    url: 'https://ddbuilding.com/news',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://ddbuilding.com',
  },
  {
    key: 'themart',
    name: 'Chicago Merchandise Mart',
    category: CAT.DESIGN_CENTER,
    url: 'https://www.themart.com/press-releases/',
    selector: 'a[href]',
    linkFilter: /\/(press|news|event|release)/i,
    baseUrl: 'https://www.themart.com',
  },
  {
    key: 'dc-dallas',
    name: 'Decorative Center Dallas',
    category: CAT.DESIGN_CENTER,
    url: 'https://dcdallas.com/latest-news/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press|latest)/i,
    baseUrl: 'https://dcdallas.com',
  },
  {
    key: 'dec-center-houston',
    name: 'Decorative Center Houston',
    category: CAT.DESIGN_CENTER,
    url: 'https://www.decorativecenterhouston.com/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press|feature|article)/i,
    baseUrl: 'https://www.decorativecenterhouston.com',
  },
  {
    key: 'adac-atlanta',
    name: 'ADAC Atlanta',
    category: CAT.DESIGN_CENTER,
    url: 'https://adacatlanta.com/adac-news/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press|adac)/i,
    baseUrl: 'https://adacatlanta.com',
  },
  {
    key: 'nydc',
    name: 'New York Design Center',
    category: CAT.DESIGN_CENTER,
    url: 'https://www.nydc.com/news/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://www.nydc.com',
  },
  {
    key: 'sf-design-center',
    name: 'San Francisco Design Center',
    category: CAT.DESIGN_CENTER,
    url: 'https://www.sfdesigncenter.com/news-events/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://www.sfdesigncenter.com',
  },
  {
    key: 'washington-design-center',
    name: 'Washington Design Center',
    category: CAT.DESIGN_CENTER,
    url: 'https://www.dcdesigncenter.com/news/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://www.dcdesigncenter.com',
  },
  {
    key: 'miami-design-district',
    name: 'Miami Design District',
    category: CAT.DESIGN_CENTER,
    url: 'https://miamidesigndistrict.net/news/',
    selector: 'a[href]',
    linkFilter: /\/(news|event|press)/i,
    baseUrl: 'https://miamidesigndistrict.net',
  },
  // --- Wallpaper Houses ---
  // Each entry has: url, baseUrl, selector, linkFilter (optional reject regex via excludeFilter)
  // Some use `parser: 'sitemap'` to read a sitemap.xml instead of HTML
  // `browserbase: true` routes through Browserbase cloud chromium (anti-bot bypass)
  {
    key: 'schumacher',
    name: 'Schumacher',
    category: CAT.WALLPAPER,
    url: 'https://schumacher.com/blog',
    selector: 'h3 a, h2 a',
    linkFilter: /\/blog\/[a-z0-9-]+\/?$/i,
    excludeFilter: /\/blog\/(category|author|search|tag|page)\//i,
    baseUrl: 'https://schumacher.com',
    // CloudFront 403s data-center IPs (Kamatera) but allows residential
    // (Mac2). Force through Browserbase so prod cron works too.
    browserbase: true,
  },
  {
    key: 'thibaut',
    name: 'Thibaut',
    category: CAT.WALLPAPER,
    url: 'https://www.thibautdesign.com/sitemaps/collections.xml',
    parser: 'sitemap',
    baseUrl: 'https://www.thibautdesign.com',
  },
  {
    key: 'phillip-jeffries',
    name: 'Phillip Jeffries',
    category: CAT.WALLPAPER,
    url: 'https://blog.phillipjeffries.com/',
    selector: 'h2 a',
    linkFilter: /blog\.phillipjeffries\.com\/[a-z0-9-]+$/i,
    excludeFilter: /\/(tag|category|author|page|search)/i,
    baseUrl: 'https://blog.phillipjeffries.com',
  },
  {
    key: 'scalamandre',
    name: 'Scalamandre',
    category: CAT.WALLPAPER,
    // Scalamandre publishes no public news/blog (their /news page literally says "No news!").
    // Their sitemap.xml is a flat urlset listing brand/collection pages with lastmod —
    // newest = newest collection drops. Use those as "what's new" headlines.
    url: 'https://www.scalamandre.com/sitemap.xml',
    parser: 'sitemap',
    linkFilter: /\/brands\/[a-z-]+\/[a-z0-9-]+\.html$/i,
    baseUrl: 'https://www.scalamandre.com',
  },
  {
    key: 'maya-romanoff',
    name: 'Maya Romanoff',
    category: CAT.WALLPAPER,
    url: 'https://www.mayaromanoff.com/news/',
    selector: 'h2 a',
    linkFilter: /\/news\/[a-z0-9-]+\/?$/i,
    excludeFilter: /\/news\/?$|\/news\/page/i,
    baseUrl: 'https://www.mayaromanoff.com',
  },
  {
    key: 'arte-international',
    name: 'Arte International',
    category: CAT.WALLPAPER,
    url: 'https://www.arte-international.com/en/stories',
    selector: 'a[href*="/stories/"]',
    linkFilter: /\/en\/stories\/[a-z0-9-]{4,}$/i,
    excludeFilter: /\?category=|\/stories\/?$/i,
    baseUrl: 'https://www.arte-international.com',
  },
  {
    key: 'brewster',
    name: 'Brewster Home Fashions',
    category: CAT.WALLPAPER,
    url: 'https://www.brewsterwallcovering.com/blog',
    selector: 'h3 a, h2 a',
    linkFilter: /brewsterwallcovering\.com\/[a-z0-9-]+$/i,
    excludeFilter: /\/(blog|news|about|contact|search|cart|account|page-not-found)$/i,
    baseUrl: 'https://www.brewsterwallcovering.com',
  },
  {
    key: 'york-wallcoverings',
    name: 'York Wallcoverings',
    category: CAT.WALLPAPER,
    url: 'https://www.yorkwallcoverings.com/blog',
    selector: 'a[href*="/blog/"]',
    linkFilter: /\/blog\/[a-z0-9-]+$/i,
    excludeFilter: /\/blog\/(tag|month|rss|category|author|page)\//i,
    baseUrl: 'https://www.yorkwallcoverings.com',
  },
  {
    key: 'wallquest',
    name: 'Wallquest',
    category: CAT.WALLPAPER,
    // Wallquest is behind Cloudflare bot protection AND publishes no blog/news posts.
    // Use the sitemap (fetched via Browserbase) and surface the most recently updated
    // collection pages with human-readable slugs (≥1 dash, not a SKU code).
    url: 'https://wallquest.com/sitemap.xml',
    parser: 'sitemap',
    linkFilter: /wallquest\.com\/[a-z][a-z-]+-[a-z][a-z0-9-]+$/i,
    excludeFilter: /\/(news|blog|cart|login|register|cms|about|contact|privacy|terms|sitemap|customer|account|search|forgot|password|category-page|residential|commercial|coloratlas|email|wp-)/i,
    baseUrl: 'https://wallquest.com',
    browserbase: true,
  },
  {
    key: 'cole-and-son',
    name: 'Cole & Son',
    category: CAT.WALLPAPER,
    url: 'https://cole-and-son.com/sitemap_blogs_1.xml',
    parser: 'sitemap',
    sitemapTitleFromImage: true,
    linkFilter: /\/blogs\/[a-z-]+\/[a-z0-9-]+$/i,
    baseUrl: 'https://cole-and-son.com',
  },
];

// Social follow links (no scraping — just reference links)
const SOCIAL_LINKS = [
  { name: 'Architectural Digest', ig: 'https://www.instagram.com/archdigest/', li: 'https://www.linkedin.com/company/architectural-digest/' },
  { name: 'Elle Decor', ig: 'https://www.instagram.com/elledecor/', li: 'https://www.linkedin.com/company/elle-decor/' },
  { name: 'Business of Home', ig: 'https://www.instagram.com/businessofhome/', li: 'https://www.linkedin.com/company/business-of-home/' },
  { name: 'Dezeen', ig: 'https://www.instagram.com/dezeen/', li: 'https://www.linkedin.com/company/dezeen/' },
  { name: 'Design Milk', ig: 'https://www.instagram.com/designmilk/', li: 'https://www.linkedin.com/company/design-milk/' },
  { name: 'Pacific Design Center', ig: 'https://www.instagram.com/pacificdesigncenter/', li: 'https://www.linkedin.com/company/pacific-design-center/' },
  { name: 'ADAC Atlanta', ig: 'https://www.instagram.com/adacatlanta/', li: 'https://www.linkedin.com/company/atlanta-decorative-arts-center/' },
  { name: 'Miami Design District', ig: 'https://www.instagram.com/miamidesigndistrict/', li: 'https://www.linkedin.com/company/miami-design-district/' },
];

module.exports = { RSS_SOURCES, SCRAPE_SOURCES, SOCIAL_LINKS, CAT };