`scripts/data-hound-press.js`

174 lines
#!/usr/bin/env node
// Data Hound #3 — Google News RSS scrape for press articles. For each business
// missing press coverage, query Google News RSS for "{name}" {city} salon and
// extract <item> entries from credible publishers. ZERO Anthropic API.
// Google News RSS endpoint:
//   https://news.google.com/rss/search?q=QUERY&hl=en-US&gl=US&ceid=US:en
//
//   node scripts/data-hound-press.js
//   LIMIT=200 node scripts/data-hound-press.js
//   DRY=1 node scripts/data-hound-press.js

import 'dotenv/config';
import { query } from '../src/lib/db.js';

const SLEEP_MS = 2500;          // baseline: ~0.4 req/sec — Google RSS safe floor
const BACKOFF_BASE = 15_000;    // 15s after first 503
const BACKOFF_MAX = 300_000;    // cap at 5 min
const CIRCUIT_OPEN_AFTER = 5;   // consecutive 503s before circuit opens
const CIRCUIT_COOLDOWN = 120_000; // 2-min full pause when circuit opens
const LIMIT = process.env.LIMIT ? parseInt(process.env.LIMIT, 10) : null;
const DRY = process.env.DRY === '1';

const ts = () => new Date().toISOString().replace('T', ' ').slice(0, 19);
const log = m => console.log(`[${ts()}] ${m}`);
const sleep = ms => new Promise(r => setTimeout(r, ms));

// Allowlist of credible publishers. Anything else is dropped.
// Matched as substring (case-insensitive) against the publisher name.
const ALLOW = [
  'la mag', 'los angeles magazine', 'la times', 'eater', 'time out',
  'voyage la', 'hollywood reporter', 'thrillist', 'goop', 'allure',
  'vogue', 'glamour', 'wwd', 'racked', 'curbed', 'la weekly',
  'beverly hills courier', 'westside today', 'hollywood patch',
  'modern salon', 'beauty launchpad', 'american salon', 'nailpro',
  'spa and beauty today', 'wellness magazine', 'dazed', 'refinery29',
  'who what wear', 'pop sugar', 'byrdie', 'into the gloss',
  'la magazine', 'sf chronicle', 'forbes', 'bloomberg', 'cnbc',
  'visit california', 'visit los angeles', 'discover los angeles',
  'discover la', 'angeleno', 'l.a. taco', 'la taco',
  'we like la', 'the daily beast', 'people', 'us weekly',
  'la confidential', 'cbs los angeles', 'nbc los angeles', 'kcrw',
  'kcet', 'kpcc', 'laist',
];

function isAllowedPublisher(name) {
  const lc = String(name || '').toLowerCase();
  return ALLOW.some(a => lc.includes(a));
}

// Fast lightweight RSS extractor — no external deps. Pulls each <item>...</item>
// block and extracts <title>, <link>, <pubDate>, <description>, <source>.
function parseRSS(xml) {
  const items = [];
  const itemRe = /<item>([\s\S]*?)<\/item>/g;
  let m;
  while ((m = itemRe.exec(xml)) !== null) {
    const block = m[1];
    const grab = (tag) => {
      const re = new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`);
      const r = re.exec(block);
      return r ? r[1].replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1').trim() : null;
    };
    const link = grab('link');
    if (!link) continue;
    items.push({
      title:     grab('title'),
      url:       link,
      published: grab('pubDate'),
      excerpt:   (grab('description') || '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').slice(0, 400),
      publisher: grab('source') || null,
    });
  }
  return items;
}

async function googleNewsSearch(name, city, attempt = 0) {
  const q = encodeURIComponent(`"${name}" ${city || 'Los Angeles'} salon`);
  const url = `https://news.google.com/rss/search?q=${q}&hl=en-US&gl=US&ceid=US:en`;
  const r = await fetch(url, {
    signal: AbortSignal.timeout(15000),
    headers: { 'user-agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' },
  });
  if (r.status === 503) {
    // Exponential backoff with jitter, capped at BACKOFF_MAX
    const delay = Math.min(BACKOFF_BASE * 2 ** attempt, BACKOFF_MAX)
      + Math.floor(Math.random() * 2000);
    throw Object.assign(new Error(`HTTP 503`), { is503: true, backoffMs: delay });
  }
  if (!r.ok) throw new Error(`HTTP ${r.status}`);
  const xml = await r.text();
  return parseRSS(xml);
}

async function run() {
  const rows = await query(
    `SELECT b.id, b.slug, b.name, b.city,
            COALESCE(NULLIF(b.source_data_json->>'dba_name',''), b.name) AS display_name,
            COUNT(a.id)::int AS article_count
       FROM businesses b
       LEFT JOIN business_articles a ON a.business_id = b.id
      WHERE b.category IN ('salon','barbershop','spa','nail')
        AND (b.website IS NOT NULL AND b.website <> '' OR b.neighborhood IS NOT NULL)
      GROUP BY b.id
     HAVING COUNT(a.id) = 0
      ORDER BY (CASE WHEN b.website IS NOT NULL AND b.website <> '' THEN 0 ELSE 1 END),
               (CASE WHEN b.neighborhood IS NOT NULL THEN 0 ELSE 1 END),
               b.name
      ${LIMIT ? `LIMIT ${LIMIT}` : ''}`
  );
  log(`Press hound: ${rows.rows.length} candidates (no existing press, eligible)`);

  let scanned = 0, found = 0, articlesAdded = 0, errs = 0;
  let consecutive503 = 0;

  for (const b of rows.rows) {
    scanned++;
    if (DRY) { log(`DRY: ${b.display_name} / ${b.city}`); await sleep(SLEEP_MS); continue; }

    // Circuit breaker: too many consecutive 503s → pause hard before continuing
    if (consecutive503 >= CIRCUIT_OPEN_AFTER) {
      log(`  CIRCUIT OPEN after ${consecutive503} consecutive 503s — cooling down ${CIRCUIT_COOLDOWN / 1000}s`);
      await sleep(CIRCUIT_COOLDOWN);
      consecutive503 = 0;
    }

    let attempt = 0;
    let success = false;
    while (attempt <= 3 && !success) {
      try {
        const items = await googleNewsSearch(b.display_name, b.city, attempt);
        consecutive503 = 0; // reset on success
        const accepted = items.filter(it => it.publisher && isAllowedPublisher(it.publisher));

        let added = 0;
        for (const it of accepted) {
          try {
            const r = await query(
              `INSERT INTO business_articles (business_id, url, title, publisher, published_at, excerpt, source)
               VALUES ($1, $2, $3, $4, $5, $6, 'google-news-rss')
               ON CONFLICT (business_id, url) DO NOTHING
               RETURNING id`,
              [b.id, it.url, it.title, it.publisher, it.published || null, it.excerpt || null]
            );
            if (r.rowCount > 0) { added++; articlesAdded++; }
          } catch (e) {
            // INSERT can fail on bad pubDate parsing — skip
          }
        }
        if (added > 0) found++;
        success = true;
      } catch (e) {
        if (e.is503) {
          consecutive503++;
          attempt++;
          log(`  503 backoff ${e.backoffMs}ms (attempt ${attempt}, consecutive=${consecutive503}) — ${b.display_name}`);
          await sleep(e.backoffMs);
        } else {
          errs++;
          if (errs % 20 === 0) log(`  ${errs} errors (last: ${e.message?.slice(0, 80)})`);
          break;
        }
      }
    }
    if (!success && attempt > 3) errs++;
    if (scanned % 25 === 0) log(`  scanned=${scanned} biz_with_press=${found} articles=${articlesAdded} errors=${errs}`);
    await sleep(SLEEP_MS);
  }

  log(`DONE — scanned=${scanned} biz_with_press=${found} articles_added=${articlesAdded} errors=${errs}`);
  process.exit(0);
}

run().catch(e => { console.error(e); process.exit(1); });