`scripts/enrich-reddit.js`

157 lines
#!/usr/bin/env node
/**
 * Reddit signal enrichment.
 *
 * For each LA-county org, query Reddit's public search API for posts mentioning
 * the org name, restricted to subreddits where LA-area recommendations
 * actually surface. Stash hits in org_signals so the pitch flow can show
 * "what Reddit thinks of you" snippets.
 *
 * Free, no auth needed (public /search.json). Rate-limited to 1 rps.
 *
 *   node scripts/enrich-reddit.js
 *   LIMIT=20 node scripts/enrich-reddit.js              # smoke
 *   ONLY_DOCTORS=1 node scripts/enrich-reddit.js        # doctor-type orgs only
 */
const { fetch } = require('undici');
const { pool, query } = require('../agents/shared/db');
const { setHostRateLimit } = require('../agents/shared/compliance');

const LIMIT = process.env.LIMIT ? Number(process.env.LIMIT) : null;
const ONLY_DOCTORS = process.env.ONLY_DOCTORS === '1';

const DOCTOR_TYPES = ['medical_group','clinic','dental_office','optometrist_office','surgery_center','hospital','fqhc','tcm_clinic','acupuncture_clinic','chiropractor_office','tcm_herbalist','rcfe','ccrc','adult_residential','home_care_agency','home_health','hospice','nursing_facility'];

// LA-area subreddits where small-biz / medical recommendations surface.
const SUBREDDITS = [
  'AskLosAngeles', 'LosAngeles', 'AskLA', 'AskDocs', 'medicine',
  'sanfernandovalley', 'Pasadena', 'longbeach', 'BeverlyHills',
  'westhollywood', 'culvercity', 'santamonica',
];

const UA = 'professional-directory/0.1 by /u/stevemdr (research)';

setHostRateLimit('www.reddit.com', 0.7);   // ~1 req/1.4s — Reddit returns 429 above ~1 rps anonymously

function safeName(name) {
  // For Reddit search we want bare org name without legal suffixes that hurt recall.
  return String(name || '')
    .replace(/\b(LLC|INC|INC\.|CORP|CORPORATION|MEDICAL GROUP|MEDICAL CENTER|HEALTH|CARE|GROUP|ASSOC|ASSOCIATES|PC|P\.C\.|PROFESSIONAL CORPORATION)\b/gi, '')
    .replace(/\s+/g, ' ').trim();
}

async function searchReddit(query) {
  const url = `https://www.reddit.com/search.json?q=${encodeURIComponent('"' + query + '"')}&limit=15&sort=relevance&type=link`;
  await new Promise(r => setTimeout(r, 1500));   // explicit pace; lib also enforces
  try {
    const res = await fetch(url, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(15_000) });
    if (res.status === 429) {
      await new Promise(r => setTimeout(r, 30_000));   // back off on rate-limit
      return null;
    }
    if (!res.ok) return null;
    const j = await res.json();
    const children = j?.data?.children || [];
    return children.map(c => c.data).filter(d => d && d.subreddit);
  } catch (_) { return null; }
}

function relevanceScore(post, orgName, orgCity) {
  if (!post) return 0;
  const haystack = ((post.title || '') + ' ' + (post.selftext || '')).toLowerCase();
  const needle = orgName.toLowerCase();
  if (!haystack.includes(needle)) return 0;
  let score = 0;
  // HARD filter: must be in an LA-relevant subreddit OR mention the org's city.
  const inLaSub = SUBREDDITS.includes(post.subreddit);
  const cityHit = orgCity && haystack.includes(orgCity.toLowerCase());
  if (!inLaSub && !cityHit) return 0;
  if (inLaSub) score += 3;
  if (cityHit) score += 2;
  // Active discussion bonus.
  if ((post.num_comments || 0) > 5) score += 1;
  // Recommendation-shape bonus.
  if (/recommend|review|experience|seen|been to|tried|happy|terrible|avoid/.test(haystack)) score += 1;
  return score;
}

async function processOrg(o) {
  // Skip if we already have signals for this org.
  const existing = await query('SELECT 1 FROM org_signals WHERE organization_id = $1 AND source = \'reddit\' LIMIT 1', [o.id]);
  if (existing.rowCount > 0) return { hits: 0, skipped: true };

  const name = safeName(o.name);
  // Require longer/more-specific names. "Nail Bar" gets too many false hits;
  // require >=12 chars OR >=2 words AND >= 8 chars.
  const wordCount = name.split(/\s+/).filter(Boolean).length;
  if (!name || (name.length < 12 && wordCount < 2) || name.length < 8) return { hits: 0, skipped: 'short-name' };

  const posts = await searchReddit(name);
  if (!posts || posts.length === 0) return { hits: 0 };

  let inserted = 0;
  for (const p of posts) {
    const score = relevanceScore(p, name, o.city);
    if (score < 3) continue;   // require strong relevance: LA-sub OR city-mention
    const url = 'https://www.reddit.com' + (p.permalink || '');
    try {
      await query(`
        INSERT INTO org_signals
          (organization_id, source, subsource, url, title, body, author, posted_at, raw_json)
        VALUES ($1, 'reddit', $2, $3, $4, $5, $6, to_timestamp($7), $8::jsonb)
        ON CONFLICT (url) DO NOTHING
      `, [
        o.id, p.subreddit, url,
        (p.title || '').slice(0, 500),
        (p.selftext || '').slice(0, 2000),
        p.author || null,
        p.created_utc || 0,
        JSON.stringify({ score: p.score, comments: p.num_comments, relevance: score }),
      ]);
      inserted++;
    } catch (_) {}
  }
  return { hits: inserted };
}

async function main() {
  const where = [`opted_out = false`, `name IS NOT NULL`];
  const params = [];
  if (ONLY_DOCTORS) { where.push('type = ANY($1)'); params.push(DOCTOR_TYPES); }
  const sql = `
    SELECT id, name, type, city FROM organizations
     WHERE ${where.join(' AND ')}
     ORDER BY lead_score DESC NULLS LAST, id
     ${LIMIT ? `LIMIT ${LIMIT}` : 'LIMIT 5000'}
  `;
  const r = await query(sql, params);
  console.log(`[reddit] candidates=${r.rowCount}`);

  let orgs = 0, totalHits = 0, withAny = 0, skipped = 0;
  for (const o of r.rows) {
    orgs++;
    try {
      const stat = await processOrg(o);
      if (stat.skipped) skipped++;
      else {
        if (stat.hits > 0) withAny++;
        totalHits += stat.hits;
      }
      if (orgs % 25 === 0 || stat.hits > 0) {
        console.log(`[reddit] orgs=${orgs}/${r.rowCount} new_hits=${stat.hits} | totals: with_any=${withAny} skipped=${skipped} hits=${totalHits}`);
      }
    } catch (e) {
      console.error(`[reddit] error org=${o.id} (${o.name}): ${e.message}`);
    }
  }

  console.log(`[reddit] DONE orgs=${orgs} hits=${totalHits} with_any=${withAny} skipped=${skipped}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[reddit] fatal:', err);
  try { await pool.end(); } catch (_) {}
  process.exit(1);
});