`scripts/scan-ad-signals.js`

178 lines
#!/usr/bin/env node
// Ad-pixel fingerprint scanner — pure regex, no LLM.
//
// For each unclaimed installer with a website, fetch homepage and detect
// tracking pixels for: Google Ads, Meta, TikTok, Pinterest, LinkedIn, Twitter,
// Reddit, Bing, Snap. Stores result as JSONB in installers.ad_signals.
//
// Usage:
//   node scripts/scan-ad-signals.js                # dry run
//   node scripts/scan-ad-signals.js --commit       # write to DB
//   node scripts/scan-ad-signals.js --max=200      # cap (default 100)
//   node scripts/scan-ad-signals.js --rescan       # re-scan even already-scanned sites

require('dotenv').config();
const db = require('../lib/db');

const UA = 'NationalPaperHangersBot/1.0 (+https://nationalpaperhangers.com/about)';
const SOURCE = 'studio_site_adscan';
const REQUEST_DELAY_MS = 3000;
const COMMIT = process.argv.includes('--commit');
const RESCAN = process.argv.includes('--rescan');
const MAX = parseInt((process.argv.find(a => a.startsWith('--max=')) || '--max=100').split('=')[1], 10);

const PIXEL_PATTERNS = {
  google_ads: [
    /googleadservices\.com/i,
    /gtag\(\s*['"]config['"]\s*,\s*['"]AW-/i,
    /google_conversion_id/i,
    /googletagmanager\.com\/gtag\/js\?id=AW-/i
  ],
  google_analytics: [
    /gtag\(\s*['"]config['"]\s*,\s*['"]G-/i,
    /www\.googletagmanager\.com\/gtm\.js/i,
    /www\.google-analytics\.com\/analytics\.js/i
  ],
  meta_pixel: [
    /connect\.facebook\.net\/[^"']*\/fbevents\.js/i,
    /fbq\(\s*['"]init['"]/i,
    /facebook\.com\/tr\?id=/i
  ],
  tiktok_pixel: [
    /analytics\.tiktok\.com/i,
    /ttq\.load\(/i
  ],
  pinterest_tag: [
    /s\.pinimg\.com\/ct\.js/i,
    /pintrk\(\s*['"]load['"]/i
  ],
  linkedin_insight: [
    /snap\.licdn\.com\/li\.lms-analytics/i,
    /_linkedin_partner_id/i
  ],
  twitter_pixel: [
    /static\.ads-twitter\.com\/uwt\.js/i,
    /twq\(\s*['"]init['"]/i
  ],
  reddit_pixel: [
    /www\.redditstatic\.com\/ads\/pixel\.js/i,
    /rdt\(\s*['"]init['"]/i
  ],
  bing_uet: [
    /bat\.bing\.com/i,
    /uetq\.push/i
  ],
  snap_pixel: [
    /sc-static\.net\/scevent/i,
    /snaptr\(\s*['"]init['"]/i
  ]
};

function detectSignals(html) {
  const out = {};
  for (const [k, patterns] of Object.entries(PIXEL_PATTERNS)) {
    out[k] = patterns.some(p => p.test(html));
  }
  // paid_ads_count excludes pure-analytics signals (GA)
  const paidPlatforms = [
    'google_ads', 'meta_pixel', 'tiktok_pixel', 'pinterest_tag',
    'linkedin_insight', 'twitter_pixel', 'reddit_pixel', 'bing_uet', 'snap_pixel'
  ];
  out.paid_ads_count = paidPlatforms.filter(k => out[k]).length;
  return out;
}

async function logRequest(url, status, bytes, durMs, error) {
  await db.query(
    `INSERT INTO scrape_log (source, url, http_status, bytes, duration_ms, error)
     VALUES ($1,$2,$3,$4,$5,$6)`,
    [SOURCE, url, status, bytes, durMs, error]
  );
}

function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

async function main() {
  console.log(`[ad-scan] start · commit=${COMMIT} · rescan=${RESCAN} · max=${MAX}`);

  const filter = RESCAN ? '' : 'AND ad_signals IS NULL';
  const targets = await db.many(
    `SELECT id, slug, business_name, website
       FROM installers
      WHERE claim_status='unclaimed'
        AND website IS NOT NULL AND website <> ''
        ${filter}
      ORDER BY id
      LIMIT $1`,
    [MAX]
  );

  console.log(`[ad-scan] ${targets.length} candidates`);

  let scanned = 0, paidCount = 0, failed = 0;
  for (const t of targets) {
    const t0 = Date.now();
    let status = 0, html = null, bytes = 0, err = null;
    try {
      const r = await fetch(t.website, {
        headers: { 'user-agent': UA, accept: 'text/html' },
        signal: AbortSignal.timeout(15000),
        redirect: 'follow'
      });
      status = r.status;
      if (r.ok) html = await r.text();
      bytes = html ? html.length : 0;
    } catch (e) {
      err = e.message;
    }
    await logRequest(t.website, status, bytes, Date.now() - t0, err);

    if (!html) {
      console.log(`  ✗ ${t.business_name}: ${status} ${err || ''}`);
      failed++;
      // Still record an empty signals object so we don't keep retrying
      if (COMMIT) {
        await db.query(
          `UPDATE installers SET ad_signals=$2, ad_signals_at=now() WHERE id=$1`,
          [t.id, JSON.stringify({ fetch_failed: true, paid_ads_count: 0 })]
        );
      }
      await sleep(REQUEST_DELAY_MS);
      continue;
    }

    const signals = detectSignals(html);
    scanned++;
    if (signals.paid_ads_count > 0) paidCount++;

    const platforms = Object.entries(signals)
      .filter(([k, v]) => v === true && !['google_analytics'].includes(k))
      .map(([k]) => k.replace(/_/g, ' '));

    if (platforms.length > 0) {
      console.log(`  💰 ${t.business_name}: [${platforms.join(', ')}]`);
    } else if (signals.google_analytics) {
      console.log(`  · ${t.business_name}: GA only (no paid ads)`);
    } else {
      console.log(`  · ${t.business_name}: no tracking detected`);
    }

    if (COMMIT) {
      await db.query(
        `UPDATE installers SET ad_signals=$2, ad_signals_at=now() WHERE id=$1`,
        [t.id, JSON.stringify(signals)]
      );
    }

    await sleep(REQUEST_DELAY_MS);
  }

  console.log(`[ad-scan] done · scanned=${scanned} runningPaidAds=${paidCount} failed=${failed}`);
  if (failed / Math.max(1, targets.length) > 0.10) {
    console.warn(`[ad-scan] FAILURE-RATE-WARNING: ${(failed/targets.length*100).toFixed(0)}%`);
  }
  await db.pool.end();
}

main().catch(e => { console.error(e); process.exit(1); });