`scripts/broker-ad-signals.mjs`

113 lines
#!/usr/bin/env node
// TK-10708  Broker intel crawl: for every CA commercial broker, mine their page code for
// (a) ANY ad-platform tracking pixel (Google/Bing/Meta/LinkedIn/TikTok/Twitter/Pinterest/
// Reddit/Snap/Amazon/Criteo/Taboola/Outbrain/AdRoll/DoubleClick) and (b) flyer / OM / PDF
// links on their site. Slow, polite, READ-ONLY public pages. Outputs JSON + CSV.
//
// Usage: node scripts/broker-ad-signals.mjs [--n 300] [--delay 2500] [--state CA]

import { execFileSync } from 'node:child_process';
import { writeFileSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const args = process.argv.slice(2);
const getArg = (f, d) => { const i = args.indexOf(f); return i >= 0 ? args[i + 1] : d; };
const N = parseInt(getArg('--n', '400'), 10);
const DELAY = parseInt(getArg('--delay', '2500'), 10);
const STATE = getArg('--state', 'CA');
const JSONF = join(ROOT, 'public', 'advertising', 'broker-ad-signals.json');
const CSVF = join(ROOT, 'public', 'advertising', 'broker-ad-signals.csv');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
const sleep = ms => new Promise(r => setTimeout(r, ms));

// ANY ad platform — signature lives in the page code (Steve). Bing included + native/programmatic.
const AD = {
  google_ads: /googleadservices\.com|AW-\d{6,}|google_conversion_id|gtag\(\s*['"]config['"]\s*,\s*['"]AW-|googleads\.g\.doubleclick/i,
  bing_ads: /bat\.bing\.com|\buetq\b|uet_bing|\/bat\.js/i,
  meta: /connect\.facebook\.net\/[^"']*fbevents\.js|fbq\(\s*['"]init['"]/i,
  linkedin: /snap\.licdn\.com\/li\.lms-analytics|_linkedin_partner_id/i,
  tiktok: /analytics\.tiktok\.com|ttq\.load/i,
  twitter: /static\.ads-twitter\.com\/uwt\.js|twq\(\s*['"]init['"]/i,
  pinterest: /s\.pinimg\.com\/ct\.js|pintrk\(\s*['"]load['"]/i,
  reddit: /redditstatic\.com\/ads\/pixel\.js|rdt\(\s*['"]init['"]/i,
  snap: /sc-static\.net\/scevent|snaptr\(\s*['"]init['"]/i,
  amazon: /amazon-adsystem\.com|amzn\.to\/ads/i,
  criteo: /criteo\.(com|net)|Criteo_/i,
  taboola: /taboola\.com\/libtrc|_tfa\b/i,
  outbrain: /outbrain\.com\/outbrain\.js|obApi\(/i,
  adroll: /adroll\.com|__adroll/i,
  doubleclick: /doubleclick\.net|dv360|fls\.doubleclick/i,
};
const google_analytics = /gtag\(\s*['"]config['"]\s*,\s*['"]G-|GTM-[A-Z0-9]+|google-analytics\.com\/analytics\.js|googletagmanager\.com\/gtag/i;
const statcounter = /statcounter\.com|sc_project|StatCounter/i;
const PLATS = Object.keys(AD);

// flyer / OM / PDF links in the page code
function flyers(html, base) {
  const out = new Set();
  const re = /href\s*=\s*["']([^"']+)["']/gi; let m;
  while ((m = re.exec(html))) {
    const href = m[1]; const low = href.toLowerCase();
    const ctx = html.slice(Math.max(0, m.index - 80), m.index + 120).toLowerCase();
    const isPdf = /\.pdf(\?|#|$)/.test(low);
    const looksFlyer = /flyer|brochure|offering[-_ ]?mem|\bom\b|marketing[-_ ]?package|e-?brochure|property[-_ ]?package|listing[-_ ]?flyer|tour[-_ ]?book|deal[-_ ]?sheet|for[-_ ]?sale/.test(low + ' ' + ctx);
    if (isPdf || (looksFlyer && /\.pdf|\/download|\/flyer|\/brochure|\/om/.test(low))) {
      try { out.add(new URL(href, base).href); } catch {}
    }
  }
  return [...out].slice(0, 8);
}

const stateClause = STATE === 'ALL' ? '' : `AND hq_state='${STATE}'`;
const rows = execFileSync('psql', ['usre', '-t', '-A', '-F', '\t', '-c',
  `SELECT coalesce(name,''), website FROM firm WHERE asset_class='commercial' AND website IS NOT NULL AND website<>'' ${stateClause} ORDER BY agent_count DESC NULLS LAST LIMIT ${N * 3}`],
  { encoding: 'utf8' }).trim().split('\n').filter(Boolean).map(l => l.split('\t'));
const domOf = u => { try { return new URL(u.startsWith('http') ? u : 'https://' + u).hostname.replace(/^www\./, ''); } catch { return null; } };
const seen = new Set(), targets = [];
for (const [name, web] of rows) { const d = domOf(web); if (d && !seen.has(d)) { seen.add(d); targets.push({ name, domain: d }); } if (targets.length >= N) break; }

console.log(`Broker intel crawl: ${targets.length} ${STATE} commercial brokers, ${DELAY}ms apart (slow).`);
const results = [];
for (const t of targets) {
  const rec = { firm: t.name, domain: t.domain };
  try {
    const res = await fetch('https://' + t.domain + '/', { headers: { 'User-Agent': UA }, redirect: 'follow', signal: AbortSignal.timeout(15000) });
    const html = await res.text();
    // GTM injects pixels client-side that never appear in raw HTML — fetch the container(s) and scan them too.
    const gtmIds = [...new Set((html.match(/GTM-[A-Z0-9]+/g) || []))].slice(0, 3);
    let gtmBlob = '';
    for (const id of gtmIds) {
      try {
        const g = await fetch(`https://www.googletagmanager.com/gtm.js?id=${id}`, { headers: { 'User-Agent': UA }, signal: AbortSignal.timeout(10000) });
        if (g.ok) gtmBlob += '\n' + await g.text();
      } catch {}
    }
    rec.gtm_ids = gtmIds;
    const scan = html + gtmBlob;
    for (const p of PLATS) { rec[p] = AD[p].test(scan); if (rec[p] && !AD[p].test(html)) (rec.via_gtm = rec.via_gtm || []).push(p); }
    rec.paid_count = PLATS.filter(p => rec[p]).length;
    rec.google_analytics = google_analytics.test(html);
    rec.statcounter = statcounter.test(html);
    rec.flyer_urls = flyers(html, 'https://' + t.domain + '/');
    rec.flyer_count = rec.flyer_urls.length;
    const on = PLATS.filter(p => rec[p]);
    console.log(`  ${rec.paid_count ? '💰' : '  '}${rec.flyer_count ? '📄' : '  '} ${t.domain.padEnd(32)} ${on.join(',') || '-'}${rec.flyer_count ? '  ('+rec.flyer_count+' flyers)' : ''}`);
  } catch (e) { rec.error = e.message.slice(0, 40); console.log(`  ⚠  ${t.domain.padEnd(32)} ${e.message.slice(0, 30)}`); }
  rec.scanned_at = new Date().toISOString();
  results.push(rec);
  writeFileSync(JSONF, JSON.stringify(results, null, 1));   // resumable
  await sleep(DELAY);
}

// CSV (easy list)
const cols = ['firm', 'domain', ...PLATS, 'paid_count', 'google_analytics', 'statcounter', 'flyer_count', 'flyer_urls'];
const esc = v => { v = v == null ? '' : Array.isArray(v) ? v.join(' ; ') : String(v); return /[",\n]/.test(v) ? '"' + v.replace(/"/g, '""') + '"' : v; };
const csv = [cols.join(','), ...results.map(r => cols.map(c => esc(c === 'google_analytics' || c === 'statcounter' || PLATS.includes(c) ? (r[c] ? 'yes' : '') : r[c])).join(','))].join('\n');
writeFileSync(CSVF, csv);

const adv = results.filter(r => r.paid_count > 0);
console.log(`\nDone. ${results.filter(r => !r.error).length} scanned · ${adv.length} run ANY paid ads · ${results.filter(r => r.google_analytics).length} GA · ${results.filter(r => r.flyer_count > 0).length} have flyers/PDFs on site.`);
console.log(`CSV -> ${CSVF.replace(ROOT + '/', '')} · JSON -> ${JSONF.replace(ROOT + '/', '')}`);