ad-social-tracker on CA CRE brokers: slow rate-limited front-page crawl + paid-ad pixel fingerprinting (Google Ads/Meta/TikTok/LinkedIn/etc) from usre.firm registry; results -> data/broker-ad-signals.json (TK-10708)

7d35a296228dca66109d4ccf57ede812f6b70a69 · 2026-08-19 16:05:45 -0700 · Steve Abrams
Files touched

A scripts/broker-ad-signals.mjs
Diff

commit 7d35a296228dca66109d4ccf57ede812f6b70a69
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 16:05:45 2026 -0700

    ad-social-tracker on CA CRE brokers: slow rate-limited front-page crawl + paid-ad pixel fingerprinting (Google Ads/Meta/TikTok/LinkedIn/etc) from usre.firm registry; results -> data/broker-ad-signals.json (TK-10708)
---
 scripts/broker-ad-signals.mjs | 68 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)

diff --git a/scripts/broker-ad-signals.mjs b/scripts/broker-ad-signals.mjs
new file mode 100644
index 0000000..c5d34bc
--- /dev/null
+++ b/scripts/broker-ad-signals.mjs
@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+// TK-10708  Ad-signal scan for CA/LA commercial brokers (ad-social-tracker Pass 1).
+// Slowly fetches each broker's front page and mines paid-ad tracking pixels (Google Ads,
+// Meta, TikTok, LinkedIn, etc.) via the ad-social-tracker regexes. READ-ONLY public pages,
+// rate-limited ("crawl slowly" — Steve). Competitive intel only (internal, not published).
+//
+// Usage: node scripts/broker-ad-signals.mjs [--n 20] [--delay 3500] [--state CA]
+
+import { execFileSync } from 'node:child_process';
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const args = process.argv.slice(2);
+const getArg = (f, d) => { const i = args.indexOf(f); return i >= 0 ? args[i + 1] : d; };
+const N = parseInt(getArg('--n', '20'), 10);
+const DELAY = parseInt(getArg('--delay', '3500'), 10);   // slow: ~3.5s between fetches
+const STATE = getArg('--state', 'CA');
+const OUT = join(ROOT, 'data', 'broker-ad-signals.json');
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+// ad-social-tracker pixel fingerprints (pure regex, no API/LLM)
+const SIG = {
+  google_ads: /googleadservices\.com|AW-\d{6,}|google_conversion_id|gtag\(\s*['"]config['"]\s*,\s*['"]AW-/i,
+  meta_pixel: /connect\.facebook\.net\/[^"']*fbevents\.js|fbq\(\s*['"]init['"]/i,
+  tiktok_pixel: /analytics\.tiktok\.com|ttq\.load/i,
+  pinterest_tag: /s\.pinimg\.com\/ct\.js|pintrk\(\s*['"]load['"]/i,
+  linkedin_insight: /snap\.licdn\.com\/li\.lms-analytics|_linkedin_partner_id/i,
+  twitter_pixel: /static\.ads-twitter\.com\/uwt\.js|twq\(\s*['"]init['"]/i,
+  reddit_pixel: /redditstatic\.com\/ads\/pixel\.js|rdt\(\s*['"]init['"]/i,
+  bing_uet: /bat\.bing\.com|\buetq\b/i,
+  snap_pixel: /sc-static\.net\/scevent|snaptr\(\s*['"]init['"]/i,
+  google_analytics: /gtag\(\s*['"]config['"]\s*,\s*['"]G-|GTM-[A-Z0-9]+|google-analytics\.com\/analytics\.js/i, // measurement, not paid
+};
+const PAID = ['google_ads', 'meta_pixel', 'tiktok_pixel', 'pinterest_tag', 'linkedin_insight', 'twitter_pixel', 'reddit_pixel', 'bing_uet', 'snap_pixel'];
+
+// distinct commercial broker domains from the usre firm registry
+const rows = execFileSync('psql', ['usre', '-t', '-A', '-c',
+  `SELECT DISTINCT website FROM firm WHERE hq_state='${STATE}' AND asset_class='commercial' AND website IS NOT NULL AND website<>'' LIMIT ${N * 3}`],
+  { encoding: 'utf8' }).trim().split('\n').filter(Boolean);
+const domainOf = u => { try { return new URL(u.startsWith('http') ? u : 'https://' + u).hostname.replace(/^www\./, ''); } catch { return null; } };
+const domains = [...new Set(rows.map(domainOf).filter(Boolean))].slice(0, N);
+
+const prior = existsSync(OUT) ? JSON.parse(readFileSync(OUT, 'utf8')) : {};
+console.log(`Ad-signal scan: ${domains.length} ${STATE} commercial brokers, ${DELAY}ms apart (slow).`);
+
+for (const d of domains) {
+  try {
+    const res = await fetch('https://' + d + '/', { headers: { 'User-Agent': UA }, redirect: 'follow', signal: AbortSignal.timeout(15000) });
+    const html = await res.text();
+    const sig = {}; for (const [k, re] of Object.entries(SIG)) sig[k] = re.test(html);
+    sig.paid_ads_count = PAID.filter(k => sig[k]).length;
+    sig.scanned_at = new Date().toISOString();
+    prior[d] = sig;
+    const on = PAID.filter(k => sig[k]).map(k => k.replace('_pixel', '').replace('_tag', '').replace('_insight', '').replace('_uet', '').replace('_ads', '-ads'));
+    console.log(`  ${sig.paid_ads_count ? '💰' : '  '} ${d.padEnd(34)} ${on.join(', ') || '(no paid pixels)'}`);
+  } catch (e) {
+    prior[d] = { error: e.message.slice(0, 40), scanned_at: new Date().toISOString() };
+    console.log(`  ⚠ ${d.padEnd(34)} ${e.message.slice(0, 30)}`);
+  }
+  writeFileSync(OUT, JSON.stringify(prior, null, 1));   // persist after each (resumable)
+  await sleep(DELAY);
+}
+const scanned = Object.values(prior).filter(v => !v.error);
+const advertisers = scanned.filter(v => v.paid_ads_count > 0).length;
+console.log(`\nDone. ${scanned.length} scanned, ${advertisers} run paid ads. -> data/broker-ad-signals.json`);
← 83ef493 California RE feeds + alt pull method: facebookexternalhit U · back to Re Flyer Aggregator · ad scanner: biggest firms first (agent_count) + Google Analy 57d51c3 →