`scripts/ad_signals_report.cjs`

154 lines
#!/usr/bin/env node
// READ-ONLY ad-social-tracker pass-1 over Ventura Corridor crawled HTML.
// No network, no DB writes, no paid APIs. Pure regex over static HTML on disk.
// Identity enriched (read-only) from /tmp/vc_firm_map.tsv (DB export: name+website only).
const fs = require('fs');
const path = require('path');

const RAW_DIR = path.join(process.env.HOME, 'Projects/ventura-corridor/data/raw');
const OUT_DIR = path.join(process.env.HOME, 'Projects/ventura-corridor/data');
const DATE = '2026-06-14';

const idMap = {};
try {
  const tsv = fs.readFileSync('/tmp/vc_firm_map.tsv', 'utf8');
  for (const line of tsv.split('\n')) {
    if (!line.trim()) continue;
    const [p, name, website] = line.split('\t');
    idMap[path.basename(p)] = { name: name || '', website: website || '' };
  }
} catch (e) {}

const PAID = {
  google_ads:    /googleadservices\.com|gtag\(\s*['"]config['"]\s*,\s*['"]AW-|google_conversion_id|googleads\.g\.doubleclick\.net|\/pagead\/conversion/i,
  meta_pixel:    /connect\.facebook\.net\/[^"']*\/fbevents\.js|fbq\(\s*['"]init['"]/i,
  tiktok_pixel:  /analytics\.tiktok\.com|ttq\.load|ttq\.page/i,
  pinterest_tag: /s\.pinimg\.com\/ct\.js|pintrk\(\s*['"]load['"]/i,
  linkedin_insight: /snap\.licdn\.com\/li\.lms-analytics|_linkedin_partner_id/i,
  twitter_pixel: /static\.ads-twitter\.com\/uwt\.js|twq\(\s*['"](init|config)['"]/i,
  reddit_pixel:  /redditstatic\.com\/ads\/pixel\.js|rdt\(\s*['"]init['"]/i,
  bing_uet:      /bat\.bing\.com|uetq|uet\.gif/i,
  snap_pixel:    /sc-static\.net\/scevent|snaptr\(\s*['"]init['"]/i,
};
const GA = /gtag\(\s*['"]config['"]\s*,\s*['"]G-|www\.googletagmanager\.com\/gtm\.js|GTM-[A-Z0-9]{4,}|google-analytics\.com\/analytics\.js/i;

const SOCIAL = {
  instagram: /https?:\/\/(?:www\.)?instagram\.com\/([A-Za-z0-9_.]+)/ig,
  facebook:  /https?:\/\/(?:www\.|web\.)?facebook\.com\/([A-Za-z0-9_.\-]+)/ig,
  twitter:   /https?:\/\/(?:www\.)?(?:twitter|x)\.com\/([A-Za-z0-9_]+)/ig,
  tiktok:    /https?:\/\/(?:www\.)?tiktok\.com\/(@[A-Za-z0-9_.]+)/ig,
  linkedin:  /https?:\/\/(?:www\.)?linkedin\.com\/(company|in)\/([A-Za-z0-9_.\-%]+)/ig,
  youtube:   /https?:\/\/(?:www\.)?youtube\.com\/(channel\/[A-Za-z0-9_\-]+|c\/[A-Za-z0-9_\-]+|@[A-Za-z0-9_\-]+|user\/[A-Za-z0-9_\-]+)/ig,
  pinterest: /https?:\/\/(?:www\.)?pinterest\.com\/([A-Za-z0-9_\-\/]+)/ig,
};
const SOCIAL_JUNK = /^(sharer|share|sharer\.php|intent|home|login|tr|plugins|p|pin|create|dialog|profile\.php|pages|hashtag|search|watch|embed|v|results|feed|policy|help|about|privacy|legal|settings|gaming|premium|tv|reel|reels|explore|accounts|status)$/i;

function extractSocials(html) {
  const out = {};
  for (const [k, re] of Object.entries(SOCIAL)) {
    re.lastIndex = 0;
    const set = new Set();
    let m;
    while ((m = re.exec(html)) !== null) {
      let handle, url;
      if (k === 'linkedin') { handle = m[2]; url = 'https://www.linkedin.com/' + m[1] + '/' + m[2]; }
      else { handle = m[1]; url = m[0]; }
      if (!handle) continue;
      const tail = handle.split('/')[0];
      if (SOCIAL_JUNK.test(tail)) continue;
      if (tail.length < 2) continue;
      set.add(url.replace(/[")'\\].*$/, '').replace(/\/$/, ''));
      if (set.size >= 3) break;
    }
    if (set.size) out[k] = [...set];
  }
  return out;
}
function decodeEntities(s) {
  return s.replace(/&amp;/g, String.fromCharCode(38)).replace(/&quot;/g, String.fromCharCode(34)).replace(/&#0?39;|&apos;|&#x27;/gi, String.fromCharCode(39)).replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&nbsp;/g, " ");
}
function titleOf(html) {
  const m = html.match(/<title[^>]*>([\s\S]{0,160}?)<\/title>/i);
  return m ? decodeEntities(m[1].replace(/\s+/g, " ").trim()) : "";
}
function canonicalOf(html) {
  let m = html.match(/<link[^>]*rel=["']canonical["'][^>]*href=["']([^"']+)["']/i)
       || html.match(/<meta[^>]*property=["']og:url["'][^>]*content=["']([^"']+)["']/i);
  if (!m) return '';
  try { return new URL(m[1]).hostname.replace(/^www\./, ''); } catch (e) { return ''; }
}

const files = fs.readdirSync(RAW_DIR).filter(f => /^firm-\d+\.html$/.test(f));
const rows = [];
const platformTotals = {}; for (const k of Object.keys(PAID)) platformTotals[k] = 0;
let withGA = 0, withAnyPaid = 0;

for (const f of files) {
  let html;
  try { html = fs.readFileSync(path.join(RAW_DIR, f), 'utf8'); } catch (e) { continue; }
  const paid = {};
  let paidCount = 0;
  for (const [k, re] of Object.entries(PAID)) {
    const hit = re.test(html);
    paid[k] = hit;
    if (hit) { paidCount++; platformTotals[k]++; }
  }
  const ga = GA.test(html);
  if (ga) withGA++;
  if (paidCount > 0) withAnyPaid++;

  const dbid = idMap[f] || {};
  const name = dbid.name || titleOf(html) || f;
  let domain = '';
  try { domain = dbid.website ? new URL(dbid.website).hostname.replace(/^www\./, '') : ''; } catch (e) {}
  if (!domain) domain = canonicalOf(html);

  rows.push({
    firm_file: f, business: name, domain,
    identity_source: dbid.name ? 'db' : 'html',
    paid_ads_count: paidCount, google_analytics: ga, ...paid,
    paid_platforms: Object.entries(paid).filter(([, v]) => v).map(([k]) => k),
    socials: extractSocials(html), bytes: Buffer.byteLength(html),
  });
}

const ranked = [...rows].sort((a, b) =>
  b.paid_ads_count - a.paid_ads_count || b.bytes - a.bytes);

const report = {
  generated: new Date().toISOString(),
  source: 'READ-ONLY pass-1 over data/raw/firm-*.html (no crawl, no API, no DB write)',
  businesses_analyzed: rows.length,
  businesses_running_paid_ads: withAnyPaid,
  businesses_with_ga_only: rows.filter(r => r.paid_ads_count === 0 && r.google_analytics).length,
  platform_breakdown: platformTotals,
  top10_ad_spenders: ranked.filter(r => r.paid_ads_count > 0).slice(0, 10).map(r => ({
    business: r.business, domain: r.domain, paid_ads_count: r.paid_ads_count,
    platforms: r.paid_platforms, socials: Object.keys(r.socials),
  })),
  businesses: rows,
};

fs.writeFileSync(path.join(OUT_DIR, 'ad-signals-' + DATE + '.json'), JSON.stringify(report, null, 2));

const cols = ['firm_file','business','domain','identity_source','paid_ads_count','google_analytics']
  .concat(Object.keys(PAID))
  .concat(['paid_platforms','instagram','facebook','twitter','tiktok','linkedin','youtube','pinterest']);
const esc = v => { const s = String(v == null ? '' : v); return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s; };
const out = [cols.join(',')];
for (const r of rows) {
  out.push(cols.map(c => {
    if (c === 'paid_platforms') return esc(r.paid_platforms.join('|'));
    if (['instagram','facebook','twitter','tiktok','linkedin','youtube','pinterest'].includes(c))
      return esc((r.socials[c] || []).join('|'));
    return esc(r[c]);
  }).join(','));
}
fs.writeFileSync(path.join(OUT_DIR, 'ad-signals-' + DATE + '.csv'), out.join('\n'));

console.log('analyzed:', rows.length);
console.log('paid-ads:', withAnyPaid, '| GA:', withGA);
console.log('platforms:', JSON.stringify(platformTotals));
console.log('top10:');
for (const t of report.top10_ad_spenders) console.log('  ' + t.paid_ads_count + '  ' + t.business + '  [' + t.platforms.join(',') + ']');