`src/enrich/gtm_followthrough.ts`

180 lines
/**
 * GTM second-stage fetcher.
 *
 * For sites where the front-page HTML reveals a GTM container but no
 * direct paid-ad pixels (gtm_container=true, paid_ads_count=0), fetch
 * the GTM bundle and re-apply the pixel patterns to the bundle's tag
 * definitions. Catches the Starbucks/Discount Tire / Planet Fitness
 * class — big retailers who load every pixel through GTM.
 *
 * No headless browser. Bundle URL is public, response is plaintext JS.
 *
 * Persisted under ad_signals.gtm_followthrough = {
 *   fetched_at, container_id, platforms_added: string[],
 *   bundle_size_kb, status: 'ok' | 'no_container_id' | 'fetch_error' | 'parse_error'
 * }
 *
 * paid_ads_count is bumped if new platforms are detected.
 */
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { query, pool } from '../../db/pool.ts';

interface Row {
  business_id: number;
  name: string;
  raw_html_path: string;
  ad_signals: any;
}

const GTM_ID_RE = /GTM-[A-Z0-9]+/i;

const PIXEL_PATTERNS: Array<[string, RegExp]> = [
  ['google_ads',       /googleadservices\.com|AW-\d|google_conversion_id/i],
  ['meta_pixel',       /connect\.facebook\.net\/[^"']*\/fbevents\.js|fbevents|facebook\.com\/tr/i],
  ['tiktok_pixel',     /analytics\.tiktok\.com|business-api\.tiktok\.com|TiktokAnalyticsObject/i],
  ['pinterest_tag',    /s\.pinimg\.com\/ct|pintrk/i],
  ['linkedin_insight', /snap\.licdn\.com\/li\.lms-analytics|_linkedin_(data_)?partner_id/i],
  ['twitter_pixel',    /static\.ads-twitter\.com|analytics\.twitter\.com\/i\/adsct|t\.co\/i\/adsct/i],
  ['reddit_pixel',     /redditstatic\.com\/ads\/pixel|RedditPixel/i],
  ['bing_uet',         /bat\.bing\.com|bat\.r\.msn\.com/i],
  ['snap_pixel',       /sc-static\.net\/scevent|snaptr/i],
];

async function fetchGtmBundle(containerId: string, timeoutMs = 8000): Promise<string | null> {
  try {
    const url = `https://www.googletagmanager.com/gtm.js?id=${containerId}`;
    const ctrl = new AbortController();
    const t = setTimeout(() => ctrl.abort(), timeoutMs);
    const r = await fetch(url, {
      signal: ctrl.signal,
      headers: { 'User-Agent': 'ventura-corridor/0.1 (steveabramsdesigns@gmail.com)' },
    });
    clearTimeout(t);
    if (!r.ok) return null;
    return await r.text();
  } catch {
    return null;
  }
}

function detectInBundle(bundle: string): string[] {
  const found: string[] = [];
  for (const [name, re] of PIXEL_PATTERNS) {
    if (re.test(bundle)) found.push(name);
  }
  return found;
}

async function main() {
  const r = await query<Row>(`
    SELECT
      be.business_id,
      b.name,
      (SELECT raw_html_path FROM front_page_audits a
         WHERE a.business_id = be.business_id AND a.raw_html_path IS NOT NULL
         ORDER BY a.audited_at DESC LIMIT 1) AS raw_html_path,
      be.ad_signals
    FROM business_enrichment be
    JOIN businesses b ON b.id = be.business_id
    WHERE (be.ad_signals->>'gtm_container')::boolean = true
      AND (be.ad_signals->>'paid_ads_count')::int = 0
    ORDER BY b.name
  `);

  console.log(`[gtm-ft] ${r.rowCount} GTM-only candidates`);

  let okCount = 0, addedCount = 0, noIdCount = 0, fetchErrCount = 0;
  const platformDeltas: Record<string, number> = {};

  for (const row of r.rows) {
    const rel = row.raw_html_path?.startsWith('data/') ? row.raw_html_path : 'data/' + row.raw_html_path;
    let html: string;
    try {
      html = readFileSync(join(process.cwd(), rel), 'utf8');
    } catch {
      continue;
    }
    const m = html.match(GTM_ID_RE);
    if (!m) {
      noIdCount++;
      await persistResult(row.business_id, row.ad_signals, {
        status: 'no_container_id',
        platforms_added: [],
        container_id: null,
        bundle_size_kb: 0,
      });
      continue;
    }
    const containerId = m[0].toUpperCase();
    const bundle = await fetchGtmBundle(containerId);
    if (!bundle) {
      fetchErrCount++;
      await persistResult(row.business_id, row.ad_signals, {
        status: 'fetch_error',
        platforms_added: [],
        container_id: containerId,
        bundle_size_kb: 0,
      });
      continue;
    }
    const platforms = detectInBundle(bundle);
    okCount++;
    if (platforms.length > 0) addedCount++;
    for (const p of platforms) platformDeltas[p] = (platformDeltas[p] || 0) + 1;
    console.log(`  ${row.name.padEnd(42)} GTM=${containerId.padEnd(15)} +[${platforms.join(',')}]`);
    await persistResult(row.business_id, row.ad_signals, {
      status: 'ok',
      platforms_added: platforms,
      container_id: containerId,
      bundle_size_kb: Math.round(bundle.length / 1024),
    });
    // Throttle a touch to be polite to GTM
    await new Promise(r => setTimeout(r, 250));
  }

  console.log(`\n[gtm-ft] processed: ${okCount} fetched OK · ${addedCount} gained at least one platform`);
  console.log(`[gtm-ft]            ${noIdCount} no_container_id · ${fetchErrCount} fetch_error`);
  console.log('[gtm-ft] platform delta (new detections):');
  for (const [p, n] of Object.entries(platformDeltas).sort((a, b) => b[1] - a[1])) {
    console.log(`  ${p.padEnd(20)} +${n}`);
  }
  await pool.end();
}

async function persistResult(
  businessId: number,
  existingSignals: any,
  followthrough: { status: string; platforms_added: string[]; container_id: string | null; bundle_size_kb: number },
) {
  const platformsToBump = followthrough.platforms_added;
  // Build a partial overlay that bumps the platform booleans + paid_ads_count + adds gtm_followthrough
  const overlay: Record<string, unknown> = {
    gtm_followthrough: { ...followthrough, fetched_at: new Date().toISOString() },
  };
  for (const p of platformsToBump) overlay[p] = true;

  // Recompute paid_ads_count: existing flags OR new flags
  const merged = { ...existingSignals, ...overlay };
  const paidKeys = ['google_ads','meta_pixel','tiktok_pixel','pinterest_tag','linkedin_insight','twitter_pixel','reddit_pixel','bing_uet','snap_pixel'];
  let paidCount = 0;
  for (const k of paidKeys) if (merged[k]) paidCount++;
  overlay.paid_ads_count = paidCount;

  await query(
    `
    INSERT INTO business_enrichment (business_id, ad_signals, signals_updated_at)
    VALUES ($1, $2, now())
    ON CONFLICT (business_id) DO UPDATE
    SET ad_signals = COALESCE(business_enrichment.ad_signals, '{}'::jsonb) || EXCLUDED.ad_signals,
        signals_updated_at = now()
    `,
    [businessId, JSON.stringify(overlay)],
  );
}

main().catch((e) => {
  console.error('[gtm-ft]', e);
  process.exit(1);
});