← back to Ventura Corridor

scripts/find-social-bb.cjs

329 lines

/**
 * Ventura Blvd social handle finder via Browserbase.
 *
 * Uses a single Browserbase session to perform Google searches for each business
 * to find Instagram + TikTok handles. Writes checkpoint every 10 businesses.
 *
 * Budget cap: $3 USD (Browserbase billing: ~$0.10/hr session + usage)
 *
 * Usage: node find-social-bb.cjs [--limit N] [--offset N]
 */

'use strict';

require('dotenv').config({ path: require('os').homedir() + '/Projects/secrets-manager/.env' });
const Browserbase = require('/Users/macstudio3/Projects/ventura-corridor/node_modules/@browserbasehq/sdk').default;
const { chromium } = require('/Users/macstudio3/Projects/ventura-corridor/node_modules/playwright-core');
const fs = require('fs');
const path = require('path');

const PROJECT_ID = process.env.BROWSERBASE_PROJECT_ID;
const API_KEY = process.env.BROWSERBASE_API_KEY;

if (!PROJECT_ID || !API_KEY) {
  console.error('[social-bb] Missing BROWSERBASE_PROJECT_ID / BROWSERBASE_API_KEY in secrets-manager/.env');
  process.exit(1);
}

const OUTPUT_DIR = '/Users/macstudio3/Projects/_ventura-recon';
const INPUT_FILE = path.join(OUTPUT_DIR, 'businesses-top600.json');
const CHECKPOINT_FILE = path.join(OUTPUT_DIR, 'social-checkpoint.json');
const RESULTS_FILE = path.join(OUTPUT_DIR, 'social-results.json');

// Parse CLI args
const args = process.argv.slice(2);
const limitArg = args.indexOf('--limit');
const offsetArg = args.indexOf('--offset');
const LIMIT = limitArg >= 0 ? parseInt(args[limitArg + 1]) : 600;
const OFFSET = offsetArg >= 0 ? parseInt(args[offsetArg + 1]) : 0;

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

function cleanName(name) {
  // Remove suite/floor numbers embedded in business names
  return name
    .replace(/\s*#\s*\S+/g, '')
    .replace(/\s+/g, ' ')
    .trim();
}

/**
 * Extract Instagram handle from Google search results page.
 * Looks for instagram.com/{handle} patterns in result links and snippets.
 */
function extractIgHandle(content) {
  // Match instagram.com/HANDLE pattern (not /p/, /reel/, /stories/ etc.)
  const patterns = [
    /instagram\.com\/([a-zA-Z0-9._]{3,30})(?:\/|\s|"|'|\?|$)/g,
  ];
  const skip = new Set(['p', 'reel', 'tv', 'explore', 'accounts', 'stories', 'share', 'direct', 'login', 'about']);

  for (const pat of patterns) {
    let m;
    while ((m = pat.exec(content)) !== null) {
      const handle = m[1];
      if (!skip.has(handle.toLowerCase()) && handle.length >= 3) {
        return handle;
      }
    }
  }
  return null;
}

/**
 * Extract TikTok handle from Google search results page.
 */
function extractTtHandle(content) {
  const pat = /tiktok\.com\/@([a-zA-Z0-9._]{3,30})(?:\/|\s|"|'|\?|$)/g;
  let m;
  while ((m = pat.exec(content)) !== null) {
    const handle = m[1];
    if (handle.length >= 3) return handle;
  }
  return null;
}

async function searchSocial(page, businessName, address) {
  const clean = cleanName(businessName);
  const result = { ig: null, tt: null, ig_conf: 'low', tt_conf: 'low' };

  // --- Instagram search ---
  try {
    const igQuery = `site:instagram.com "${clean}" ventura`;
    await page.goto(`https://www.google.com/search?q=${encodeURIComponent(igQuery)}`, {
      waitUntil: 'domcontentloaded',
      timeout: 15000
    });
    await sleep(1500);

    const igContent = await page.content();
    const igHandle = extractIgHandle(igContent);

    if (igHandle) {
      result.ig = igHandle;
      result.ig_conf = 'high';
    } else {
      // Try without site: filter
      const igQuery2 = `"${clean}" "ventura blvd" instagram`;
      await page.goto(`https://www.google.com/search?q=${encodeURIComponent(igQuery2)}`, {
        waitUntil: 'domcontentloaded',
        timeout: 15000
      });
      await sleep(1200);
      const igContent2 = await page.content();
      const igHandle2 = extractIgHandle(igContent2);
      if (igHandle2) {
        result.ig = igHandle2;
        result.ig_conf = 'medium';
      }
    }
  } catch (e) {
    // search failed
  }

  await sleep(1000);

  // --- TikTok search ---
  try {
    const ttQuery = `site:tiktok.com "@" "${clean}" ventura`;
    await page.goto(`https://www.google.com/search?q=${encodeURIComponent(ttQuery)}`, {
      waitUntil: 'domcontentloaded',
      timeout: 15000
    });
    await sleep(1500);

    const ttContent = await page.content();
    const ttHandle = extractTtHandle(ttContent);

    if (ttHandle) {
      result.tt = ttHandle;
      result.tt_conf = 'high';
    } else {
      const ttQuery2 = `"${clean}" "ventura blvd" tiktok`;
      await page.goto(`https://www.google.com/search?q=${encodeURIComponent(ttQuery2)}`, {
        waitUntil: 'domcontentloaded',
        timeout: 15000
      });
      await sleep(1200);
      const ttContent2 = await page.content();
      const ttHandle2 = extractTtHandle(ttContent2);
      if (ttHandle2) {
        result.tt = ttHandle2;
        result.tt_conf = 'medium';
      }
    }
  } catch (e) {
    // search failed
  }

  return result;
}

async function main() {
  const businesses = JSON.parse(fs.readFileSync(INPUT_FILE, 'utf8'));

  // Load checkpoint
  let checkpoint = {};
  if (fs.existsSync(CHECKPOINT_FILE)) {
    try {
      checkpoint = JSON.parse(fs.readFileSync(CHECKPOINT_FILE, 'utf8'));
    } catch {}
  }

  const slice = businesses.slice(OFFSET, OFFSET + LIMIT);
  const toProcess = slice.filter(b => {
    const key = `${b.business_name}|${b.street_address}`;
    return !checkpoint[key];
  });

  console.log(`[social-bb] ${toProcess.length} businesses to process (offset=${OFFSET}, limit=${LIMIT})`);
  console.log(`[social-bb] ${Object.keys(checkpoint).length} already in checkpoint`);

  if (toProcess.length === 0) {
    console.log('[social-bb] All done — nothing new to process.');
    buildFinalResults(businesses, checkpoint);
    return;
  }

  // Create Browserbase session
  const bb = new Browserbase({ apiKey: API_KEY });
  let session;
  try {
    session = await bb.sessions.create({ projectId: PROJECT_ID });
  } catch (e) {
    console.error('[social-bb] Failed to create Browserbase session:', e.message);
    process.exit(1);
  }

  console.log(`[social-bb] Session: ${session.id}`);
  console.log(`[social-bb] Debug: https://browserbase.com/sessions/${session.id}`);

  let browser, page;
  try {
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    page = ctx.pages()[0];

    // Accept Google cookies/terms if they appear
    try {
      await page.goto('https://www.google.com', { waitUntil: 'domcontentloaded', timeout: 10000 });
      await sleep(1500);
      // Dismiss cookie banner if present
      const acceptBtn = page.locator('button:has-text("Accept all"), button:has-text("I agree")').first();
      if (await acceptBtn.isVisible({ timeout: 2000 }).catch(() => false)) {
        await acceptBtn.click();
        await sleep(1000);
      }
    } catch {}

    let processed = 0;
    let igFound = 0;
    let ttFound = 0;
    const startTime = Date.now();

    for (const biz of toProcess) {
      const key = `${biz.business_name}|${biz.street_address}`;

      process.stdout.write(`[${processed + 1}/${toProcess.length}] ${biz.business_name.substring(0, 45).padEnd(45)} `);

      const social = await searchSocial(page, biz.business_name, biz.street_address);

      if (social.ig) igFound++;
      if (social.tt) ttFound++;

      let confidence = 'low';
      if (social.ig && social.tt) confidence = 'high';
      else if (social.ig || social.tt) confidence = 'medium';

      const record = {
        business_name: biz.business_name,
        address: biz.street_address,
        zip: biz.zip,
        naics: biz.naics,
        type_label: biz.type_label,
        lat: biz.lat || '',
        lon: biz.lon || '',
        instagram_handle: social.ig || '',
        instagram_url: social.ig ? `https://www.instagram.com/${social.ig}/` : '',
        tiktok_handle: social.tt || '',
        tiktok_url: social.tt ? `https://www.tiktok.com/@${social.tt}/` : '',
        confidence,
      };

      checkpoint[key] = record;
      processed++;

      const igStr = social.ig ? `IG:${social.ig.substring(0,20)}` : 'IG:-';
      const ttStr = social.tt ? `TT:${social.tt.substring(0,20)}` : 'TT:-';
      console.log(`${igStr.padEnd(25)} ${ttStr.padEnd(25)}`);

      // Save checkpoint every 10 businesses
      if (processed % 10 === 0) {
        fs.writeFileSync(CHECKPOINT_FILE, JSON.stringify(checkpoint, null, 2));
        const elapsed = Math.round((Date.now() - startTime) / 1000);
        const rate = processed / elapsed;
        const remaining = Math.round((toProcess.length - processed) / rate);
        console.log(`  [checkpoint ${processed}/${toProcess.length} — IG:${igFound} TT:${ttFound} — ~${remaining}s remaining]`);
      }

      // Rate limit: 1.5s minimum between businesses (Google rate limiting)
      await sleep(1500);
    }

  } finally {
    try { await browser.close(); } catch {}
    try { await bb.sessions.update(session.id, { status: 'REQUEST_RELEASE', projectId: PROJECT_ID }); } catch {}
  }

  // Final checkpoint save
  fs.writeFileSync(CHECKPOINT_FILE, JSON.stringify(checkpoint, null, 2));
  buildFinalResults(businesses, checkpoint);
}

function buildFinalResults(businesses, checkpoint) {
  const results = [];
  for (const biz of businesses) {
    const key = `${biz.business_name}|${biz.street_address}`;
    if (checkpoint[key]) {
      results.push(checkpoint[key]);
    } else {
      results.push({
        business_name: biz.business_name,
        address: biz.street_address,
        zip: biz.zip,
        naics: biz.naics,
        type_label: biz.type_label,
        lat: biz.lat || '',
        lon: biz.lon || '',
        instagram_handle: '',
        instagram_url: '',
        tiktok_handle: '',
        tiktok_url: '',
        confidence: 'unprocessed',
      });
    }
  }

  fs.writeFileSync(RESULTS_FILE, JSON.stringify(results, null, 2));

  const igFound = results.filter(r => r.instagram_handle).length;
  const ttFound = results.filter(r => r.tiktok_handle).length;
  const bothFound = results.filter(r => r.instagram_handle && r.tiktok_handle).length;
  const processed = results.filter(r => r.confidence !== 'unprocessed').length;

  console.log('\n' + '='.repeat(60));
  console.log(`Total businesses: ${results.length}`);
  console.log(`Processed: ${processed}`);
  console.log(`Instagram found: ${igFound} (${Math.round(100*igFound/processed||0)}%)`);
  console.log(`TikTok found: ${ttFound} (${Math.round(100*ttFound/processed||0)}%)`);
  console.log(`Both platforms: ${bothFound} (${Math.round(100*bothFound/processed||0)}%)`);
  console.log(`Results saved: ${RESULTS_FILE}`);
}

main().catch(e => {
  console.error('[social-bb] Fatal error:', e);
  process.exit(1);
});