`scripts/check_prospect_domains.js`

158 lines
#!/usr/bin/env node
/**
 * Domain availability check for small-biz prospects.
 *
 * For each org with no website, slugify the name → candidate.com, check
 * availability via Verisign RDAP (free, no key, public). Persist back to
 * organizations.suggested_domain / domain_available / domain_checked_at.
 *
 * RDAP semantics:
 *   404 → available (or never registered; unallocated)
 *   200 → registered
 *   429 → rate-limited (retry with backoff)
 *
 * Run:
 *   node scripts/check_prospect_domains.js                  # all prospects, all categories
 *   LIMIT=200 node scripts/check_prospect_domains.js        # small batch
 *   CATEGORY=cosmetology node scripts/check_prospect_domains.js
 */
const { fetch } = require('undici');
const { pool, query } = require('../agents/shared/db');

const CATEGORY_TYPES = {
  cosmetology:     ['salon','barbershop','nail_salon','beauty_supply','spa'],
  chiropractic:    ['chiropractor_office'],
  chinese_medicine:['acupuncture_clinic','tcm_clinic','tcm_herbalist'],
  dental:          ['dental_office'],
  optometry:       ['optometrist_office','optical_shop'],
  fitness:         ['yoga_studio','pilates_studio','gym'],
  tattoo:          ['tattoo_studio','piercing_studio'],
  pets:            ['vet_clinic','pet_grooming','pet_shop'],
};

const RDAP_COM = 'https://rdap.verisign.com/com/v1/domain';
const RPS = parseFloat(process.env.RDAP_RPS || '5');   // 5 req/s — Verisign tolerates it
const USER_AGENT = process.env.USER_AGENT
  || 'ProfessionalDirectoryBot/0.1 (small-biz prospect tool; contact: steve@designerwallcoverings.com)';

const STOP_WORDS = new Set([
  'the','and','of','for','llc','inc','co','company','corp','corporation',
  'studio','group','center','centre','clinic','salon','spa','salon','shop','services','office',
]);

function slugify(name) {
  if (!name) return null;
  // 1) take alphanumerics only, treat hyphen/space as boundary
  let s = name.toLowerCase()
    .replace(/&/g, ' and ')
    .replace(/[^a-z0-9\s-]/g, ' ')
    .trim();
  // 2) drop common stop words but keep meaningful tokens
  const tokens = s.split(/[\s-]+/).filter(t => t && !STOP_WORDS.has(t));
  if (tokens.length === 0) return null;
  // 3) join, drop the leading 'a'/'i' digits etc
  let joined = tokens.join('');
  // 4) cap at 63 chars (RFC 1035 label limit)
  if (joined.length < 3 || joined.length > 63) return null;
  // 5) start/end alphanumeric
  if (!/^[a-z0-9]/.test(joined) || !/[a-z0-9]$/.test(joined)) return null;
  return joined;
}

let lastFetch = 0;
async function gate() {
  const minInterval = 1000 / RPS;
  const wait = Math.max(0, lastFetch + minInterval - Date.now());
  if (wait > 0) await new Promise(r => setTimeout(r, wait));
  lastFetch = Date.now();
}

async function checkRdap(slug) {
  await gate();
  const url = `${RDAP_COM}/${slug}.com`;
  try {
    const res = await fetch(url, {
      headers: { 'User-Agent': USER_AGENT, Accept: 'application/json' },
      signal: AbortSignal.timeout(10000),
    });
    if (res.status === 404) return { available: true };
    if (res.status === 200) return { available: false };
    if (res.status === 429) {
      await new Promise(r => setTimeout(r, 5000));
      return null; // signal retry-later
    }
    return null;
  } catch (e) {
    return null;
  }
}

async function loadProspects(limit, category) {
  const where = ["(website IS NULL OR website = '')",
                 "name IS NOT NULL",
                 "domain_checked_at IS NULL",
                 "opted_out = FALSE"];
  const params = []; let i = 1;
  if (category && CATEGORY_TYPES[category]) {
    where.push(`type = ANY($${i}::text[])`); params.push(CATEGORY_TYPES[category]); i++;
  } else {
    // Default: only the small-biz types
    const allTypes = Object.values(CATEGORY_TYPES).flat();
    where.push(`type = ANY($${i}::text[])`); params.push(allTypes); i++;
  }
  const r = await query(
    `SELECT id, name FROM organizations WHERE ${where.join(' AND ')} ORDER BY id LIMIT ${Number(limit) || 1000000}`,
    params
  );
  return r.rows;
}

async function persist(orgId, slug, available) {
  await query(`
    UPDATE organizations
       SET suggested_domain  = $2,
           domain_available  = $3,
           domain_checked_at = NOW()
     WHERE id = $1
  `, [orgId, slug ? `${slug}.com` : null, available]);
}

async function main() {
  const limit = process.env.LIMIT ? Number(process.env.LIMIT) : null;
  const category = process.env.CATEGORY || null;
  console.log(`[domains] mode=${category || 'all'} limit=${limit || 'all'} rps=${RPS}`);

  const prospects = await loadProspects(limit, category);
  console.log(`[domains] candidates=${prospects.length}`);

  let checked = 0, available = 0, taken = 0, skipped = 0, errors = 0;
  for (const p of prospects) {
    const slug = slugify(p.name);
    if (!slug) { await persist(p.id, null, null); skipped++; continue; }

    let result = await checkRdap(slug);
    if (result === null) {
      // One retry on transient.
      await new Promise(r => setTimeout(r, 1000));
      result = await checkRdap(slug);
    }
    if (result === null) { errors++; continue; }
    await persist(p.id, slug, result.available);
    checked++;
    if (result.available) available++; else taken++;

    if (checked % 50 === 0) {
      console.log(`[domains] checked=${checked} available=${available} taken=${taken} skipped=${skipped} errors=${errors}`);
    }
  }

  console.log(`[domains] done. checked=${checked} available=${available} taken=${taken} skipped=${skipped} errors=${errors}`);
  await pool.end();
}

main().catch(async (err) => {
  console.error('[domains] fatal:', err);
  try { await pool.end(); } catch (_) {}
  process.exit(1);
});