`src/enrich/website_via_ddg.ts`

184 lines
/**
 * Website discovery via DuckDuckGo HTML endpoint (html.duckduckgo.com/html).
 *
 * No API key, no monthly quota — polite alternative when Brave's free
 * tier is exhausted. Lower precision than Brave's API (parses raw HTML),
 * but we apply the same domain-credibility filter so false positives
 * collapse to "no winner found".
 *
 * Run:
 *   npm run enrich:websites:ddg                       # default 30, high-value categories
 *   tsx src/enrich/website_via_ddg.ts -- --limit=100
 *   tsx src/enrich/website_via_ddg.ts -- --limit=50 --all   # any BTRC row
 *
 * Polite: 2.5s gap between queries, 8s timeout, custom UA.
 */
import * as cheerio from 'cheerio';
import { query, pool } from '../../db/pool.ts';

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 ventura-corridor/0.1 (website-discovery; +https://leads.venturaclaw.com)';
const GAP_MS = parseInt(process.env.DDG_GAP_MS || '2500', 10);
const TIMEOUT_MS = parseInt(process.env.DDG_TIMEOUT_MS || '8000', 10);

const BLOCK = new Set([
  // Social
  'yelp.com', 'facebook.com', 'instagram.com', 'twitter.com', 'x.com',
  'linkedin.com', 'tiktok.com', 'youtube.com', 'pinterest.com', 'reddit.com',
  // Maps / aggregators / dirs
  'mapquest.com', 'google.com', 'maps.google.com', 'foursquare.com',
  'tripadvisor.com', 'opentable.com', 'doordash.com', 'ubereats.com',
  'grubhub.com', 'wikipedia.org', 'wikidata.org',
  'angi.com', 'angieslist.com', 'thumbtack.com', 'alignable.com',
  'yellowpages.com', 'whitepages.com', 'apollo.io', 'zoominfo.com',
  'crunchbase.com', 'spokeo.com', 'fastpeoplesearch.com',
  // Real estate / business records
  'zillow.com', 'realtor.com', 'redfin.com', 'apartments.com',
  'bbb.org', 'manta.com', 'bizapedia.com', 'opencorporates.com',
  'dnb.com', 'corporationwiki.com',
  // Reviews / health / legal
  'avvo.com', 'lawyers.com', 'martindale.com', 'findlaw.com',
  'healthgrades.com', 'zocdoc.com', 'vitals.com', 'webmd.com',
  // Jobs / news (we want the biz site, not a press hit)
  'glassdoor.com', 'indeed.com', 'duckduckgo.com',
  // Gov
  'lacity.org', 'data.lacity.org', 'lavote.gov', 'lacounty.gov', 'dca.lacity.org'
]);

function host(u: string): string | null {
  try { return new URL(u).hostname.replace(/^www\./, '').toLowerCase(); } catch { return null; }
}

function credible(u: string): boolean {
  const h = host(u);
  if (!h) return false;
  if (BLOCK.has(h)) return false;
  // Block any subdomain of a blocked root.
  for (const root of BLOCK) if (h === root || h.endsWith('.' + root)) return false;
  if (h.endsWith('.gov') || h.endsWith('.gov.us')) return false;
  return true;
}

function nameTokens(name: string): string[] {
  return name.toLowerCase().replace(/[^a-z0-9 ]/g, ' ').split(/\s+/).filter(t => t.length >= 3 && !['the','and','for','inc','llc','corp','of'].includes(t));
}

function domainMatchesName(host: string, name: string): boolean {
  const stem = host.split('.').slice(0, -1).join('').replace(/[^a-z0-9]/g, '');
  const tokens = nameTokens(name);
  if (!tokens.length) return false;
  // Strict: at least one name-token of >=4 chars appears in the domain stem
  return tokens.some(t => t.length >= 4 && stem.includes(t));
}

const HIGH_VALUE_CATEGORY_PREFIXES = [
  'amenity: restaurant', 'amenity: fast_food', 'amenity: cafe',
  'amenity: bar', 'amenity: pub', 'amenity: nightclub',
  'shop: hairdresser', 'shop: clothes', 'shop: shoes', 'shop: jewelry',
  'shop: bakery', 'shop: butcher', 'shop: coffee', 'shop: tea',
  'shop: cosmetics', 'shop: beauty', 'shop: tattoo', 'shop: pet',
  'shop: bicycle', 'shop: bookstore', 'shop: convenience',
  'shop: department_store', 'shop: gift', 'shop: hardware',
  'shop: hifi', 'shop: optician', 'shop: photo', 'shop: stationery',
  'shop: supermarket', 'shop: variety_store',
  'leisure: fitness_centre', 'leisure: dance', 'leisure: bowling_alley',
  'amenity: dentist', 'amenity: pharmacy', 'amenity: veterinary',
  'tourism: hotel', 'tourism: motel', 'tourism: apartment',
  'amenity: car_wash', 'shop: car_parts',
];

interface Row { id: number; name: string; city: string | null }

async function pick(limit: number, mode: 'high_value' | 'all'): Promise<Row[]> {
  if (mode === 'high_value') {
    const conds = HIGH_VALUE_CATEGORY_PREFIXES.map((_, i) => `category ILIKE $${i + 2}`).join(' OR ');
    const params: any[] = [limit, ...HIGH_VALUE_CATEGORY_PREFIXES.map(p => p + '%')];
    const r = await query<Row>(`
      SELECT id, name, city FROM businesses
      WHERE on_corridor AND website IS NULL
        AND length(name) >= 5
        AND name NOT LIKE '% LLC'
        AND name NOT ILIKE 'POST OFFICE BOX%'
        AND (${conds})
      ORDER BY id
      LIMIT $1
    `, params);
    return r.rows;
  }
  const r = await query<Row>(`
    SELECT id, name, city FROM businesses
    WHERE on_corridor AND website IS NULL
      AND length(name) >= 5
      AND name NOT LIKE '% LLC'
      AND name NOT ILIKE 'POST OFFICE BOX%'
    ORDER BY id LIMIT $1
  `, [limit]);
  return r.rows;
}

async function ddgSearch(q: string): Promise<string[]> {
  // html.duckduckgo.com renders <a class="result__a"> with the target URL in
  // the href (as a redirect through /l/?uddg=...). We unwrap that.
  const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(q);
  const res = await fetch(url, {
    headers: { 'User-Agent': UA, 'Accept': 'text/html', 'Accept-Language': 'en-US,en;q=0.9' },
    redirect: 'follow',
    signal: AbortSignal.timeout(TIMEOUT_MS)
  });
  if (!res.ok) throw new Error(`ddg ${res.status}`);
  const html = await res.text();
  const $ = cheerio.load(html);
  const out: string[] = [];
  $('a.result__a').each((_, el) => {
    const href = String($(el).attr('href') || '');
    if (!href) return;
    // DDG sometimes returns absolute URLs, sometimes wraps them in /l/?uddg=...
    let abs = href;
    try {
      const u = new URL(href, 'https://duckduckgo.com');
      const uddg = u.searchParams.get('uddg');
      if (uddg) abs = decodeURIComponent(uddg);
    } catch {}
    if (/^https?:\/\//i.test(abs)) out.push(abs);
  });
  return out.slice(0, 12);
}

async function main() {
  const argLimit = process.argv.find(a => a.startsWith('--limit='));
  const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 30;
  const mode = process.argv.includes('--all') ? 'all' : 'high_value';
  const queue = await pick(limit, mode);
  console.log(`[ddg] enriching ${queue.length} rows · category=${mode} · throttling ${GAP_MS}ms/query`);
  let ok = 0, miss = 0, err = 0;
  for (let i = 0; i < queue.length; i++) {
    const row = queue[i];
    const q = `"${row.name}" Ventura Blvd ${row.city || 'Sherman Oaks'}`;
    try {
      const urls = await ddgSearch(q);
      const winner = urls.find(u => {
        if (!credible(u)) return false;
        const h = host(u);
        return h && domainMatchesName(h, row.name);
      });
      if (winner) {
        const h = host(winner);
        const website = 'https://' + h;
        await query(`UPDATE businesses SET website = $1, last_seen_at = now() WHERE id = $2`, [website, row.id]);
        ok++;
        console.log(`  [${String(i + 1).padStart(3)}/${queue.length}] ✓ ${row.name.padEnd(40).slice(0, 40)} → ${website}`);
      } else {
        miss++;
        console.log(`  [${String(i + 1).padStart(3)}/${queue.length}] · ${row.name.padEnd(40).slice(0, 40)} (no credible match)`);
      }
    } catch (e: any) {
      err++;
      console.log(`  [${String(i + 1).padStart(3)}/${queue.length}] ⚠ ${row.name.slice(0, 40)} ${e.message?.slice(0, 60)}`);
    }
    await new Promise(r => setTimeout(r, GAP_MS));
  }
  console.log(`[ddg] done · ${ok} found · ${miss} no-result · ${err} errors`);
  await pool.end();
}

main().catch(e => { console.error(e); pool.end(); process.exit(1); });