`scripts/ingest/local-portals.js`

121 lines
'use strict';

// Local town-portal scrapers for static-HTML CR business directories.
// URL list from EXA-agent research 2026-05-07. Browserbase-required
// portals (TripAdvisor / Booking / Yelp) live in their own scripts.

const { fetchText, slug, sleep, regionMap, resolveRegion, upsertPlace, startRun, finishRun, pool } = require('./_lib');

const PORTALS = [
  { url: 'https://tamarindoguide.com/businesses/',                       region: 'tamarindo',     vertical: 'service_retail', label: 'tamarindoguide',   maxPages: 5 },
  { url: 'https://www.alltama.com/listings',                              region: 'tamarindo',     vertical: 'service_retail', label: 'alltama' },
  { url: 'https://www.tamarindorentals.com/tamarindo-area-directory/',    region: 'tamarindo',     vertical: 'service_retail', label: 'tamarindo-rentals-dir' },
  { url: 'https://caribeconnect.com/table/',                              region: 'puerto-viejo',  vertical: 'service_retail', label: 'caribeconnect' },
  { url: 'https://www.puertoviejosatellite.com/services.php',             region: 'puerto-viejo',  vertical: 'service_retail', label: 'pv-satellite' },
  { url: 'https://www.welcometojaco.com/costa-rica',                      region: 'jaco',          vertical: 'service_retail', label: 'welcometojaco' },
  { url: 'https://www.jacocostaricadirectory.com/',                       region: 'jaco',          vertical: 'service_retail', label: 'jaco-directory' },
  { url: 'https://www.touristinmonteverde.com/tours/?cat=11',             region: 'monteverde',    vertical: 'tourism_tour',   label: 'tourist-monteverde' },
  { url: 'https://camaraturismosamara.com/businesses/',                   region: 'samara',        vertical: 'service_retail', label: 'camara-samara' },
  { url: 'https://www.searchsamara.com/copy-of-leisure-services-1',       region: 'samara',        vertical: 'service_retail', label: 'searchsamara' },
  { url: 'https://nosaranow.com/business-directory/',                     region: 'nosara',        vertical: 'service_retail', label: 'nosaranow' },
];
// Candidate URLs that 404'd or had DNS errors — kept here for posterity / future verification:
// quepolandia.com/category/businesses, arenal.net/business-directory, santateresacostarica.com/businesses,
// thecostaricanews.com/business-directory, govisitcostarica.com/region/business-directory,
// nosaratravel.com/local-businesses, manuelantonioestates.com/businesses, uvitainformationcenter.com,
// cocosbeachcostarica.com/business-directory, dominicaldays.com/business-directory, escazunews.com/category/local-business


const KEYWORD_BLOCK = /(^#|mailto:|tel:|javascript:|^\/?$|\bhome\b|\bcontact\b|\babout\b|\blogin\b|\bregister\b|\bcart\b|\bcheckout\b|\bsearch\b|\bsubscribe\b|\bprivacy\b|\bterms\b|\bsitemap\b|\bblog\b|\bnews\b|\barchive\b)/i;
const KEYWORD_KEEP  = /(business|directory|listing|company|empresa|negocio|hotel|tour|restaurant|restaurante|salon|spa|fitness|surf|yoga|store|tienda|cabin|villa|rental|alquiler|service|servicio|gym|cafe|bar)/i;

function extractFromArticles(html, base) {
  const out = [];
  // Try common WP/listing patterns: <article>, <div class="...listing|business|card...">
  const blockRe = /<(article|div)[^>]+class="[^"]*(listing|business|card|entry|item|post|directory)[^"]*"[^>]*>([\s\S]*?)<\/\1>/gi;
  let m;
  while ((m = blockRe.exec(html)) !== null) {
    const block = m[3];
    const a = block.match(/<a[^>]+href="([^"#]+)"[^>]*>\s*([^<]{2,90})\s*<\/a>/i);
    if (a) {
      const href = a[1].trim();
      const text = a[2].replace(/\s+/g,' ').trim();
      if (text && !KEYWORD_BLOCK.test(href + ' ' + text)) out.push({ href, text });
    }
  }
  if (out.length) return out;
  // Fallback: all anchors with keep-keyword in text or href
  const re = /<a[^>]+href="([^"#]+)"[^>]*>\s*([^<]{3,90})\s*<\/a>/gi;
  while ((m = re.exec(html)) !== null) {
    const href = m[1].trim();
    const text = m[2].replace(/\s+/g,' ').trim();
    if (KEYWORD_BLOCK.test(href + ' ' + text)) continue;
    if (!KEYWORD_KEEP.test(href + ' ' + text)) continue;
    out.push({ href, text });
  }
  return out;
}

function paginatedUrls(p) {
  if (!p.maxPages) return [p.url];
  const urls = [p.url];
  for (let i = 2; i <= p.maxPages; i++) {
    if (/\/$/.test(p.url)) urls.push(`${p.url}page/${i}/`);
    else urls.push(`${p.url}/page/${i}/`);
  }
  return urls;
}

(async () => {
  const runId = await startRun('local-portals', 'Static-HTML CR town-portal directory scrape (EXA-vetted URLs)');
  const rmap = await regionMap();
  let total = 0, added = 0, updated = 0, ok = 0, fail = 0;
  const seen = new Set();

  for (const p of PORTALS) {
    const region = resolveRegion(rmap, p.region);
    let portalRows = 0;
    try {
      for (const url of paginatedUrls(p)) {
        let html;
        try { html = await fetchText(url); }
        catch (e) { console.warn(`[local-portals] ${p.label} ${url} → ${e.message}`); continue; }
        const anchors = extractFromArticles(html, url);
        const host = new URL(p.url).host;

        for (const a of anchors) {
          let absHref;
          try { absHref = new URL(a.href, url).toString(); } catch { continue; }
          if (new URL(absHref).host !== host) continue;
          const placeSlug = slug(`${p.label}-${a.text}`);
          if (seen.has(placeSlug)) continue;
          seen.add(placeSlug);
          const r = await upsertPlace({
            slug: placeSlug,
            name: a.text,
            category: p.vertical.startsWith('tourism_') ? 'tourism' : 'service',
            vertical: p.vertical,
            region_id: region.id,
            description: `Listed on ${p.label} (${region.name}).`,
            source: p.label,
            source_url: absHref,
          });
          if (r.inserted) added++; else updated++;
          total++;
          portalRows++;
        }
        await sleep(700);
      }
      ok++;
      console.log(`[local-portals] ${p.label} rows=${portalRows}`);
    } catch (e) {
      fail++;
      console.warn(`[local-portals] ${p.label} err: ${e.message}`);
    }
  }

  await finishRun(runId, { rows_in: total, rows_added: added, rows_updated: updated, status: fail ? 'partial' : 'ok', notes: `portals_ok=${ok} portals_fail=${fail}` });
  console.log(`[local-portals] DONE in=${total} added=${added} updated=${updated} portals_ok=${ok} portals_fail=${fail}`);
  await pool.end();
})();