← back to Costa Rica
scripts/cr-portal-crawl.js
227 lines
#!/usr/bin/env node
// cr-portal-crawl.js — FREE ($0) enrichment: extract real business website/phone/email
// from the local-portal detail pages our places were scraped from.
// Sources: tamarindoguide (per-listing pages), pv-satellite (/en/<area>/<biz>/ detail
// pages only — category/nav URLs skipped), welcometojaco (detail pages; site embeds
// the business URL in JS, so we also raw-scan the HTML).
// Polite: 1.2s between requests, honest UA, 20s timeout, skip non-200. No auth walls.
// Writes: UPDATE places SET website=$1, phone=COALESCE(phone,$2),
// email=COALESCE(email,$3), website_source='portal' WHERE id=$4 (fill NULLs only)
const { Client } = require('pg');
const UA = 'CR-Directory/1.0 (costarica.agentabrams.com; info@agentabrams.com)';
const DELAY_MS = 1200;
const TIMEOUT_MS = 20000;
const SOURCES = ['tamarindoguide', 'pv-satellite', 'welcometojaco'];
// Generic platforms / assets / share widgets — never a "business website".
const BLOCK_HOST_RE = new RegExp(
[
'booking\\.com', 'airbnb\\.', 'tripadvisor\\.', 'expedia\\.', 'hotels\\.com',
'google\\.', 'gstatic\\.', 'googleapis\\.', 'goo\\.gl', 'youtube\\.', 'youtu\\.be',
'twitter\\.com', '(^|\\.)x\\.com', 'pinterest\\.', 'linkedin\\.', 'tiktok\\.',
'yelp\\.', 'whatsapp\\.com', 'wa\\.me', 'waze\\.com', 'messenger\\.com',
'apple\\.com', 'paypal\\.', 'gravatar\\.', 'wordpress\\.org', 'wp\\.com',
'optimizecdn\\.com', 'cloudflare', 'jsdelivr', 'bootstrapcdn', 'fontawesome',
'fonts\\.', 'schema\\.org', 'w3\\.org', 'addtoany\\.', 'sharethis\\.',
'jquery', 'polyfill\\.', 'recaptcha', 'archive\\.org',
'^welcometo[a-z]+\\.com$', // welcometojaco sister-portal network
].join('|'),
'i'
);
const SOCIAL_HOST_RE = /(facebook\.com|instagram\.com|fb\.com)/i;
const SOCIAL_JUNK_RE = /(\/sharer|\/share\.php|\/share\?|\/dialog\/|\/intent\/|\/plugins\/|\/tr\?|facebook\.com\/?$|instagram\.com\/?$)/i;
const ASSET_EXT_RE = /\.(png|jpe?g|gif|webp|svg|ico|css|js|woff2?|ttf|eot|pdf|xml|json|mp4|webm)(\?|$)/i;
function hostOf(u) {
try { return new URL(u).hostname.replace(/^www\./i, '').toLowerCase(); }
catch { return null; }
}
function normalizeUrl(u) {
try {
const url = new URL(u.trim());
if (!/^https?:$/.test(url.protocol)) return null;
url.hash = '';
return url.href;
} catch { return null; }
}
// pv-satellite: only /en/<area>/<biz>/ (3 path segments, no query) are detail pages.
function isPvsDetail(u) {
try {
const url = new URL(u);
if (url.search) return false;
const segs = url.pathname.split('/').filter(Boolean);
return segs.length === 3 && (segs[0] === 'en' || segs[0] === 'es');
} catch { return false; }
}
function fetchWithTimeout(url) {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
return fetch(url, {
headers: { 'User-Agent': UA, Accept: 'text/html,application/xhtml+xml' },
redirect: 'follow',
signal: ctrl.signal,
}).finally(() => clearTimeout(t));
}
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
// Extract candidate outbound links + tel/mailto from a page.
function extractLinks(html, portalHost) {
const out = { web: [], social: [], tels: [], mails: [] };
const seen = new Set();
const consider = (raw) => {
const norm = normalizeUrl(raw);
if (!norm) return;
const host = hostOf(norm);
if (!host || host === portalHost || host.endsWith('.' + portalHost)) return;
if (ASSET_EXT_RE.test(norm)) return;
if (BLOCK_HOST_RE.test(host)) return;
if (seen.has(norm)) return;
seen.add(norm);
if (SOCIAL_HOST_RE.test(host)) {
if (!SOCIAL_JUNK_RE.test(norm)) out.social.push(norm);
} else {
out.web.push(norm);
}
};
// href / tel / mailto
const hrefRe = /href=["']([^"']+)["']/gi;
let m;
while ((m = hrefRe.exec(html))) {
const v = m[1].replace(/&/g, '&').trim();
if (/^tel:/i.test(v)) {
const tel = v.replace(/^tel:/i, '').replace(/[^\d+]/g, '');
if (tel.replace(/\D/g, '').length >= 7) out.tels.push(tel);
} else if (/^mailto:/i.test(v)) {
const em = v.replace(/^mailto:/i, '').split('?')[0].trim().toLowerCase();
if (/^[^@\s]+@[^@\s]+\.[a-z]{2,}$/i.test(em)) out.mails.push(em);
} else if (/^https?:\/\//i.test(v)) {
consider(v);
}
}
// Raw-scan for JS-embedded URLs (welcometojaco pattern) — same filters apply.
const rawRe = /https?:\/\/[a-zA-Z0-9][a-zA-Z0-9.-]*\.[a-zA-Z]{2,}(?:\/[^\s"'<>\\)]*)?/g;
while ((m = rawRe.exec(html))) consider(m[0]);
return out;
}
async function main() {
const db = new Client({ connectionString: process.env.DATABASE_URL });
await db.connect();
const { rows } = await db.query(
`SELECT id, name, slug, source, source_url FROM places
WHERE website IS NULL AND source = ANY($1) AND source_url LIKE 'http%'
ORDER BY source, id`,
[SOURCES]
);
const stats = {
total_rows: rows.length, attempted: 0, skipped_nondetail: 0,
fetched_ok: 0, fetch_fail: 0, websites: 0, phones: 0, emails: 0,
per_source: {},
};
for (const s of SOURCES) stats.per_source[s] = { rows: 0, websites: 0, phones: 0, emails: 0, skipped: 0 };
// Rows to actually crawl (pv-satellite category/nav pages skipped).
const targets = [];
for (const r of rows) {
stats.per_source[r.source].rows++;
if (r.source === 'pv-satellite' && !isPvsDetail(r.source_url)) {
stats.skipped_nondetail++;
stats.per_source[r.source].skipped++;
continue;
}
targets.push(r);
}
// Fetch each unique URL once.
const urls = [...new Set(targets.map((r) => r.source_url))];
const pages = new Map(); // url -> html|null
console.log(`rows=${rows.length} targets=${targets.length} unique_urls=${urls.length}`);
for (let i = 0; i < urls.length; i++) {
const url = urls[i];
try {
const res = await fetchWithTimeout(url);
if (res.status === 200) {
pages.set(url, await res.text());
stats.fetched_ok++;
} else {
pages.set(url, null);
stats.fetch_fail++;
console.log(` [${res.status}] ${url}`);
}
} catch (e) {
pages.set(url, null);
stats.fetch_fail++;
console.log(` [ERR ${e.name}] ${url}`);
}
if (i < urls.length - 1) await sleep(DELAY_MS);
}
// Chrome detection: per source, outbound URLs present on >=60% of pages (and >=2)
// are site chrome (portal's own socials, sister links) — never a business website.
const chrome = {}; // source -> Set(url)
for (const s of SOURCES) {
const srcUrls = urls.filter((u) => targets.some((r) => r.source === s && r.source_url === u) && pages.get(u));
const counts = new Map();
for (const u of srcUrls) {
const portalHost = hostOf(u);
const links = extractLinks(pages.get(u), portalHost);
for (const l of new Set([...links.web, ...links.social])) counts.set(l, (counts.get(l) || 0) + 1);
}
chrome[s] = new Set();
const threshold = Math.max(2, Math.ceil(srcUrls.length * 0.6));
for (const [l, c] of counts) if (c >= threshold && srcUrls.length >= 2) chrome[s].add(l);
if (chrome[s].size) console.log(`chrome[${s}]:`, [...chrome[s]].join(' | '));
}
// Per-row extraction + update.
for (const r of targets) {
const html = pages.get(r.source_url);
if (!html) continue;
stats.attempted++;
const portalHost = hostOf(r.source_url);
const links = extractLinks(html, portalHost);
const webC = links.web.filter((l) => !chrome[r.source].has(l));
const socC = links.social.filter((l) => !chrome[r.source].has(l));
const website = webC[0] || socC[0] || null;
const phone = links.tels[0] || null;
const email = links.mails[0] || null;
if (!website && !phone && !email) continue;
if (website) {
await db.query(
`UPDATE places SET website=$1, phone=COALESCE(phone,$2), email=COALESCE(email,$3),
website_source='portal', updated_at=now() WHERE id=$4 AND website IS NULL`,
[website, phone, email, r.id]
);
stats.websites++; stats.per_source[r.source].websites++;
} else {
await db.query(
`UPDATE places SET phone=COALESCE(phone,$1), email=COALESCE(email,$2), updated_at=now() WHERE id=$3`,
[phone, email, r.id]
);
}
if (phone) { stats.phones++; stats.per_source[r.source].phones++; }
if (email) { stats.emails++; stats.per_source[r.source].emails++; }
console.log(` #${r.id} ${r.name} -> ${website || '-'} ${phone || ''} ${email || ''}`);
}
console.log('\nSUMMARY ' + JSON.stringify(stats, null, 2));
await db.end();
}
main().catch((e) => { console.error(e); process.exit(1); });