← back to Costa Rica
Portal detail-page crawler: 79 websites + 61 phones + 14 emails from tamarindoguide/pv-satellite/welcometojaco (website_source='portal')
7bcb527cd7f38e73adfc56403a98147fd5af4791 · 2026-07-22 21:27:16 -0700 · Steve
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Files touched
A scripts/cr-portal-crawl.js
Diff
commit 7bcb527cd7f38e73adfc56403a98147fd5af4791
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jul 22 21:27:16 2026 -0700
Portal detail-page crawler: 79 websites + 61 phones + 14 emails from tamarindoguide/pv-satellite/welcometojaco (website_source='portal')
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
scripts/cr-portal-crawl.js | 226 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 226 insertions(+)
diff --git a/scripts/cr-portal-crawl.js b/scripts/cr-portal-crawl.js
new file mode 100644
index 0000000..b8917cb
--- /dev/null
+++ b/scripts/cr-portal-crawl.js
@@ -0,0 +1,226 @@
+#!/usr/bin/env node
+// cr-portal-crawl.js — FREE ($0) enrichment: extract real business website/phone/email
+// from the local-portal detail pages our places were scraped from.
+// Sources: tamarindoguide (per-listing pages), pv-satellite (/en/<area>/<biz>/ detail
+// pages only — category/nav URLs skipped), welcometojaco (detail pages; site embeds
+// the business URL in JS, so we also raw-scan the HTML).
+// Polite: 1.2s between requests, honest UA, 20s timeout, skip non-200. No auth walls.
+// Writes: UPDATE places SET website=$1, phone=COALESCE(phone,$2),
+// email=COALESCE(email,$3), website_source='portal' WHERE id=$4 (fill NULLs only)
+
+const { Client } = require('pg');
+
+const UA = 'CR-Directory/1.0 (costarica.agentabrams.com; info@agentabrams.com)';
+const DELAY_MS = 1200;
+const TIMEOUT_MS = 20000;
+
+const SOURCES = ['tamarindoguide', 'pv-satellite', 'welcometojaco'];
+
+// Generic platforms / assets / share widgets — never a "business website".
+const BLOCK_HOST_RE = new RegExp(
+ [
+ 'booking\\.com', 'airbnb\\.', 'tripadvisor\\.', 'expedia\\.', 'hotels\\.com',
+ 'google\\.', 'gstatic\\.', 'googleapis\\.', 'goo\\.gl', 'youtube\\.', 'youtu\\.be',
+ 'twitter\\.com', '(^|\\.)x\\.com', 'pinterest\\.', 'linkedin\\.', 'tiktok\\.',
+ 'yelp\\.', 'whatsapp\\.com', 'wa\\.me', 'waze\\.com', 'messenger\\.com',
+ 'apple\\.com', 'paypal\\.', 'gravatar\\.', 'wordpress\\.org', 'wp\\.com',
+ 'optimizecdn\\.com', 'cloudflare', 'jsdelivr', 'bootstrapcdn', 'fontawesome',
+ 'fonts\\.', 'schema\\.org', 'w3\\.org', 'addtoany\\.', 'sharethis\\.',
+ 'jquery', 'polyfill\\.', 'recaptcha', 'archive\\.org',
+ '^welcometo[a-z]+\\.com$', // welcometojaco sister-portal network
+ ].join('|'),
+ 'i'
+);
+
+const SOCIAL_HOST_RE = /(facebook\.com|instagram\.com|fb\.com)/i;
+const SOCIAL_JUNK_RE = /(\/sharer|\/share\.php|\/share\?|\/dialog\/|\/intent\/|\/plugins\/|\/tr\?|facebook\.com\/?$|instagram\.com\/?$)/i;
+const ASSET_EXT_RE = /\.(png|jpe?g|gif|webp|svg|ico|css|js|woff2?|ttf|eot|pdf|xml|json|mp4|webm)(\?|$)/i;
+
+function hostOf(u) {
+ try { return new URL(u).hostname.replace(/^www\./i, '').toLowerCase(); }
+ catch { return null; }
+}
+
+function normalizeUrl(u) {
+ try {
+ const url = new URL(u.trim());
+ if (!/^https?:$/.test(url.protocol)) return null;
+ url.hash = '';
+ return url.href;
+ } catch { return null; }
+}
+
+// pv-satellite: only /en/<area>/<biz>/ (3 path segments, no query) are detail pages.
+function isPvsDetail(u) {
+ try {
+ const url = new URL(u);
+ if (url.search) return false;
+ const segs = url.pathname.split('/').filter(Boolean);
+ return segs.length === 3 && (segs[0] === 'en' || segs[0] === 'es');
+ } catch { return false; }
+}
+
+function fetchWithTimeout(url) {
+ const ctrl = new AbortController();
+ const t = setTimeout(() => ctrl.abort(), TIMEOUT_MS);
+ return fetch(url, {
+ headers: { 'User-Agent': UA, Accept: 'text/html,application/xhtml+xml' },
+ redirect: 'follow',
+ signal: ctrl.signal,
+ }).finally(() => clearTimeout(t));
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// Extract candidate outbound links + tel/mailto from a page.
+function extractLinks(html, portalHost) {
+ const out = { web: [], social: [], tels: [], mails: [] };
+ const seen = new Set();
+
+ const consider = (raw) => {
+ const norm = normalizeUrl(raw);
+ if (!norm) return;
+ const host = hostOf(norm);
+ if (!host || host === portalHost || host.endsWith('.' + portalHost)) return;
+ if (ASSET_EXT_RE.test(norm)) return;
+ if (BLOCK_HOST_RE.test(host)) return;
+ if (seen.has(norm)) return;
+ seen.add(norm);
+ if (SOCIAL_HOST_RE.test(host)) {
+ if (!SOCIAL_JUNK_RE.test(norm)) out.social.push(norm);
+ } else {
+ out.web.push(norm);
+ }
+ };
+
+ // href / tel / mailto
+ const hrefRe = /href=["']([^"']+)["']/gi;
+ let m;
+ while ((m = hrefRe.exec(html))) {
+ const v = m[1].replace(/&/g, '&').trim();
+ if (/^tel:/i.test(v)) {
+ const tel = v.replace(/^tel:/i, '').replace(/[^\d+]/g, '');
+ if (tel.replace(/\D/g, '').length >= 7) out.tels.push(tel);
+ } else if (/^mailto:/i.test(v)) {
+ const em = v.replace(/^mailto:/i, '').split('?')[0].trim().toLowerCase();
+ if (/^[^@\s]+@[^@\s]+\.[a-z]{2,}$/i.test(em)) out.mails.push(em);
+ } else if (/^https?:\/\//i.test(v)) {
+ consider(v);
+ }
+ }
+
+ // Raw-scan for JS-embedded URLs (welcometojaco pattern) — same filters apply.
+ const rawRe = /https?:\/\/[a-zA-Z0-9][a-zA-Z0-9.-]*\.[a-zA-Z]{2,}(?:\/[^\s"'<>\\)]*)?/g;
+ while ((m = rawRe.exec(html))) consider(m[0]);
+
+ return out;
+}
+
+async function main() {
+ const db = new Client({ connectionString: process.env.DATABASE_URL });
+ await db.connect();
+
+ const { rows } = await db.query(
+ `SELECT id, name, slug, source, source_url FROM places
+ WHERE website IS NULL AND source = ANY($1) AND source_url LIKE 'http%'
+ ORDER BY source, id`,
+ [SOURCES]
+ );
+
+ const stats = {
+ total_rows: rows.length, attempted: 0, skipped_nondetail: 0,
+ fetched_ok: 0, fetch_fail: 0, websites: 0, phones: 0, emails: 0,
+ per_source: {},
+ };
+ for (const s of SOURCES) stats.per_source[s] = { rows: 0, websites: 0, phones: 0, emails: 0, skipped: 0 };
+
+ // Rows to actually crawl (pv-satellite category/nav pages skipped).
+ const targets = [];
+ for (const r of rows) {
+ stats.per_source[r.source].rows++;
+ if (r.source === 'pv-satellite' && !isPvsDetail(r.source_url)) {
+ stats.skipped_nondetail++;
+ stats.per_source[r.source].skipped++;
+ continue;
+ }
+ targets.push(r);
+ }
+
+ // Fetch each unique URL once.
+ const urls = [...new Set(targets.map((r) => r.source_url))];
+ const pages = new Map(); // url -> html|null
+ console.log(`rows=${rows.length} targets=${targets.length} unique_urls=${urls.length}`);
+ for (let i = 0; i < urls.length; i++) {
+ const url = urls[i];
+ try {
+ const res = await fetchWithTimeout(url);
+ if (res.status === 200) {
+ pages.set(url, await res.text());
+ stats.fetched_ok++;
+ } else {
+ pages.set(url, null);
+ stats.fetch_fail++;
+ console.log(` [${res.status}] ${url}`);
+ }
+ } catch (e) {
+ pages.set(url, null);
+ stats.fetch_fail++;
+ console.log(` [ERR ${e.name}] ${url}`);
+ }
+ if (i < urls.length - 1) await sleep(DELAY_MS);
+ }
+
+ // Chrome detection: per source, outbound URLs present on >=60% of pages (and >=2)
+ // are site chrome (portal's own socials, sister links) — never a business website.
+ const chrome = {}; // source -> Set(url)
+ for (const s of SOURCES) {
+ const srcUrls = urls.filter((u) => targets.some((r) => r.source === s && r.source_url === u) && pages.get(u));
+ const counts = new Map();
+ for (const u of srcUrls) {
+ const portalHost = hostOf(u);
+ const links = extractLinks(pages.get(u), portalHost);
+ for (const l of new Set([...links.web, ...links.social])) counts.set(l, (counts.get(l) || 0) + 1);
+ }
+ chrome[s] = new Set();
+ const threshold = Math.max(2, Math.ceil(srcUrls.length * 0.6));
+ for (const [l, c] of counts) if (c >= threshold && srcUrls.length >= 2) chrome[s].add(l);
+ if (chrome[s].size) console.log(`chrome[${s}]:`, [...chrome[s]].join(' | '));
+ }
+
+ // Per-row extraction + update.
+ for (const r of targets) {
+ const html = pages.get(r.source_url);
+ if (!html) continue;
+ stats.attempted++;
+ const portalHost = hostOf(r.source_url);
+ const links = extractLinks(html, portalHost);
+ const webC = links.web.filter((l) => !chrome[r.source].has(l));
+ const socC = links.social.filter((l) => !chrome[r.source].has(l));
+ const website = webC[0] || socC[0] || null;
+ const phone = links.tels[0] || null;
+ const email = links.mails[0] || null;
+ if (!website && !phone && !email) continue;
+
+ if (website) {
+ await db.query(
+ `UPDATE places SET website=$1, phone=COALESCE(phone,$2), email=COALESCE(email,$3),
+ website_source='portal', updated_at=now() WHERE id=$4 AND website IS NULL`,
+ [website, phone, email, r.id]
+ );
+ stats.websites++; stats.per_source[r.source].websites++;
+ } else {
+ await db.query(
+ `UPDATE places SET phone=COALESCE(phone,$1), email=COALESCE(email,$2), updated_at=now() WHERE id=$3`,
+ [phone, email, r.id]
+ );
+ }
+ if (phone) { stats.phones++; stats.per_source[r.source].phones++; }
+ if (email) { stats.emails++; stats.per_source[r.source].emails++; }
+ console.log(` #${r.id} ${r.name} -> ${website || '-'} ${phone || ''} ${email || ''}`);
+ }
+
+ console.log('\nSUMMARY ' + JSON.stringify(stats, null, 2));
+ await db.end();
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });
← 2f8854c Free website enrichment via OpenStreetMap/Overpass: 89 place
·
back to Costa Rica
·
OSM matcher: honest provenance — social profiles tagged -soc 1a69c32 →