← back to Costa Rica
MEIC PYMEs: full XLSX import (25,661 rows) + auto-region create per cantón; xlsx parser + binary fetch fix; updated local-portals URLs to EXA-vetted set (200 town-portal rows)
c91cf7b39ec37023d9fbc65cc584ac21976f4155 · 2026-05-08 00:03:05 -0700 · Steve
Files touched
M package-lock.jsonM package.jsonM scripts/ingest/_lib.jsM scripts/ingest/local-portals.jsM scripts/ingest/meic-pymes.js
Diff
commit c91cf7b39ec37023d9fbc65cc584ac21976f4155
Author: Steve <steve@designerwallcoverings.com>
Date: Fri May 8 00:03:05 2026 -0700
MEIC PYMEs: full XLSX import (25,661 rows) + auto-region create per cantón; xlsx parser + binary fetch fix; updated local-portals URLs to EXA-vetted set (200 town-portal rows)
---
package-lock.json | 106 +++++++++++++++++++++++++-
package.json | 3 +-
scripts/ingest/_lib.js | 30 +++++++-
scripts/ingest/local-portals.js | 126 +++++++++++++++++++------------
scripts/ingest/meic-pymes.js | 160 +++++++++++++++++++++++++---------------
5 files changed, 314 insertions(+), 111 deletions(-)
diff --git a/package-lock.json b/package-lock.json
index ffc7edb..43a98f9 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,7 +11,8 @@
"dotenv": "^16.4.5",
"express": "^4.21.2",
"express-basic-auth": "^1.2.1",
- "pg": "^8.13.1"
+ "pg": "^8.13.1",
+ "xlsx": "^0.18.5"
}
},
"node_modules/accepts": {
@@ -27,6 +28,15 @@
"node": ">= 0.6"
}
},
+ "node_modules/adler-32": {
+ "version": "1.3.1",
+ "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz",
+ "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -128,6 +138,28 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/cfb": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz",
+ "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "adler-32": "~1.3.0",
+ "crc-32": "~1.2.0"
+ },
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
+ "node_modules/codepage": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz",
+ "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/content-disposition": {
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz",
@@ -164,6 +196,18 @@
"integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==",
"license": "MIT"
},
+ "node_modules/crc-32": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz",
+ "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==",
+ "license": "Apache-2.0",
+ "bin": {
+ "crc32": "bin/crc32.njs"
+ },
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -360,6 +404,15 @@
"node": ">= 0.6"
}
},
+ "node_modules/frac": {
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz",
+ "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
@@ -959,6 +1012,18 @@
"node": ">= 10.x"
}
},
+ "node_modules/ssf": {
+ "version": "0.11.2",
+ "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz",
+ "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "frac": "~1.1.2"
+ },
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -1017,6 +1082,45 @@
"node": ">= 0.8"
}
},
+ "node_modules/wmf": {
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz",
+ "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
+ "node_modules/word": {
+ "version": "0.3.0",
+ "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz",
+ "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==",
+ "license": "Apache-2.0",
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
+ "node_modules/xlsx": {
+ "version": "0.18.5",
+ "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz",
+ "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "adler-32": "~1.3.0",
+ "cfb": "~1.2.1",
+ "codepage": "~1.15.0",
+ "crc-32": "~1.2.1",
+ "ssf": "~0.11.2",
+ "wmf": "~1.0.1",
+ "word": "~0.3.0"
+ },
+ "bin": {
+ "xlsx": "bin/xlsx.njs"
+ },
+ "engines": {
+ "node": ">=0.8"
+ }
+ },
"node_modules/xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
diff --git a/package.json b/package.json
index 3ee1e57..91d413a 100644
--- a/package.json
+++ b/package.json
@@ -13,6 +13,7 @@
"dotenv": "^16.4.5",
"express": "^4.21.2",
"express-basic-auth": "^1.2.1",
- "pg": "^8.13.1"
+ "pg": "^8.13.1",
+ "xlsx": "^0.18.5"
}
}
diff --git a/scripts/ingest/_lib.js b/scripts/ingest/_lib.js
index 578fa9d..1e075ed 100644
--- a/scripts/ingest/_lib.js
+++ b/scripts/ingest/_lib.js
@@ -17,6 +17,12 @@ async function fetchText(url, opts = {}) {
return await res.text();
}
+async function fetchBuffer(url, opts = {}) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA, ...(opts.headers||{}) }, redirect: 'follow', signal: opts.signal });
+ if (!res.ok) throw new Error(`HTTP ${res.status} on ${url}`);
+ return Buffer.from(await res.arrayBuffer());
+}
+
async function fetchJson(url, opts = {}) {
const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json', ...(opts.headers||{}) }, redirect: 'follow', signal: opts.signal });
if (!res.ok) throw new Error(`HTTP ${res.status} on ${url}`);
@@ -51,6 +57,28 @@ function resolveRegion(rmap, hint) {
return rmap.fallback;
}
+async function ensureRegion(rmap, name, province) {
+ if (!name) return rmap.fallback;
+ const cleanName = String(name).trim();
+ const cleanProv = String(province||'').trim() || 'CR';
+ const key = cleanName.toLowerCase();
+ if (rmap.byName.has(key)) return rmap.byName.get(key);
+ const s = slug(cleanName);
+ if (rmap.bySlug.has(s)) return rmap.bySlug.get(s);
+ // INSERT new cantón
+ const { rows } = await pool.query(
+ `INSERT INTO regions (slug, name, province, region_type) VALUES ($1, $2, $3, 'canton')
+ ON CONFLICT (slug) DO UPDATE SET province = EXCLUDED.province
+ RETURNING id, slug, name, province, lat, lng`,
+ [s, cleanName, cleanProv]
+ );
+ const created = rows[0];
+ rmap.bySlug.set(created.slug, created);
+ rmap.byName.set(created.name.toLowerCase(), created);
+ rmap.rows.push(created);
+ return created;
+}
+
async function upsertPlace(p) {
const cols = ['slug','name','category','vertical','region_id','description','address','phone','email','website','price_range','rating','image_url','tags','lat','lng','cedula_juridica','source','source_url'];
const vals = cols.map(c => p[c] ?? null);
@@ -80,4 +108,4 @@ async function finishRun(id, { rows_in=0, rows_added=0, rows_updated=0, status='
);
}
-module.exports = { pool, UA, fetchText, fetchJson, slug, sleep, regionMap, resolveRegion, upsertPlace, startRun, finishRun };
+module.exports = { pool, UA, fetchText, fetchJson, fetchBuffer, slug, sleep, regionMap, resolveRegion, ensureRegion, upsertPlace, startRun, finishRun };
diff --git a/scripts/ingest/local-portals.js b/scripts/ingest/local-portals.js
index a5a4d55..7d0fca7 100644
--- a/scripts/ingest/local-portals.js
+++ b/scripts/ingest/local-portals.js
@@ -1,80 +1,110 @@
'use strict';
-// Local town-portal scrapers. Each portal has its own listing layout; this
-// runner is a generic "fetch + extract anchor names" pass that tags origin.
-// Browserbase-required portals (TripAdvisor, Booking, Yelp) live in their
-// own scripts; this one targets static-HTML town directories.
+// Local town-portal scrapers for static-HTML CR business directories.
+// URL list from EXA-agent research 2026-05-07. Browserbase-required
+// portals (TripAdvisor / Booking / Yelp) live in their own scripts.
const { fetchText, slug, sleep, regionMap, resolveRegion, upsertPlace, startRun, finishRun, pool } = require('./_lib');
const PORTALS = [
- // { url, region_hint, vertical_hint, name_selector_regex }
- { url: 'https://nosaranow.com/business-directory/', region: 'nosara', vertical: 'service_retail', label: 'nosaranow' },
- { url: 'https://www.tamarindo.com/directory/', region: 'tamarindo', vertical: 'service_retail', label: 'tamarindo-com' },
- { url: 'https://monteverdeinfo.com/businesses/', region: 'monteverde', vertical: 'service_retail', label: 'monteverdeinfo' },
- { url: 'https://www.quepos.com/business-directory', region: 'quepos', vertical: 'service_retail', label: 'quepos-com' },
- { url: 'https://thehowlermag.com/category/directory/', region: 'tamarindo', vertical: 'service_retail', label: 'howler-tamarindo' },
+ { url: 'https://tamarindoguide.com/businesses/', region: 'tamarindo', vertical: 'service_retail', label: 'tamarindoguide', maxPages: 5 },
+ { url: 'https://www.alltama.com/listings', region: 'tamarindo', vertical: 'service_retail', label: 'alltama' },
+ { url: 'https://www.tamarindorentals.com/tamarindo-area-directory/', region: 'tamarindo', vertical: 'service_retail', label: 'tamarindo-rentals-dir' },
+ { url: 'https://caribeconnect.com/table/', region: 'puerto-viejo', vertical: 'service_retail', label: 'caribeconnect' },
+ { url: 'https://www.puertoviejosatellite.com/services.php', region: 'puerto-viejo', vertical: 'service_retail', label: 'pv-satellite' },
+ { url: 'https://www.welcometojaco.com/costa-rica', region: 'jaco', vertical: 'service_retail', label: 'welcometojaco' },
+ { url: 'https://www.jacocostaricadirectory.com/', region: 'jaco', vertical: 'service_retail', label: 'jaco-directory' },
+ { url: 'https://www.touristinmonteverde.com/tours/?cat=11', region: 'monteverde', vertical: 'tourism_tour', label: 'tourist-monteverde' },
+ { url: 'https://camaraturismosamara.com/businesses/', region: 'samara', vertical: 'service_retail', label: 'camara-samara' },
+ { url: 'https://www.searchsamara.com/copy-of-leisure-services-1', region: 'samara', vertical: 'service_retail', label: 'searchsamara' },
+ { url: 'https://nosaranow.com/business-directory/', region: 'nosara', vertical: 'service_retail', label: 'nosaranow' },
];
-function extractAnchors(html) {
+const KEYWORD_BLOCK = /(^#|mailto:|tel:|javascript:|^\/?$|\bhome\b|\bcontact\b|\babout\b|\blogin\b|\bregister\b|\bcart\b|\bcheckout\b|\bsearch\b|\bsubscribe\b|\bprivacy\b|\bterms\b|\bsitemap\b|\bblog\b|\bnews\b|\barchive\b)/i;
+const KEYWORD_KEEP = /(business|directory|listing|company|empresa|negocio|hotel|tour|restaurant|restaurante|salon|spa|fitness|surf|yoga|store|tienda|cabin|villa|rental|alquiler|service|servicio|gym|cafe|bar)/i;
+
+function extractFromArticles(html, base) {
const out = [];
- const re = /<a[^>]+href="([^"]+)"[^>]*>([^<]{2,80})<\/a>/gi;
+ // Try common WP/listing patterns: <article>, <div class="...listing|business|card...">
+ const blockRe = /<(article|div)[^>]+class="[^"]*(listing|business|card|entry|item|post|directory)[^"]*"[^>]*>([\s\S]*?)<\/\1>/gi;
let m;
+ while ((m = blockRe.exec(html)) !== null) {
+ const block = m[3];
+ const a = block.match(/<a[^>]+href="([^"#]+)"[^>]*>\s*([^<]{2,90})\s*<\/a>/i);
+ if (a) {
+ const href = a[1].trim();
+ const text = a[2].replace(/\s+/g,' ').trim();
+ if (text && !KEYWORD_BLOCK.test(href + ' ' + text)) out.push({ href, text });
+ }
+ }
+ if (out.length) return out;
+ // Fallback: all anchors with keep-keyword in text or href
+ const re = /<a[^>]+href="([^"#]+)"[^>]*>\s*([^<]{3,90})\s*<\/a>/gi;
while ((m = re.exec(html)) !== null) {
const href = m[1].trim();
const text = m[2].replace(/\s+/g,' ').trim();
- if (!text) continue;
- if (/(^#|mailto:|tel:|javascript:|^\/$|^\/$|\bhome\b|\bcontact\b|\babout\b|\blogin\b|\bregister\b)/i.test(href + ' ' + text)) continue;
- if (/^\d+$/.test(text) || text.length < 3) continue;
+ if (KEYWORD_BLOCK.test(href + ' ' + text)) continue;
+ if (!KEYWORD_KEEP.test(href + ' ' + text)) continue;
out.push({ href, text });
}
return out;
}
+function paginatedUrls(p) {
+ if (!p.maxPages) return [p.url];
+ const urls = [p.url];
+ for (let i = 2; i <= p.maxPages; i++) {
+ if (/\/$/.test(p.url)) urls.push(`${p.url}page/${i}/`);
+ else urls.push(`${p.url}/page/${i}/`);
+ }
+ return urls;
+}
+
(async () => {
- const runId = await startRun('local-portals', 'Static-HTML CR town-portal directory scrape');
+ const runId = await startRun('local-portals', 'Static-HTML CR town-portal directory scrape (EXA-vetted URLs)');
const rmap = await regionMap();
let total = 0, added = 0, updated = 0, ok = 0, fail = 0;
+ const seen = new Set();
for (const p of PORTALS) {
+ const region = resolveRegion(rmap, p.region);
+ let portalRows = 0;
try {
- const html = await fetchText(p.url);
- const anchors = extractAnchors(html);
- const seen = new Set();
- const region = resolveRegion(rmap, p.region);
- // Heuristic: keep only anchors whose href looks like a deeper listing page on the same domain
- const host = new URL(p.url).host;
- const candidates = anchors.filter(a => {
- try {
- const u = new URL(a.href, p.url);
- if (u.host !== host) return false;
- if (u.pathname === '/' || u.pathname === new URL(p.url).pathname) return false;
- return /(business|directory|listing|company|empresa|negocio|hotel|tour|restaurant|salon|spa|fitness|surf|yoga|store|tienda)/i.test(u.pathname + ' ' + a.text);
- } catch { return false; }
- });
+ for (const url of paginatedUrls(p)) {
+ let html;
+ try { html = await fetchText(url); }
+ catch (e) { console.warn(`[local-portals] ${p.label} ${url} → ${e.message}`); continue; }
+ const anchors = extractFromArticles(html, url);
+ const host = new URL(p.url).host;
- for (const c of candidates.slice(0, 200)) {
- const placeSlug = slug(`${p.label}-${c.text}-${region.slug}`);
- if (seen.has(placeSlug)) continue; seen.add(placeSlug);
- const r = await upsertPlace({
- slug: placeSlug,
- name: c.text,
- category: p.vertical.startsWith('tourism_') ? 'tourism' : 'service',
- vertical: p.vertical,
- region_id: region.id,
- description: `Listed on ${p.label} (${region.name}).`,
- source: p.label,
- source_url: new URL(c.href, p.url).toString(),
- });
- if (r.inserted) added++; else updated++;
- total++;
+ for (const a of anchors) {
+ let absHref;
+ try { absHref = new URL(a.href, url).toString(); } catch { continue; }
+ if (new URL(absHref).host !== host) continue;
+ const placeSlug = slug(`${p.label}-${a.text}`);
+ if (seen.has(placeSlug)) continue;
+ seen.add(placeSlug);
+ const r = await upsertPlace({
+ slug: placeSlug,
+ name: a.text,
+ category: p.vertical.startsWith('tourism_') ? 'tourism' : 'service',
+ vertical: p.vertical,
+ region_id: region.id,
+ description: `Listed on ${p.label} (${region.name}).`,
+ source: p.label,
+ source_url: absHref,
+ });
+ if (r.inserted) added++; else updated++;
+ total++;
+ portalRows++;
+ }
+ await sleep(700);
}
ok++;
- console.log(`[local-portals] ${p.label} in=${candidates.length}`);
- await sleep(800);
+ console.log(`[local-portals] ${p.label} rows=${portalRows}`);
} catch (e) {
fail++;
- console.warn(`[local-portals] ${p.label} err:`, e.message);
+ console.warn(`[local-portals] ${p.label} err: ${e.message}`);
}
}
diff --git a/scripts/ingest/meic-pymes.js b/scripts/ingest/meic-pymes.js
index 3af1a46..4665a8b 100644
--- a/scripts/ingest/meic-pymes.js
+++ b/scripts/ingest/meic-pymes.js
@@ -1,107 +1,147 @@
'use strict';
// MEIC PYMEs Activas — bulk national list of registered SMEs.
-// Source: datos.go.cr CKAN dataset "Empresas Activas" (XLSX).
-// Discovers latest CSV/XLSX resource via CKAN API and ingests rows.
+// Source: datos.go.cr CKAN — package "Lista de Pymes activas - Empresas Activas".
-const { fetchJson, fetchText, slug, regionMap, resolveRegion, upsertPlace, startRun, finishRun, pool } = require('./_lib');
+const { fetchJson, fetchBuffer, slug, regionMap, ensureRegion, upsertPlace, startRun, finishRun, pool } = require('./_lib');
const fs = require('fs');
const path = require('path');
+const XLSX = require('xlsx');
-const CKAN_BASE = process.env.CKAN_BASE || 'https://datosabiertos.presidencia.go.cr/api/3/action';
+const CKAN_HOSTS = (process.env.CKAN_BASE
+ ? [process.env.CKAN_BASE]
+ : [
+ 'https://www.datosabiertos.go.cr/api/3/action',
+ 'https://datos.go.cr/api/3/action',
+ 'https://datosabiertos.presidencia.go.cr/api/3/action',
+ ]);
const SEARCH_QS = 'q=pymes+activas+OR+empresas+activas&rows=10';
const CACHE_DIR = path.join(__dirname, '..', '..', 'data', 'cache');
+const ROW_CAP = parseInt(process.env.MEIC_ROW_CAP || '15000', 10);
function findResource(pkg) {
const resources = pkg.result?.resources || [];
- const csv = resources.find(r => /csv$/i.test(r.format||r.url||''));
- const xlsx = resources.find(r => /xlsx?$/i.test(r.format||r.url||''));
- return csv || xlsx || resources[0];
+ return resources.find(r => /xlsx?$/i.test(r.format||r.url||''))
+ || resources.find(r => /csv$/i.test(r.format||r.url||''))
+ || resources[0];
+}
+
+function findCol(headers, ...needles) {
+ for (let i = 0; i < headers.length; i++) {
+ const h = String(headers[i] || '').toLowerCase().trim();
+ if (needles.some(n => h.includes(n))) return i;
+ }
+ return -1;
}
(async () => {
- const runId = await startRun('meic-pymes', 'Fetch MEIC PYMEs Activas via datos.go.cr CKAN');
+ const runId = await startRun('meic-pymes', 'Fetch MEIC PYMEs Activas via CKAN + parse XLSX');
let added = 0, updated = 0, total = 0;
try {
fs.mkdirSync(CACHE_DIR, { recursive: true });
- const search = await fetchJson(`${CKAN_BASE}/package_search?${SEARCH_QS}`);
- const pkgs = search.result?.results || [];
- if (!pkgs.length) throw new Error('no PYMEs package found in datos.go.cr CKAN search');
+ let CKAN_BASE = null, search = null;
+ for (const host of CKAN_HOSTS) {
+ try { search = await fetchJson(`${host}/package_search?${SEARCH_QS}`); CKAN_BASE = host; break; }
+ catch (e) { console.warn(`[meic-pymes] ${host} → ${e.message}`); }
+ }
+ if (!search) throw new Error('all CKAN hosts unreachable');
- const pkg = pkgs[0];
+ const pkg = (search.result?.results || [])[0];
+ if (!pkg) throw new Error('no PYMEs package in CKAN');
const pkgFull = await fetchJson(`${CKAN_BASE}/package_show?id=${encodeURIComponent(pkg.id)}`);
const res = findResource(pkgFull);
- if (!res?.url) throw new Error('no downloadable resource on package');
+ if (!res?.url) throw new Error('no resource on package');
- console.log(`[meic-pymes] package=${pkg.title} resource=${res.format} url=${res.url}`);
+ const ext = (path.extname(res.url) || '.xlsx').toLowerCase();
+ const cachePath = path.join(CACHE_DIR, `meic-pymes${ext}`);
+ let buf;
+ if (fs.existsSync(cachePath) && Buffer.byteLength(fs.readFileSync(cachePath)) > 100000) {
+ buf = fs.readFileSync(cachePath);
+ console.log(`[meic-pymes] using cache (${(buf.length/1024).toFixed(1)} KB)`);
+ } else {
+ buf = await fetchBuffer(res.url);
+ fs.writeFileSync(cachePath, buf);
+ console.log(`[meic-pymes] downloaded ${(buf.length/1024).toFixed(1)} KB → ${cachePath}`);
+ }
- const cachePath = path.join(CACHE_DIR, `meic-pymes${path.extname(res.url)||'.csv'}`);
- const body = await fetchText(res.url);
- fs.writeFileSync(cachePath, body);
- console.log(`[meic-pymes] cached → ${cachePath} (${(body.length/1024).toFixed(1)} KB)`);
+ const wb = XLSX.read(buf, { type: 'buffer' });
+ const sheetName = wb.SheetNames[0];
+ const ws = wb.Sheets[sheetName];
+ const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null });
+ if (!rows.length) throw new Error('XLSX has no rows');
- if (!/csv$/i.test(res.format||res.url||'')) {
- await finishRun(runId, { status: 'partial', notes: `XLSX cached; CSV-parsing path only — XLSX requires xlsx pkg (not in deps)` });
- console.log('[meic-pymes] XLSX cached; install `xlsx` to parse, or convert manually.');
- return;
+ // Header is usually row 0 or 1 (sometimes a title banner is at row 0).
+ let headerIdx = 0;
+ for (let i = 0; i < Math.min(5, rows.length); i++) {
+ const r = rows[i] || [];
+ const lower = r.map(c => String(c||'').toLowerCase());
+ if (lower.some(c => /(nombre|raz[oó]n)/i.test(c)) && lower.some(c => /(canton|cantón|provincia|distrito)/i.test(c))) {
+ headerIdx = i; break;
+ }
}
+ const headers = rows[headerIdx] || [];
+ console.log(`[meic-pymes] sheet=${sheetName} headerIdx=${headerIdx} cols=${headers.length}`);
+ console.log(`[meic-pymes] headers: ${headers.map(h => String(h||'').trim()).join(' | ')}`);
- const lines = body.split(/\r?\n/).filter(Boolean);
- if (!lines.length) throw new Error('empty CSV');
+ const iName = findCol(headers, 'nombre','razon','razón','denominación','empresa');
+ const iAct = findCol(headers, 'actividad','rubro','sector','codiiso','ciiu');
+ const iCedula = findCol(headers, 'cedula','cédula','identif','id_juri');
+ const iProv = findCol(headers, 'provincia');
+ const iCanton = findCol(headers, 'canton','cantón');
+ const iDist = findCol(headers, 'distrito');
+ const iEmail = findCol(headers, 'email','correo');
+ const iPhone = findCol(headers, 'telefono','teléfono');
+ const iSize = findCol(headers, 'tamaño','tamano','tamaño_empresa','classification');
- const header = lines[0].split(/[;,]/).map(h => h.trim().toLowerCase());
- const idx = (...names) => header.findIndex(h => names.some(n => h.includes(n)));
- const iName = idx('nombre','razon');
- const iCanton = idx('canton','cantón');
- const iProv = idx('provincia');
- const iAct = idx('actividad','rubro','sector');
- const iCedula = idx('cedula','cédula','identificacion');
- const iDir = idx('direccion','dirección','domicilio');
- const iEmail = idx('email','correo');
- const iPhone = idx('telefono','teléfono','phone');
+ if (iName < 0) throw new Error(`no name column found; headers=${headers.slice(0,12)}`);
+ console.log(`[meic-pymes] mapping: name=${iName} cedula=${iCedula} canton=${iCanton} prov=${iProv} act=${iAct}`);
const rmap = await regionMap();
+ const dataRows = rows.slice(headerIdx + 1).filter(r => Array.isArray(r) && r[iName]);
- for (const line of lines.slice(1, 5001)) { // cap per-run to 5k for first pass
- const cols = line.split(/[;,](?=(?:[^"]*"[^"]*")*[^"]*$)/).map(c => c.replace(/^"|"$/g,'').trim());
- const name = cols[iName]; if (!name) continue;
- const canton = cols[iCanton] || cols[iProv];
- const region = resolveRegion(rmap, canton);
- const act = (cols[iAct]||'').toLowerCase();
+ for (const r of dataRows.slice(0, ROW_CAP)) {
+ const name = String(r[iName] || '').trim(); if (!name) continue;
+ const canton = String(r[iCanton] || '').trim();
+ const province = String(r[iProv] || '').trim();
+ const region = canton ? await ensureRegion(rmap, canton, province) : rmap.fallback;
+ const act = String(r[iAct] || '').toLowerCase();
const vertical =
- /restaurante|gastronom|comida|cafeter|bar|panade/.test(act) ? 'service_food' :
- /hotel|hospeda|cabin|alojami|villa/.test(act) ? 'tourism_hotel' :
- /tour|turismo|aventura|operador/.test(act) ? 'tourism_tour' :
- /belleza|spa|peluquer|barber|estética/.test(act) ? 'service_beauty' :
- /tienda|comerc|venta|retail/.test(act) ? 'service_retail' :
- /gimna|fitness|crossfit|yoga/.test(act) ? 'service_fitness' :
- /mascot|veterin|pet/.test(act) ? 'service_pet' :
- /taller|mecan|automotr|car wash/.test(act) ? 'service_auto' :
- /limpie|aseo|janitor/.test(act) ? 'service_cleaning' :
- /diseñ|publicid|fotograf|estudio/.test(act) ? 'service_creative' :
+ /restaurante|gastronom|comida|cafeter|bar|panade|repostería|alimentos/.test(act) ? 'service_food' :
+ /hotel|hospeda|cabin|alojami|villa|posada|lodge/.test(act) ? 'tourism_hotel' :
+ /tour|turismo|aventura|operador|recreac/.test(act) ? 'tourism_tour' :
+ /belleza|spa|peluquer|barber|estética|estetica/.test(act) ? 'service_beauty' :
+ /tienda|comerc|venta|retail|supermerc|abarrot|ferreter/.test(act) ? 'service_retail' :
+ /gimna|fitness|crossfit|yoga|deporte/.test(act) ? 'service_fitness' :
+ /mascot|veterin|pet|animal/.test(act) ? 'service_pet' :
+ /taller|mecan|automotr|car wash|llanter|repuesto/.test(act) ? 'service_auto' :
+ /limpie|aseo|janitor|lavander/.test(act) ? 'service_cleaning' :
+ /diseñ|publicid|fotograf|estudio|art|creativ|imprent/.test(act) ? 'service_creative' :
+ /alquiler|rental|arrendami|inmobil/.test(act) ? 'rentals_realestate' :
'service_retail';
- const category = vertical.startsWith('tourism_') ? 'tourism' : 'service';
- const placeSlug = slug(`meic-${cols[iCedula]||name}-${region.slug}`);
- const r = await upsertPlace({
+ const category = vertical.startsWith('tourism_') ? 'tourism' : (vertical.startsWith('rentals_') ? 'rentals' : 'service');
+ const cedula = iCedula >= 0 ? String(r[iCedula] || '').trim() : '';
+ const placeSlug = cedula ? `meic-${cedula}` : slug(`meic-${name}-${region.slug}`);
+ const upsertResult = await upsertPlace({
slug: placeSlug,
name,
category, vertical,
region_id: region.id,
- description: act ? `MEIC-registered PYME: ${act}` : 'MEIC-registered PYME.',
- address: cols[iDir] || canton || '',
- phone: cols[iPhone] || null,
- email: cols[iEmail] || null,
- cedula_juridica: cols[iCedula] || null,
+ description: act ? `MEIC-registered: ${act}` : 'MEIC-registered PYME.',
+ address: [iDist >= 0 ? r[iDist] : null, canton].filter(Boolean).join(', '),
+ phone: iPhone >= 0 ? (String(r[iPhone] || '').trim() || null) : null,
+ email: iEmail >= 0 ? (String(r[iEmail] || '').trim() || null) : null,
+ cedula_juridica: cedula || null,
source: 'meic-pymes',
source_url: res.url,
});
- if (r.inserted) added++; else updated++;
+ if (upsertResult.inserted) added++; else updated++;
total++;
+ if (total % 1000 === 0) console.log(`[meic-pymes] progress: ${total}`);
}
await finishRun(runId, { rows_in: total, rows_added: added, rows_updated: updated, status: 'ok' });
- console.log(`[meic-pymes] in=${total} added=${added} updated=${updated}`);
+ console.log(`[meic-pymes] DONE in=${total} added=${added} updated=${updated}`);
} catch (e) {
await finishRun(runId, { status: 'error', notes: e.message });
console.error('[meic-pymes] FAIL', e.message);
← e517d11 ingest pipeline: ict-cst + local-portals + meic-pymes + goog
·
back to Costa Rica
·
yolo tick 1: place detail page (/p/:slug) + region landing r 1c4ce34 →