← back to Costa Rica
tick 4 wip: Hacienda (CR MoF) enricher + migration (HACIENDA_BATCH+throttle env-tunable, idempotent retry, ingest_runs logging) — gated on Steve external-API approval
4cbc62e9f8c451895145fbf4ece8b1f7cc8110f6 · 2026-05-08 09:10:30 -0700 · Steve
Files touched
A scripts/ingest/hacienda-enricher.jsA scripts/migrate_003_hacienda.sql
Diff
commit 4cbc62e9f8c451895145fbf4ece8b1f7cc8110f6
Author: Steve <steve@designerwallcoverings.com>
Date: Fri May 8 09:10:30 2026 -0700
tick 4 wip: Hacienda (CR MoF) enricher + migration (HACIENDA_BATCH+throttle env-tunable, idempotent retry, ingest_runs logging) — gated on Steve external-API approval
---
scripts/ingest/hacienda-enricher.js | 127 ++++++++++++++++++++++++++++++++++++
scripts/migrate_003_hacienda.sql | 12 ++++
2 files changed, 139 insertions(+)
diff --git a/scripts/ingest/hacienda-enricher.js b/scripts/ingest/hacienda-enricher.js
new file mode 100644
index 0000000..8eab661
--- /dev/null
+++ b/scripts/ingest/hacienda-enricher.js
@@ -0,0 +1,127 @@
+'use strict';
+// Hacienda (Ministerio de Hacienda CR) enrichment pass.
+// Endpoint: https://api.hacienda.go.cr/fe/ae?identificacion={cedula}
+// Returns: { nombre, tipoIdentificacion, regimen.codigo/descripcion, situacion.moroso/omiso/estado, actividades:[{estado,codigo,tipo,descripcion}] }
+// Throttle: 1500ms between calls. Cap per tick: env HACIENDA_BATCH (default 200).
+//
+// Usage:
+// node scripts/ingest/hacienda-enricher.js # process default batch
+// HACIENDA_BATCH=10 node scripts/ingest/hacienda-enricher.js # quick test
+// HACIENDA_RETRY_ERRORS=1 ... # also re-try rows with prior errors
+//
+// Re-runs are safe: rows with hacienda_enriched_at IS NOT NULL are skipped (unless HACIENDA_RETRY_ERRORS=1
+// AND they have a stored error).
+
+const { fetchJson, sleep, pool } = require('./_lib');
+
+const BATCH = parseInt(process.env.HACIENDA_BATCH || '200', 10);
+const THROTTLE_MS = parseInt(process.env.HACIENDA_THROTTLE_MS || '1500', 10);
+const RETRY_ERRORS = process.env.HACIENDA_RETRY_ERRORS === '1';
+const ENDPOINT = 'https://api.hacienda.go.cr/fe/ae?identificacion=';
+
+function normalizeCedula(c) {
+ if (!c) return null;
+ return String(c).replace(/[^0-9]/g, '');
+}
+
+(async () => {
+ const t0 = Date.now();
+ const { rows: runIns } = await pool.query(
+ `INSERT INTO ingest_runs (source, status) VALUES ('hacienda-enricher','running') RETURNING id`
+ );
+ const runId = runIns[0].id;
+
+ let q;
+ if (RETRY_ERRORS) {
+ q = `SELECT id, slug, cedula_juridica
+ FROM places
+ WHERE cedula_juridica IS NOT NULL
+ AND (hacienda_enriched_at IS NULL OR hacienda_error IS NOT NULL)
+ ORDER BY hacienda_enriched_at NULLS FIRST, id
+ LIMIT $1`;
+ } else {
+ q = `SELECT id, slug, cedula_juridica
+ FROM places
+ WHERE cedula_juridica IS NOT NULL
+ AND hacienda_enriched_at IS NULL
+ ORDER BY id
+ LIMIT $1`;
+ }
+ const { rows: targets } = await pool.query(q, [BATCH]);
+ console.log(`[hacienda] queued ${targets.length} rows (batch=${BATCH}, throttle=${THROTTLE_MS}ms, retry=${RETRY_ERRORS})`);
+
+ let ok = 0, miss = 0, err = 0;
+
+ for (const r of targets) {
+ const ced = normalizeCedula(r.cedula_juridica);
+ if (!ced || ced.length < 9) {
+ await pool.query(
+ `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
+ ['invalid-cedula-format', r.id]
+ );
+ err++;
+ continue;
+ }
+
+ try {
+ const j = await fetchJson(ENDPOINT + encodeURIComponent(ced), {
+ headers: { 'Accept': 'application/json' },
+ });
+
+ // Defensive: API may return {nombre:null} for non-registered cedulas
+ const officialName = j.nombre || null;
+ const tipoId = j.tipoIdentificacion || null;
+ const situacion = j.situacion?.estado || null;
+ const regimen = j.regimen?.descripcion || null;
+ const actividades = Array.isArray(j.actividades) ? j.actividades : null;
+
+ if (!officialName && !situacion && !actividades?.length) {
+ await pool.query(
+ `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
+ ['no-data-returned', r.id]
+ );
+ miss++;
+ } else {
+ await pool.query(
+ `UPDATE places
+ SET hacienda_official_name=$1,
+ hacienda_tipo_id=$2,
+ hacienda_situacion=$3,
+ hacienda_regimen=$4,
+ hacienda_actividades=$5,
+ hacienda_enriched_at=NOW(),
+ hacienda_error=NULL
+ WHERE id=$6`,
+ [officialName, tipoId, situacion, regimen,
+ actividades ? JSON.stringify(actividades) : null,
+ r.id]
+ );
+ ok++;
+ }
+ } catch (e) {
+ const msg = String(e.message || e).slice(0, 200);
+ await pool.query(
+ `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
+ [`fetch-error: ${msg}`, r.id]
+ );
+ err++;
+ }
+
+ if ((ok + miss + err) % 25 === 0) {
+ console.log(`[hacienda] progress ${ok + miss + err}/${targets.length} ok=${ok} miss=${miss} err=${err}`);
+ }
+ await sleep(THROTTLE_MS);
+ }
+
+ const dt = ((Date.now() - t0) / 1000).toFixed(1);
+ console.log(`[hacienda] done in ${dt}s — ok=${ok} miss=${miss} err=${err}`);
+
+ await pool.query(
+ `UPDATE ingest_runs
+ SET finished_at=NOW(), rows_in=$1, rows_updated=$2, status='ok',
+ notes=$3
+ WHERE id=$4`,
+ [targets.length, ok, `miss=${miss} err=${err} batch=${BATCH}`, runId]
+ );
+ await pool.end();
+})();
diff --git a/scripts/migrate_003_hacienda.sql b/scripts/migrate_003_hacienda.sql
new file mode 100644
index 0000000..b58ce28
--- /dev/null
+++ b/scripts/migrate_003_hacienda.sql
@@ -0,0 +1,12 @@
+-- Hacienda (Costa Rica MoF) enrichment columns
+ALTER TABLE places
+ ADD COLUMN IF NOT EXISTS hacienda_official_name TEXT,
+ ADD COLUMN IF NOT EXISTS hacienda_tipo_id TEXT,
+ ADD COLUMN IF NOT EXISTS hacienda_situacion TEXT,
+ ADD COLUMN IF NOT EXISTS hacienda_regimen TEXT,
+ ADD COLUMN IF NOT EXISTS hacienda_actividades JSONB,
+ ADD COLUMN IF NOT EXISTS hacienda_enriched_at TIMESTAMPTZ,
+ ADD COLUMN IF NOT EXISTS hacienda_error TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_places_hacienda_situacion ON places(hacienda_situacion);
+CREATE INDEX IF NOT EXISTS idx_places_hacienda_enriched ON places(hacienda_enriched_at);
← a3854c7 track YOLO loop progress + flag prod-deploy + fallback-scrip
·
back to Costa Rica
·
yolo tick 4: /provinces overview page (7 provinces, top-6 ca 3d52097 →