`scripts/ingest/hacienda-enricher.js`

128 lines
'use strict';
// Hacienda (Ministerio de Hacienda CR) enrichment pass.
// Endpoint: https://api.hacienda.go.cr/fe/ae?identificacion={cedula}
// Returns: { nombre, tipoIdentificacion, regimen.codigo/descripcion, situacion.moroso/omiso/estado, actividades:[{estado,codigo,tipo,descripcion}] }
// Throttle: 1500ms between calls. Cap per tick: env HACIENDA_BATCH (default 200).
//
// Usage:
//   node scripts/ingest/hacienda-enricher.js                # process default batch
//   HACIENDA_BATCH=10 node scripts/ingest/hacienda-enricher.js   # quick test
//   HACIENDA_RETRY_ERRORS=1 ...                             # also re-try rows with prior errors
//
// Re-runs are safe: rows with hacienda_enriched_at IS NOT NULL are skipped (unless HACIENDA_RETRY_ERRORS=1
// AND they have a stored error).

const { fetchJson, sleep, pool } = require('./_lib');

const BATCH = parseInt(process.env.HACIENDA_BATCH || '200', 10);
const THROTTLE_MS = parseInt(process.env.HACIENDA_THROTTLE_MS || '1500', 10);
const RETRY_ERRORS = process.env.HACIENDA_RETRY_ERRORS === '1';
const ENDPOINT = 'https://api.hacienda.go.cr/fe/ae?identificacion=';

function normalizeCedula(c) {
  if (!c) return null;
  return String(c).replace(/[^0-9]/g, '');
}

(async () => {
  const t0 = Date.now();
  const { rows: runIns } = await pool.query(
    `INSERT INTO ingest_runs (source, status) VALUES ('hacienda-enricher','running') RETURNING id`
  );
  const runId = runIns[0].id;

  let q;
  if (RETRY_ERRORS) {
    q = `SELECT id, slug, cedula_juridica
           FROM places
          WHERE cedula_juridica IS NOT NULL
            AND (hacienda_enriched_at IS NULL OR hacienda_error IS NOT NULL)
       ORDER BY hacienda_enriched_at NULLS FIRST, id
          LIMIT $1`;
  } else {
    q = `SELECT id, slug, cedula_juridica
           FROM places
          WHERE cedula_juridica IS NOT NULL
            AND hacienda_enriched_at IS NULL
       ORDER BY id
          LIMIT $1`;
  }
  const { rows: targets } = await pool.query(q, [BATCH]);
  console.log(`[hacienda] queued ${targets.length} rows (batch=${BATCH}, throttle=${THROTTLE_MS}ms, retry=${RETRY_ERRORS})`);

  let ok = 0, miss = 0, err = 0;

  for (const r of targets) {
    const ced = normalizeCedula(r.cedula_juridica);
    if (!ced || ced.length < 9) {
      await pool.query(
        `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
        ['invalid-cedula-format', r.id]
      );
      err++;
      continue;
    }

    try {
      const j = await fetchJson(ENDPOINT + encodeURIComponent(ced), {
        headers: { 'Accept': 'application/json' },
      });

      // Defensive: API may return {nombre:null} for non-registered cedulas
      const officialName = j.nombre || null;
      const tipoId = j.tipoIdentificacion || null;
      const situacion = j.situacion?.estado || null;
      const regimen = j.regimen?.descripcion || null;
      const actividades = Array.isArray(j.actividades) ? j.actividades : null;

      if (!officialName && !situacion && !actividades?.length) {
        await pool.query(
          `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
          ['no-data-returned', r.id]
        );
        miss++;
      } else {
        await pool.query(
          `UPDATE places
              SET hacienda_official_name=$1,
                  hacienda_tipo_id=$2,
                  hacienda_situacion=$3,
                  hacienda_regimen=$4,
                  hacienda_actividades=$5,
                  hacienda_enriched_at=NOW(),
                  hacienda_error=NULL
            WHERE id=$6`,
          [officialName, tipoId, situacion, regimen,
           actividades ? JSON.stringify(actividades) : null,
           r.id]
        );
        ok++;
      }
    } catch (e) {
      const msg = String(e.message || e).slice(0, 200);
      await pool.query(
        `UPDATE places SET hacienda_enriched_at=NOW(), hacienda_error=$1 WHERE id=$2`,
        [`fetch-error: ${msg}`, r.id]
      );
      err++;
    }

    if ((ok + miss + err) % 25 === 0) {
      console.log(`[hacienda] progress ${ok + miss + err}/${targets.length} ok=${ok} miss=${miss} err=${err}`);
    }
    await sleep(THROTTLE_MS);
  }

  const dt = ((Date.now() - t0) / 1000).toFixed(1);
  console.log(`[hacienda] done in ${dt}s — ok=${ok} miss=${miss} err=${err}`);

  await pool.query(
    `UPDATE ingest_runs
        SET finished_at=NOW(), rows_in=$1, rows_updated=$2, status='ok',
            notes=$3
      WHERE id=$4`,
    [targets.length, ok, `miss=${miss} err=${err} batch=${BATCH}`, runId]
  );
  await pool.end();
})();