`scripts/ingest.js`

55 lines
// Runs every enabled adapter, normalizes results, and UPSERTs into products.
// Safe to run repeatedly (cron): the (network, external_id) key means re-ingest
// updates prices/stock in place instead of duplicating.
//
// Usage:  node scripts/ingest.js [--limit=500] [--only=cj,amazon]
// Env is read from the process (pm2 ecosystem / shell). dotenv is optional:
try { require('dotenv').config(); } catch (_) { /* dotenv not installed — env comes from the shell */ }
const db = require('../lib/db');
const adapters = require('../lib/adapters');
const { normalizeProduct, UPSERT_SQL, upsertParams } = require('../lib/normalize');

function arg(name, def) {
  const hit = process.argv.find((a) => a.startsWith(`--${name}=`));
  return hit ? hit.split('=')[1] : def;
}

async function main() {
  const env = process.env;
  const limit = parseInt(arg('limit', '500'), 10);
  const only = (arg('only', '') || '').split(',').filter(Boolean);

  await db.query('SELECT 1');
  const summary = [];

  for (const a of adapters) {
    if (only.length && !only.includes(a.network)) continue;
    if (!a.enabled(env)) { summary.push(`${a.network}: skipped (no creds)`); continue; }

    let raws = [];
    try {
      raws = await a.fetch(env, { limit });
    } catch (e) {
      summary.push(`${a.network}: ERROR ${e.message}`);
      continue;
    }

    let ok = 0, skip = 0;
    for (const raw of raws) {
      const norm = normalizeProduct(raw, a.network);
      if (!norm) { skip++; continue; }
      try {
        await db.query(UPSERT_SQL, upsertParams(norm));
        ok++;
      } catch (e) { skip++; }
    }
    summary.push(`${a.network}: ${ok} upserted, ${skip} skipped (of ${raws.length})`);
  }

  console.log('--- ingest summary ---');
  summary.forEach((s) => console.log('  ' + s));
  await db.pool.end();
}

main().catch((e) => { console.error(e); process.exit(1); });