`crawl/run.js`

55 lines
'use strict';
// CRUnifiedDB crawl dispatcher.
//   node crawl/run.js <slug>        run one firm agent
//   node crawl/run.js all           run every crawl-enabled, low/medium-ToS firm
//   node crawl/run.js <slug> --force run even a high-ToS firm (LoopNet/CoStar) — you accept the risk
const fs = require('fs');
const path = require('path');
const base = require('./base-crawler');

const FIRMS_DIR = path.join(__dirname, 'firms');
const force = process.argv.includes('--force');
const arg = process.argv[2];

function firmModule(slug) {
  const f = path.join(FIRMS_DIR, `${slug}.js`);
  return fs.existsSync(f) ? require(f) : null;
}

async function runOne(slug) {
  const sponsor = await base.getSponsor(slug);
  if (!sponsor) { console.log(`✗ ${slug}: not a sponsor in CRUnifiedDB`); return; }
  if (sponsor.role === 'financing_partner') { console.log(`— ${slug}: financing partner (no listings to crawl)`); return; }
  const mod = firmModule(slug);
  if (!mod || typeof mod.crawl !== 'function') { console.log(`— ${slug}: no crawler module yet (crawl/firms/${slug}.js)`); return; }
  if (sponsor.tos_risk === 'high' && !force) {
    console.log(`⚠ ${slug}: ToS risk = HIGH (${sponsor.notes || ''}). Skipped. Re-run with --force to accept the risk.`);
    return;
  }
  const runId = await base.startRun(sponsor.id);
  console.log(`▶ ${slug}: crawling ${sponsor.listings_url || sponsor.website} …`);
  let found = 0, added = 0, cost = 0;
  try {
    const res = await mod.crawl({ base, sponsor,
      upsert: async (L) => { found++; if (await base.upsertListing(sponsor.id, L)) added++; } });
    cost = res?.cost || 0;
    await base.finishRun(runId, { status: 'ok', found, added, cost, notes: res?.notes || '' });
    console.log(`✓ ${slug}: ${found} found, ${added} new  ($${cost.toFixed(2)})`);
  } catch (e) {
    await base.finishRun(runId, { status: 'error', found, added, cost, notes: String(e).slice(0, 400) });
    console.log(`✗ ${slug}: ${e}`);
  }
}

(async () => {
  if (!arg) { console.log('usage: node crawl/run.js <slug|all> [--force]'); process.exit(1); }
  if (arg === 'all') {
    const { rows } = await base.pool.query(
      `SELECT slug FROM sponsors WHERE crawl_enabled AND role='broker' AND tos_risk <> 'high' ORDER BY slug`);
    for (const r of rows) await runOne(r.slug);
  } else {
    await runOne(arg);
  }
  await base.pool.end();
})();