`src/jobs/refresh_all.ts`

47 lines
/**
 * Monthly refresh orchestrator (launchd com.steve.usre-refresh):
 * zillow -> redfin -> fhfa -> acs -> derive -> score -> alerts.
 * Every job is an idempotent upsert, so re-running any month is safe; jobs run
 * as child processes because each script owns its pool lifecycle (pool.end()).
 * A failed job logs + continues (per-source failure shows red in ingest_runs;
 * one dead source must not starve the others).
 */
import { execFileSync } from 'node:child_process';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = join(__dirname, '..', '..');

const JOBS: Array<[string, string, string[]?]> = [
  ['zillow', 'src/ingest/zillow_research.ts'],
  ['redfin', 'src/ingest/redfin_tracker.ts'],
  ['fhfa', 'src/ingest/fhfa_hpi.ts'],
  ['acs', 'src/ingest/census_acs.ts'],
  ['derive', 'src/ingest/derive_metrics.ts'],
  ['score', 'src/score/opportunity.ts'],
  ['alerts', 'src/jobs/alerts_check.ts'],
  // TK-10155: the listings ingest + lifecycle sweep run on THIS existing cadence (monthly,
  // com.steve.usre-refresh Day 25) rather than a new launchd job (installing one is gated).
  // (a) refresh the broker listings feed, then (b) sweep lifecycle states so gone-from-feed
  // withdrawals + withdrawal-snapshots are captured. Both are appended and, like every job
  // above, wrapped so a failure logs + continues — it must never break zillow/redfin/etc.
  // NOTE: listings churn in DAYS, so a DAILY sweep is the real fix — drafted separately as a
  // gated launchd job (fix 3). This monthly wiring is the reversible, no-new-job baseline.
  ['listings', 'src/ingest/listings/engine.ts', ['all']],
  ['lifecycle-sweep', 'src/jobs/listing-lifecycle-sweep.ts'],
];

let failed = 0;
for (const [name, script, extraArgs] of JOBS) {
  console.log(`\n[refresh] ── ${name} ──`);
  try {
    execFileSync('npx', ['tsx', script, ...(extraArgs ?? [])], { cwd: ROOT, stdio: 'inherit', timeout: 45 * 60_000 });
  } catch (e: any) {
    failed++;
    console.error(`[refresh] ${name} FAILED: ${e.message}`);
  }
}
console.log(`\n[refresh] complete, ${failed} job(s) failed`);
process.exit(failed > 0 ? 1 : 0);