`scripts/ingest-propublica-ca-orgs.mjs`

55 lines
// Ingest California nonprofits from the ProPublica Nonprofit Explorer API (free, no key).
// Proof-of-pipeline: paginates CA orgs into charitable_orgs. MAX_PAGES bounds it for v1;
// the full CA org universe (~150k) is reached by the org-index backfill node.
// Docs: https://projects.propublica.org/nonprofits/api
import { q, startRun, upsertSource } from '../lib/db.js';

const BASE = 'https://projects.propublica.org/nonprofits/api/v2/search.json';
const MAX_PAGES = parseInt(process.env.MAX_PAGES || '5', 10); // 100 orgs/page
const SLUG = 'propublica';

async function main() {
  await upsertSource({
    slug: SLUG, name: 'ProPublica Nonprofit Explorer', url: BASE,
    jurisdiction: 'federal', family: 'charitable', granularity: 'org', access: 'api',
    notes: 'CA org index + EINs; free, no key. Full corpus via IRS 990 XML join.',
  });
  const run = await startRun(SLUG);
  let seen = 0, upserted = 0;
  try {
    for (let page = 0; page < MAX_PAGES; page++) {
      const url = `${BASE}?state%5Bid%5D=CA&page=${page}`;
      const res = await fetch(url, { headers: { 'User-Agent': 'ca-donations/0.1 (public-records)' } });
      if (!res.ok) throw new Error(`ProPublica HTTP ${res.status} on page ${page}`);
      const json = await res.json();
      const orgs = json.organizations || [];
      if (!orgs.length) break;
      for (const o of orgs) {
        seen++;
        const ein = o.strein || String(o.ein);
        const sub = o.subseccd ? `501(c)(${o.subseccd})` : null;
        const r = await q(
          `INSERT INTO charitable_orgs (ein,name,city,state,ntee_code,subsection,source_slug)
           VALUES ($1,$2,$3,$4,$5,$6,$7)
           ON CONFLICT (ein) DO UPDATE SET name=EXCLUDED.name, city=EXCLUDED.city,
             state=EXCLUDED.state, ntee_code=EXCLUDED.ntee_code, subsection=EXCLUDED.subsection
           RETURNING id`,
          [ein, o.name, o.city, o.state, o.ntee_code || o.raw_ntee_code || null, sub, SLUG],
        );
        if (r.length) upserted++;
      }
      process.stdout.write(`  page ${page}: ${orgs.length} orgs (running total ${seen})\n`);
      await new Promise((r) => setTimeout(r, 500)); // be polite
    }
    await run.done(seen, upserted, 'ok');
    console.log(`ProPublica CA orgs: ${seen} seen, ${upserted} upserted. $0 (free API).`);
  } catch (e) {
    await run.done(seen, upserted, 'error', String(e.message));
    console.error('FAILED:', e.message);
    process.exit(1);
  } finally {
    await (await import('../lib/db.js')).pool.end();
  }
}
main();