`scripts/fetch-dins.js`

177 lines
#!/usr/bin/env node
// fetch-dins.js — Paginate the CAL FIRE master DINS dataset from ArcGIS and save to data/dins.json
// Service: https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/POSTFIRE_MASTER_DATA_SHARE/FeatureServer/0
// 132,522 total records (as of Aug 2026), maxRecordCount=2000 per page

const https = require('https');
const fs = require('fs');
const path = require('path');

const BASE_URL = 'https://services1.arcgis.com/jUJYIo9tSA7EHvfZ/arcgis/rest/services/POSTFIRE_MASTER_DATA_SHARE/FeatureServer/0/query';
const DATA_DIR = path.join(__dirname, '..', 'data');
const OUT_FILE = path.join(DATA_DIR, 'dins.json');

const args = process.argv.slice(2);
const limitArg = args.find(a => a.startsWith('--limit='));
const MAX_LIMIT = limitArg ? parseInt(limitArg.split('=')[1], 10) : Infinity;
const PAGE_SIZE = 2000;

// Fields we care about for the viewer
const FIELDS = 'OBJECTID,DAMAGE,STRUCTURETYPE,CITY,STATE,COUNTY,COMMUNITY,INCIDENTNAME,INCIDENTSTARTDATE,STRUCTURECATEGORY,LATITUDE,LONGITUDE';

function fetchPage(offset) {
  return new Promise((resolve, reject) => {
    const params = new URLSearchParams({
      where: '1=1',
      outFields: FIELDS,
      resultOffset: offset,
      resultRecordCount: PAGE_SIZE,
      orderByFields: 'OBJECTID ASC',
      f: 'json'
    });
    const url = `${BASE_URL}?${params.toString()}`;
    const req = https.get(url, (res) => {
      let body = '';
      res.on('data', chunk => body += chunk);
      res.on('end', () => {
        try {
          const d = JSON.parse(body);
          if (d.error) reject(new Error(JSON.stringify(d.error)));
          else resolve(d);
        } catch (e) { reject(e); }
      });
    });
    req.on('error', reject);
    req.setTimeout(30000, () => { req.destroy(); reject(new Error('timeout at offset ' + offset)); });
  });
}

async function fetchCount() {
  return new Promise((resolve, reject) => {
    const params = new URLSearchParams({ where: '1=1', returnCountOnly: 'true', f: 'json' });
    const url = `${BASE_URL}?${params.toString()}`;
    https.get(url, (res) => {
      let body = '';
      res.on('data', c => body += c);
      res.on('end', () => {
        try { resolve(JSON.parse(body).count); } catch (e) { reject(e); }
      });
    }).on('error', reject);
  });
}

async function main() {
  if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });

  console.log('Fetching total count from CAL FIRE master DINS service...');
  const total = await fetchCount();
  const fetchUpTo = Math.min(total, MAX_LIMIT);
  console.log(`Total records: ${total.toLocaleString()} — fetching up to ${fetchUpTo.toLocaleString()}`);

  const records = [];
  let offset = 0;
  let page = 0;

  while (offset < fetchUpTo) {
    const thisPage = Math.min(PAGE_SIZE, fetchUpTo - offset);
    process.stdout.write(`  Page ${++page}: offset=${offset} (${records.length.toLocaleString()} so far)...\r`);

    let result;
    for (let attempt = 0; attempt < 3; attempt++) {
      try {
        result = await fetchPage(offset);
        break;
      } catch (e) {
        console.warn(`\n  Error on page ${page} attempt ${attempt+1}: ${e.message}`);
        if (attempt === 2) throw e;
        await new Promise(r => setTimeout(r, 2000 * (attempt + 1)));
      }
    }

    const features = result.features || [];
    for (const f of features) {
      const a = f.attributes || {};
      records.push({
        id: a.OBJECTID,
        damage: a.DAMAGE,
        structureType: a.STRUCTURETYPE,
        structureCategory: a.STRUCTURECATEGORY,
        city: a.CITY,
        county: a.COUNTY,
        state: a.STATE,
        community: a.COMMUNITY,
        incident: a.INCIDENTNAME,
        incidentDate: a.INCIDENTSTARTDATE ? new Date(a.INCIDENTSTARTDATE).toISOString().slice(0, 10) : null,
        lat: a.LATITUDE,
        lng: a.LONGITUDE
      });
    }

    if (features.length < PAGE_SIZE) break; // last page
    offset += features.length;

    // Small pause to be polite to the public API
    await new Promise(r => setTimeout(r, 100));
  }

  console.log(`\nFetched ${records.length.toLocaleString()} records total.`);

  // Compute community stats
  const byCity = {};
  for (const r of records) {
    const key = (r.city || r.community || 'Unknown').trim();
    if (!byCity[key]) {
      byCity[key] = {
        city: key,
        county: r.county,
        state: r.state,
        incidents: new Set(),
        firstDate: r.incidentDate,
        lastDate: r.incidentDate,
        total: 0,
        damaged: 0, // includes all damage categories
        counts: {}
      };
    }
    const c = byCity[key];
    c.total++;
    if (r.incident) c.incidents.add(r.incident);
    if (r.incidentDate) {
      if (!c.firstDate || r.incidentDate < c.firstDate) c.firstDate = r.incidentDate;
      if (!c.lastDate || r.incidentDate > c.lastDate) c.lastDate = r.incidentDate;
    }
    const dmg = r.damage || 'Unknown';
    c.counts[dmg] = (c.counts[dmg] || 0) + 1;
    if (dmg !== 'No Damage') c.damaged++;
  }

  // Serialize (convert Set to Array)
  const communities = Object.values(byCity).map(c => ({
    ...c,
    incidents: [...c.incidents].sort()
  })).sort((a, b) => {
    // Sort by destroyed count desc, then total desc
    const ad = a.counts['Destroyed (>50%)'] || 0;
    const bd = b.counts['Destroyed (>50%)'] || 0;
    return bd - ad || b.total - a.total;
  });

  const destroyed = records.filter(r => r.damage === 'Destroyed (>50%)').length;

  const output = {
    fetchedAt: new Date().toISOString(),
    source: 'CAL FIRE DINS Master Data (services1.arcgis.com/jUJYIo9tSA7EHvfZ/POSTFIRE_MASTER_DATA_SHARE)',
    totalRecords: records.length,
    totalCities: communities.length,
    destroyedCount: destroyed,
    communities,
    records
  };

  fs.writeFileSync(OUT_FILE, JSON.stringify(output, null, 2));
  console.log(`Saved ${OUT_FILE}`);
  console.log(`  ${communities.length} communities, ${destroyed.toLocaleString()} destroyed structures`);
}

main().catch(e => { console.error('FATAL:', e.message); process.exit(1); });