← back to Permit Radar

scripts/ingest-stub.js

151 lines

// permit-radar / scripts/ingest-stub.js
//
// REAL PERMIT FEED SEAM
// ─────────────────────────────────────────────────────────────────────────────
// Replace data/permits.json sample data with real county permit records here.
//
// HOW A REAL FEED WORKS (per-county options):
//
// 1. MUNICIPAL / COUNTY OPEN DATA PORTALS (free, Socrata/ArcGIS APIs)
//    Many counties publish permit data via Socrata: e.g.
//    - Los Angeles County: data.lacounty.gov  (search "building permits")
//    - San Francisco:      data.sfgov.org      /resource/i98e-djp9.json
//    - Chicago:            data.cityofchicago.org  /resource/ydr8-5enu.json
//    Use the Socrata SODA API — free, JSON, filterable by date & type.
//    Example:
//      const { SodaClient } = require('soda-js');
//      const client = new SodaClient('data.lacounty.gov');
//      client.query().withDataset('permit_dataset_id').where('permit_type_desc', 'REMODEL')
//            .getRows().on('row', row => permits.push(normalizeRow(row)));
//
// 2. BUILDFAX / PERMITZIP / SALESDRIVE (paid commercial feeds)
//    These aggregate nationwide permit data behind a subscription API.
//    - BuildFax: https://buildfax.com/api  (REST, JSON, $$ per county)
//    - PermitZip: bulk CSV download or webhook per jurisdiction
//    Wire credentials via env: BUILDFAX_API_KEY=... or PERMITZIP_API_KEY=...
//
// 3. DIRECT COUNTY PORTAL SCRAPING (inconsistent, often paywalled)
//    Some counties use iParc / EnerGov / Accela portals. Most require auth
//    or session tokens. NOT recommended for production without legal review.
//
// NORMALIZATION CONTRACT
// ─────────────────────────────────────────────────────────────────────────────
// Every upstream record must be normalized to this schema before writing to
// data/permits.json so the server + frontend work without changes:
//
//   {
//     id: string,           // unique e.g. "LA-2024-100001"
//     address: string,      // street address
//     city: string,
//     jurisdiction: string, // "Los Angeles County"
//     state: string,        // "CA"
//     zip: string,
//     date_filed: string,   // "YYYY-MM-DD"
//     permit_type: string,  // machine key: kitchen_remodel | bath_remodel | addition | full_remodel | adu | reroof
//     permit_type_label: string, // human label
//     valuation: number,    // dollar amount
//     contractor: string,   // may be ""
//     contractor_license: string,
//     status: string,       // "approved" | "pending" | "issued"
//     description: string,
//     bedrooms: number,
//     sqft_affected: number,
//     created_at: string,   // ISO timestamp of when WE ingested it
//   }
//
// USAGE (to be run as a cron job or manual refresh):
//   node scripts/ingest-stub.js --county "los-angeles" --days 7
//   node scripts/ingest-stub.js --source buildfax --zips 90210,90401
//
// The stub below shows the structure; replace with real API calls.
// ─────────────────────────────────────────────────────────────────────────────

'use strict';
const fs   = require('fs');
const path = require('path');

const DATA_FILE = path.join(__dirname, '..', 'data', 'permits.json');

// ── Normalization helper ──────────────────────────────────────────────────────
function normalizeRow(raw, jurisdiction, source) {
  // Adapt field names from whichever source is feeding this.
  // This is a skeleton — fill in per-source mappings.
  return {
    id: source + '-' + (raw.permit_number || raw.id || Date.now()),
    address: raw.address || raw.street_address || '',
    city: raw.city || raw.city_name || '',
    jurisdiction: jurisdiction,
    state: raw.state || '',
    zip: raw.zip_code || raw.zip || '',
    date_filed: (raw.issue_date || raw.filed_date || raw.date_filed || '').slice(0, 10),
    permit_type: mapPermitType(raw.permit_type || raw.permit_type_desc || ''),
    permit_type_label: raw.permit_type_desc || raw.permit_type || '',
    valuation: parseFloat(raw.valuation || raw.estimated_cost || 0) || 0,
    contractor: raw.contractor_name || raw.contractor || '',
    contractor_license: raw.contractor_license || '',
    status: (raw.status || 'pending').toLowerCase(),
    description: raw.description || raw.work_description || '',
    bedrooms: parseInt(raw.bedrooms || 0, 10) || 0,
    sqft_affected: parseInt(raw.sqft || raw.sq_ft_affected || 0, 10) || 0,
    created_at: new Date().toISOString(),
  };
}

function mapPermitType(raw) {
  const r = (raw || '').toLowerCase();
  if (r.includes('kitchen'))                      return 'kitchen_remodel';
  if (r.includes('bath'))                         return 'bath_remodel';
  if (r.includes('addition') || r.includes('adu') || r.includes('accessory')) return r.includes('adu') || r.includes('accessory') ? 'adu' : 'addition';
  if (r.includes('reroof') || r.includes('roof')) return 'reroof';
  if (r.includes('remodel') || r.includes('renovation') || r.includes('interior')) return 'full_remodel';
  return 'full_remodel';
}

// ── Stub: fake fetch from a Socrata endpoint ──────────────────────────────────
async function fetchFromSocrata(domain, dataset, jurisdiction) {
  // Real impl: replace URL below with actual county Socrata dataset ID.
  // const url = `https://${domain}/resource/${dataset}.json?$where=date_filed>='${since}'&$limit=1000`;
  // const rows = await fetch(url).then(r => r.json());
  // return rows.map(r => normalizeRow(r, jurisdiction, domain));
  console.log(`[ingest-stub] STUB: would fetch from https://${domain}/resource/${dataset}.json`);
  return [];
}

// ── Main ──────────────────────────────────────────────────────────────────────
async function main() {
  const args = process.argv.slice(2);
  const dryRun = args.includes('--dry-run');
  console.log('[ingest-stub] Real-feed seam — replace stubs with live API calls.');
  console.log('[ingest-stub] Source options: Socrata open data, BuildFax, PermitZip — see comments above.');
  if (dryRun) { console.log('[ingest-stub] --dry-run: no writes.'); return; }

  // Example county feed registrations:
  const sources = [
    { domain: 'data.lacounty.gov',      dataset: 'PLACEHOLDER_DATASET_ID', jurisdiction: 'Los Angeles County' },
    { domain: 'data.sfgov.org',         dataset: 'i98e-djp9',              jurisdiction: 'San Francisco County' },
    { domain: 'data.cityofchicago.org', dataset: 'ydr8-5enu',              jurisdiction: 'Cook County' },
  ];

  let all = [];
  for (const s of sources) {
    const rows = await fetchFromSocrata(s.domain, s.dataset, s.jurisdiction);
    all = all.concat(rows);
  }

  if (!all.length) {
    console.log('[ingest-stub] No records fetched (stubs return empty). Wire real APIs to populate.');
    return;
  }

  // Merge with existing — deduplicate by id
  let existing = [];
  try { existing = JSON.parse(fs.readFileSync(DATA_FILE, 'utf8')); } catch {}
  const existingIds = new Set(existing.map(p => p.id));
  const newRecords = all.filter(p => !existingIds.has(p.id));
  const merged = [...existing, ...newRecords].sort((a, b) => b.date_filed.localeCompare(a.date_filed));
  fs.writeFileSync(DATA_FILE, JSON.stringify(merged, null, 2), 'utf8');
  console.log(`[ingest-stub] Wrote ${merged.length} total permits (${newRecords.length} new) to data/permits.json`);
}

main().catch(err => { console.error('[ingest-stub] Error:', err.message); process.exit(1); });