`scripts/ingest-osm-salons.js`

250 lines
#!/usr/bin/env node
// Ingest salon-adjacent businesses from OpenStreetMap via Overpass API.
//
// Why this complements the BBCB ingest:
//   - OSM has shops that BBCB doesn't (massage parlors, day spas, lash/brow
//     studios, blowout bars not registered as full establishments)
//   - OSM provides lat/lng — BBCB only gives addresses, so we use OSM coords
//     to backfill geocoding on shops that match by name+zip
//   - Free, no API key, single bulk query
//
// Targets: 5 counties — LA, Orange, Riverside, San Bernardino, Ventura.
//
// OSM tags we treat as salons:
//   shop=hairdresser  → barbershop or salon
//   shop=beauty       → salon (catch-all: nails, lashes, brows, waxing, esthetics)
//   shop=massage      → spa
//   amenity=spa       → spa
//   leisure=spa       → spa
//
//   node scripts/ingest-osm-salons.js               # full ingest
//   DRY=1 node scripts/ingest-osm-salons.js         # parse only, no DB writes
//   COUNTY=LA node scripts/ingest-osm-salons.js     # one county at a time

import 'dotenv/config';
import fs from 'node:fs';
import path from 'node:path';
import { query, one } from '../src/lib/db.js';
import { slugify } from '../src/lib/slug.js';

const TMP_DIR = '/tmp/osm-ingest';
const OVERPASS_URL = process.env.OVERPASS_URL || 'https://overpass-api.de/api/interpreter';

const COUNTIES = {
  LA:           { name: 'Los Angeles County',      key: 'LOS ANGELES' },
  ORANGE:       { name: 'Orange County',           key: 'ORANGE' },
  RIVERSIDE:    { name: 'Riverside County',        key: 'RIVERSIDE' },
  SAN_BERNARDINO:{ name: 'San Bernardino County',  key: 'SAN BERNARDINO' },
  VENTURA:      { name: 'Ventura County',          key: 'VENTURA' },
};

const TAG_TO_CATEGORY = {
  hairdresser: 'salon',  // OSM uses hairdresser for both barbers AND hair salons; refine via name
  beauty:      'salon',
  massage:     'spa',
};

const DRY = process.env.DRY === '1';
const ONLY = process.env.COUNTY ? process.env.COUNTY.toUpperCase() : null;

function ts() { return new Date().toISOString().replace('T', ' ').slice(0, 19); }
function log(m) { console.log(`[${ts()}] ${m}`); }

// Build Overpass QL for one county.
function buildQuery(countyName) {
  return `[out:json][timeout:180];
area["name"="${countyName}"]["admin_level"="6"]->.a;
(
  node["shop"="hairdresser"](area.a);
  way["shop"="hairdresser"](area.a);
  node["shop"="beauty"](area.a);
  way["shop"="beauty"](area.a);
  node["shop"="massage"](area.a);
  way["shop"="massage"](area.a);
  node["amenity"="spa"](area.a);
  way["amenity"="spa"](area.a);
  node["leisure"="spa"](area.a);
  way["leisure"="spa"](area.a);
);
out center tags;`;
}

// Cache the Overpass response per county so re-runs don't hammer the public API.
async function fetchCounty(countyKey, countyName) {
  const cachePath = path.join(TMP_DIR, `osm_${countyKey.toLowerCase().replace(/_/g, '-')}.json`);
  if (fs.existsSync(cachePath) && fs.statSync(cachePath).size > 1000) {
    log(`  ${countyKey}: reusing cache (${(fs.statSync(cachePath).size / 1024).toFixed(0)} KB)`);
    return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
  }
  log(`  ${countyKey}: querying Overpass (~30-60s)…`);
  // Overpass 406s without explicit Accept + User-Agent (anti-abuse heuristic).
  const body = 'data=' + encodeURIComponent(buildQuery(countyName));
  const res = await fetch(OVERPASS_URL, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/x-www-form-urlencoded',
      'Accept': 'application/json',
      'User-Agent': 'la-salon-directory/1.0 (+steveabramsdesigns@gmail.com)',
    },
    body,
  });
  if (!res.ok) throw new Error(`Overpass HTTP ${res.status} for ${countyKey}`);
  const j = await res.json();
  fs.writeFileSync(cachePath, JSON.stringify(j));
  log(`  ${countyKey}: ${j.elements?.length || 0} elements, cached → ${cachePath}`);
  return j;
}

// Heuristic: "barbershop" if name contains barber/cuts/fade keywords AND tag is hairdresser.
function refineCategory(rawTag, name) {
  const n = String(name || '').toLowerCase();
  if (rawTag === 'hairdresser' && /\b(barber|barbers|barbershop|fade|cuts?|men.?s)\b/.test(n)) return 'barbershop';
  return TAG_TO_CATEGORY[rawTag] || 'salon';
}

function makeSlug(name, zip, osmId) {
  const base = slugify(name) || 'salon';
  if (zip && /^\d{5}/.test(zip)) return `${base}-${zip.slice(0, 5)}`;
  return `osm-${osmId}-${base}`.slice(0, 200);
}

// === Per-element processor =========================================
function processElement(el, countyKey) {
  const t = el.tags || {};
  const name = (t.name || '').trim();
  if (!name) return null;

  // Coordinates: nodes have lat/lon directly; ways carry .center.lat / .center.lon
  const lat = el.lat ?? el.center?.lat ?? null;
  const lon = el.lon ?? el.center?.lon ?? null;

  // Determine the raw tag we matched on (priority order matches the Overpass query)
  let rawTag = null;
  if (t.shop === 'hairdresser')  rawTag = 'hairdresser';
  else if (t.shop === 'beauty')  rawTag = 'beauty';
  else if (t.shop === 'massage') rawTag = 'massage';
  else if (t.amenity === 'spa')  rawTag = 'massage';
  else if (t.leisure === 'spa')  rawTag = 'massage';
  else return null;

  const category = refineCategory(rawTag, name);
  const street = [t['addr:housenumber'], t['addr:street']].filter(Boolean).join(' ').trim();
  const city = (t['addr:city'] || '').trim();
  const state = (t['addr:state'] || 'CA').trim();
  const zip = (t['addr:postcode'] || '').trim().slice(0, 10);
  const phone = (t.phone || t['contact:phone'] || '').trim().slice(0, 32) || null;
  const website = (t.website || t['contact:website'] || '').trim() || null;

  return {
    slug: makeSlug(name, zip, el.id),
    name,
    category,
    address: street || null,
    city: city || null,
    state,
    zip: zip || null,
    neighborhood: countyKey === 'LA' ? null : COUNTIES[countyKey].key, // leave LA neighborhoods blank for now (real LA neighborhoods need geocoding)
    latitude: lat,
    longitude: lon,
    phone,
    website,
    source_data_json: {
      source: 'osm',
      source_id: `${el.type}/${el.id}`,
      source_url: `https://www.openstreetmap.org/${el.type}/${el.id}`,
      tier: 'B',
      retrieved_at: new Date().toISOString(),
      raw_tag: rawTag,
      county_search: COUNTIES[countyKey].key,
      tags: t,
    },
  };
}

async function upsert(row) {
  // Use COALESCE to NEVER overwrite a non-null with a null on update — this
  // matters when an existing BBCB row gets matched by slug and we just want
  // to fill in lat/lng + phone + website without nuking the BBCB metadata.
  const r = await query(
    `INSERT INTO businesses
       (slug, name, category, address, city, state, zip, neighborhood,
        latitude, longitude, phone, website, source_data_json, tier)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,'free')
     ON CONFLICT (slug) DO UPDATE SET
       latitude       = COALESCE(businesses.latitude, EXCLUDED.latitude),
       longitude      = COALESCE(businesses.longitude, EXCLUDED.longitude),
       phone          = COALESCE(businesses.phone, EXCLUDED.phone),
       website        = COALESCE(businesses.website, EXCLUDED.website),
       address        = COALESCE(NULLIF(businesses.address,''), EXCLUDED.address),
       city           = COALESCE(NULLIF(businesses.city,''), EXCLUDED.city),
       zip            = COALESCE(NULLIF(businesses.zip,''), EXCLUDED.zip),
       updated_at     = now()
     RETURNING (xmax = 0) AS was_insert`,
    [row.slug, row.name, row.category, row.address, row.city, row.state, row.zip,
     row.neighborhood, row.latitude, row.longitude, row.phone, row.website,
     JSON.stringify(row.source_data_json)]
  );
  return r.rows[0]?.was_insert;
}

// === Main ==========================================================
async function main() {
  fs.mkdirSync(TMP_DIR, { recursive: true });
  log(`mode=${DRY ? 'DRY' : 'WRITE'}${ONLY ? ` only=${ONLY}` : ''}`);

  const counties = ONLY ? [[ONLY, COUNTIES[ONLY]]] : Object.entries(COUNTIES);
  if (ONLY && !COUNTIES[ONLY]) { console.error('unknown COUNTY:', ONLY); process.exit(2); }

  const totals = { matched: 0, inserted: 0, updated: 0, skipped: 0 };
  const byCategory = {};

  for (const [key, def] of counties) {
    const data = await fetchCounty(key, def.name);
    let countyMatched = 0, countyInserted = 0, countyUpdated = 0;

    for (const el of data.elements || []) {
      const row = processElement(el, key);
      if (!row) { totals.skipped++; continue; }
      countyMatched++;
      byCategory[row.category] = (byCategory[row.category] || 0) + 1;

      if (DRY) {
        if (countyMatched <= 5) console.log(`  ${row.slug.padEnd(50)} ${row.name} · ${row.city || '?'} · ${row.category} · ${row.latitude?.toFixed(4) || '—'},${row.longitude?.toFixed(4) || '—'}`);
        continue;
      }
      try {
        const inserted = await upsert(row);
        if (inserted) countyInserted++;
        else countyUpdated++;
      } catch (e) {
        if (totals.matched < 5) log(`  ERR ${row.slug}: ${e.message.slice(0, 200)}`);
      }
    }
    totals.matched += countyMatched;
    totals.inserted += countyInserted;
    totals.updated += countyUpdated;
    log(`  ${key}: matched=${countyMatched} inserted=${countyInserted} updated=${countyUpdated}`);
  }

  log(`TOTAL: matched=${totals.matched} inserted=${totals.inserted} updated=${totals.updated} skipped=${totals.skipped}`);
  log(`BY CATEGORY: ${Object.entries(byCategory).map(([k,v])=>`${k}=${v}`).join(' ')}`);

  if (!DRY) {
    const tally = await one(`
      SELECT
        COUNT(*) AS total,
        COUNT(*) FILTER (WHERE source_data_json->>'source' = 'dca_bbcb') AS bbcb,
        COUNT(*) FILTER (WHERE source_data_json->>'source' = 'osm') AS osm,
        COUNT(*) FILTER (WHERE latitude IS NOT NULL) AS geocoded,
        COUNT(*) FILTER (WHERE category = 'spa') AS spas,
        COUNT(*) FILTER (WHERE category = 'salon') AS salons,
        COUNT(*) FILTER (WHERE category = 'barbershop') AS barbershops
      FROM businesses
    `);
    log(`DB now: total=${tally.total} bbcb=${tally.bbcb} osm=${tally.osm} geocoded=${tally.geocoded} salons=${tally.salons} barbershops=${tally.barbershops} spas=${tally.spas}`);
  }
  process.exit(0);
}

main().catch(e => { console.error(e); process.exit(2); });