`scripts/ingest-wilshire-corridor.js`

328 lines
#!/usr/bin/env node
// Wilshire Blvd Corridor ingest — every registered business on the ~16-mile
// stretch of Wilshire Boulevard from Downtown LA to the Pacific Ocean.
//
// Wilshire Blvd is essentially LA's "main street" — 16 miles of nearly flat,
// straight east-west corridor from Wilshire/Grand (DTLA) west through:
//   - MacArthur Park / Westlake (City of Los Angeles)
//   - Koreatown
//   - Mid-Wilshire / Hancock Park
//   - Miracle Mile (LACMA, Petersen Automotive Museum)
//   - Beverly Hills (Wilshire crosses through 90211/90212/90210)
//   - Westwood (UCLA, Federal Building)
//   - Brentwood
//   - Santa Monica → ocean
//
// "Registered with the city" = LA Office of Finance BTRC for City-of-LA
// stretches, plus Beverly Hills + Santa Monica business licenses for those
// jurisdictions. Same approximation pattern as Sunset/Ventura: pull every
// commercial OSM presence on/near Wilshire Blvd, tag them
// `corridor: 'wilshire-blvd-16mi'` for filtering.
//
//   node scripts/ingest-wilshire-corridor.js          # full corridor ingest
//   DRY=1 node scripts/ingest-wilshire-corridor.js    # parse only

import 'dotenv/config';
import fs from 'node:fs';
import path from 'node:path';
import { query, one } from '../src/lib/db.js';
import { slugify } from '../src/lib/slug.js';

const TMP_DIR = '/tmp/osm-ingest';
const OVERPASS_URL = process.env.OVERPASS_URL || 'https://overpass-api.de/api/interpreter';
const CORRIDOR_TAG = 'wilshire-blvd-16mi';

// Bounding box covering Wilshire Blvd from DTLA (east end, ~ -118.255) to the
// ocean at Santa Monica (west end, ~ -118.510). Wilshire is essentially flat
// east-west, so the latitude band is narrow (34.040 - 34.075).
// Overpass uses "south,west,north,east".
const BBOX = '34.040,-118.510,34.075,-118.235';

// Reverse-mapping OSM tag → our category. Matches Sunset corridor exactly so
// the editorial UI behaves the same.
const SHOP_TAG_TO_CATEGORY = {
  hairdresser: 'salon', beauty: 'salon', massage: 'spa', cosmetics: 'salon',
  // Eat/drink
  restaurant: 'restaurant', cafe: 'cafe', bakery: 'bakery', deli: 'deli',
  bar: 'bar', pub: 'bar', fast_food: 'restaurant', ice_cream: 'cafe',
  // Retail
  clothes: 'retail', shoes: 'retail', jewelry: 'retail', books: 'retail',
  florist: 'retail', gift: 'retail', wine: 'retail', alcohol: 'retail',
  optician: 'retail', furniture: 'retail', boutique: 'retail',
  supermarket: 'grocery', convenience: 'grocery',
  // Services
  dry_cleaning: 'service', laundry: 'service', tailor: 'service',
  car_repair: 'service', car: 'service',
  // Health / wellness
  pharmacy: 'pharmacy', chemist: 'pharmacy',
  // Pets
  pet: 'retail',
};
const AMENITY_TAG_TO_CATEGORY = {
  restaurant: 'restaurant', cafe: 'cafe', bar: 'bar', pub: 'bar',
  fast_food: 'restaurant', ice_cream: 'cafe', food_court: 'restaurant',
  bank: 'finance', atm: 'finance',
  pharmacy: 'pharmacy', dentist: 'health', doctors: 'health', clinic: 'health', veterinary: 'health',
  spa: 'spa',
  gym: 'fitness', fitness_centre: 'fitness',
  cinema: 'entertainment', theatre: 'entertainment', nightclub: 'entertainment',
  fuel: 'service', car_wash: 'service',
};

const DRY = process.env.DRY === '1';

// BBCB zip codes that fall along Wilshire Blvd. Used in the SQL post-step to
// retroactively flag DCA-licensed shops that are already in the DB but
// weren't tagged with the corridor (their `addr:street` from BBCB usually
// mentions "Wilshire" if they're actually on the strip).
//
// City of LA portion: 90017 (DTLA), 90057 (MacArthur Park / Westlake),
// 90020 (Koreatown), 90010 (Mid-Wilshire / Park Mile),
// 90005 (Koreatown south), 90004 (Hancock Park),
// 90019 (Mid-Wilshire west), 90036 (Miracle Mile),
// 90035 (south Beverly Hills border), 90048 (Beverly Grove).
// Beverly Hills: 90211, 90212, 90210.
// Westwood: 90024. Brentwood: 90049.
// Santa Monica: 90401, 90403, 90404.
const WILSHIRE_ZIPS = [
  '90017', // DTLA (eastern end)
  '90057', // MacArthur Park / Westlake
  '90020', // Koreatown
  '90010', // Mid-Wilshire / Park Mile
  '90005', // Koreatown south
  '90004', // Hancock Park
  '90019', // Mid-Wilshire west
  '90036', // Miracle Mile (LACMA, Petersen)
  '90035', // south Beverly Hills border
  '90048', // Beverly Grove
  '90211', // Beverly Hills
  '90212', // Beverly Hills
  '90210', // Beverly Hills (Wilshire crosses through 90210)
  '90024', // Westwood (UCLA, Federal Building)
  '90049', // Brentwood
  '90401', // Santa Monica (oceanside)
  '90403', // Santa Monica
  '90404', // Santa Monica
];

function ts() { return new Date().toISOString().replace('T', ' ').slice(0, 19); }
function log(m) { console.log(`[${ts()}] ${m}`); }

function buildQuery() {
  return `[out:json][timeout:240];
(
  node["shop"](${BBOX});
  way["shop"](${BBOX});
  node["amenity"~"^(restaurant|cafe|bar|pub|fast_food|ice_cream|food_court|bank|atm|pharmacy|dentist|doctors|clinic|veterinary|spa|gym|fitness_centre|cinema|theatre|nightclub|fuel|car_wash)$"](${BBOX});
  way["amenity"~"^(restaurant|cafe|bar|pub|fast_food|ice_cream|food_court|bank|atm|pharmacy|dentist|doctors|clinic|veterinary|spa|gym|fitness_centre|cinema|theatre|nightclub|fuel|car_wash)$"](${BBOX});
  node["leisure"="spa"](${BBOX});
  way["leisure"="spa"](${BBOX});
  node["office"](${BBOX});
  way["office"](${BBOX});
);
out center tags;`;
}

async function fetchData() {
  const cachePath = path.join(TMP_DIR, 'wilshire-corridor.json');
  if (fs.existsSync(cachePath) && fs.statSync(cachePath).size > 1000) {
    log(`reusing cache (${(fs.statSync(cachePath).size / 1024).toFixed(0)} KB)`);
    return JSON.parse(fs.readFileSync(cachePath, 'utf8'));
  }
  log('querying Overpass for the entire Wilshire Blvd corridor (~30-90s)…');
  const body = 'data=' + encodeURIComponent(buildQuery());
  const res = await fetch(OVERPASS_URL, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/x-www-form-urlencoded',
      'Accept': 'application/json',
      'User-Agent': 'la-salon-directory/1.0 (+steveabramsdesigns@gmail.com)',
    },
    body,
  });
  if (!res.ok) throw new Error(`Overpass HTTP ${res.status}`);
  const j = await res.json();
  fs.writeFileSync(cachePath, JSON.stringify(j));
  log(`${j.elements?.length || 0} elements, cached → ${cachePath}`);
  return j;
}

// Determine category from OSM tags. Priority: shop → amenity → leisure → office.
function categorizeFromTags(t) {
  if (t.shop && SHOP_TAG_TO_CATEGORY[t.shop])         return { cat: SHOP_TAG_TO_CATEGORY[t.shop], rawTag: 'shop=' + t.shop };
  if (t.amenity && AMENITY_TAG_TO_CATEGORY[t.amenity]) return { cat: AMENITY_TAG_TO_CATEGORY[t.amenity], rawTag: 'amenity=' + t.amenity };
  if (t.leisure === 'spa')                            return { cat: 'spa', rawTag: 'leisure=spa' };
  if (t.office)                                       return { cat: 'office', rawTag: 'office=' + t.office };
  // Catch-all: any shop=* tag we didn't enumerate
  if (t.shop)                                         return { cat: 'retail', rawTag: 'shop=' + t.shop };
  return null;
}

// Address-based filter: only keep elements where addr:street contains
// "Wilshire" (Wilshire Blvd, Wilshire Boulevard, etc.) — this excludes
// everything within the bbox that's on a side street.
function isOnWilshireBlvd(t) {
  const street = (t['addr:street'] || '').toLowerCase();
  return /wilshire/.test(street);
}

function refineSalonCategory(rawTag, name, base) {
  if (base !== 'salon') return base;
  if (/\b(barber|barbers|barbershop|fade|cuts?|men.?s)\b/i.test(name || '')) return 'barbershop';
  return 'salon';
}

function makeSlug(name, zip, osmId) {
  const base = slugify(name) || 'biz';
  if (zip && /^\d{5}/.test(zip)) return `${base}-${zip.slice(0, 5)}`;
  return `osm-${osmId}-${base}`.slice(0, 200);
}

function processElement(el) {
  const t = el.tags || {};
  const name = (t.name || '').trim();
  if (!name) return null;
  const cat = categorizeFromTags(t);
  if (!cat) return null;
  if (!isOnWilshireBlvd(t)) return null;

  const lat = el.lat ?? el.center?.lat ?? null;
  const lon = el.lon ?? el.center?.lon ?? null;
  const street = [t['addr:housenumber'], t['addr:street']].filter(Boolean).join(' ').trim();
  const city = (t['addr:city'] || '').trim();
  const state = (t['addr:state'] || 'CA').trim();
  const zip = (t['addr:postcode'] || '').trim().slice(0, 10);
  const phone = (t.phone || t['contact:phone'] || '').trim().slice(0, 32) || null;
  const website = (t.website || t['contact:website'] || '').trim() || null;

  const finalCat = refineSalonCategory(cat.rawTag, name, cat.cat);

  return {
    slug: makeSlug(name, zip, el.id),
    name,
    category: finalCat,
    address: street || null,
    city: city || null,
    state,
    zip: zip || null,
    neighborhood: city || null,
    latitude: lat,
    longitude: lon,
    phone,
    website,
    source_data_json: {
      source: 'osm-corridor',
      source_id: `${el.type}/${el.id}`,
      source_url: `https://www.openstreetmap.org/${el.type}/${el.id}`,
      tier: 'B',
      retrieved_at: new Date().toISOString(),
      raw_tag: cat.rawTag,
      corridor: CORRIDOR_TAG,
      tags: t,
    },
  };
}

async function upsert(row) {
  const r = await query(
    `INSERT INTO businesses
       (slug, name, category, address, city, state, zip, neighborhood,
        latitude, longitude, phone, website, source_data_json, tier)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,'free')
     ON CONFLICT (slug) DO UPDATE SET
       latitude       = COALESCE(businesses.latitude, EXCLUDED.latitude),
       longitude      = COALESCE(businesses.longitude, EXCLUDED.longitude),
       phone          = COALESCE(businesses.phone, EXCLUDED.phone),
       website        = COALESCE(businesses.website, EXCLUDED.website),
       neighborhood   = COALESCE(EXCLUDED.neighborhood, businesses.neighborhood),
       address        = COALESCE(NULLIF(businesses.address,''), EXCLUDED.address),
       source_data_json = jsonb_set(
         COALESCE(businesses.source_data_json, '{}'::jsonb),
         '{corridor}', to_jsonb(${"'" + CORRIDOR_TAG + "'"}::text)
       ),
       updated_at     = now()
     RETURNING (xmax = 0) AS was_insert`,
    [row.slug, row.name, row.category, row.address, row.city, row.state, row.zip,
     row.neighborhood, row.latitude, row.longitude, row.phone, row.website,
     JSON.stringify(row.source_data_json)]
  );
  return r.rows[0]?.was_insert;
}

// Tag any DCA-licensed BBCB shop whose zip falls in the Wilshire corridor zip
// list AND whose address contains "wilshire". Mirrors the Sunset post-step.
async function tagBbcbAlongWilshire() {
  const zipPattern = '^(' + WILSHIRE_ZIPS.join('|') + ')';
  const r = await query(
    `UPDATE businesses
        SET source_data_json = jsonb_set(
              COALESCE(source_data_json, '{}'::jsonb),
              '{corridor}',
              to_jsonb($2::text)
            ),
            updated_at = now()
      WHERE source_data_json->>'source' = 'dca_bbcb'
        AND zip ~ $1
        AND address ~* 'wilshire'
      RETURNING slug`,
    [zipPattern, CORRIDOR_TAG]
  );
  return r.rowCount;
}

async function main() {
  fs.mkdirSync(TMP_DIR, { recursive: true });
  log(`mode=${DRY ? 'DRY' : 'WRITE'} corridor=${CORRIDOR_TAG} bbox=${BBOX}`);

  const data = await fetchData();
  let matched = 0, inserted = 0, updated = 0, skippedNotOnWilshire = 0, skippedNoCat = 0;
  const byCategory = {};
  const byCity = {};

  for (const el of data.elements || []) {
    const t = el.tags || {};
    const cat = categorizeFromTags(t);
    if (!cat || !t.name) { skippedNoCat++; continue; }
    if (!isOnWilshireBlvd(t)) { skippedNotOnWilshire++; continue; }

    const row = processElement(el);
    if (!row) continue;
    matched++;
    byCategory[row.category] = (byCategory[row.category] || 0) + 1;
    if (row.city) byCity[row.city] = (byCity[row.city] || 0) + 1;

    if (DRY) {
      if (matched <= 12) console.log(`  ${(row.slug || '').padEnd(60)} ${row.name} · ${row.category} · ${row.city || '?'}`);
      continue;
    }
    try {
      const ins = await upsert(row);
      if (ins) inserted++; else updated++;
    } catch (e) {
      log(`  ERR ${row.slug}: ${e.message.slice(0, 200)}`);
    }
  }

  log(`MATCHED=${matched} inserted=${inserted} updated=${updated}`);
  log(`SKIPPED: not_on_wilshire=${skippedNotOnWilshire} no_category=${skippedNoCat}`);
  log(`BY CATEGORY: ${Object.entries(byCategory).sort((a,b)=>b[1]-a[1]).slice(0,15).map(([k,v])=>`${k}=${v}`).join(' ')}`);
  log(`BY CITY: ${Object.entries(byCity).sort((a,b)=>b[1]-a[1]).slice(0,10).map(([k,v])=>`${k}=${v}`).join(' ')}`);

  let bbcbTagged = 0;
  if (!DRY) {
    bbcbTagged = await tagBbcbAlongWilshire();
    log(`BBCB-along-Wilshire tagged: ${bbcbTagged}`);

    const tally = await one(`
      SELECT COUNT(*) AS n,
             COUNT(*) FILTER (WHERE latitude IS NOT NULL) AS geocoded
        FROM businesses
       WHERE source_data_json->>'corridor' = $1
    `, [CORRIDOR_TAG]).catch(() => null);
    if (tally) log(`DB now: corridor=${tally.n} geocoded=${tally.geocoded}`);
  }
  process.exit(0);
}

main().catch(e => { console.error(e); process.exit(2); });