`scripts/refresh-deals.mjs`

118 lines
#!/usr/bin/env node
// refresh-deals.mjs — rebuild data/deals-sample.json from the CURRENT CRCP corpus.
// Sources (read-only): ~/Projects/commercialrealestate/data/ranked.json (scored deals, no coords)
//                      ~/Projects/commercialrealestate/data/map-points.json (coords, keyed listing-<id>)
// Deals are scored with the deliverable's OWN transparent model (public/deal-score.js) so the
// Live Deal Desk, map color ramp, and cards all rank identically. Demographics are PRESERVED
// (Census median data — doesn't move month to month). $0 / local. Idempotent.
import { readFileSync, writeFileSync, copyFileSync, existsSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createRequire } from 'node:module';

const HERE = dirname(fileURLToPath(import.meta.url));
const ROOT = join(HERE, '..');
const DATA = join(ROOT, 'data');
const CRCP = join(process.env.HOME, 'Projects', 'commercialrealestate', 'data');
const require = createRequire(import.meta.url);
const DealScore = require(join(ROOT, 'public', 'deal-score.js'));

const readJson = (p, fb) => { try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return fb; } };
const asArray = (v) => Array.isArray(v) ? v : (v && (v.deals || v.ranked || v.items || v.points || v.map || Object.values(v).find(Array.isArray))) || [];
// Normalize ALL-CAPS vendor strings to Title Case (keep short tokens like directionals uppercased where sane).
const titleCase = (s) => typeof s === 'string'
  ? s.replace(/\w\S*/g, w => w.length <= 2 && /^[NSEW]{1,2}$/i.test(w) ? w.toUpperCase() : w.charAt(0).toUpperCase() + w.slice(1).toLowerCase())
  : s;
// LA-metro bounds — keep the map tight (drops Antelope Valley / far-outlier pins).
const inLA = (lat, lng) => lat >= 33.5 && lat <= 34.5 && lng >= -119 && lng <= -117.5;

const ranked = asArray(readJson(join(CRCP, 'ranked.json'), []));
const points = asArray(readJson(join(CRCP, 'map-points.json'), []));
const prev   = readJson(join(DATA, 'deals-sample.json'), {});

if (!ranked.length) { console.error('FATAL: ranked.json empty/unreadable — aborting, leaving deals-sample.json untouched'); process.exit(1); }

// coord index: strip "listing-"/"deal-" prefixes so ranked ids join
const coord = new Map();
for (const p of points) {
  if (!(p.lat && p.lng)) continue;
  const k = String(p.id || '').replace(/^(listing|deal)-/, '');
  if (k) coord.set(k, { lat: p.lat, lng: p.lng });
}

// score every ranked deal with the deliverable's model, join coords
const scored = [];
for (const d of ranked) {
  const f = d.finance || {};
  const s = DealScore.compute({
    grossYieldPro: f.grossYield ?? f.grossAnnual ?? null,
    cap_rate: d.cap_rate, verified: d.verified === true,
    dscr: f.dscr ?? null, proxyDscr: f.proxyDscr ?? null,
  });
  if (s.score == null) continue;
  const c = coord.get(String(d.id));
  scored.push({
    id: d.id, address: titleCase(d.address), city: titleCase(d.city), type: d.type,
    price: d.price ?? null, units: d.units ?? null,
    cap_rate: d.cap_rate ?? null, dscr: f.dscr ?? null, coc: f.coc ?? null,
    score: Math.round(s.score),
    lat: c ? c.lat : null, lng: c ? c.lng : null,
  });
}

// deals[] = top-scored WITH geo (so the map desk is populated), highest first.
// Guards (audit findings): drop priceless cards, the implausible single-unit mega-price
// (land/dev mispriced as multifamily), and out-of-LA outliers; then dedupe CRCP near-dupes.
const seenAddr = new Set();
const withGeo = scored
  .filter(d => d.lat && d.lng && d.price != null)
  .filter(d => inLA(d.lat, d.lng))
  .filter(d => !(d.units === 1 && d.price > 15e6))
  .sort((a, b) => b.score - a.score)
  .filter(d => {
    const k = String(d.address || '').toLowerCase().replace(/\s+/g, ' ').trim() + '@' + d.lat.toFixed(4) + ',' + d.lng.toFixed(4);
    if (seenAddr.has(k)) return false; seenAddr.add(k); return true;
  });
const DEALS_N = 60;
const deals = withGeo.slice(0, DEALS_N);
const usedIds = new Set(deals.map(d => d.id));

// extraPoints[] = fresh map density (map-points not already a scored card), capped
const EXTRA_N = 180;
const extraPoints = [];
for (const p of points) {
  if (!(p.lat && p.lng)) continue;
  const k = String(p.id || '').replace(/^(listing|deal)-/, '');
  if (usedIds.has(k)) continue;
  if (!inLA(p.lat, p.lng)) continue;
  extraPoints.push({ address: titleCase(p.address), city: titleCase(p.city), price: p.price ?? null, type: p.type || 'Commercial', lat: p.lat, lng: p.lng });
  if (extraPoints.length >= EXTRA_N) break;
}

// demographics — PRESERVE the existing Census block (curated; not in CRCP feed)
const demographics = Array.isArray(prev.demographics) && prev.demographics.length
  ? prev.demographics
  : [];

const out = {
  meta: {
    source: 'CRCP ranked.json + map-points.json (~/Projects/commercialrealestate)',
    sampled: '2026-08-06',
    scored_by: 'public/deal-score.js (transparent weighted model)',
    corpus_deals: ranked.length,
    corpus_points: points.length,
    deals: deals.length,
    withGeo: withGeo.length,
    note: 'Refreshed from the live CRCP corpus; deals re-scored by the deliverable model. Demographics preserved from prior Census sample.',
  },
  deals,
  withGeo: withGeo.length,
  extraPoints,
  demographics,
};

if (existsSync(join(DATA, 'deals-sample.json'))) copyFileSync(join(DATA, 'deals-sample.json'), join(DATA, 'deals-sample.json.bak'));
writeFileSync(join(DATA, 'deals-sample.json'), JSON.stringify(out, null, 1));
console.log(`OK deals-sample.json refreshed: ${deals.length} scored+geo deals (top of ${withGeo.length} geo / ${scored.length} scored / ${ranked.length} corpus), ${extraPoints.length} extra map points, ${demographics.length} demographics preserved.`);
console.log(`   score range: ${deals[deals.length-1]?.score}..${deals[0]?.score}`);