`scripts/sources/firms.js`

280 lines
// sources/firms.js — multi-firm source registry + shared normalizer for the SFV CRE tracker.
//
// WHY THIS EXISTS: refresh-listings.js was hard-coupled to Crexi (it sniffed api.crexi.com and
// hand-mapped that one schema). Adding more brokerage/marketplace "firms" by cloning that whole
// file per firm doesn't scale. This module inverts the control: ONE shared normalizer + dedup,
// plus a declarative list of FIRMS. The orchestrator (refresh-all.js) drives every firm through
// the same capture → normalize → dedup → append path, so a new firm is a ~10-line config, not a
// new scraper.
//
// VERIFIED vs NEEDS-DISCOVERY: crexi's extractor is the proven one (ported verbatim from the
// working refresh-listings.js). The other firms ship with nav configs + the GENERIC listing-shape
// sniffer (genericExtract), which auto-detects address/price/city-bearing JSON from any XHR on the
// firm's domain. That gets real coverage for many sites, but each firm should be locked with one
// `node scripts/discover-firm.js <key>` run (cheap Browserbase session) before you trust its yield.
// Do not represent a NEEDS-DISCOVERY firm as "verified working" until its discovery pass confirms.

'use strict';

const { LA_COUNTY, inLACounty } = require('./la-county');

// ── Shared market filter ─────────────────────────────────────────────────────────────────────────
// Scope is now ALL of Los Angeles County (per Steve, 2026-06-27 "all LA county frankly"). The geo
// gate is the canonical LA_COUNTY place set in la-county.js (88 incorporated cities + major LA
// neighborhoods + unincorporated communities). `SFV` is kept as an alias so any older import that
// references it still works — it now points at the full county set.
const SFV = LA_COUNTY;

// Capture band — WIDE by default (Steve: "add all properties", 2026-06-27). We capture the whole LA
// County commercial market and let the VIEWER filter by budget ($400k-down affordability) on top of
// the full set. MINP just rejects obvious non-prices/typos; MAXP is effectively uncapped. Override
// via CC_MINP / CC_MAXP to re-narrow a sweep.
const MINP = +(process.env.CC_MINP || 50000);
const MAXP = +(process.env.CC_MAXP || 1000000000);

// Address-normalizer used as the cross-firm dedup key (identical rules to the original refresh).
const norm = s => String(s || '').toLowerCase()
  .replace(/[^a-z0-9 ]/g, '')
  .replace(/\b(ave|avenue|st|street|blvd|boulevard|dr|drive|rd|road|ln|lane|pl|place|ct|court)\b/g, '')
  .replace(/\s+/g, ' ').trim();

// All commercial asset classes we navigate, with each firm's URL slug. Expanded from multifamily+
// retail to the full set (Steve: "all properties") so office/industrial/land/hospitality are captured.
const TYPES = [
  ['multifamily', 'Multifamily'], ['retail', 'Retail'], ['office', 'Office'],
  ['industrial', 'Industrial'], ['mixed-use', 'Mixed-use'], ['land', 'Land'],
  ['hospitality', 'Hospitality']
];

const today = () => new Date().toISOString().slice(0, 10);

// ── Generic listing-shape sniffer ───────────────────────────────────────────────────────────────
// Given any parsed JSON body from a firm's API, dig out an array of "asset-like" objects and map
// each to the common intermediate shape { id, price, city, address, cap, types, desc, urlSlug }.
// Heuristic but conservative: it only keeps objects that have BOTH a plausible price and a city.
const ARRAY_KEYS = ['data', 'results', 'assets', 'items', 'listings', 'properties', 'hits', 'records', 'docs'];
const pickArray = (j) => {
  if (Array.isArray(j)) return j;
  if (!j || typeof j !== 'object') return [];
  // Case-insensitive match against known container keys (M&M's .NET API uses "Results", etc.).
  const lower = {}; for (const k of Object.keys(j)) lower[k.toLowerCase()] = k;
  for (const want of ARRAY_KEYS) {
    const k = lower[want]; if (k == null) continue;
    const v = j[k];
    if (Array.isArray(v)) return v;
    if (v && typeof v === 'object') {                         // {Results:{Items:[...]}} / {data:{items:[...]}}
      const inner = pickArray(v); if (inner.length) return inner;
    }
  }
  // Last resort: first non-empty array-of-objects value on the top level.
  for (const v of Object.values(j)) {
    if (Array.isArray(v) && v.length && typeof v[0] === 'object') return v;
  }
  return [];
};

const num = (v) => { const n = +String(v == null ? '' : v).replace(/[^0-9.]/g, ''); return Number.isFinite(n) && n > 0 ? n : null; };

// Case-insensitive field access — many firm APIs (M&M's .NET, etc.) capitalize keys (ListPrice, City).
const ciIndex = (o) => { const m = {}; if (o && typeof o === 'object') for (const k of Object.keys(o)) m[k.toLowerCase()] = o[k]; return m; };
const ciStr = (o, names) => { const m = ciIndex(o); for (const n of names) { const v = m[n.toLowerCase()]; if (typeof v === 'string' && v.trim()) return v.trim(); } return ''; };
const ciVal = (o, names) => { const m = ciIndex(o); for (const n of names) { const v = m[n.toLowerCase()]; if (v != null) return v; } return null; };

// Pull a {city,address} out of an asset that may nest them under locations[0] / address{} / fields.
const locOf = (a) => {
  const m = ciIndex(a);
  const loc = (Array.isArray(m.locations) && m.locations[0]) || m.location || (m.address && typeof m.address === 'object' ? m.address : null) || m.geo || a;
  const city = ciStr(loc, ['city', 'cityName', 'town']) || ciStr(a, ['city', 'cityName', 'town']);
  const address = ciStr(loc, ['address', 'addressLine1', 'street', 'line1', 'fullAddress', 'streetAddress', 'name'])
    || ciStr(a, ['address', 'streetAddress', 'addressLine1', 'name', 'title', 'displayAddress']);
  return { city, address };
};

function genericExtract(json) {
  const out = [];
  for (const a of pickArray(json)) {
    if (!a || typeof a !== 'object') continue;
    const price = num(ciVal(a, ['askingPrice', 'price', 'listPrice', 'salePrice', 'priceValue', 'amount']));
    const { city, address } = locOf(a);
    if (!price || !city || !address) continue;
    const id = String(ciVal(a, ['id', 'assetId', 'listingId', 'propertyId', 'slug']) ?? address).slice(0, 64);
    const capRaw = ciVal(a, ['capRate', 'cap', 'capRateValue']);
    out.push({
      id, price, city, address,
      cap: typeof capRaw === 'number' ? capRaw : num(capRaw),
      urlSlug: ciStr(a, ['urlSlug', 'slug', 'url', 'detailUrl', 'permalink']),
      desc: ciStr(a, ['description', 'summary', 'remarks', 'subtitle']),
      types: ciVal(a, ['types', 'propertyTypes']) || (ciVal(a, ['propertyType']) ? [ciVal(a, ['propertyType'])] : [])
    });
  }
  return out;
}

// Crexi's verified extractor — its API matches genericExtract anyway, but we keep it explicit so the
// reference source is never silently changed by a heuristic tweak.
function crexiExtract(json) {
  const arr = Array.isArray(json) ? json : (json.data || json.assets || json.results || []);
  if (!Array.isArray(arr)) return [];
  return arr.map(a => {
    const loc = (a.locations && a.locations[0]) || {};
    const price = a.askingPrice || a.price;
    if (!(a.id || a.assetId) || !price || !loc.city) return null;
    return {
      id: a.id || a.assetId, price: +price, city: loc.city, address: loc.address || (a.name || ''),
      cap: typeof a.capRate === 'number' ? a.capRate : null, urlSlug: a.urlSlug || '',
      desc: String(a.description || ''), types: a.types || []
    };
  }).filter(Boolean);
}

// ── Firm registry ────────────────────────────────────────────────────────────────────────────────
// key         stable id used in CC_FIRMS, the listing id prefix, and discover-firm.
// name        human label.
// idPrefix    prepended to each listing id so cross-firm ids never collide.
// apiHosts    substrings; the orchestrator captures any JSON response whose URL contains one.
// navUrls     (city, typeSlug) -> array of pages to visit to make the firm's search XHRs fire.
// extract     (json) -> intermediate assets. Defaults to genericExtract.
// status      'verified' (proven) or 'needs-discovery' (wired, run discover-firm to confirm yield).
const FIRMS = [
  {
    key: 'crexi', name: 'Crexi', idPrefix: 'crx', status: 'verified',
    apiHosts: ['api.crexi.com'],
    navUrls: (city, slug) => [`https://www.crexi.com/properties/CA/${city.replace(/\s+/g, '-')}/${slug}`],
    extract: crexiExtract,
    sourceUrl: (a) => 'https://www.crexi.com/properties/' + a.id + (a.urlSlug ? '/' + a.urlSlug : '')
  },
  {
    key: 'marcusmillichap', name: 'Marcus & Millichap', idPrefix: 'mm', status: 'needs-discovery',
    // M&M is the dominant SFV multifamily brokerage — highest-value add. DISCOVERY (2026-06-27): it
    // HAS a real JSON API — POST /api/contentsearch/properties (and /mapproperties) returning
    // {Results:[...]} with capitalized .NET field names (now handled by the case-insensitive sniffer).
    // Remaining gap: the search returns empty Results unless the POST body carries the location filter
    // (lat/lng/polygon). A search URL nav alone didn't populate it. TODO: capture & replay the POST
    // payload (a discover2-style request-body log) to get populated results.
    apiHosts: ['marcusmillichap.com', 'api.marcusmillichap'],
    navUrls: (city) => [`https://www.marcusmillichap.com/properties?searchText=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'lee-associates', name: 'Lee & Associates', idPrefix: 'lee', status: 'needs-discovery',
    apiHosts: ['lee-associates.com'],
    navUrls: (city) => [`https://www.lee-associates.com/properties/?keyword=${encodeURIComponent(city + ' CA')}`],
    extract: genericExtract
  },
  {
    key: 'colliers', name: 'Colliers', idPrefix: 'col', status: 'needs-discovery',
    apiHosts: ['colliers.com'],
    navUrls: (city) => [`https://www.colliers.com/en-us/properties?searchTerm=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'cbre', name: 'CBRE', idPrefix: 'cbre', status: 'needs-discovery',
    apiHosts: ['cbre.com'],
    navUrls: (city) => [`https://www.cbre.com/properties/properties-for-sale?location=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'jll', name: 'JLL', idPrefix: 'jll', status: 'needs-discovery',
    apiHosts: ['jll.com', 'property.jll.com'],
    navUrls: (city) => [`https://property.jll.com/search?q=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  // ── Additional commercial brokerages active in the San Fernando Valley (2026-06-27) ──
  // Major multifamily/commercial shops that list SFV deals. All start needs-discovery: run
  // `node scripts/discover-firm.js <key>` (or just include them in a refresh-all sweep) to confirm
  // each one's public property-search emits listing JSON the generic sniffer can read.
  {
    key: 'matthews', name: 'Matthews REIS', idPrefix: 'mat', status: 'needs-discovery',
    apiHosts: ['matthews.com'],
    navUrls: (city) => [`https://www.matthews.com/listings/?location=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'kidder', name: 'Kidder Mathews', idPrefix: 'kid', status: 'needs-discovery',
    apiHosts: ['kidder.com'],
    navUrls: (city) => [`https://kidder.com/properties/?search=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'naicapital', name: 'NAI Capital', idPrefix: 'nai', status: 'needs-discovery',
    apiHosts: ['naicapital.com'],
    navUrls: (city) => [`https://www.naicapital.com/properties/?keyword=${encodeURIComponent(city + ' CA')}`],
    extract: genericExtract
  },
  {
    key: 'cushwake', name: 'Cushman & Wakefield', idPrefix: 'cw', status: 'needs-discovery',
    apiHosts: ['cushmanwakefield.com'],
    navUrls: (city) => [`https://www.cushmanwakefield.com/en/united-states/properties?q=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'newmark', name: 'Newmark', idPrefix: 'nmrk', status: 'needs-discovery',
    apiHosts: ['nmrk.com'],
    navUrls: (city) => [`https://www.nmrk.com/properties?location=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'berkadia', name: 'Berkadia', idPrefix: 'berk', status: 'needs-discovery',
    apiHosts: ['berkadia.com'],
    navUrls: (city) => [`https://www.berkadia.com/listings/?location=${encodeURIComponent(city + ', CA')}`],
    extract: genericExtract
  },
  {
    key: 'stepp', name: 'Stepp Commercial', idPrefix: 'stp', status: 'needs-discovery',
    apiHosts: ['steppcommercial.com'],
    navUrls: (city) => [`https://www.steppcommercial.com/listings/?q=${encodeURIComponent(city)}`],
    extract: genericExtract
  },
  {
    // DISCOVERY (2026-06-27): LoopNet (CoStar-owned) emitted 0 JSON on its apiHosts — it server-renders
    // and is heavily anti-bot; Browserbase sees no listing API. Treat as a dead end without a much
    // deeper anti-bot approach. Crexi already covers most of the same inventory.
    key: 'loopnet', name: 'LoopNet', idPrefix: 'ln', status: 'needs-discovery',
    apiHosts: ['loopnet.com', 'api.loopnet.com'],
    navUrls: (city, slug) => [`https://www.loopnet.com/search/${slug === 'retail' ? 'retail' : 'multifamily'}/${city.replace(/\s+/g, '-').toLowerCase()}-ca/for-sale/`],
    extract: genericExtract
  }
];

const byKey = (key) => FIRMS.find(f => f.key === key);

// ── Shared normalizer: intermediate asset (+ its firm) -> the canonical listings.json record. ─────
// This is the EXACT field shape the original refresh produced, so analyze.js / ranked.json need no
// changes. We add `firm` and `firm_key` so the viewer can facet "who's listing what".
function classifyType(types) {
  const s = (Array.isArray(types) ? types.join(' ') : String(types || '')).toLowerCase();
  if (/multi.?family|apartment|duplex|triplex|fourplex/.test(s)) return 'Multifamily';
  if (/mixed.?use/.test(s)) return 'Mixed-use';
  if (/retail|shopping|storefront|net.?lease/.test(s)) return 'Retail';
  if (/office|medical office/.test(s)) return 'Office';
  if (/industrial|warehouse|flex|manufactur|distribution/.test(s)) return 'Industrial';
  if (/hospitality|hotel|motel|lodging/.test(s)) return 'Hospitality';
  if (/\bland\b|development|lot/.test(s)) return 'Land';
  if (/self.?storage/.test(s)) return 'Self-storage';
  if (/mobile.?home|manufactured housing/.test(s)) return 'Mobile-home park';
  return 'Multifamily';
}
function toListing(a, firm) {
  const type = classifyType(a.types);
  const units = (() => {
    const m = String(a.desc || '').match(/(?:\|\s*)?\b(\d{1,3})\s*units?\b/i);
    if (m) { const n = +m[1]; if (n >= 1 && n <= 500 && a.price / n >= 50000) return n; }
    return null;
  })();
  const src = firm.sourceUrl ? firm.sourceUrl(a)
    : (a.urlSlug && /^https?:/.test(a.urlSlug) ? a.urlSlug : '');
  return {
    id: firm.idPrefix + a.id, address: a.address, city: a.city, zip: '', type,
    price: a.price, units: units || 1, sqft: null, cap_rate: a.cap ?? null, verified: false,
    year_built: null, rent_control: 'verify', status: 'Active',
    firm: firm.name, firm_key: firm.key,
    upside_note: `Auto-added by ${firm.name} refresh ${today()}. ${a.cap ? 'Broker cap ' + a.cap + '% (unverified).' : 'Cap not disclosed.'} Verify status, rent roll, and financials before acting.`,
    source: src
  };
}

// Apply the SFV + price-band + plausibility gate to a batch of intermediate assets.
const inBand = (a) => a && a.city && SFV.has(a.city.toLowerCase()) && a.price >= MINP && a.price <= MAXP && a.address;

module.exports = { SFV, MINP, MAXP, TYPES, norm, today, FIRMS, byKey, genericExtract, toListing, inBand };