`scripts/harvest-fd-lee.js`

87 lines
// harvest-fd-lee.js — $0 firm-direct harvest for Lee & Associates (TK-10081 Phase 2).
// Lee runs Buildout; the public inventory.json is plain-fetchable (no browser, no key, no bot wall).
// For each of OUR crexi-sourced Lee deals, resolve the firm-direct listing URL + full broker roster
// so the site can link the broker's OWN property page instead of Crexi. Writes data/raw/fd-lee.json.
// READ-ONLY against ranked.json — never mutates it. $0 (plain fetch only).
'use strict';
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');
const FEED = 'https://buildout.com/plugins/9a64a93980aeae8db347e72cdfa8ca61017acc9a/inventory.json';

// Address normalizer — strip unit/suite, standardize street-type words, collapse to a match key.
const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) {
  let s = String(a || '').toLowerCase().split(',')[0];            // drop city/state tail
  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');  // strip only the unit token, not the whole tail
  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
const priceOf = (item) => {
  const pair = (item.index_attributes || []).find(p => /price/i.test(p[0]));
  if (!pair) return null;
  const n = Number(String(pair[1]).replace(/[^\d.]/g, ''));
  return Number.isFinite(n) && n > 0 ? n : null;
};

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'ranked.json'), 'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u) => { try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lee-associates'))
                       .map(d => ({ id: d.id, address: d.address, city: d.city, key: norm(d.address) }));
  console.log(`Lee targets: ${targets.length}`);

  // Build normalized-address → firm-direct listing index by paging the Buildout feed.
  const index = new Map();
  let page = 1, total = Infinity, seen = 0;
  while (seen < total && page <= 400) {
    let json;
    try {
      const res = await fetch(`${FEED}?page=${page}`, { headers: { 'accept': 'application/json' } });
      if (res.status === 429) { console.log(`  429 at page ${page} — backing off 3s`); await sleep(3000); continue; }
      if (!res.ok) { console.log(`  page ${page} HTTP ${res.status} — stop`); break; }
      json = await res.json();
    } catch (e) { console.log(`  page ${page} error ${e.message} — stop`); break; }
    const inv = json.inventory || [];
    total = (json.meta && json.meta.total) || total;
    if (!inv.length) break;
    for (const it of inv) {
      const k = norm(it.address_one_line);
      if (k && !index.has(k)) index.set(k, {
        listing_url: it.link_target || null,
        price: priceOf(it),
        brokers: (it.broker_contacts || []).map(b => ({ name: b.name, email: b.email || null }))
                  .filter(b => b.name) || [],
        broker_str: it.broker || null,
        sub_type: it.property_sub_type_name || null,
        addr: it.address_one_line || null
      });
    }
    seen += inv.length;
    if (page % 25 === 0) console.log(`  paged ${seen}/${total} (index ${index.size})`);
    page++;
    await sleep(150);
  }
  console.log(`Indexed ${index.size} Lee listings from ${seen} feed rows.`);

  const rows = targets.map(t => {
    const hit = index.get(t.key);
    if (!hit) return { deal_id: t.id, address: t.address, matched: false };
    return { deal_id: t.id, address: t.address, matched: true,
      listing_url: hit.listing_url, price: hit.price,
      brokers: hit.brokers.length ? hit.brokers : (hit.broker_str ? [{ name: hit.broker_str, email: null }] : []),
      firm_addr: hit.addr, sub_type: hit.sub_type, source_firm: 'Lee & Associates' };
  });
  const matched = rows.filter(r => r.matched).length;
  const out = { firm: 'Lee & Associates', generated: 'PENDING_STAMP', total_targets: targets.length, matched, rows };
  fs.mkdirSync(path.join(ROOT, 'data', 'raw'), { recursive: true });
  fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'fd-lee.json'), JSON.stringify(out, null, 2));
  console.log(`\nMATCHED ${matched}/${targets.length}  → data/raw/fd-lee.json`);
  console.log(JSON.stringify(rows.filter(r => r.matched).slice(0, 3), null, 2));
  const misses = rows.filter(r => !r.matched).map(r => r.address);
  if (misses.length) console.log(`\nUnmatched (${misses.length}):`, misses.slice(0, 10).join(' | '));
})();