`scripts/harvest-fd-lyonstahl.js`

102 lines
// harvest-fd-lyonstahl.js — $0 firm-direct harvest for Lyon Stahl (TK-10081 Phase 2).
// Lyon Stahl = WordPress; property PDP URLs carry the full address slug, so we match OUR crexi-sourced
// Lyon Stahl deals against the SITEMAP slugs (cheap) and only deep-fetch the matched PDPs (<=74) to pull
// price + broker roster from each PDP's JSON-LD. Writes data/raw/fd-lyonstahl.json. READ-ONLY on ranked.
// $0 (plain fetch only), polite (~300ms between PDP fetches).
'use strict';
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');

const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) {
  let s = String(a || '').toLowerCase().split(',')[0];
  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');  // strip only the unit token, not the whole tail
  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));

// Pull JSON-LD @graph nodes out of a PDP's HTML and extract listing price/url + agent roster.
function parsePdp(html) {
  const out = { price: null, listing_url: null, brokers: [] };
  const blocks = [...html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
  for (const m of blocks) {
    let data; try { data = JSON.parse(m[1].trim()); } catch { continue; }
    const nodes = Array.isArray(data) ? data : (data['@graph'] || [data]);
    for (const n of nodes) {
      const type = Array.isArray(n['@type']) ? n['@type'].join(',') : (n['@type'] || '');
      if (/RealEstateListing|Product|Residence|Offer/i.test(type)) {
        if (out.price == null) {
          const p = n.price || (n.offers && (n.offers.price || (n.offers[0] && n.offers[0].price)));
          const num = Number(String(p == null ? '' : p).replace(/[^\d.]/g, ''));
          if (Number.isFinite(num) && num > 0) out.price = num;
        }
        if (!out.listing_url && n.url) out.listing_url = n.url;
      }
      const agents = [].concat(n.agent || n.author || []);
      for (const a of agents) if (a && a.name) out.brokers.push({ name: a.name, title: a.jobTitle || null });
    }
  }
  return out;
}

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'ranked.json'), 'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u) => { try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lyonstahl'))
                       .map(d => ({ id: d.id, address: d.address, key: norm(d.address) }));
  console.log(`Lyon Stahl targets: ${targets.length}`);

  // 1) collect all PDP URLs from the 12 property sitemaps
  const urls = new Set();
  for (let i = 1; i <= 12; i++) {
    try {
      const res = await fetch(`https://lyonstahl.com/properties-sitemap${i}.xml`);
      if (!res.ok) { if (i > 1) break; continue; }
      const xml = await res.text();
      for (const m of xml.matchAll(/<loc>([^<]+\/properties\/[^<]+)<\/loc>/g)) {
        const u = m[1].trim();
        if (!/\/properties\/?$/.test(u)) urls.add(u);
      }
    } catch { break; }
    await sleep(120);
  }
  console.log(`Sitemap PDP URLs: ${urls.size}`);
  // slug index: normalized "street ..." → url
  const slugIndex = [...urls].map(u => {
    const slug = (u.match(/\/properties\/([^/]+)\/?$/) || [])[1] || '';
    return { u, s: slug.replace(/-/g, ' ').toLowerCase().replace(/\s+/g, ' ').trim() };
  });
  // Require a street number + at least one word (>=2 tokens) so a bare "524" can never false-match.
  const findUrl = (key) => (key.split(' ').length < 2) ? null
    : (slugIndex.find(x => x.s === key || x.s.startsWith(key + ' ')) || {}).u || null;

  // 2) match + deep-fetch only matched PDPs
  const rows = [];
  let matched = 0;
  for (const t of targets) {
    const url = findUrl(t.key);
    if (!url) { rows.push({ deal_id: t.id, address: t.address, matched: false }); continue; }
    let pdp = { price: null, listing_url: url, brokers: [] };
    try {
      const res = await fetch(url, { headers: { 'user-agent': 'Mozilla/5.0' } });
      if (res.ok) { const parsed = parsePdp(await res.text()); pdp = { ...parsed, listing_url: parsed.listing_url || url }; }
    } catch { /* keep url-only */ }
    matched++;
    rows.push({ deal_id: t.id, address: t.address, matched: true,
      listing_url: pdp.listing_url, price: pdp.price,
      brokers: pdp.brokers, source_firm: 'Lyon Stahl' });
    await sleep(300);
  }
  const out = { firm: 'Lyon Stahl', generated: 'PENDING_STAMP', total_targets: targets.length, matched, rows };
  fs.mkdirSync(path.join(ROOT, 'data', 'raw'), { recursive: true });
  fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'fd-lyonstahl.json'), JSON.stringify(out, null, 2));
  console.log(`\nMATCHED ${matched}/${targets.length}  → data/raw/fd-lyonstahl.json`);
  console.log(JSON.stringify(rows.filter(r => r.matched).slice(0, 3), null, 2));
  const misses = rows.filter(r => !r.matched).map(r => r.address);
  if (misses.length) console.log(`\nUnmatched (${misses.length}):`, misses.slice(0, 10).join(' | '));
})();