`scripts/harvest-fd-lyonstahl-links.js`

56 lines
// harvest-fd-lyonstahl-links.js — $0 firm-direct DEEP-LINK harvest for Lyon Stahl.
// Lyon Stahl exposes property URLs in its public sitemap, so the firm-direct deep link IS the matched
// sitemap URL — no PDP fetch, no Browserbase needed (plain-fetch sitemaps returned 200). Matches OUR
// crexi-sourced Lyon Stahl deals to their sitemap URL by normalized address (min-2-token guard prevents
// the bare-number false-positive). URL-only (no roster overwrite) — address-reliable, zero contamination.
// Writes data/raw/fd-lyonstahl-verified.json. $0. Run: node scripts/harvest-fd-lyonstahl-links.js
'use strict';
const fs = require('fs');
const path = require('path');
const ROOT = path.join(__dirname, '..');
const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) {
  let s = String(a || '').toLowerCase().split(',')[0];
  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lyonstahl'))
                       .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
  console.log(`Lyon Stahl targets: ${targets.length}`);

  const urls = new Set();
  for (let i = 1; i <= 12; i++) {
    try {
      const res = await fetch(`https://lyonstahl.com/properties-sitemap${i}.xml`);
      if (!res.ok) { if (i > 1) break; continue; }
      for (const m of (await res.text()).matchAll(/<loc>([^<]+\/properties\/[^<]+)<\/loc>/g)) {
        const u = m[1].trim(); if (!/\/properties\/?$/.test(u)) urls.add(u);
      }
    } catch { break; }
    await sleep(120);
  }
  const slugIndex = [...urls].map(u => ({ u, s: ((u.match(/\/properties\/([^/]+)\/?$/)||[])[1]||'').replace(/-/g,' ').toLowerCase().replace(/\s+/g,' ').trim() }));
  console.log(`Sitemap PDP URLs: ${urls.size}`);
  // min-2-token guard: a bare street number can never match
  const findUrl = (key) => (key.split(' ').length < 2) ? null
    : (slugIndex.find(x => x.s === key || x.s.startsWith(key + ' ')) || {}).u || null;

  const rows = targets.map(t => { const url = findUrl(t.key);
    return url ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:url, source_firm:'Lyon Stahl' }
               : { deal_id:t.id, address:t.address, matched:false }; });
  const m = rows.filter(r=>r.matched);
  const out = { firm:'Lyon Stahl', via:'$0 sitemap', generated:'PENDING_STAMP', total_targets:targets.length, matched:m.length, rows };
  fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
  fs.writeFileSync(path.join(ROOT,'data','raw','fd-lyonstahl-verified.json'), JSON.stringify(out,null,2));
  console.log(`\nMATCHED ${m.length}/${targets.length} deep-links (\$0) → data/raw/fd-lyonstahl-verified.json`);
  console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });