crcp: $0 Lyon Stahl sitemap deep-link harvester — 25/74 clean matches, staged for approval (TK-10081)

68f40f95b0d615ad3b5d04d8fbde33582633508d · 2026-08-01 20:05:40 -0700 · Steve Abrams
Files touched

A scripts/harvest-fd-lyonstahl-links.js
Diff

commit 68f40f95b0d615ad3b5d04d8fbde33582633508d
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Aug 1 20:05:40 2026 -0700

    crcp: $0 Lyon Stahl sitemap deep-link harvester — 25/74 clean matches, staged for approval (TK-10081)
---
 scripts/harvest-fd-lyonstahl-links.js | 55 +++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/scripts/harvest-fd-lyonstahl-links.js b/scripts/harvest-fd-lyonstahl-links.js
new file mode 100644
index 0000000..6028ab5
--- /dev/null
+++ b/scripts/harvest-fd-lyonstahl-links.js
@@ -0,0 +1,55 @@
+// harvest-fd-lyonstahl-links.js — $0 firm-direct DEEP-LINK harvest for Lyon Stahl.
+// Lyon Stahl exposes property URLs in its public sitemap, so the firm-direct deep link IS the matched
+// sitemap URL — no PDP fetch, no Browserbase needed (plain-fetch sitemaps returned 200). Matches OUR
+// crexi-sourced Lyon Stahl deals to their sitemap URL by normalized address (min-2-token guard prevents
+// the bare-number false-positive). URL-only (no roster overwrite) — address-reliable, zero contamination.
+// Writes data/raw/fd-lyonstahl-verified.json. $0. Run: node scripts/harvest-fd-lyonstahl-links.js
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const ROOT = path.join(__dirname, '..');
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) {
+  let s = String(a || '').toLowerCase().split(',')[0];
+  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
+  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+(async () => {
+  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
+  const deals = ranked.ranked || ranked;
+  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lyonstahl'))
+                       .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
+  console.log(`Lyon Stahl targets: ${targets.length}`);
+
+  const urls = new Set();
+  for (let i = 1; i <= 12; i++) {
+    try {
+      const res = await fetch(`https://lyonstahl.com/properties-sitemap${i}.xml`);
+      if (!res.ok) { if (i > 1) break; continue; }
+      for (const m of (await res.text()).matchAll(/<loc>([^<]+\/properties\/[^<]+)<\/loc>/g)) {
+        const u = m[1].trim(); if (!/\/properties\/?$/.test(u)) urls.add(u);
+      }
+    } catch { break; }
+    await sleep(120);
+  }
+  const slugIndex = [...urls].map(u => ({ u, s: ((u.match(/\/properties\/([^/]+)\/?$/)||[])[1]||'').replace(/-/g,' ').toLowerCase().replace(/\s+/g,' ').trim() }));
+  console.log(`Sitemap PDP URLs: ${urls.size}`);
+  // min-2-token guard: a bare street number can never match
+  const findUrl = (key) => (key.split(' ').length < 2) ? null
+    : (slugIndex.find(x => x.s === key || x.s.startsWith(key + ' ')) || {}).u || null;
+
+  const rows = targets.map(t => { const url = findUrl(t.key);
+    return url ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:url, source_firm:'Lyon Stahl' }
+               : { deal_id:t.id, address:t.address, matched:false }; });
+  const m = rows.filter(r=>r.matched);
+  const out = { firm:'Lyon Stahl', via:'$0 sitemap', generated:'PENDING_STAMP', total_targets:targets.length, matched:m.length, rows };
+  fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
+  fs.writeFileSync(path.join(ROOT,'data','raw','fd-lyonstahl-verified.json'), JSON.stringify(out,null,2));
+  console.log(`\nMATCHED ${m.length}/${targets.length} deep-links (\$0) → data/raw/fd-lyonstahl-verified.json`);
+  console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
← 5f8c270 crcp: apply 31 Lee firm-direct deep-links to ranked.json (St · back to Commercialrealestate · crcp: apply 25 Lyon Stahl firm-direct deep-links (Steve-appr 57b49b8 →