← back to Commercialrealestate
crcp: WIP $0 firm-direct harvesters (Lee/Buildout + Lyon Stahl) — prove path; bot-walled at scale, correctness fixes applied (TK-10081 Phase 2)
0d5f4716ba21e8f6366e7121de619b7137425998 · 2026-07-31 15:23:51 -0700 · Steve Abrams
Files touched
A scripts/harvest-fd-lee.jsA scripts/harvest-fd-lyonstahl.js
Diff
commit 0d5f4716ba21e8f6366e7121de619b7137425998
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Jul 31 15:23:51 2026 -0700
crcp: WIP $0 firm-direct harvesters (Lee/Buildout + Lyon Stahl) — prove path; bot-walled at scale, correctness fixes applied (TK-10081 Phase 2)
---
scripts/harvest-fd-lee.js | 86 ++++++++++++++++++++++++++++++++++
scripts/harvest-fd-lyonstahl.js | 101 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 187 insertions(+)
diff --git a/scripts/harvest-fd-lee.js b/scripts/harvest-fd-lee.js
new file mode 100644
index 0000000..80cdbc1
--- /dev/null
+++ b/scripts/harvest-fd-lee.js
@@ -0,0 +1,86 @@
+// harvest-fd-lee.js — $0 firm-direct harvest for Lee & Associates (TK-10081 Phase 2).
+// Lee runs Buildout; the public inventory.json is plain-fetchable (no browser, no key, no bot wall).
+// For each of OUR crexi-sourced Lee deals, resolve the firm-direct listing URL + full broker roster
+// so the site can link the broker's OWN property page instead of Crexi. Writes data/raw/fd-lee.json.
+// READ-ONLY against ranked.json — never mutates it. $0 (plain fetch only).
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const ROOT = path.join(__dirname, '..');
+const FEED = 'https://buildout.com/plugins/9a64a93980aeae8db347e72cdfa8ca61017acc9a/inventory.json';
+
+// Address normalizer — strip unit/suite, standardize street-type words, collapse to a match key.
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+ place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) {
+ let s = String(a || '').toLowerCase().split(',')[0]; // drop city/state tail
+ s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,''); // strip only the unit token, not the whole tail
+ s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+ return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+const priceOf = (item) => {
+ const pair = (item.index_attributes || []).find(p => /price/i.test(p[0]));
+ if (!pair) return null;
+ const n = Number(String(pair[1]).replace(/[^\d.]/g, ''));
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+
+(async () => {
+ const ranked = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'ranked.json'), 'utf8'));
+ const deals = ranked.ranked || ranked;
+ const hostOf = (u) => { try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+ const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lee-associates'))
+ .map(d => ({ id: d.id, address: d.address, city: d.city, key: norm(d.address) }));
+ console.log(`Lee targets: ${targets.length}`);
+
+ // Build normalized-address → firm-direct listing index by paging the Buildout feed.
+ const index = new Map();
+ let page = 1, total = Infinity, seen = 0;
+ while (seen < total && page <= 400) {
+ let json;
+ try {
+ const res = await fetch(`${FEED}?page=${page}`, { headers: { 'accept': 'application/json' } });
+ if (res.status === 429) { console.log(` 429 at page ${page} — backing off 3s`); await sleep(3000); continue; }
+ if (!res.ok) { console.log(` page ${page} HTTP ${res.status} — stop`); break; }
+ json = await res.json();
+ } catch (e) { console.log(` page ${page} error ${e.message} — stop`); break; }
+ const inv = json.inventory || [];
+ total = (json.meta && json.meta.total) || total;
+ if (!inv.length) break;
+ for (const it of inv) {
+ const k = norm(it.address_one_line);
+ if (k && !index.has(k)) index.set(k, {
+ listing_url: it.link_target || null,
+ price: priceOf(it),
+ brokers: (it.broker_contacts || []).map(b => ({ name: b.name, email: b.email || null }))
+ .filter(b => b.name) || [],
+ broker_str: it.broker || null,
+ sub_type: it.property_sub_type_name || null,
+ addr: it.address_one_line || null
+ });
+ }
+ seen += inv.length;
+ if (page % 25 === 0) console.log(` paged ${seen}/${total} (index ${index.size})`);
+ page++;
+ await sleep(150);
+ }
+ console.log(`Indexed ${index.size} Lee listings from ${seen} feed rows.`);
+
+ const rows = targets.map(t => {
+ const hit = index.get(t.key);
+ if (!hit) return { deal_id: t.id, address: t.address, matched: false };
+ return { deal_id: t.id, address: t.address, matched: true,
+ listing_url: hit.listing_url, price: hit.price,
+ brokers: hit.brokers.length ? hit.brokers : (hit.broker_str ? [{ name: hit.broker_str, email: null }] : []),
+ firm_addr: hit.addr, sub_type: hit.sub_type, source_firm: 'Lee & Associates' };
+ });
+ const matched = rows.filter(r => r.matched).length;
+ const out = { firm: 'Lee & Associates', generated: 'PENDING_STAMP', total_targets: targets.length, matched, rows };
+ fs.mkdirSync(path.join(ROOT, 'data', 'raw'), { recursive: true });
+ fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'fd-lee.json'), JSON.stringify(out, null, 2));
+ console.log(`\nMATCHED ${matched}/${targets.length} → data/raw/fd-lee.json`);
+ console.log(JSON.stringify(rows.filter(r => r.matched).slice(0, 3), null, 2));
+ const misses = rows.filter(r => !r.matched).map(r => r.address);
+ if (misses.length) console.log(`\nUnmatched (${misses.length}):`, misses.slice(0, 10).join(' | '));
+})();
diff --git a/scripts/harvest-fd-lyonstahl.js b/scripts/harvest-fd-lyonstahl.js
new file mode 100644
index 0000000..074052c
--- /dev/null
+++ b/scripts/harvest-fd-lyonstahl.js
@@ -0,0 +1,101 @@
+// harvest-fd-lyonstahl.js — $0 firm-direct harvest for Lyon Stahl (TK-10081 Phase 2).
+// Lyon Stahl = WordPress; property PDP URLs carry the full address slug, so we match OUR crexi-sourced
+// Lyon Stahl deals against the SITEMAP slugs (cheap) and only deep-fetch the matched PDPs (<=74) to pull
+// price + broker roster from each PDP's JSON-LD. Writes data/raw/fd-lyonstahl.json. READ-ONLY on ranked.
+// $0 (plain fetch only), polite (~300ms between PDP fetches).
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const ROOT = path.join(__dirname, '..');
+
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+ place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) {
+ let s = String(a || '').toLowerCase().split(',')[0];
+ s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,''); // strip only the unit token, not the whole tail
+ s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+ return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+// Pull JSON-LD @graph nodes out of a PDP's HTML and extract listing price/url + agent roster.
+function parsePdp(html) {
+ const out = { price: null, listing_url: null, brokers: [] };
+ const blocks = [...html.matchAll(/<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)];
+ for (const m of blocks) {
+ let data; try { data = JSON.parse(m[1].trim()); } catch { continue; }
+ const nodes = Array.isArray(data) ? data : (data['@graph'] || [data]);
+ for (const n of nodes) {
+ const type = Array.isArray(n['@type']) ? n['@type'].join(',') : (n['@type'] || '');
+ if (/RealEstateListing|Product|Residence|Offer/i.test(type)) {
+ if (out.price == null) {
+ const p = n.price || (n.offers && (n.offers.price || (n.offers[0] && n.offers[0].price)));
+ const num = Number(String(p == null ? '' : p).replace(/[^\d.]/g, ''));
+ if (Number.isFinite(num) && num > 0) out.price = num;
+ }
+ if (!out.listing_url && n.url) out.listing_url = n.url;
+ }
+ const agents = [].concat(n.agent || n.author || []);
+ for (const a of agents) if (a && a.name) out.brokers.push({ name: a.name, title: a.jobTitle || null });
+ }
+ }
+ return out;
+}
+
+(async () => {
+ const ranked = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'ranked.json'), 'utf8'));
+ const deals = ranked.ranked || ranked;
+ const hostOf = (u) => { try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+ const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lyonstahl'))
+ .map(d => ({ id: d.id, address: d.address, key: norm(d.address) }));
+ console.log(`Lyon Stahl targets: ${targets.length}`);
+
+ // 1) collect all PDP URLs from the 12 property sitemaps
+ const urls = new Set();
+ for (let i = 1; i <= 12; i++) {
+ try {
+ const res = await fetch(`https://lyonstahl.com/properties-sitemap${i}.xml`);
+ if (!res.ok) { if (i > 1) break; continue; }
+ const xml = await res.text();
+ for (const m of xml.matchAll(/<loc>([^<]+\/properties\/[^<]+)<\/loc>/g)) {
+ const u = m[1].trim();
+ if (!/\/properties\/?$/.test(u)) urls.add(u);
+ }
+ } catch { break; }
+ await sleep(120);
+ }
+ console.log(`Sitemap PDP URLs: ${urls.size}`);
+ // slug index: normalized "street ..." → url
+ const slugIndex = [...urls].map(u => {
+ const slug = (u.match(/\/properties\/([^/]+)\/?$/) || [])[1] || '';
+ return { u, s: slug.replace(/-/g, ' ').toLowerCase().replace(/\s+/g, ' ').trim() };
+ });
+ // Require a street number + at least one word (>=2 tokens) so a bare "524" can never false-match.
+ const findUrl = (key) => (key.split(' ').length < 2) ? null
+ : (slugIndex.find(x => x.s === key || x.s.startsWith(key + ' ')) || {}).u || null;
+
+ // 2) match + deep-fetch only matched PDPs
+ const rows = [];
+ let matched = 0;
+ for (const t of targets) {
+ const url = findUrl(t.key);
+ if (!url) { rows.push({ deal_id: t.id, address: t.address, matched: false }); continue; }
+ let pdp = { price: null, listing_url: url, brokers: [] };
+ try {
+ const res = await fetch(url, { headers: { 'user-agent': 'Mozilla/5.0' } });
+ if (res.ok) { const parsed = parsePdp(await res.text()); pdp = { ...parsed, listing_url: parsed.listing_url || url }; }
+ } catch { /* keep url-only */ }
+ matched++;
+ rows.push({ deal_id: t.id, address: t.address, matched: true,
+ listing_url: pdp.listing_url, price: pdp.price,
+ brokers: pdp.brokers, source_firm: 'Lyon Stahl' });
+ await sleep(300);
+ }
+ const out = { firm: 'Lyon Stahl', generated: 'PENDING_STAMP', total_targets: targets.length, matched, rows };
+ fs.mkdirSync(path.join(ROOT, 'data', 'raw'), { recursive: true });
+ fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'fd-lyonstahl.json'), JSON.stringify(out, null, 2));
+ console.log(`\nMATCHED ${matched}/${targets.length} → data/raw/fd-lyonstahl.json`);
+ console.log(JSON.stringify(rows.filter(r => r.matched).slice(0, 3), null, 2));
+ const misses = rows.filter(r => !r.matched).map(r => r.address);
+ if (misses.length) console.log(`\nUnmatched (${misses.length}):`, misses.slice(0, 10).join(' | '));
+})();
← ff6c52b crcp: doctrine link-scrub — never link CREXi/aggregators; li
·
back to Commercialrealestate
·
auto-save: 2026-07-31T17:59:58 (2 files) — data/crcp-drift-s 6baa491 →