← back to Commercialrealestate

firm-site scraper: scroll for lazy-loaded grids + try common listings subpaths (/listings,/properties,/for-sale,/inventory), keep best-yielding page — unlocks firms that yielded 0

eab941cf6e5276dc24a7768c0140bc4311e03795 · 2026-08-19 10:56:59 -0700 · Steve Abrams

Files touched

• M scripts/backfill-openclaw-llm.js

Diff

commit eab941cf6e5276dc24a7768c0140bc4311e03795
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 10:56:59 2026 -0700

    firm-site scraper: scroll for lazy-loaded grids + try common listings subpaths (/listings,/properties,/for-sale,/inventory), keep best-yielding page — unlocks firms that yielded 0
---
 scripts/backfill-openclaw-llm.js | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/scripts/backfill-openclaw-llm.js b/scripts/backfill-openclaw-llm.js
index 88dee7a..5f5d43f 100644
--- a/scripts/backfill-openclaw-llm.js
+++ b/scripts/backfill-openclaw-llm.js
@@ -47,19 +47,31 @@ async function llmExtract(text) {
   const seen = new Set(); return out.filter(l => { const k = l.address.toLowerCase(); return !seen.has(k) && seen.add(k); });
 }
 
+async function pageText() { // scroll to trigger lazy-loaded listing grids, then read
+  for (let i = 0; i < 3; i++) { oc(['browser', 'evaluate', '--fn', '() => window.scrollTo(0, document.body.scrollHeight)']); await sleep(1200); }
+  const txt = ocEval(`() => document.body.innerText`);
+  return typeof txt === 'string' ? txt : (txt && (txt.result || txt.value)) || '';
+}
 async function scrapeDomain(url) {
-  if (!oc(['browser', 'navigate', /^https?:/i.test(url) ? url : 'https://' + url])) return { err: 'nav-fail' };
-  // find the best listings page link
+  const base = /^https?:/i.test(url) ? url : 'https://' + url;
+  if (!oc(['browser', 'navigate', base])) return { err: 'nav-fail' };
+  // discover the listings page link on the homepage
   const links = ocEval(`() => [...document.querySelectorAll('a')].map(a=>({t:(a.textContent||'').trim().slice(0,40),h:a.href})).filter(l=>/listing|propert|for.?sale|inventory|available|our.?deals/i.test(l.t+' '+l.h)).slice(0,10)`);
   const cand = (Array.isArray(links) ? links : []).map(l => l.h).filter(Boolean);
-  // prefer an /listings or /properties page; else stay on the homepage
   const listUrl = cand.find(h => /listing|propert|for-?sale|inventory/i.test(h)) || null;
-  if (listUrl) oc(['browser', 'navigate', listUrl]);
-  const txt = ocEval(`() => document.body.innerText`);
-  const text = typeof txt === 'string' ? txt : (txt && (txt.result || txt.value)) || '';
-  if (!text || text.length < 400) return { err: 'no-text' };
-  const listings = await llmExtract(text);
-  return { listUrl: listUrl || url, listings };
+  const origin = (() => { try { return new URL(base).origin; } catch { return base.replace(/\/$/, ''); } })();
+  // try: discovered link + common subpaths + homepage; keep the best-yielding page
+  const tryUrls = [...new Set([listUrl, origin + '/listings', origin + '/properties', origin + '/for-sale', origin + '/inventory', base].filter(Boolean))].slice(0, 4);
+  let best = { listings: [], listUrl: base };
+  for (const u of tryUrls) {
+    if (u !== base && !oc(['browser', 'navigate', u])) continue;
+    const text = await pageText();
+    if (!text || text.length < 400) continue;
+    const listings = await llmExtract(text);
+    if (listings.length > best.listings.length) best = { listings, listUrl: u };
+    if (best.listings.length >= 6) break; // good enough
+  }
+  return best.listings.length ? best : { err: 'no-listings' };
 }
 
 (async () => {

← c189e73 CRCP: harden per-broker grind — auto-dismiss JS dialogs + al  ·  back to Commercialrealestate  ·  direct-listings viewer: /api/direct-listings + public/direct 8f9ce55 →