crawl-broker-listings: read sale[]/lease[] payloads + honor totalCount (no 12-page truncation) + accurate other-listing count (codex review)

3a8c16880f9642d3fb04bbcc2afc364d5593fe6a · 2026-08-19 08:50:17 -0700 · Steve Abrams
Files touched

M scripts/crawl-broker-listings.js
Diff

commit 3a8c16880f9642d3fb04bbcc2afc364d5593fe6a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 08:50:17 2026 -0700

    crawl-broker-listings: read sale[]/lease[] payloads + honor totalCount (no 12-page truncation) + accurate other-listing count (codex review)
---
 scripts/crawl-broker-listings.js | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/scripts/crawl-broker-listings.js b/scripts/crawl-broker-listings.js
index 49e2413..923e7ca 100644
--- a/scripts/crawl-broker-listings.js
+++ b/scripts/crawl-broker-listings.js
@@ -84,7 +84,10 @@ const norm = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').tr
       if (!u.includes('api.crexi.com')) return;
       if ((resp.headers()['content-type'] || '').indexOf('json') < 0) return;
       let j; try { j = await resp.json(); } catch { return; }
-      const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || j.items || null);
+      // Crexi broker-profile payloads carry the book as `sale[]` / `lease[]` arrays (confirmed via
+      // public scrapers) — that IS the correct source. Also handle the generic list envelopes.
+      const arr = Array.isArray(j) ? j
+        : [].concat(j.data || [], j.assets || [], j.results || [], j.items || [], j.sale || [], j.lease || [], j.forSale || [], (j.broker && [].concat(j.broker.sale || [], j.broker.lease || [])) || []);
       if (Array.isArray(arr) && arr.length && arr[0] && (arr[0].id || arr[0].assetId)) {
         let p; try { p = new URL(u).pathname; } catch { p = u; }
         sniffed.push({ reqPath: p, assets: arr });
@@ -115,18 +118,23 @@ const norm = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').tr
             (o) => `https://api.crexi.com/brokers/${id}/assets?pageSize=60&pageNumber=${o}`,
             (o) => `https://api.crexi.com/assets/search?brokerId=${id}&pageSize=60&pageNumber=${o}`
           ];
-          const out = []; let picked = null;
+          const listOf = (j) => Array.isArray(j) ? j
+            : [].concat(j.data || [], j.assets || [], j.results || [], j.items || [], j.sale || [], j.lease || [], j.forSale || []);
+          const totalOf = (j) => (j && (j.totalCount || j.total || j.count || (j.pagination && (j.pagination.totalCount || j.pagination.total)))) || 0;
+          const out = []; let picked = null, total = 0;
           for (const mk of tries) {
             try { const r = await fetch(mk(1), { headers: { authorization: auth, accept: 'application/json' } });
-              if (r.ok) { const j = await r.json(); const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || j.items || []);
-                if (Array.isArray(arr) && arr.length && (arr[0].id || arr[0].assetId)) { picked = mk; out.push(...arr); break; } } } catch {}
+              if (r.ok) { const j = await r.json(); const arr = listOf(j);
+                if (arr.length && (arr[0].id || arr[0].assetId)) { picked = mk; total = totalOf(j); out.push(...arr); break; } } } catch {}
           }
-          if (picked) for (let o = 2; o <= 12; o++) {   // up to ~720 assets/broker
+          // Honor totalCount so big books (e.g. 2,784) aren't truncated. Cap at 60 pages (~3,600) as a backstop.
+          const maxPage = total ? Math.min(60, Math.ceil(total / 60)) : 12;
+          if (picked) for (let o = 2; o <= maxPage; o++) {
             try { const r = await fetch(picked(o), { headers: { authorization: auth, accept: 'application/json' } });
-              if (!r.ok) break; const j = await r.json(); const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || j.items || []);
+              if (!r.ok) break; const arr = listOf(await r.json());
               if (!arr.length) break; out.push(...arr); } catch { break; }
           }
-          return { endpoint: picked ? picked(1).split('?')[0] : null, assets: out };
+          return { endpoint: picked ? picked(1).split('?')[0] : null, total, assets: out };
         }, { id: b.crexi_id, auth });
         if (pageAssets.endpoint) { const d = discovery.find(x => x.path === '[fetch] ' + pageAssets.endpoint); d ? (d.count += pageAssets.assets.length) : discovery.push({ path: '[fetch] ' + pageAssets.endpoint, count: pageAssets.assets.length }); }
         for (const a of pageAssets.assets) { const id = a.id || a.assetId; if (id) assetsById.set(id, a); }
@@ -144,7 +152,7 @@ const norm = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').tr
             `INSERT INTO broker_other_listing(broker_id, ext_id, title, address, city, state, price, asset_type, url, source, found_at)
              VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,'crexi-broker-crawl', now())
              ON CONFLICT DO NOTHING`,
-            [b.id, String(a.id), a.address, a.address, a.city, null, a.price || null, type, url]).then(() => otherRows++).catch(() => {});
+            [b.id, String(a.id), a.address, a.address, a.city, null, a.price || null, type, url]).then(r => { otherRows += r.rowCount; }).catch(() => {});
           // graph: listing + broker_listing edge
           await db.upsertListing({ id: 'crx' + a.id, address: a.address, city: a.city, type, price: a.price, cap_rate: a.cap, firm_name: b.firm, source: url }).catch(() => {});
           await db.link(b.id, 'crx' + a.id, 'book').catch(() => {});
← ce630d7 Add crawl-broker-listings.js: Crexi broker→full-book expansi · back to Commercialrealestate · CRCP refine: add optional async CFG.detail hook to lending-g 6e1d7c5 →