backfill firm-site: add --offset to sweep beyond the top 40 firms

5205cc5846a08180b37b5240239331703ab0dc19 · 2026-08-19 10:40:28 -0700 · Steve Abrams

Files touched

M scripts/backfill-openclaw-llm.js

Diff

commit 5205cc5846a08180b37b5240239331703ab0dc19
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 10:40:28 2026 -0700

    backfill firm-site: add --offset to sweep beyond the top 40 firms
---
 scripts/backfill-openclaw-llm.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/backfill-openclaw-llm.js b/scripts/backfill-openclaw-llm.js
index 887c2f8..7ba8881 100644
--- a/scripts/backfill-openclaw-llm.js
+++ b/scripts/backfill-openclaw-llm.js
@@ -12,6 +12,7 @@ const db = require('./db/brokers-db');
 const OC = process.env.HOME + '/.npm-global/bin/openclaw';
 const OLLAMA = 'http://127.0.0.1:11434/api/generate';
 const LIMIT = +(process.argv.find(a => a.startsWith('--limit='))?.split('=')[1]) || 5;
+const OFFSET = +(process.argv.find(a => a.startsWith('--offset='))?.split('=')[1]) || 0;
 const APPLY = process.argv.includes('--apply');
 const BIG = /cbre|kw\.com|yourkwoffice|kellerwilliams|marcusmillichap|kidder|coldwell|compass|remax|century21|colliers|jll|cushman|berkshire/i;
 const domainOf = u => { try { return new URL(/^https?:/i.test(u) ? u : 'https://' + u).hostname.replace(/^www\./, ''); } catch { return u; } };
@@ -70,7 +71,7 @@ async function scrapeDomain(url) {
        FROM broker b JOIN firm f ON f.id=b.firm_id
       WHERE b.website IS NOT NULL AND b.website !~* 'crexi' AND b.website !~* $1
       GROUP BY b.firm_id, f.name HAVING count(*) >= 2
-      ORDER BY count(*) DESC LIMIT $2`, [BIG.source, LIMIT])).rows;
+      ORDER BY count(*) DESC LIMIT $2 OFFSET $3`, [BIG.source, LIMIT, OFFSET])).rows;
   console.log(`\n== Option-3 openclaw+LLM backfill · ${rows.length} firm domains · ${APPLY ? 'APPLY' : 'DRY-RUN'} ==\n`);
   let totFound = 0, wrote = 0;
   for (const r of rows) {

← e4bc315 CRCP: sync rsync now includes broker-phone-overlay.json to p · back to Commercialrealestate · brokers.html: node 'View full profile' -> /agent.html (our p b9d6c4c →