enrich: --shard=i/N + --ollama/--model for max-it fan-out across Mac2+Mac1 Ollama

daf3111959cad118959ec06d58e9ce5d46872a6d · 2026-08-12 08:10:46 -0700 · steve

Files touched

M scripts/enrich-contractor.js

Diff

commit daf3111959cad118959ec06d58e9ce5d46872a6d
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Aug 12 08:10:46 2026 -0700

    enrich: --shard=i/N + --ollama/--model for max-it fan-out across Mac2+Mac1 Ollama
---
 scripts/enrich-contractor.js | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scripts/enrich-contractor.js b/scripts/enrich-contractor.js
index 8d94663..01abcf0 100644
--- a/scripts/enrich-contractor.js
+++ b/scripts/enrich-contractor.js
@@ -10,15 +10,17 @@
 // Usage: node scripts/enrich-contractor.js [--loop] [--batch=N] [--all]
 import { q, pool } from '../src/db.js';
 
-const OLLAMA = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
-const MODEL = process.env.ENRICH_MODEL || 'qwen2.5:latest'; // fast local model
-const DELAY = Number(process.env.ENRICH_DELAY_MS || 1200);   // polite delay between DDG hits
-const sleep = ms => new Promise(r => setTimeout(r, ms));
-
 const flags = new Set(process.argv.slice(2).filter(a => a.startsWith('--') && !a.includes('=')));
 const kv = Object.fromEntries(process.argv.slice(2).filter(a => a.includes('=')).map(a => a.slice(2).split('=')));
+const OLLAMA = kv.ollama || process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';
+const MODEL = kv.model || process.env.ENRICH_MODEL || 'qwen2.5:latest'; // fast local model
+const DELAY = Number(process.env.ENRICH_DELAY_MS || 1200);   // polite delay between DDG hits
+const sleep = ms => new Promise(r => setTimeout(r, ms));
 const BATCH = Number(kv.batch || 40);
 const LA_ONLY = !flags.has('--all');
+// --shard=i/N : this worker only takes rows where hashtext(LicenseNo) % N == i (max-it fan-out)
+const [SHARD_I, SHARD_N] = (kv.shard || '0/1').split('/').map(Number);
+const SHARD = SHARD_N > 1 ? `AND (abs(hashtext(c."LicenseNo")) % ${SHARD_N}) = ${SHARD_I}` : '';
 
 // ---- free DuckDuckGo HTML search ----
 async function ddg(query) {
@@ -67,7 +69,7 @@ async function pick() {
     )
     SELECT c."LicenseNo", c."BusinessName", c."City", c."ZIPCode", c."BusinessPhone", c."Classifications(s)" AS cls
     FROM cslb_raw c LEFT JOIN hot ON hot.zip = left(c."ZIPCode",5)
-    WHERE c."PrimaryStatus"='CLEAR' AND c."BusinessName" IS NOT NULL AND c.enriched_at IS NULL ${laFilter}
+    WHERE c."PrimaryStatus"='CLEAR' AND c."BusinessName" IS NOT NULL AND c.enriched_at IS NULL ${laFilter} ${SHARD}
     ORDER BY COALESCE(hot.activity,0) DESC, c."Classifications(s)" LIKE 'B%' DESC
     LIMIT ${BATCH}`)).rows;
 }

← 5fc3e57 Contractor enrichment: deterministic directory blocklist (nu · back to La Socrata Ingester · enrich: don't lose DDG-rate-limited rows (mark ddg-blocked, 14a832a →