← back to Homesonspec
build-loop: max-it — fan out 4 adapters in parallel (per-domain), 30rpm, bigger enrich batches
e118a5bd374ef5df1e05059d2253cc1545854cb1 · 2026-07-28 07:26:16 -0700 · Steve
Files touched
Diff
commit e118a5bd374ef5df1e05059d2253cc1545854cb1
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jul 28 07:26:16 2026 -0700
build-loop: max-it — fan out 4 adapters in parallel (per-domain), 30rpm, bigger enrich batches
---
scripts/build-loop.sh | 36 +++++++++++++++++++++++++-----------
1 file changed, 25 insertions(+), 11 deletions(-)
diff --git a/scripts/build-loop.sh b/scripts/build-loop.sh
index 902213d..3888cbf 100644
--- a/scripts/build-loop.sh
+++ b/scripts/build-loop.sh
@@ -42,6 +42,22 @@ fullname() { case "$1" in
lc() { echo "$1" | tr '[:upper:]' '[:lower:]'; }
slug() { fullname "$1" | tr '[:upper:] ' '[:lower:]-'; } # "North Carolina" -> north-carolina
+# max-it: one worker per adapter, sweeping ALL states. The 4 run CONCURRENTLY —
+# different domains, so each stays polite within its own per-source rate limit while
+# 4x the inventory lands per wall-clock second. $0 (no LLM anywhere in ingestion).
+sweep_adapter() {
+ local kind="$1"
+ for S in $STATES; do
+ [ -f "$STOP" ] && break
+ case "$kind" in
+ kb) KBHOME_STATE="$S" CLI kb-home-site | sed "s/^/ kb $S: /" ;;
+ drh) DRHORTON_STATE="$(slug "$S")" DRHORTON_PAGE_LIMIT=$CAP CLI dr-horton-site | sed "s/^/ drh $S: /" ;;
+ plt) PULTE_STATE="$(fullname "$S")" PULTE_PAGE_LIMIT=$CAP CLI pultegroup-site | sed "s/^/ plt $S: /" ;;
+ tri) TRIPOINTE_STATE="$(lc "$S")" TRIPOINTE_PAGE_LIMIT=$CAP CLI tri-pointe-site | sed "s/^/ tri $S: /" ;;
+ esac
+ done
+}
+
# wait until no other adapter run is in flight (avoid same-source races)
while pgrep -f "cli.ts --adapter" 2>/dev/null | grep -qv "^$$\$"; do sleep 20; done
@@ -55,20 +71,18 @@ while [ ! -f "$STOP" ] && [ "$sweep" -lt "$MAX_SWEEPS" ]; do
s0=$(PSQL "select count(*) from \"InventoryHome\" where status='PUBLISHED';")
b0=$(PSQL "select count(distinct \"builderId\") from \"InventoryHome\" where status='PUBLISHED';")
LOG "===== SWEEP $sweep/$MAX_SWEEPS start: $s0 homes / $b0 builders ====="
- for S in $STATES; do
- [ -f "$STOP" ] && break
- LOG "-- $S --"
- KBHOME_STATE="$S" CLI kb-home-site | sed "s/^/ kb $S: /"
- DRHORTON_STATE="$(slug "$S")" DRHORTON_PAGE_LIMIT=$CAP CLI dr-horton-site | sed "s/^/ drh $S: /"
- PULTE_STATE="$(fullname "$S")" PULTE_PAGE_LIMIT=$CAP CLI pultegroup-site | sed "s/^/ plt $S: /"
- TRIPOINTE_STATE="$(lc "$S")" TRIPOINTE_PAGE_LIMIT=$CAP CLI tri-pointe-site | sed "s/^/ tri $S: /"
- done
- # drain enrichment queue
- for i in $(seq 1 40); do
+ LOG "fanning out 4 adapters in parallel across ${CAP}-deep states (West Coast first)"
+ sweep_adapter kb &
+ sweep_adapter drh &
+ sweep_adapter plt &
+ sweep_adapter tri &
+ wait
+ # drain enrichment queue (bigger batches for speed)
+ for i in $(seq 1 60); do
rem=$(PSQL "select count(*) from \"Community\" where lat is not null and lon is not null and amenities is null;")
[ "${rem:-0}" -eq 0 ] && { LOG "enrich: queue drained"; break; }
LOG "enrich pass $i: $rem left"
- NEARBY_LIMIT=40 npx tsx scripts/enrich-nearby.ts >/dev/null 2>&1
+ NEARBY_LIMIT=80 npx tsx scripts/enrich-nearby.ts >/dev/null 2>&1
done
s1=$(PSQL "select count(*) from \"InventoryHome\" where status='PUBLISHED';")
b1=$(PSQL "select count(distinct \"builderId\") from \"InventoryHome\" where status='PUBLISHED';")
← 168b21d build-loop: West Coast first + deeper per-state cap (CAP=120
·
back to Homesonspec
·
deploy-kamatera: rewrite for homesonspec self-host (correct be37128 →