← back to Homesonspec
collectors: add Perry, Brookfield, Holt adapters (+3 builders, yolo iter-1)
33db5031840a435ebcbbaff2cba44a8553dec3d6 · 2026-07-28 20:41:32 -0700 · Steve Abrams
DTD-panel-approved (5/5 verdict A) increment. All facts-only/honest-UA/robots-respected,
DB-verified per-home coverage:
- Perry (perry-homes-site): Algolia inventory index, TX+FL (~1637 homes; 100% price/beds/baths/sqft).
- Brookfield (brookfield-site): Sitecore Discover type=Lot filter, 9 states (~412; 97.5% price,
100% beds/baths/sqft+geo). New builder -> created Builder + SourceRegistry rows.
- Holt (holt-homes-site): Yoast per-home sitemap, OR/WA West-Coast (~128; 100% incl lat/lon).
Perry/Brookfield every-sweep (fast APIs); Holt throttled with heavy crawlers. City Ventures
skipped (robots Disallow: /). Built via 3 parallel isolated subagents + serial gated integration.
Files touched
Diff
commit 33db5031840a435ebcbbaff2cba44a8553dec3d6
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 28 20:41:32 2026 -0700
collectors: add Perry, Brookfield, Holt adapters (+3 builders, yolo iter-1)
DTD-panel-approved (5/5 verdict A) increment. All facts-only/honest-UA/robots-respected,
DB-verified per-home coverage:
- Perry (perry-homes-site): Algolia inventory index, TX+FL (~1637 homes; 100% price/beds/baths/sqft).
- Brookfield (brookfield-site): Sitecore Discover type=Lot filter, 9 states (~412; 97.5% price,
100% beds/baths/sqft+geo). New builder -> created Builder + SourceRegistry rows.
- Holt (holt-homes-site): Yoast per-home sitemap, OR/WA West-Coast (~128; 100% incl lat/lon).
Perry/Brookfield every-sweep (fast APIs); Holt throttled with heavy crawlers. City Ventures
skipped (robots Disallow: /). Built via 3 parallel isolated subagents + serial gated integration.
---
scripts/build-loop.sh | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/scripts/build-loop.sh b/scripts/build-loop.sh
index b403e4d5..98a54ebb 100644
--- a/scripts/build-loop.sh
+++ b/scripts/build-loop.sh
@@ -116,6 +116,13 @@ sweep_beazer() { BEAZER_PAGE_LIMIT=250 CLI beazer-site | sed "s/^/
# Drees: sitemap of ~649 per-home (pageType=Home) pages; floorplan pages gated out.
sweep_drees() { DREES_PAGE_LIMIT=720 CLI drees-homes-site | sed "s/^/ drs NATIONAL: /"; }
+# Perry = single Algolia inventory index (TX+FL, ~1637 real homes), paginated GET, fast — every sweep.
+sweep_perry() { PERRY_PAGE_LIMIT=30 CLI perry-homes-site | sed "s/^/ per NATIONAL: /"; }
+# Brookfield = single Sitecore-Discover POST (9 states, ~412 homes), paginated, fast — every sweep.
+sweep_brookfield() { BROOKFIELD_PAGE_LIMIT=10 CLI brookfield-site | sed "s/^/ brk NATIONAL: /"; }
+# Holt = per-home sitemap crawl (OR/WA, ~128 homes = 128 rate-limited GETs) — throttle w/ heavy crawlers.
+sweep_holt() { HOLT_PAGE_LIMIT=140 CLI holt-homes-site | sed "s/^/ hlt REGIONAL: /"; }
+
# metro-iterating adapters (footprint = METROS, not states) — scoped to served metros only
# (Cody WAF-lesson from the start). Ashton Woods publishes per-metro quick-move-in homes.
sweep_metros() {
@@ -134,7 +141,7 @@ while pgrep -f "cli.ts --adapter" 2>/dev/null | grep -qv "^$$\$"; do sleep 20; d
# a transient failure shouldn't permanently sideline a chosen builder, and genuine breakage is
# surfaced by the dashboard's per-source failed_runs + the freshness canary (not silent). To
# retire a builder, remove it from THIS list (don't rely on auto-pause surviving a restart).
-for k in dr-horton-site kb-home-site pultegroup-site tri-pointe-site lennar-site toll-brothers-site david-weekley-site taylor-morrison-site ashton-woods-site discovery-homes-site meritage-homes-site ryan-homes-site highland-homes-site landsea-homes-site shea-homes-site lgi-homes-site century-communities-site beazer-site drees-homes-site; do
+for k in dr-horton-site kb-home-site pultegroup-site tri-pointe-site lennar-site toll-brothers-site david-weekley-site taylor-morrison-site ashton-woods-site discovery-homes-site meritage-homes-site ryan-homes-site highland-homes-site landsea-homes-site shea-homes-site lgi-homes-site century-communities-site beazer-site drees-homes-site perry-homes-site brookfield-site holt-homes-site; do
PSQL "update \"SourceRegistry\" set active=true, health='HEALTHY', \"consecutiveFailures\"=0 where key='$k';" >/dev/null
done
@@ -152,8 +159,10 @@ while [ ! -f "$STOP" ] && [ "$sweep" -lt "$MAX_SWEEPS" ]; do
sweep_highland &
sweep_landsea &
sweep_shea &
+ sweep_perry &
+ sweep_brookfield &
# heavy crawlers: throttled to every 4th sweep (hundreds of GETs each, slow-changing data)
- if [ $((sweep % 4)) -eq 1 ]; then sweep_lgi & sweep_century & sweep_beazer & sweep_drees & fi
+ if [ $((sweep % 4)) -eq 1 ]; then sweep_lgi & sweep_century & sweep_beazer & sweep_drees & sweep_holt & fi
sweep_adapter drh &
sweep_adapter plt &
sweep_adapter tri &
← 590c9393 auto-save: 2026-07-28T20:33:03 (1 files) — collectors/ashton
·
back to Homesonspec
·
brookfield: drop shouldcountasquickmovein filter (contrarian 955307e6 →