Lennar adapter: LENNAR_STATE_FILTER for targeted per-state crawls + drop promo/event non-community URLs (alphabetical full-sweep is too slow to reach late-alphabet states like TX)

c642140836cb688a9185dd094dcc8624f2b40936 · 2026-07-22 23:13:59 -0700 · Steve

Files touched

M collectors/lennar/src/index.ts

Diff

commit c642140836cb688a9185dd094dcc8624f2b40936
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jul 22 23:13:59 2026 -0700

    Lennar adapter: LENNAR_STATE_FILTER for targeted per-state crawls + drop promo/event non-community URLs (alphabetical full-sweep is too slow to reach late-alphabet states like TX)
---
 collectors/lennar/src/index.ts | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/collectors/lennar/src/index.ts b/collectors/lennar/src/index.ts
index f356737..4e52801 100644
--- a/collectors/lennar/src/index.ts
+++ b/collectors/lennar/src/index.ts
@@ -122,9 +122,15 @@ export const lennarAdapter: SourceAdapter = {
     }
     const fetcher = new LiveFetcher(ctx.registry);
     const sitemap = await fetcher.fetch(SITEMAP_URL);
-    const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+    let urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
       .map((m) => m[1]!)
       .filter(isCommunityUrl);
+    // Drop obvious non-community length-5 URLs (promo/event pages waste fetches).
+    urls = urls.filter((u) => !/\/(promo|event)\//i.test(u));
+    // LENNAR_STATE_FILTER=texas restricts the sweep to one state so coverage can
+    // be filled targeted + fast instead of grinding the whole sitemap alphabetically.
+    const stateFilter = process.env.LENNAR_STATE_FILTER?.trim().toLowerCase();
+    if (stateFilter) urls = urls.filter((u) => u.toLowerCase().includes(`/new-homes/${stateFilter}/`));
     urls.sort();
     for (const url of urls.slice(0, PAGE_LIMIT)) {
       // A dead/stale sitemap URL (404) or a transient blip must SKIP, never abort

← ef19ec0 Fix broken search: when a query doesn't resolve to a locatio · back to Homesonspec · Listings contact: capture per-community sales-office phone f 0d7bc77 →