[object Object]

← back to Homesonspec

fix(collectors): raise DRH+Lennar default PAGE_LIMIT 8→300 for statewide coverage

08c6d176c8022aa106111c8f7514afd6489b4f53 · 2026-07-23 01:08:26 -0700 · Steve Abrams

The 8-page default stranded >96% of inventory: there is no cursor to
advance past the sorted head, so successive runs re-hit the same 8
communities forever. DRH's CA sitemap alone has 226 communities.
300 covers a whole state per run; contentHash dedup keeps unchanged
communities cheap on recurring cron passes. Matches Pulte/TriPointe (200).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 08c6d176c8022aa106111c8f7514afd6489b4f53
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Jul 23 01:08:26 2026 -0700

    fix(collectors): raise DRH+Lennar default PAGE_LIMIT 8→300 for statewide coverage
    
    The 8-page default stranded >96% of inventory: there is no cursor to
    advance past the sorted head, so successive runs re-hit the same 8
    communities forever. DRH's CA sitemap alone has 226 communities.
    300 covers a whole state per run; contentHash dedup keeps unchanged
    communities cheap on recurring cron passes. Matches Pulte/TriPointe (200).
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 collectors/dr-horton/src/index.ts |  6 +++++-
 collectors/lennar/src/index.ts    | 10 ++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/collectors/dr-horton/src/index.ts b/collectors/dr-horton/src/index.ts
index 2b112cf..312e6ab 100644
--- a/collectors/dr-horton/src/index.ts
+++ b/collectors/dr-horton/src/index.ts
@@ -22,7 +22,11 @@ const SITEMAP_URL = "https://www.drhorton.com/sitemaps/website.xml";
 const ORIGIN = "https://www.drhorton.com";
 const BUILDER_SLUG = "dr-horton";
 const STATE = (process.env.DRHORTON_STATE ?? "california").toLowerCase();
-const PAGE_LIMIT = Number(process.env.DRHORTON_PAGE_LIMIT ?? 8);
+// Statewide default: DRH's CA sitemap alone has ~226 communities. A low cap
+// silently surfaces <4% of inventory (no cursor exists to advance past the
+// head), so the default must cover a whole state in one run. contentHash
+// snapshot-dedup keeps unchanged communities cheap on recurring runs.
+const PAGE_LIMIT = Number(process.env.DRHORTON_PAGE_LIMIT ?? 300);
 
 function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
   return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
diff --git a/collectors/lennar/src/index.ts b/collectors/lennar/src/index.ts
index 1223530..3608b31 100644
--- a/collectors/lennar/src/index.ts
+++ b/collectors/lennar/src/index.ts
@@ -17,14 +17,16 @@ import {
  * status, address, lat/lon, planRef). robots.txt permits the /new-homes
  * tree (only fragment paths disallowed).
  *
- * Controlled batching: LENNAR_PAGE_LIMIT community pages per run
- * (default 8) at the registry rate limit; change detection makes
- * re-visits cheap (unchanged bytes stage nothing).
+ * Statewide batching: LENNAR_PAGE_LIMIT community pages per run
+ * (default 300 — enough to cover a whole state in one pass; there is no
+ * cursor to advance past the head, so a low cap silently strands the tail)
+ * at the registry rate limit; change detection makes re-visits cheap
+ * (unchanged bytes stage nothing).
  */
 
 const SITEMAP_URL = "https://www.lennar.com/api/images/sitemapfe.xml";
 const BUILDER_SLUG = "lennar";
-const PAGE_LIMIT = Number(process.env.LENNAR_PAGE_LIMIT ?? 8);
+const PAGE_LIMIT = Number(process.env.LENNAR_PAGE_LIMIT ?? 300);
 
 function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
   return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };

← aa61dfd fix(live-fetch): broaden Accept header so D.R. Horton WAF st  ·  back to Homesonspec  ·  feat(collectors): add Taylor Morrison adapter (Sitecore fedm 86433ff →