resilience: live adapters skip per-URL fetch failures (404/blip) instead of aborting the whole sweep — a single stale Lennar /promo/ URL killed a 4.4k-page run after 15,974 homes

5c8576904e7227915f71eebbf91d7d3edc416d4a · 2026-07-22 13:20:19 -0700 · Steve Abrams

Files touched

M collectors/dream-finders/src/index.ts
M collectors/lennar/src/index.ts

Diff

commit 5c8576904e7227915f71eebbf91d7d3edc416d4a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Jul 22 13:20:19 2026 -0700

    resilience: live adapters skip per-URL fetch failures (404/blip) instead of aborting the whole sweep — a single stale Lennar /promo/ URL killed a 4.4k-page run after 15,974 homes
---
 collectors/dream-finders/src/index.ts | 6 +++++-
 collectors/lennar/src/index.ts        | 8 +++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/collectors/dream-finders/src/index.ts b/collectors/dream-finders/src/index.ts
index 28afa42..2900257 100644
--- a/collectors/dream-finders/src/index.ts
+++ b/collectors/dream-finders/src/index.ts
@@ -130,7 +130,11 @@ export const dreamFindersAdapter: SourceAdapter = {
       .filter((url) => parseSpecUrl(url) !== null);
     urls.sort();
     for (const url of urls.slice(0, PAGE_LIMIT)) {
-      yield await fetcher.fetch(url);
+      try {
+        yield await fetcher.fetch(url);
+      } catch (error) {
+        console.warn(`  skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+      }
     }
   },
 
diff --git a/collectors/lennar/src/index.ts b/collectors/lennar/src/index.ts
index fe04203..e924766 100644
--- a/collectors/lennar/src/index.ts
+++ b/collectors/lennar/src/index.ts
@@ -86,7 +86,13 @@ export const lennarAdapter: SourceAdapter = {
       .filter(isCommunityUrl);
     urls.sort();
     for (const url of urls.slice(0, PAGE_LIMIT)) {
-      yield await fetcher.fetch(url);
+      // A dead/stale sitemap URL (404) or a transient blip must SKIP, never abort
+      // the whole sweep — large builder sitemaps always carry some rot.
+      try {
+        yield await fetcher.fetch(url);
+      } catch (error) {
+        console.warn(`  skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+      }
     }
   },

← e8366c4 auto-save: 2026-07-22T13:14:23 (1 files) — ROLLBACK-DNS.txt · back to Homesonspec · HARD RULE: demo/synthetic inventory NEVER surfaces live — is f895480 →