[object Object]

← back to Homesonspec

auto-save: 2026-07-27T20:54:34 (4 files) — apps/workers/package.json apps/workers/src/cli.ts pnpm-lock.yaml collectors/discovery/

c61273589a909f2ce5d68500feb43d7b4350048f · 2026-07-27 20:54:41 -0700 · Steve Abrams

Files touched

Diff

commit c61273589a909f2ce5d68500feb43d7b4350048f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 27 20:54:41 2026 -0700

    auto-save: 2026-07-27T20:54:34 (4 files) — apps/workers/package.json apps/workers/src/cli.ts pnpm-lock.yaml collectors/discovery/
---
 apps/workers/package.json          |   3 +-
 apps/workers/src/cli.ts            |   2 +
 collectors/discovery/package.json  |  15 ++++
 collectors/discovery/src/index.ts  | 173 +++++++++++++++++++++++++++++++++++++
 collectors/discovery/tsconfig.json |   1 +
 pnpm-lock.yaml                     |  25 ++++++
 6 files changed, 218 insertions(+), 1 deletion(-)

diff --git a/apps/workers/package.json b/apps/workers/package.json
index 0243f88..1f105a3 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -26,7 +26,8 @@
     "@homesonspec/collector-kb-home": "workspace:*",
     "@homesonspec/collector-pulte": "workspace:*",
     "@homesonspec/collector-tri-pointe": "workspace:*",
-    "@homesonspec/collector-taylor-morrison": "workspace:*"
+    "@homesonspec/collector-taylor-morrison": "workspace:*",
+    "@homesonspec/collector-discovery": "workspace:*"
   },
   "devDependencies": {
     "tsx": "^4.19.2",
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index 121b3b9..3da42da 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -7,6 +7,7 @@ import { kbHomeAdapter } from "@homesonspec/collector-kb-home";
 import { pulteAdapter } from "@homesonspec/collector-pulte";
 import { triPointeAdapter } from "@homesonspec/collector-tri-pointe";
 import { taylorMorrisonAdapter } from "@homesonspec/collector-taylor-morrison";
+import { discoveryAdapter } from "@homesonspec/collector-discovery";
 import { runPipeline } from "./pipeline";
 import { recordSourceRun } from "./verify";
 
@@ -23,6 +24,7 @@ const ADAPTERS = {
   [pulteAdapter.key]: pulteAdapter,
   [triPointeAdapter.key]: triPointeAdapter,
   [taylorMorrisonAdapter.key]: taylorMorrisonAdapter,
+  [discoveryAdapter.key]: discoveryAdapter,
 };
 
 async function main() {
diff --git a/collectors/discovery/package.json b/collectors/discovery/package.json
new file mode 100644
index 0000000..bd66f4a
--- /dev/null
+++ b/collectors/discovery/package.json
@@ -0,0 +1,15 @@
+{
+  "name": "@homesonspec/collector-discovery",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+  "dependencies": {
+    "@homesonspec/collectors-common": "workspace:*",
+    "@homesonspec/schemas": "workspace:*",
+    "@homesonspec/shared": "workspace:*"
+  },
+  "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/discovery/src/index.ts b/collectors/discovery/src/index.ts
new file mode 100644
index 0000000..f28947f
--- /dev/null
+++ b/collectors/discovery/src/index.ts
@@ -0,0 +1,173 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import {
+  fetchFixtures,
+  LiveFetcher,
+  type ExtractionOutput,
+  type FetchContext,
+  type RawPage,
+  type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Discovery Homes adapter — first REGIONAL builder (recon 2026-07-27).
+ * discoveryhomes.com exposes a clean JSON API:
+ *   GET /api/communities            → JSON array of communities (name/city/state/zip/lat/lon/builder)
+ *   GET /api/homes?community_id=<id> → {renderedList: "<qmi-card html>"} of quick-move-in homes
+ * The site hosts several Seeno-family brands (builder = discovery|seeno|jmscc|sterling);
+ * this adapter scopes to DISCOVERY_BUILDER (default "discovery" → builder slug discovery-homes).
+ * Plain HTTP, no anti-bot; robots allows /api/. Facts-only (images intentionally omitted).
+ * Community context is threaded to the homes page via ignored _* query params so extract()
+ * stays a pure per-page function.
+ */
+const ORIGIN = "https://www.discoveryhomes.com";
+const COMMUNITIES_URL = `${ORIGIN}/api/communities`;
+const BUILDER_CODE = (process.env.DISCOVERY_BUILDER ?? "discovery").toLowerCase();
+const BUILDER_SLUG = process.env.DISCOVERY_BUILDER_SLUG ?? "discovery-homes";
+const PAGE_LIMIT = Number(process.env.DISCOVERY_PAGE_LIMIT ?? 200);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+  return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+const num = (v: unknown): number | null => {
+  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
+  return Number.isFinite(n) && n > 0 ? n : null;
+};
+const str = (v: unknown): string | null => (v == null ? null : String(v).trim() ? String(v).trim() : null);
+
+interface Community { id: number; name?: string; builder?: string; city?: string; state?: string; zip?: unknown;
+  county?: string; latitude?: unknown; longitude?: unknown; phone?: string; is_active?: unknown; availability?: string; }
+
+export const discoveryAdapter: SourceAdapter = {
+  key: "discovery-homes-site",
+  version: "1.0.0",
+
+  async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+    if (ctx.mode === "fixture") {
+      yield* fetchFixtures(ctx);
+      return;
+    }
+    const fetcher = new LiveFetcher(ctx.registry);
+    const commPage = await fetcher.fetch(COMMUNITIES_URL);
+    yield commPage; // extract() emits community records from this
+
+    let all: Community[] = [];
+    try { all = JSON.parse(commPage.body.toString("utf8")); } catch { all = []; }
+    const mine = all.filter((c) => (c.builder ?? "").toLowerCase() === BUILDER_CODE).slice(0, PAGE_LIMIT);
+    for (const c of mine) {
+      // thread community context via ignored params (server returns the same homes payload)
+      const p = new URLSearchParams({
+        community_id: String(c.id),
+        _comm: str(c.name) ?? "",
+        _city: str(c.city) ?? "",
+        _state: (str(c.state) ?? "").toUpperCase(),
+        _zip: str(c.zip) ?? "",
+      });
+      try { yield await fetcher.fetch(`${ORIGIN}/api/homes?${p.toString()}`); }
+      catch (e) { /* skip a community whose homes page errors/blocks */ }
+    }
+  },
+
+  extract(page: RawPage): ExtractionOutput {
+    try {
+      if (page.url.includes("/api/communities")) return extractCommunities(page);
+      if (page.url.includes("/api/homes")) return extractHomes(page);
+      return { records: [], errors: [{ url: page.url, reason: "unrecognized page" }] };
+    } catch (error) {
+      return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+    }
+  },
+};
+
+function extractCommunities(page: RawPage): ExtractionOutput {
+  let all: Community[] = [];
+  try { all = JSON.parse(page.body.toString("utf8")); } catch { return { records: [], errors: [{ url: page.url, reason: "communities JSON parse failed" }] }; }
+  const records: ExtractedRecord[] = [];
+  for (const c of all) {
+    if ((c.builder ?? "").toLowerCase() !== BUILDER_CODE) continue;
+    const name = str(c.name);
+    if (!name) continue;
+    const phone = str(c.phone);
+    records.push({
+      entityType: "community",
+      canonicalHints: { builderSlug: BUILDER_SLUG, communityName: name },
+      fields: {
+        name: fv(name, name, page.url),
+        street: fv<string>(null, null, page.url),
+        city: fv(str(c.city), str(c.city), page.url),
+        state: fv((str(c.state) ?? "").toUpperCase() || null, str(c.state), page.url),
+        zip: fv(str(c.zip), str(c.zip), page.url),
+        county: fv(str(c.county), str(c.county), page.url),
+        metro: fv<string>(null, null, page.url),
+        lat: fv(num(c.latitude), str(c.latitude), page.url),
+        lon: fv(num(c.longitude), str(c.longitude), page.url),
+        hoaFeeMonthly: fv<number>(null, null, page.url),
+        schoolDistrict: fv<string>(null, null, page.url),
+        ageRestricted: fv<boolean>(null, null, page.url),
+        salesPhone: fv(phone ? phone.replace(/[^0-9+]/g, "") : null, phone, page.url, "community sales phone"),
+      },
+    });
+  }
+  return { records, errors: [] };
+}
+
+function extractHomes(page: RawPage): ExtractionOutput {
+  const json = (() => { try { return JSON.parse(page.body.toString("utf8")); } catch { return null; } })();
+  const html: string = json?.renderedList ?? "";
+  const u = new URL(page.url);
+  const communityName = decodeURIComponent(u.searchParams.get("_comm") ?? "") || null;
+  const city = decodeURIComponent(u.searchParams.get("_city") ?? "") || null;
+  const state = (u.searchParams.get("_state") ?? "").toUpperCase() || null;
+  const zip = u.searchParams.get("_zip") || null;
+  if (!html || !communityName) return { records: [], errors: [] };
+
+  const records: ExtractedRecord[] = [];
+  const cards = html.split("qmi-card__container").slice(1);
+  for (const card of cards) {
+    const priceM = card.match(/qmi-card__price[^>]*>\s*([^<]+)/i);
+    const priceRaw = priceM ? priceM[1]!.trim() : "";
+    const price = num(priceRaw.startsWith("$") ? priceRaw : (/\$[\d,]/.test(priceRaw) ? priceRaw : null));
+    if (!price) continue; // skip sold / unpriced — only current for-sale inventory
+
+    const href = (card.match(/href="([^"]+quick-move-ins[^"]+)"/i)?.[1]) ?? "";
+    const homeId = href.match(/-(\d{5,})(?:\?|$)/)?.[1] ?? null;
+    const plan = card.match(/qmi-card__name[^>]*>[\s\S]*?<a[^>]*>\s*([^<]+?)\s*</i)?.[1]?.trim() ?? null;
+    const beds = num(card.match(/class="beds"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
+    const baths = num(card.match(/class="baths"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
+    const sqft = num(card.match(/class="sqft"[^>]*>\s*([\d,]+)/i)?.[1] ?? null);
+    const addrBlock = card.match(/qmi-card__address[^>]*>\s*([^<]+)/i)?.[1]?.replace(/\s+/g, " ").trim() ?? "";
+    // strip "City, ST ZIP" tail (and a bare city) to isolate the street
+    let street: string | null = addrBlock || null;
+    if (street) {
+      street = street.replace(/,?\s*[A-Z]{2}\s+\d{5}(?:-\d{4})?\s*$/, "").trim();
+      if (city) street = street.replace(new RegExp("\\s*" + city.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\s*$", "i"), "").trim();
+      if (!street) street = addrBlock;
+    }
+    if (!street || !homeId) continue;
+
+    records.push({
+      entityType: "inventory_home",
+      canonicalHints: { builderSlug: BUILDER_SLUG, communityName, address: street, builderInventoryId: homeId, planName: plan },
+      fields: {
+        street: fv(street, addrBlock, page.url),
+        city: fv(city, city, page.url),
+        state: fv(state, state, page.url),
+        zip: fv(zip, zip, page.url),
+        price: fv(price, priceRaw, page.url, `price ${priceRaw}`),
+        beds: fv(beds, null, page.url),
+        bathsTotal: fv(baths, null, page.url),
+        sqft: fv(sqft, null, page.url),
+        stories: fv<number>(null, null, page.url),
+        garageSpaces: fv<number>(null, null, page.url),
+        homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Discovery Homes quick move-in"),
+        constructionStatus: fv("MOVE_IN_READY" as never, priceRaw, page.url),
+        estCompletionDate: fv<string>(null, null, page.url),
+        lotNumber: fv<string>(null, null, page.url),
+        builderInventoryId: fv(homeId, homeId, page.url),
+        planName: fv(plan, plan, page.url),
+        availabilityStatus: fv("Quick Move-In", null, page.url),
+        images: fv<string[]>([], null, page.url), // facts-only — mediaRights=NONE
+      },
+    });
+  }
+  return { records, errors: [] };
+}
diff --git a/collectors/discovery/tsconfig.json b/collectors/discovery/tsconfig.json
new file mode 100644
index 0000000..e9bfc48
--- /dev/null
+++ b/collectors/discovery/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 18659fc..d8a370d 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -142,6 +142,9 @@ importers:
 
   apps/workers:
     dependencies:
+      '@homesonspec/collector-discovery':
+        specifier: workspace:*
+        version: link:../../collectors/discovery
       '@homesonspec/collector-dr-horton':
         specifier: workspace:*
         version: link:../../collectors/dr-horton
@@ -223,6 +226,28 @@ importers:
         specifier: ^4.0.0
         version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
 
+  collectors/discovery:
+    dependencies:
+      '@homesonspec/collectors-common':
+        specifier: workspace:*
+        version: link:../common
+      '@homesonspec/schemas':
+        specifier: workspace:*
+        version: link:../../packages/schemas
+      '@homesonspec/shared':
+        specifier: workspace:*
+        version: link:../../packages/shared
+    devDependencies:
+      '@types/node':
+        specifier: ^22.10.5
+        version: 22.20.1
+      typescript:
+        specifier: ^5.7.2
+        version: 5.9.3
+      vitest:
+        specifier: ^4.0.0
+        version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
   collectors/dr-horton:
     dependencies:
       '@homesonspec/collectors-common':

← fbd5fc6 workers: register 29 Pacific-region regional builders as onb  ·  back to Homesonspec  ·  collectors: add Discovery Homes adapter (first regional; See 444eee6 →