← back to Homesonspec
collectors: add Discovery Homes adapter (first regional; Seeno-family feed)
444eee611ebb7d6ee93ba4d2a3b4207a1f7c21ea · 2026-07-27 20:58:29 -0700 · Steve
discoveryhomes.com JSON API (/api/communities + a single site-wide /api/homes).
Joins each home card to its community by href-slug in fetch(), emits a normalized
page so extract() stays pure. Onboards 4 Seeno-family brands from one feed
(discovery/seeno/jmc/sterling). Facts-only (images omitted), robots-enforced,
sold/unpriced skipped. Fixed a location-mislabel bug (community_id param is
ignored by the server; derive location from the home's own href+address). Live:
discovery 14 homes + sterling 3, +109 communities. Registered seeno/sterling builders.
Files touched
M apps/workers/scripts/register-pacific-regionals.tsM collectors/discovery/src/index.ts
Diff
commit 444eee611ebb7d6ee93ba4d2a3b4207a1f7c21ea
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jul 27 20:58:29 2026 -0700
collectors: add Discovery Homes adapter (first regional; Seeno-family feed)
discoveryhomes.com JSON API (/api/communities + a single site-wide /api/homes).
Joins each home card to its community by href-slug in fetch(), emits a normalized
page so extract() stays pure. Onboards 4 Seeno-family brands from one feed
(discovery/seeno/jmc/sterling). Facts-only (images omitted), robots-enforced,
sold/unpriced skipped. Fixed a location-mislabel bug (community_id param is
ignored by the server; derive location from the home's own href+address). Live:
discovery 14 homes + sterling 3, +109 communities. Registered seeno/sterling builders.
---
apps/workers/scripts/register-pacific-regionals.ts | 3 +
collectors/discovery/src/index.ts | 214 ++++++++++++---------
2 files changed, 124 insertions(+), 93 deletions(-)
diff --git a/apps/workers/scripts/register-pacific-regionals.ts b/apps/workers/scripts/register-pacific-regionals.ts
index 50fe584..9804392 100644
--- a/apps/workers/scripts/register-pacific-regionals.ts
+++ b/apps/workers/scripts/register-pacific-regionals.ts
@@ -41,6 +41,9 @@ const ROWS: Row[] = [
{ name: "Sekisui House US", site: "sekisuihouse.com", states: "WA;OR" },
// Nevada
{ name: "Harmony Homes", site: null, states: "NV", note: "recon-pending: verify NV Las Vegas domain before crawl" },
+ // Seeno-family brands sharing the discoveryhomes.com feed (onboarded via discovery-homes-site adapter)
+ { name: "Seeno Homes", site: "discoveryhomes.com", states: "CA;NV", note: "Seeno-family; served by discovery-homes-site adapter" },
+ { name: "Sterling Homes", site: "discoveryhomes.com", states: "CA;ID", note: "Seeno-family; served by discovery-homes-site adapter" },
// Hawaii
{ name: "Castle & Cooke Homes", site: "castlecooke.com", states: "HI" },
{ name: "Gentry Homes", site: "gentryhawaii.com", states: "HI" },
diff --git a/collectors/discovery/src/index.ts b/collectors/discovery/src/index.ts
index f28947f..ee82ca1 100644
--- a/collectors/discovery/src/index.ts
+++ b/collectors/discovery/src/index.ts
@@ -2,6 +2,7 @@ import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
+ sha256,
type ExtractionOutput,
type FetchContext,
type RawPage,
@@ -10,20 +11,27 @@ import {
/**
* Discovery Homes adapter — first REGIONAL builder (recon 2026-07-27).
- * discoveryhomes.com exposes a clean JSON API:
- * GET /api/communities → JSON array of communities (name/city/state/zip/lat/lon/builder)
- * GET /api/homes?community_id=<id> → {renderedList: "<qmi-card html>"} of quick-move-in homes
- * The site hosts several Seeno-family brands (builder = discovery|seeno|jmscc|sterling);
- * this adapter scopes to DISCOVERY_BUILDER (default "discovery" → builder slug discovery-homes).
- * Plain HTTP, no anti-bot; robots allows /api/. Facts-only (images intentionally omitted).
- * Community context is threaded to the homes page via ignored _* query params so extract()
- * stays a pure per-page function.
+ * discoveryhomes.com hosts several Seeno-family brands on one JSON API:
+ * GET /api/communities → JSON array (name/city/state/zip/lat/lon/builder/seo_name/url)
+ * GET /api/homes → {renderedList:"<qmi-card html>"} — a SINGLE site-wide feed
+ * (the community_id param is IGNORED by the server).
+ * So we fetch /api/homes ONCE and join each card back to its community by the
+ * href slug (/ST/city/communities/<slug>/...), which yields the home's true
+ * builder + location. The join happens in fetch(); extract() stays pure by
+ * reading a normalized JSON page we synthesize. Plain HTTP, robots allows /api/.
+ * Facts-only (images intentionally omitted). Sold/unpriced homes are skipped.
*/
const ORIGIN = "https://www.discoveryhomes.com";
const COMMUNITIES_URL = `${ORIGIN}/api/communities`;
-const BUILDER_CODE = (process.env.DISCOVERY_BUILDER ?? "discovery").toLowerCase();
-const BUILDER_SLUG = process.env.DISCOVERY_BUILDER_SLUG ?? "discovery-homes";
-const PAGE_LIMIT = Number(process.env.DISCOVERY_PAGE_LIMIT ?? 200);
+const HOMES_URL = `${ORIGIN}/api/homes`;
+const NORMALIZED_URL = `${HOMES_URL}#normalized`;
+// feed builder code -> our Builder.slug (all must exist as Builder rows)
+const BUILDER_MAP: Record<string, string> = {
+ discovery: "discovery-homes",
+ jmscc: "jmc-homes",
+ seeno: "seeno-homes",
+ sterling: "sterling-homes",
+};
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
@@ -33,13 +41,70 @@ const num = (v: unknown): number | null => {
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (v == null ? null : String(v).trim() ? String(v).trim() : null);
+const slugify = (s: string) => (s || "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
-interface Community { id: number; name?: string; builder?: string; city?: string; state?: string; zip?: unknown;
- county?: string; latitude?: unknown; longitude?: unknown; phone?: string; is_active?: unknown; availability?: string; }
+interface Community {
+ id: number; name?: string; builder?: string; city?: string; state?: string; zip?: unknown;
+ county?: string; latitude?: unknown; longitude?: unknown; phone?: string; seo_name?: string; url?: string;
+}
+interface NormHome {
+ builderSlug: string; communityName: string; street: string; city: string | null; state: string | null;
+ zip: string | null; price: number; priceRaw: string; beds: number | null; baths: number | null;
+ sqft: number | null; plan: string | null; homeId: string;
+}
+
+/** index communities by every slug a home href might reference */
+function buildIndex(comms: Community[]): Map<string, Community> {
+ const idx = new Map<string, Community>();
+ for (const c of comms) {
+ const keys = [c.seo_name, typeof c.url === "string" ? c.url.replace(/\/+$/, "").split("/").pop() : null, slugify(c.name ?? "")];
+ for (const k of keys) if (k) idx.set(String(k).toLowerCase(), c);
+ }
+ return idx;
+}
+
+function parseHomes(html: string, idx: Map<string, Community>): NormHome[] {
+ const out: NormHome[] = [];
+ for (const card of html.split("qmi-card__container").slice(1)) {
+ const href = card.match(/href="\/([A-Z]{2})\/([^/]+)\/communities\/([^/]+)\/[^"]*?-(\d{5,})/i);
+ if (!href) continue;
+ const hrefState = href[1]!.toUpperCase();
+ const commSlug = href[3]!.toLowerCase();
+ const homeId = href[4]!;
+ const community = idx.get(commSlug);
+ if (!community) continue;
+ const builderSlug = BUILDER_MAP[(community.builder ?? "").toLowerCase()];
+ if (!builderSlug) continue; // builder not registered / out of scope
+
+ const priceRaw = (card.match(/qmi-card__price[^>]*>\s*([^<]+)/i)?.[1] ?? "").trim();
+ const price = /\$[\d,]/.test(priceRaw) ? num(priceRaw) : null;
+ if (!price) continue; // sold / unpriced → not current for-sale inventory
+
+ const beds = num(card.match(/class="beds"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
+ const baths = num(card.match(/class="baths"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
+ const sqft = num(card.match(/class="sqft"[^>]*>\s*([\d,]+)/i)?.[1] ?? null);
+ const plan = card.match(/qmi-card__name[^>]*>[\s\S]*?<a[^>]*>\s*([^<]+?)\s*</i)?.[1]?.trim() ?? null;
+ const addrBlock = (card.match(/qmi-card__address[^>]*>\s*([^<]+)/i)?.[1] ?? "").replace(/\s+/g, " ").trim();
+
+ // location: state from href (authoritative), zip from address block, city from the matched community
+ const zip = addrBlock.match(/\b(\d{5})(?:-\d{4})?\b/)?.[1] ?? str(community.zip);
+ const city = str(community.city);
+ const state = hrefState || (str(community.state) ?? "")?.toUpperCase() || null;
+ let street = addrBlock.replace(/,?\s*[A-Z]{2}\s+\d{5}(?:-\d{4})?\s*$/, "").trim();
+ if (city) street = street.replace(new RegExp("\\s*" + city.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\s*$", "i"), "").trim();
+ if (!street) continue;
+
+ out.push({
+ builderSlug, communityName: str(community.name) ?? commSlug, street, city, state, zip,
+ price, priceRaw, beds, baths, sqft, plan, homeId,
+ });
+ }
+ return out;
+}
export const discoveryAdapter: SourceAdapter = {
key: "discovery-homes-site",
- version: "1.0.0",
+ version: "1.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
@@ -48,30 +113,24 @@ export const discoveryAdapter: SourceAdapter = {
}
const fetcher = new LiveFetcher(ctx.registry);
const commPage = await fetcher.fetch(COMMUNITIES_URL);
- yield commPage; // extract() emits community records from this
+ yield commPage; // → community records
- let all: Community[] = [];
- try { all = JSON.parse(commPage.body.toString("utf8")); } catch { all = []; }
- const mine = all.filter((c) => (c.builder ?? "").toLowerCase() === BUILDER_CODE).slice(0, PAGE_LIMIT);
- for (const c of mine) {
- // thread community context via ignored params (server returns the same homes payload)
- const p = new URLSearchParams({
- community_id: String(c.id),
- _comm: str(c.name) ?? "",
- _city: str(c.city) ?? "",
- _state: (str(c.state) ?? "").toUpperCase(),
- _zip: str(c.zip) ?? "",
- });
- try { yield await fetcher.fetch(`${ORIGIN}/api/homes?${p.toString()}`); }
- catch (e) { /* skip a community whose homes page errors/blocks */ }
- }
+ let comms: Community[] = [];
+ try { comms = JSON.parse(commPage.body.toString("utf8")); } catch { comms = []; }
+ const idx = buildIndex(comms);
+ const homesPage = await fetcher.fetch(HOMES_URL);
+ let html = "";
+ try { html = JSON.parse(homesPage.body.toString("utf8")).renderedList ?? ""; } catch { html = ""; }
+ const normalized = parseHomes(html, idx);
+ const body = Buffer.from(JSON.stringify(normalized), "utf8");
+ yield { url: NORMALIZED_URL, retrievedAt: new Date().toISOString(), contentType: "application/json", body, contentHash: sha256(body) };
},
extract(page: RawPage): ExtractionOutput {
try {
+ if (page.url.includes("#normalized")) return extractNormalizedHomes(page);
if (page.url.includes("/api/communities")) return extractCommunities(page);
- if (page.url.includes("/api/homes")) return extractHomes(page);
- return { records: [], errors: [{ url: page.url, reason: "unrecognized page" }] };
+ return { records: [], errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
@@ -83,13 +142,14 @@ function extractCommunities(page: RawPage): ExtractionOutput {
try { all = JSON.parse(page.body.toString("utf8")); } catch { return { records: [], errors: [{ url: page.url, reason: "communities JSON parse failed" }] }; }
const records: ExtractedRecord[] = [];
for (const c of all) {
- if ((c.builder ?? "").toLowerCase() !== BUILDER_CODE) continue;
+ const builderSlug = BUILDER_MAP[(c.builder ?? "").toLowerCase()];
+ if (!builderSlug) continue;
const name = str(c.name);
if (!name) continue;
const phone = str(c.phone);
records.push({
entityType: "community",
- canonicalHints: { builderSlug: BUILDER_SLUG, communityName: name },
+ canonicalHints: { builderSlug, communityName: name },
fields: {
name: fv(name, name, page.url),
street: fv<string>(null, null, page.url),
@@ -110,64 +170,32 @@ function extractCommunities(page: RawPage): ExtractionOutput {
return { records, errors: [] };
}
-function extractHomes(page: RawPage): ExtractionOutput {
- const json = (() => { try { return JSON.parse(page.body.toString("utf8")); } catch { return null; } })();
- const html: string = json?.renderedList ?? "";
- const u = new URL(page.url);
- const communityName = decodeURIComponent(u.searchParams.get("_comm") ?? "") || null;
- const city = decodeURIComponent(u.searchParams.get("_city") ?? "") || null;
- const state = (u.searchParams.get("_state") ?? "").toUpperCase() || null;
- const zip = u.searchParams.get("_zip") || null;
- if (!html || !communityName) return { records: [], errors: [] };
-
- const records: ExtractedRecord[] = [];
- const cards = html.split("qmi-card__container").slice(1);
- for (const card of cards) {
- const priceM = card.match(/qmi-card__price[^>]*>\s*([^<]+)/i);
- const priceRaw = priceM ? priceM[1]!.trim() : "";
- const price = num(priceRaw.startsWith("$") ? priceRaw : (/\$[\d,]/.test(priceRaw) ? priceRaw : null));
- if (!price) continue; // skip sold / unpriced — only current for-sale inventory
-
- const href = (card.match(/href="([^"]+quick-move-ins[^"]+)"/i)?.[1]) ?? "";
- const homeId = href.match(/-(\d{5,})(?:\?|$)/)?.[1] ?? null;
- const plan = card.match(/qmi-card__name[^>]*>[\s\S]*?<a[^>]*>\s*([^<]+?)\s*</i)?.[1]?.trim() ?? null;
- const beds = num(card.match(/class="beds"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
- const baths = num(card.match(/class="baths"[^>]*>\s*([\d.]+)/i)?.[1] ?? null);
- const sqft = num(card.match(/class="sqft"[^>]*>\s*([\d,]+)/i)?.[1] ?? null);
- const addrBlock = card.match(/qmi-card__address[^>]*>\s*([^<]+)/i)?.[1]?.replace(/\s+/g, " ").trim() ?? "";
- // strip "City, ST ZIP" tail (and a bare city) to isolate the street
- let street: string | null = addrBlock || null;
- if (street) {
- street = street.replace(/,?\s*[A-Z]{2}\s+\d{5}(?:-\d{4})?\s*$/, "").trim();
- if (city) street = street.replace(new RegExp("\\s*" + city.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + "\\s*$", "i"), "").trim();
- if (!street) street = addrBlock;
- }
- if (!street || !homeId) continue;
-
- records.push({
- entityType: "inventory_home",
- canonicalHints: { builderSlug: BUILDER_SLUG, communityName, address: street, builderInventoryId: homeId, planName: plan },
- fields: {
- street: fv(street, addrBlock, page.url),
- city: fv(city, city, page.url),
- state: fv(state, state, page.url),
- zip: fv(zip, zip, page.url),
- price: fv(price, priceRaw, page.url, `price ${priceRaw}`),
- beds: fv(beds, null, page.url),
- bathsTotal: fv(baths, null, page.url),
- sqft: fv(sqft, null, page.url),
- stories: fv<number>(null, null, page.url),
- garageSpaces: fv<number>(null, null, page.url),
- homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Discovery Homes quick move-in"),
- constructionStatus: fv("MOVE_IN_READY" as never, priceRaw, page.url),
- estCompletionDate: fv<string>(null, null, page.url),
- lotNumber: fv<string>(null, null, page.url),
- builderInventoryId: fv(homeId, homeId, page.url),
- planName: fv(plan, plan, page.url),
- availabilityStatus: fv("Quick Move-In", null, page.url),
- images: fv<string[]>([], null, page.url), // facts-only — mediaRights=NONE
- },
- });
- }
+function extractNormalizedHomes(page: RawPage): ExtractionOutput {
+ let homes: NormHome[] = [];
+ try { homes = JSON.parse(page.body.toString("utf8")); } catch { return { records: [], errors: [{ url: page.url, reason: "normalized homes parse failed" }] }; }
+ const records: ExtractedRecord[] = homes.map((h) => ({
+ entityType: "inventory_home" as const,
+ canonicalHints: { builderSlug: h.builderSlug, communityName: h.communityName, address: h.street, builderInventoryId: h.homeId, planName: h.plan },
+ fields: {
+ street: fv(h.street, h.street, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(h.state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ price: fv(h.price, h.priceRaw, page.url, `price ${h.priceRaw}`),
+ beds: fv(h.beds, null, page.url),
+ bathsTotal: fv(h.baths, null, page.url),
+ sqft: fv(h.sqft, null, page.url),
+ stories: fv<number>(null, null, page.url),
+ garageSpaces: fv<number>(null, null, page.url),
+ homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Discovery Homes quick move-in"),
+ constructionStatus: fv("MOVE_IN_READY" as never, h.priceRaw, page.url),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv<string>(null, null, page.url),
+ builderInventoryId: fv(h.homeId, h.homeId, page.url),
+ planName: fv(h.plan, h.plan, page.url),
+ availabilityStatus: fv("Quick Move-In", null, page.url),
+ images: fv<string[]>([], null, page.url), // facts-only — mediaRights=NONE
+ },
+ }));
return { records, errors: [] };
}
← c612735 auto-save: 2026-07-27T20:54:34 (4 files) — apps/workers/pack
·
back to Homesonspec
·
recon: classify 8 regional builders' plain-fetch inventory f 8550e53 →