← back to Homesonspec
collectors/drees-homes/src/index.ts
379 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Drees Homes adapter — SSR EMBEDDED_JSON source (recon 2026-07-28, EPiServer).
*
* dreeshomes.com is server-rendered EPiServer/Optimizely. Its single flat
* sitemap.xml (4,202 URLs) enumerates TWO very different page types that share
* an identical-looking trailing shape — this is exactly the Dream Finders
* "*-floorplan/ is not one home" trap, and Drees actually has BOTH:
*
* • pageType:"Plan" (~2,559 `.../<plan>-floorplan/` model pages) — a generic
* floorplan/model page carrying RANGES, not a home:
* priceLow:522900, priceHigh:555900, bedLow:4, bedHigh:5, discountedPrice:0,
* zipCode:null, mlsNumber:null. There is no single address/price here.
* Parsing one of these as "one home" is precisely the hollow-data failure
* (identical sqft, 0% real price). → SKIPPED.
*
* • pageType:"Home" (~646 street-address-slug pages, e.g.
* .../trescott-60s/6338-rippling-rock-drive/) — a genuine single spec /
* inventory / model home: ONE entityName address, ONE discountedPrice, ONE
* sqFtLow/bedLow/bathLow, plus zipCode + lat/lon + mlsNumber. → PARSED.
*
* The facts are NOT in ld+json (there is none) and NOT visibly in the DOM until
* a Vue bundle hydrates — but the canonical fact object IS present in the plain
* GET HTML as an HTML-entity-encoded JSON blob (…"discountedPrice"…).
* So a plain GET is sufficient; no browser, no /api call, no login is needed.
*
* The gate that avoids the trap: extract the entity object, and EMIT A HOME ONLY
* WHEN pageType === "Home". Everything else (Plan, Neighborhood, Community …) is
* reported and skipped — never coerced into a home row.
*
* Facts-only: the object also carries an `images[]` array; it is intentionally
* dropped (POLICY: no images).
*
* robots.txt (dreeshomes.com) allows the inventory tree (only /episerver/,
* /utils/, brochure paths etc. are disallowed); the sitemap is advertised there.
*
* Batch control: DREES_PAGE_LIMIT (home pages fetched, default 10).
*/
const SITEMAP_URL = "https://www.dreeshomes.com/sitemap.xml";
const BUILDER_SLUG = "drees-homes";
const PAGE_LIMIT = Number(process.env.DREES_PAGE_LIMIT ?? 10);
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
// (Drees uses 0 as a sentinel for "no value" across price/sqft/bed/bath.)
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (half-baths / garages may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
/** The canonical Drees per-page fact object (a subset of its keys we consume). */
interface DreesEntity {
pageType?: string; // "Home" | "Plan" | "Neighborhood" | "Community" | …
entityName?: string; // "7262 Murrel Drive, Franklin, TN 37064" on Home pages
neighborhoodName?: string;
communityName?: string;
cityName?: string;
stateInitials?: string;
zipCode?: string;
homeType?: string; // "Single Family"
planName?: string;
elevation?: string;
// Single-home pages carry the *Low fields as the real value; *High is 0/null.
priceLow?: number;
priceHigh?: number;
originalPrice?: number;
discountedPrice?: number;
sqFtLow?: number;
sqFtHigh?: number;
bedLow?: number;
bedHigh?: number;
bathLow?: number;
bathHigh?: number;
halfBathLow?: number;
halfBathHigh?: number;
storiesLow?: number;
storiesHigh?: number;
garagesLow?: number;
garagesHigh?: number;
isModelHome?: boolean;
mlsNumber?: string;
// Note: latitude/longitude are NOT on this object — they live on the
// home-plan-highlight-product-block element and are read via extractLatLon().
}
/** Decode the HTML-entity encoding EPiServer uses inside the SSR blob. */
function decodeEntities(s: string): string {
return s
.replace(/"/g, '"')
.replace(/'/g, "'")
.replace(/'/g, "'")
.replace(/&/g, "&");
}
/**
* Pull the page's canonical fact object out of the SSR HTML. It is the object
* that contains `"entityName"` AND `"pageType"` AND `"discountedPrice"` — the
* home-highlight product-block model. We scan for the "entityName" key, walk
* back to its enclosing `{`, then brace-match forward and JSON.parse the slice.
*/
function extractEntity(html: string): DreesEntity | null {
const h = decodeEntities(html);
// Find an entityName whose object also carries discountedPrice — that pins us
// to the fact object rather than an unrelated block that happens to name an
// entity. We try each entityName occurrence until one parses with the markers.
let searchFrom = 0;
for (;;) {
const keyIdx = h.indexOf('"entityName"', searchFrom);
if (keyIdx < 0) return null;
searchFrom = keyIdx + 12;
// Walk back to the enclosing "{".
let start = -1;
let depth = 0;
for (let i = keyIdx; i >= 0; i--) {
const c = h[i];
if (c === "}") depth++;
else if (c === "{") {
if (depth === 0) {
start = i;
break;
}
depth--;
}
}
if (start < 0) continue;
// Brace-match forward to the matching "}".
let end = -1;
depth = 0;
for (let i = start; i < h.length; i++) {
const c = h[i];
if (c === "{") depth++;
else if (c === "}") {
depth--;
if (depth === 0) {
end = i + 1;
break;
}
}
}
if (end < 0) continue;
const slice = h.slice(start, end);
// Cheap marker check before the (relatively expensive) parse.
if (!slice.includes('"pageType"') || !slice.includes('"discountedPrice"')) continue;
try {
const obj = JSON.parse(slice) as DreesEntity;
if (obj && typeof obj === "object" && obj.pageType) return obj;
} catch {
// Not clean JSON (an outer wrapper leaked in) — keep looking.
}
}
}
/** entityName is "7262 Murrel Drive, Franklin, TN 37064" — take the street part. */
function streetFromEntityName(entityName: string | null): string | null {
if (!entityName) return null;
const street = entityName.split(",")[0]?.trim();
return street ? street : null;
}
/**
* lat/lon do NOT live in the page-model fact object — they're on the
* `home-plan-highlight-product-block` web-component attribute. Extract them from
* the decoded HTML independently; many pages legitimately omit them (→ null,
* never guessed). US longitude is negative — signs are preserved. 0 is a null
* sentinel. Key order isn't guaranteed, so we read each key separately.
*/
function extractLatLon(html: string): { lat: number | null; lon: number | null } {
const h = decodeEntities(html);
const block = h.match(/<home-plan-highlight-product-block[^>]*>/);
const scope = block ? block[0] : h;
const latM = scope.match(/"latitude"\s*:\s*(-?\d+(?:\.\d+)?)/);
const lonM = scope.match(/"longitude"\s*:\s*(-?\d+(?:\.\d+)?)/);
const lat = latM ? Number(latM[1]) : NaN;
const lon = lonM ? Number(lonM[1]) : NaN;
return {
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
lon: Number.isFinite(lon) && lon !== 0 ? lon : null,
};
}
export const dreesHomesAdapter: SourceAdapter = {
key: "drees-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let sitemap: RawPage;
try {
sitemap = await fetcher.fetch(SITEMAP_URL);
} catch (error) {
console.warn(` drees sitemap: ${error instanceof Error ? error.message : String(error)}`);
return; // no sitemap → nothing to collect (source marked degraded upstream)
}
// Candidate home URLs: the trailing segment is a street address (starts with
// a digit, e.g. `6338-rippling-rock-drive`) and it is NOT a `*-floorplan/`
// model page. This is a CHEAP prefilter to avoid fetching all 2,559 Plan
// pages — the AUTHORITATIVE gate is still pageType==="Home" in extract().
const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.filter((url) => {
const path = url.replace(/^https?:\/\/[^/]+/, "");
if (path.includes("-floorplan/")) return false;
const segs = path.split("/").filter(Boolean);
const last = segs[segs.length - 1] ?? "";
return /^\d/.test(last); // street-address slug
});
urls.sort();
for (const url of urls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const entity = extractEntity(html);
if (!entity) {
return { records: [], errors: [{ url: page.url, reason: "no Drees entity fact object in SSR HTML" }] };
}
// THE TRAP GATE. A `pageType:"Plan"` page is a floorplan MODEL with price/
// bed/sqft RANGES (priceLow≠priceHigh, discountedPrice:0), not a single
// home — emitting it as one home is the Dream Finders hollow-data failure.
// Only a `pageType:"Home"` page is a real individual spec/inventory home.
if (entity.pageType !== "Home") {
return {
records: [],
errors: [{ url: page.url, reason: `pageType=${entity.pageType} (not a single Home) — skipped, not coerced into a home row` }],
};
}
const url = page.url;
const address = streetFromEntityName(str(entity.entityName));
const community = str(entity.neighborhoodName) ?? str(entity.communityName);
const city = str(entity.cityName);
const state = normalizeStateCode(str(entity.stateInitials));
const zip = zip5(entity.zipCode);
const plan = str(entity.planName);
// lat/lon are on the highlight block, not the fact object — and often absent.
const { lat, lon } = extractLatLon(html);
// Single-home pages report the value in the *Low field (discountedPrice for
// price). We take discountedPrice → originalPrice → priceLow, first > 0.
const price = posNum(entity.discountedPrice) ?? posNum(entity.originalPrice) ?? posNum(entity.priceLow);
const beds = posNum(entity.bedLow);
const full = nonNegNum(entity.bathLow);
const half = nonNegNum(entity.halfBathLow);
const bathsTotal = full === null ? null : full + (half ?? 0) * 0.5;
const sqft = posNum(entity.sqFtLow);
const stories = posNum(entity.storiesLow);
const garages = nonNegNum(entity.garagesLow);
const mls = str(entity.mlsNumber);
const isModel = entity.isModelHome === true;
if (!address) {
return { records: [], errors: [{ url, reason: "Home page missing entityName address — skipped" }] };
}
// The publisher requires an inventory home to hang off a community (FK).
// If Drees gives neither a neighborhood nor a community name, we can't
// attach it — skip and log honestly rather than stage a crash-at-publish row.
if (!community) {
return { records: [], errors: [{ url, reason: `home ${address} has no neighborhood/community name — cannot attach, skipped` }] };
}
const records: ExtractedRecord[] = [
// Community FIRST — publish creates/refreshes the FK target the home needs.
{
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "Drees neighborhoodName/communityName"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, str(entity.stateInitials), url),
zip: fv(zip, str(entity.zipCode), url),
county: fv<string>(null, null, url),
metro: fv<string>(null, null, url),
lat: fv(lat, null, url),
lon: fv(lon, null, url),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
},
{
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
builderInventoryId: mls ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, url, "Drees entityName street segment"),
city: fv(city, city, url),
state: fv(state, str(entity.stateInitials), url),
zip: fv(zip, str(entity.zipCode), url),
price: fv(price, price === null ? null : String(price), url, price === null ? null : `Drees discountedPrice ${price}`),
beds: fv(beds, beds === null ? null : String(beds), url),
bathsTotal: fv(
bathsTotal,
bathsTotal === null ? null : String(bathsTotal),
url,
bathsTotal === null ? null : `${full} full + ${half ?? 0} half`,
),
sqft: fv(sqft, sqft === null ? null : String(sqft), url),
stories: fv(stories, stories === null ? null : String(stories), url),
garageSpaces: fv(garages, garages === null ? null : String(garages), url),
homeType: fv("SINGLE_FAMILY" as const, str(entity.homeType), url, str(entity.homeType) ?? "Drees single-family home"),
// Drees single-home pages don't expose a granular construction stage;
// isModelHome only distinguishes a model (display) home from a spec.
// We do not guess a construction status → null (never fabricated).
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(null, null, url),
estCompletionDate: fv<string>(null, null, url),
lotNumber: fv<string>(null, null, url),
builderInventoryId: fv(mls, mls, url, mls ? `Drees MLS# ${mls}` : null),
lat: fv(lat, null, url, lat === null ? null : "Drees latitude"),
lon: fv(lon, null, url, lon === null ? null : "Drees longitude"),
planName: fv(plan, plan, url, isModel ? `${plan ?? "?"} (model home)` : plan),
// facts-only: the entity carries images[] but they are intentionally dropped.
images: fv<string[]>([], null, url),
},
},
];
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};