← back to Homesonspec
collectors/david-weekley/src/index.ts
320 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* David Weekley Homes adapter — national builder (recon 2026-07-22, built 2026-07-27).
*
* davidweekleyhomes.com serves per-metro "homes-ready-soon" list pages that are
* FULLY server-rendered (no browser, no bot-wall). robots.txt only blocks
* querystring dupes (?planId=/?communityId=), NOT the list pages we read.
*
* Sitemap https://www.davidweekleyhomes.com/sitemap.xml
* -> ~855 real /new-homes/{st}/{market}/homes-ready-soon URLs (drop the
* `%7B...%7D` template placeholders).
* Each list page renders per-home QMI cards as `.single-tabbed-listing`
* showcase blocks with:
* - link /homes-ready-soon/{st}/{mkt}/{city}/{community}/{jobNumber}
* - h3 .plan-title (plan name, e.g. "The Heron"), .plan-id ("plan A225")
* - span.address "2445 Jackson Burn Drive, Royse City, TX 75189"
* - .starting-price .value "$362,990"
* - .square-footage .value "2190"
* - .feature.story-count/.bedroom-count/.number-baths/.garage-count .value
* - .label.ready "Ready Now" (green -> MOVE_IN_READY) | "Ready m/d/yyyy"
* (gray -> UNDER_CONSTRUCTION w/ est completion date)
*
* Plain HTTP HTML_PARSE. Captures per-home ElevationRendering photos from each card
* block (images enabled 2026-07-28 per Steve's hotlink approval).
* State scope via DAVIDWEEKLEY_STATE (2-letter, e.g. "tx"); page cap via
* DAVIDWEEKLEY_PAGE_LIMIT. No per-home geo in v1 (list pages carry no lat/lon).
*/
const BUILDER_SLUG = "david-weekley";
const SITEMAP = "https://www.davidweekleyhomes.com/sitemap.xml";
const STATE_SEG = (process.env.DAVIDWEEKLEY_STATE ?? "").toLowerCase().trim();
const PAGE_LIMIT = Number(process.env.DAVIDWEEKLEY_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => (v == null ? null : String(v).trim() ? String(v).trim() : null);
const decodeEntities = (s: string): string =>
s
.replace(/&#(\d+);/g, (_, d) => String.fromCharCode(Number(d)))
.replace(/&/g, "&")
.replace(/ /g, " ")
.replace(/"/g, '"')
.replace(/'|'/g, "'")
.replace(/—|–|–|—/g, "-")
.replace(/&[a-z]+;/gi, " ");
/** first captured group of a class-scoped `.value` div, whitespace-collapsed */
function featureValue(block: string, featureClass: string): string | null {
// e.g. class="... feature bedroom-count"> ... <div class="value"> 3 </div>
const re = new RegExp(featureClass + '"[\\s\\S]{0,240}?class="value"[^>]*>\\s*([^<]+?)\\s*<', "i");
const m = block.match(re);
return m ? m[1]!.replace(/\s+/g, " ").trim() : null;
}
/**
* COMMUNITY coordinate from the page JSON — city-gated, warranty-safe. [yolo iter-6]
*
* DW pages embed several distinct coords and picking the wrong one wrong-pins a home
* tens of miles from its city (worse than null), so this only trusts the ONE coord we
* can prove is the home's own community:
* - REJECTED: the warranty/regional-HQ coord — it lives in a `"MapCenter":{...}` sibling
* of `WarrantyEmail`/`WarrantyPhone`/`AfterHoursPhone`. Never a `MapCenter`.
* - REJECTED: the regional SALES-OFFICE coords — `"MapPoint"` objects nested inside an
* `"Offices":[{OfficeType,Address,DrivingDirections,MapImage,PhoneNumber,MapPoint}]`
* array. One office serves a whole metro (e.g. all Phoenix homes → one Tempe office),
* so its coord is 10-40mi from most homes. Its Address.City is the office city, which
* will NOT match the home's city — the city-gate below rejects it.
* - ACCEPTED: the community coord — a community object carrying `"Grouping":"<name>"` +
* `"Geolocation":N` + an `Address` whose City/StateAbbreviation is the COMMUNITY's own
* city, immediately followed by `"MapPoint":{Latitude,Longitude}`. This only appears on
* the per-community DETAIL page (the per-metro `homes-ready-soon` LIST pages carry only
* the warranty + office coords, so this returns null there — correct, no wrong pin).
*
* Gate: the matched community MapPoint's OWN Address.City+State must equal the home's
* city+state, AND the coord must sit in a coarse US bbox. City-equality is the strong gate
* that distinguishes the community coord from every regional-office coord on the same page.
*/
function communityGeo(html: string, city: string | null, state: string | null): { lat: number; lon: number } | null {
if (!city || !state) return null;
const re = /"Grouping":"[^"]*","Geolocation":\d+,"Address":\{"Line1":"[^"]*","Line2":[^,]*,"City":"([^"]*)","StateAbbreviation":"([^"]*)"[^}]*\},"MapPoint":\{"Latitude":(-?\d+(?:\.\d+)?),"Longitude":(-?\d+(?:\.\d+)?)/g;
const wantCity = city.toLowerCase().trim();
const wantState = state.toUpperCase().trim();
for (const m of html.matchAll(re)) {
if (m[1]!.toLowerCase().trim() !== wantCity) continue; // office/HQ coord → City mismatch → reject
if (m[2]!.toUpperCase().trim() !== wantState) continue;
const lat = Number(m[3]), lon = Number(m[4]);
if (!Number.isFinite(lat) || !Number.isFinite(lon) || lat === 0 || lon === 0) continue;
if (lat < 15 || lat > 72 || lon < -180 || lon > -60) continue; // coarse US bbox (lon must be west)
return { lat, lon };
}
return null;
}
/** MOVE_IN_READY when "Ready Now"; else UNDER_CONSTRUCTION (dated) */
function statusOf(label: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" {
if (label && /ready\s*now/i.test(label)) return "MOVE_IN_READY";
return "UNDER_CONSTRUCTION";
}
/** parse a "Ready 8/12/2026" label into ISO date, else null */
function estDate(label: string | null): string | null {
const m = label?.match(/ready\s+(\d{1,2})\/(\d{1,2})\/(\d{4})/i);
if (!m) return null;
const [, mm, dd, yyyy] = m;
return `${yyyy}-${String(mm).padStart(2, "0")}-${String(dd).padStart(2, "0")}`;
}
interface ParsedHome {
jobNumber: string;
planName: string | null;
planId: string | null;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
price: number | null;
priceRaw: string | null;
sqft: number | null;
beds: number | null;
baths: number | null;
stories: number | null;
garages: number | null;
statusLabel: string | null;
community: string | null;
images: string[];
}
/** parse all per-home QMI showcase cards out of one homes-ready-soon list page */
export function parseHomes(html: string): ParsedHome[] {
const out: ParsedHome[] = [];
const seenJobs = new Set<string>();
// split on the per-home showcase card container
const blocks = html.split(/class="single-tabbed-listing/);
for (const raw of blocks.slice(1)) {
// a per-home card links to /homes-ready-soon/{st}/{mkt}/{city}/{community}/{jobNumber}
const jobM = raw.match(/href="\/homes-ready-soon\/([a-z]{2})\/[^"/]+\/[^"/]+\/([^"/]+)\/(\d{4,})"/i);
if (!jobM) continue;
const hrefState = jobM[1]!.toUpperCase();
const communitySlug = jobM[2]!;
const jobNumber = jobM[3]!;
if (seenJobs.has(jobNumber)) continue;
// full address "2445 Jackson Burn Drive, Royse City, TX 75189"
const addrRaw = decodeEntities(raw.match(/class="address"[^>]*>\s*([^<]+?)\s*</i)?.[1] ?? "").replace(/\s+/g, " ").trim();
if (!addrRaw) continue; // no street address -> not a resolvable per-home listing
seenJobs.add(jobNumber);
// split "street, city, ST zip"
let street: string | null = addrRaw;
let city: string | null = null;
let state: string | null = hrefState;
let zip: string | null = null;
const am = addrRaw.match(/^(.*?),\s*(.*?),\s*([A-Z]{2})\s+(\d{5})(?:-\d{4})?$/);
if (am) {
street = am[1]!.trim();
city = am[2]!.trim();
state = am[3]!.toUpperCase();
zip = am[4]!;
}
const priceRaw = raw.match(/class="[^"]*starting-price"[\s\S]{0,160}?class="value"[^>]*>\s*([^<]+?)\s*</i)?.[1]?.trim() ?? null;
const price = priceRaw && /\$[\d,]/.test(priceRaw) ? num(priceRaw) : null;
if (!price) continue; // sold / unpriced -> not current for-sale inventory
const sqft = num(raw.match(/class="[^"]*square-footage"[\s\S]{0,160}?class="value"[^>]*>\s*([^<]+?)\s*</i)?.[1] ?? null);
const beds = num(featureValue(raw, "bedroom-count"));
const baths = num(featureValue(raw, "number-baths"));
const stories = num(featureValue(raw, "story-count"));
const garages = num(featureValue(raw, "garage-count"));
const planName = decodeEntities(raw.match(/class="plan-title"[^>]*>\s*([^<]+?)\s*</i)?.[1] ?? "").replace(/\s+/g, " ").trim() || null;
const planId = raw.match(/class="plan-id"[^>]*>\s*([^<]+?)\s*</i)?.[1]?.replace(/\s+/g, " ").trim() || null;
const statusLabel = decodeEntities(raw.match(/class="label ready[^"]*"[^>]*>\s*([^<]+?)\s*</i)?.[1] ?? "").replace(/\s+/g, " ").trim() || null;
const community = decodeEntities(raw.match(/class="plan-title"/i) ? raw.match(/class="blue">\s*<a[^>]*>\s*([^<]+?)\s*</i)?.[1] ?? "" : "")
.replace(/\s+/g, " ")
.trim() || communitySlug.replace(/-/g, " ");
// Per-home imagery: ONLY the home's own ElevationRendering photo. CommunityImage is a
// shared community stock photo (same aerial across ~40 homes in a subdivision), so it
// misrepresents the specific home — better to emit nothing and let the gradient fallback
// do its job than show a wrong hero. [Cody-gate fix 2026-07-28; images per hotlink approval]
const imgMatches = [...raw.matchAll(/https:\/\/www\.davidweekleyhomes\.com\/media\/ElevationRendering\/[^"?\s]+\.(?:jpe?g|png)(?:\?[^"\s]*)?/gi)].map((m) => m[0]);
const seenImg = new Set<string>();
const images: string[] = [];
for (const u of imgMatches) {
const path = u.split("?")[0]!;
if (seenImg.has(path)) continue;
seenImg.add(path);
images.push(u);
}
out.push({
jobNumber, planName, planId, street, city, state, zip,
price, priceRaw, sqft, beds, baths, stories, garages, statusLabel, community,
images: images.slice(0, 6),
});
}
return out;
}
export const davidWeekleyAdapter: SourceAdapter = {
key: "david-weekley-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const sm = await fetcher.fetch(SITEMAP);
let urls = [...sm.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.filter((u) => /\/homes-ready-soon$/i.test(u) && !u.includes("%7B")); // drop {state}/{market} template rows
if (STATE_SEG) urls = urls.filter((u) => u.toLowerCase().includes(`/new-homes/${STATE_SEG}/`));
urls.sort();
for (const url of urls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const homes = parseHomes(html);
if (homes.length === 0) return { records: [], errors: [] }; // market page with no listed QMIs
const records: ExtractedRecord[] = [];
// Community coord (city-gated, warranty/office-safe) — present on per-community DETAIL
// pages, null on the per-metro LIST pages (they carry only warranty + office coords). A
// home inherits its own community's coord only. [yolo iter-6]
const cgeoFor = (city: string | null, state: string | null) => communityGeo(html, city, state);
// one community record per distinct community on the page (facts-only)
const communitiesSeen = new Set<string>();
for (const h of homes) {
const cname = h.community ?? "Unknown";
if (!communitiesSeen.has(cname)) {
communitiesSeen.add(cname);
const cg = cgeoFor(h.city, h.state);
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(cg?.lat ?? null, cg ? String(cg.lat) : null, page.url, cg ? "community MapPoint (city-gated)" : null),
lon: fv(cg?.lon ?? null, cg ? String(cg.lon) : null, page.url, cg ? "community MapPoint (city-gated)" : null),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv<string>(null, null, page.url),
},
});
}
}
for (const h of homes) {
if (!h.street) continue;
const cg = cgeoFor(h.city, h.state); // home inherits its own community's city-gated coord (null on list pages)
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: h.community ?? "Unknown",
address: h.street,
builderInventoryId: h.jobNumber,
planName: h.planName,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
lat: fv(cg?.lat ?? null, cg ? String(cg.lat) : null, page.url, cg ? "community MapPoint (home inherits community coord, city-gated)" : null),
lon: fv(cg?.lon ?? null, cg ? String(cg.lon) : null, page.url, cg ? "community MapPoint (home inherits community coord, city-gated)" : null),
price: fv(h.price, h.priceRaw, page.url, h.priceRaw ? `price ${h.priceRaw}` : null),
beds: fv(h.beds, null, page.url),
bathsTotal: fv(h.baths, null, page.url),
sqft: fv(h.sqft, null, page.url),
stories: fv(h.stories != null ? Math.round(h.stories) : null, null, page.url), // schema wants int (1.5-story → 2)
garageSpaces: fv(h.garages != null ? Math.round(h.garages) : null, null, page.url),
homeType: fv("SINGLE_FAMILY" as never, null, page.url, "David Weekley quick move-in"),
constructionStatus: fv(statusOf(h.statusLabel) as never, h.statusLabel, page.url),
estCompletionDate: fv(estDate(h.statusLabel), h.statusLabel, page.url),
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv(h.jobNumber, h.jobNumber, page.url),
planName: fv(h.planName, h.planName, page.url),
availabilityStatus: fv(h.statusLabel, h.statusLabel, page.url, "David Weekley ready label"),
images: fv<string[]>(h.images, null, page.url, h.images.length ? "builder listing photo" : null),
},
});
}
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};