← back to Homesonspec
collectors/ashton-woods/src/index.ts
289 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Ashton Woods adapter — national builder (recon 2026-07-22 "ld+json QMI sitemaps",
* built 2026-07-28). ashtonwoods.com publishes per-metro quick-move-in (QMI)
* sitemaps, and each QMI page is FULLY server-rendered with a clean JSON-LD
* `SingleFamilyResidence` + `Product`/`Offer` pair — no browser, no bot-wall.
* robots.txt disallows only /content/, /input/, /walk-in/, /sales_center/,
* /interactive-floorplan, /realtor-program/, /welcome — the metro QMI pages we
* read are allowed.
*
* Sitemap index https://www.ashtonwoods.com/sitemap.xml
* -> per-metro `{metro}-sitemap-qmi.xml` files (atlanta, austin, dallas,
* houston, san-antonio, phoenix, orlando, tampa, raleigh, nashville,
* charleston, jacksonville, myrtle-beach)
* -> each lists per-home QMI URLs `/{metro}/{community}/{plan}/{street-address}`.
* Each QMI page carries (facts-only, from JSON-LD):
* SingleFamilyResidence: name, address{streetAddress,addressLocality,
* addressRegion,postalCode}, ContainedIn (community), Floorsize ("2260FTK"),
* AmenityFeature ["4 Bedrooms","3 Baths | 1 Half Baths","2 Stories",
* "2 Car Garage"], geo{latitude,longitude}, telephone.
* Product.offers: price (number), priceCurrency, availability
* (schema.org/InStock -> for-sale).
*
* Plain HTTP; one page == one inventory_home. Captures per-home Widen CDN photos
* from JSON-LD + gallery (images enabled 2026-07-28 per Steve's hotlink approval).
* Metro scope via ASHTONWOODS_METRO (comma-list,
* e.g. "austin,dallas"; default = all metros); per-metro page cap via
* ASHTONWOODS_PAGE_LIMIT. HAS per-home geo (homes appear on the map).
*/
const BUILDER_SLUG = "ashton-woods";
const SITEMAP = "https://www.ashtonwoods.com/sitemap.xml";
const METRO_FILTER = (process.env.ASHTONWOODS_METRO ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.ASHTONWOODS_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
/** signed decimal (geo lat/lon can be negative — must NOT strip the minus) */
const coord = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.\-]/g, "")) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).replace(/\s+/g, " ").trim();
return s || null;
};
interface ParsedHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
sqft: number | null;
beds: number | null;
bathsTotal: number | null; // full + half
stories: number | null;
garages: number | null;
lat: number | null;
lon: number | null;
phone: string | null;
price: number | null;
available: boolean;
name: string | null;
}
/** pull the first parsed JSON-LD object of a given @type out of the page HTML */
function ldByType(html: string, wanted: string): Record<string, unknown> | null {
const blocks = [
...html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi),
];
for (const b of blocks) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const items = Array.isArray(data) ? data : [data];
for (const it of items) {
if (it && typeof it === "object" && (it as Record<string, unknown>)["@type"] === wanted) {
return it as Record<string, unknown>;
}
}
}
return null;
}
/** "3 Baths | 1 Half Baths" -> 3.5 ; "2 Baths" -> 2 */
function bathsFrom(amenities: string[]): number | null {
const line = amenities.find((a) => /bath/i.test(a));
if (!line) return null;
const full = Number(line.match(/(\d+)\s*Baths?\b/i)?.[1] ?? "0");
const half = Number(line.match(/(\d+)\s*Half\s*Baths?/i)?.[1] ?? "0");
const total = full + half * 0.5;
return total > 0 ? total : null;
}
const amenityNum = (amenities: string[], re: RegExp): number | null => {
const line = amenities.find((a) => re.test(a));
const m = line?.match(/(\d+)/);
return m ? num(m[1]) : null;
};
/** parse ONE Ashton Woods QMI page (JSON-LD driven) into a home, or null */
export function parseHome(html: string): ParsedHome | null {
const res = ldByType(html, "SingleFamilyResidence");
if (!res) return null;
const prod = ldByType(html, "Product");
const addr = (res.address ?? {}) as Record<string, unknown>;
const street = clean(addr.streetAddress);
if (!street) return null; // not a resolvable per-home listing
const amenities = (Array.isArray(res.AmenityFeature) ? res.AmenityFeature : [])
.map((a) => String(a))
.filter(Boolean);
const geo = (res.geo ?? {}) as Record<string, unknown>;
const offers = ((prod?.offers ?? {}) as Record<string, unknown>) || {};
const availStr = String(offers.availability ?? "");
return {
url: clean(res.url) ?? "",
street,
city: clean(addr.addressLocality),
state: clean(addr.addressRegion),
zip: (clean(addr.postalCode) ?? "").match(/\d{5}/)?.[0] ?? null,
community: clean(res.ContainedIn),
sqft: num(res.Floorsize), // "2260FTK" -> 2260
beds: amenityNum(amenities, /bedroom/i),
bathsTotal: bathsFrom(amenities),
stories: amenityNum(amenities, /stor(y|ies)/i),
garages: amenityNum(amenities, /car\s*garage/i),
lat: coord(geo.latitude),
lon: coord(geo.longitude),
phone: clean(res.telephone),
price: num(offers.price),
available: /InStock|LimitedAvailability|PreOrder|BackOrder/i.test(availStr) || !availStr,
name: clean(res.name),
};
}
export const ashtonWoodsAdapter: SourceAdapter = {
key: "ashton-woods-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let qmiSitemaps = [...index.body.toString("utf8").matchAll(/<loc>([^<]+-sitemap-qmi\.xml)<\/loc>/g)].map(
(m) => m[1]!,
);
if (METRO_FILTER.length) {
qmiSitemaps = qmiSitemaps.filter((u) => METRO_FILTER.some((m) => u.includes(`/${m}-sitemap-qmi.xml`)));
}
for (const smUrl of qmiSitemaps) {
let homeUrls: string[] = [];
try {
const sm = await fetcher.fetch(smUrl);
homeUrls = [...sm.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]!);
} catch (error) {
console.warn(` skip sitemap ${smUrl}: ${error instanceof Error ? error.message : String(error)}`);
continue;
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html);
if (!h) return { records: [], errors: [] }; // not a QMI/home page
if (!h.price || !h.available) return { records: [], errors: [] }; // sold / unpriced -> not current inventory
const cname = h.community ?? "Unknown";
// per-home Widen CDN photos: JSON-LD `image` is the primary elevation; the page
// also carries gallery shots on awh.widen.net. Decode &/& first so query
// strings survive, dedup by path, cap. [images enabled 2026-07-28 per hotlink approval]
const decoded = html.replace(/\\u0026/gi, "&").replace(/&/g, "&");
const ldImg = decoded.match(/"image"\s*:\s*"(https:\/\/awh\.widen\.net[^"]+)"/)?.[1];
const widenAll = [...decoded.matchAll(/https:\/\/awh\.widen\.net\/content\/[^\s"'<>\\)]+/gi)].map((m) => m[0]);
// The page-wide Widen sweep also picks up brand marketing assets that are NOT this
// home's photos — collection thumbnails, the "most-trustworthy-companies" award badge,
// logos. Blocklist them so a home's gallery only shows the home. [Cody-gate fix 2026-07-28]
const BADIMG = /collection|thumbnail|trustworthy|companies|award|badge|logo|download=true|transfer_/i;
const seenImg = new Set<string>();
const images: string[] = [];
for (const u of [ldImg, ...widenAll].filter((x): x is string => !!x)) {
if (BADIMG.test(u)) continue; // drop marketing/award/collection assets
const key = u.split("?")[0]!;
if (seenImg.has(key)) continue;
seenImg.add(key);
images.push(u);
}
const records: ExtractedRecord[] = [];
// community record (carries per-home geo so the community also maps)
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
address: h.street!,
builderInventoryId: h.url || h.street!, // no numeric job id — page URL is the stable per-home key
planName: null,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
price: fv(h.price, h.price != null ? `$${h.price}` : null, page.url, h.price != null ? `price $${h.price}` : null),
beds: fv(h.beds, null, page.url),
bathsTotal: fv(h.bathsTotal, null, page.url),
sqft: fv(h.sqft, null, page.url),
stories: fv(h.stories, null, page.url),
garageSpaces: fv(h.garages, null, page.url),
homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Ashton Woods quick move-in"),
constructionStatus: fv("MOVE_IN_READY" as never, null, page.url, "Ashton Woods QMI"),
estCompletionDate: fv<string>(null, null, page.url),
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv(h.url || h.street, h.url || h.street, page.url),
planName: fv<string>(null, null, page.url),
availabilityStatus: fv("InStock", "InStock", page.url, "schema.org availability InStock"),
images: fv<string[]>(images.slice(0, 8), null, page.url, images.length ? "builder listing photo" : null),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};