← back to Homesonspec
collectors/century-communities/src/index.ts
599 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Century Communities adapter — JSON-API list + SERVER-RENDERED per-home source
* (recon 2026-07-28).
*
* centurycommunities.com runs an EPiServer/Optimizely CMS. Two layers:
*
* LAYER 1 — community list.
* GET https://www.centurycommunities.com/api/search/findcommunities?…&latitude=…&longitude=…&radius=…
* No auth. Returns { communities:[ { name, cityStateZip, startingPrice, url,
* quickMoveInAvailable, latitude, longitude, zipCode, communityId, bed, bath,
* sqft, address1 … } ], total, showLoadMore }. The community rows are
* AGGREGATES (startingPrice; bed/bath/sqft/address1 are null) — NOT per-home.
* VERIFIED CAP: this endpoint always returns the same fixed 12 rows regardless
* of latitude/longitude/radius/page/skip/offset/pageSize (total=438,
* showLoadMore=true forever) — server-side paging is not exposed on it. So the
* API alone can only seed 12 communities. To reach the full ~490-community
* universe we ALSO read the CMS sitemap (/sitemap.xml, 8k+ URLs), whose
* community pages match /find-your-new-home/{state}/{metro}/{city}/{community}[/{collection}]/
* (not /lots/, /plans/, /promo). Every findcommunities url is already a subset
* of the sitemap, so the two are merged + de-duped.
*
* LAYER 2 — per-home quick-move-in (QMI) inventory, SERVER-RENDERED on each
* community page (community.url). Each available spec home is one
* <li class="floor_plan_contain card quick-move-in-card"
* data-price data-sqft data-date data-template="QuickMoveInCommerceBlockCard">
* carrying: data-price ($), data-sqft, an <h4 .street-number><a href=".../lots/…">
* <span .street-number-text>"2641 Gibraltar Drive | Lot 0187"</span></a>,
* <span .title> plan name, <p .home-type> ("Single Family Home"), a
* <span .est-complete-date> status ("Move-in Ready!" / "Est. Completion: Aug…"),
* and Bedrooms / Bathrooms / Square Footage icon rows (real ints/decimals).
* The community page's ld+json (Place / HomeAndConstructionBusiness) carries the
* community geo (GeoCoordinates lat/lng) + PostalAddress (city/region/postal),
* which is authoritative and injected onto each home.
*
* robots.txt (centurycommunities.com) is `User-agent: *` with a Disallow list that
* covers /EPiServer/*, /util/, /purchase/*, /site-data/* etc. — NONE of which
* touch /api/search/findcommunities, /sitemap.xml, or /find-your-new-home/* → all
* our paths are allowed. Facts-only: the images[] feed + card carousels are dropped.
*
* Batch control: CENTURY_PAGE_LIMIT (community pages to crawl, default 10)
*/
const ORIGIN = "https://www.centurycommunities.com";
const SITEMAP_URL = `${ORIGIN}/sitemap.xml`;
const BUILDER_SLUG = "century-communities";
const PAGE_LIMIT = Number(process.env.CENTURY_PAGE_LIMIT ?? 10);
const META_MARKER = "century-community:";
// National geo center + max radius — the endpoint ignores these, but they keep the
// request well-formed and self-documenting.
const FINDCOMMUNITIES_URL =
`${ORIGIN}/api/search/findcommunities?minPrice=0&maxPrice=999999999999&bedrooms=0` +
`&bathrooms=0&minSquareFeet=0&maxSquareFeet=99999&latitude=39.8&longitude=-98.5&radius=99999`;
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths may legitimately be small; 0 allowed), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
/** Preserved signs: US lon is negative; a stringified "-121.9…" must stay negative. */
const toCoord = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
// ---------------------------------------------------------------------------
// LAYER 1 — community list (findcommunities API + sitemap)
// ---------------------------------------------------------------------------
interface FindCommunity {
name?: string;
cityStateZip?: string;
startingPrice?: number;
url?: string;
quickMoveInAvailable?: boolean;
latitude?: string | number;
longitude?: string | number;
zipCode?: string | number;
communityId?: number;
}
/** The community metadata injected into a community-page RawPage so extract()
* can attach the API's quick-move-in flag / starting price / community id. */
interface InjectedMeta {
apiName: string | null;
startingPrice: number | null;
quickMoveInAvailable: boolean | null;
communityId: string | null;
apiLat: number | null;
apiLon: number | null;
apiZip: string | null;
}
/** Parse the findcommunities JSON body → community rows. Returns [] on any failure. */
export function parseFindCommunities(body: string): FindCommunity[] {
try {
const j = JSON.parse(body) as { communities?: FindCommunity[] };
return Array.isArray(j.communities) ? j.communities : [];
} catch {
return [];
}
}
/** Absolute community-page URL from a relative `communities[].url`. */
function absUrl(relOrAbs: string): string {
if (/^https?:\/\//.test(relOrAbs)) return relOrAbs;
return `${ORIGIN}${relOrAbs.startsWith("/") ? "" : "/"}${relOrAbs}`;
}
/** A community page path (not a lot/plan/promo detail):
* /find-your-new-home/{state}/{metro}/{city}/{community}[/{collection}]/ */
function isCommunityPageUrl(u: string): boolean {
let path: string;
try {
path = new URL(u).pathname;
} catch {
return false;
}
if (!path.startsWith("/find-your-new-home/")) return false;
if (path.includes("/lots/") || path.includes("/plans/") || path.includes("/promo")) return false;
const segs = path.replace(/\/+$/, "").split("/").filter(Boolean);
// ["find-your-new-home", state, metro, city, community] (5) or +collection (6)
return segs.length === 5 || segs.length === 6;
}
/** Pull community-page <loc> URLs out of the CMS sitemap XML. */
export function parseSitemapCommunityUrls(xml: string): string[] {
const out: string[] = [];
const seen = new Set<string>();
for (const m of xml.matchAll(/<loc>([^<]+)<\/loc>/g)) {
const u = m[1]!.trim();
if (isCommunityPageUrl(u) && !seen.has(u)) {
seen.add(u);
out.push(u);
}
}
return out;
}
/** Build the ordered, de-duped community-page crawl list: findcommunities rows
* first (they carry the QMI flag + geo), then any sitemap community pages not
* already seeded. `metaByUrl` maps a normalized URL → the API meta to inject. */
export function buildCrawlList(
apiCommunities: FindCommunity[],
sitemapUrls: string[],
): { urls: string[]; metaByUrl: Map<string, InjectedMeta> } {
const urls: string[] = [];
const seen = new Set<string>();
const metaByUrl = new Map<string, InjectedMeta>();
const norm = (u: string) => u.replace(/\/+$/, "/").toLowerCase();
// Prefer communities the API says HAVE quick-move-in inventory (crawl budget is
// scarce), then the rest of the API rows, then the sitemap universe.
const apiWithQmi = apiCommunities.filter((c) => str(c.url) && c.quickMoveInAvailable);
const apiRest = apiCommunities.filter((c) => str(c.url) && !c.quickMoveInAvailable);
for (const c of [...apiWithQmi, ...apiRest]) {
const u = absUrl(String(c.url));
const key = norm(u);
if (seen.has(key)) continue;
seen.add(key);
urls.push(u);
const zipFromCsz = str(c.cityStateZip)?.match(/\b(\d{5})\b/)?.[1] ?? null;
metaByUrl.set(key, {
apiName: str(c.name),
startingPrice: posNum(c.startingPrice),
quickMoveInAvailable: c.quickMoveInAvailable ?? null,
communityId: str(c.communityId),
apiLat: toCoord(c.latitude),
apiLon: toCoord(c.longitude),
apiZip: zip5(c.zipCode) ?? zipFromCsz,
});
}
for (const u of sitemapUrls) {
const key = norm(u);
if (seen.has(key)) continue;
seen.add(key);
urls.push(u);
}
return { urls, metaByUrl };
}
function encodeMeta(meta: InjectedMeta | undefined): string {
return JSON.stringify(meta ?? null);
}
function readInjectedMeta(html: string): InjectedMeta | null {
const m = html.match(new RegExp(`<!--\\s*${META_MARKER}\\s*([\\s\\S]*?)\\s*-->`));
if (!m) return null;
try {
const v = JSON.parse(m[1]!);
return v && typeof v === "object" ? (v as InjectedMeta) : null;
} catch {
return null;
}
}
// ---------------------------------------------------------------------------
// LAYER 2 — per-home QMI cards + community ld+json geo (parsed from page HTML)
// ---------------------------------------------------------------------------
interface CommunityLd {
name: string | null;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
metro: string | null;
lat: number | null;
lon: number | null;
}
/** Read the community's geo/address + metro from the page's ld+json (Place /
* HomeAndConstructionBusiness carry geo+address; a custom block carries the
* metro/city names). Any field absent → null. */
export function parseCommunityLd(html: string): CommunityLd {
const out: CommunityLd = {
name: null, street: null, city: null, state: null, zip: null, metro: null, lat: null, lon: null,
};
for (const m of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/g)) {
let j: any;
try {
j = JSON.parse(m[1]!.trim());
} catch {
continue;
}
const type = j?.["@type"];
if ((type === "Place" || type === "HomeAndConstructionBusiness") && j.geo) {
out.lat ??= toCoord(j.geo.latitude);
out.lon ??= toCoord(j.geo.longitude);
if (j.address) {
out.street ??= str(j.address.streetAddress);
out.city ??= str(j.address.addressLocality);
out.state ??= normalizeStateCode(str(j.address.addressRegion));
out.zip ??= zip5(j.address.postalCode);
}
out.name ??= str(j.name);
}
// The custom EPiServer block: { CommunityName, CityName, StateName, MetroName }
if (j && (j.CommunityName || j.MetroName)) {
out.name ??= str(j.CommunityName);
out.metro ??= str(j.MetroName);
out.city ??= str(j.CityName);
out.state ??= normalizeStateCode(str(j.StateName));
}
}
return out;
}
interface QmiHome {
price: number | null;
sqft: number | null;
address: string | null; // street portion, "Lot NNNN" stripped
lotNumber: string | null;
lotUrl: string | null;
plan: string | null;
homeType: string | null;
status: string | null;
beds: number | null;
baths: number | null;
}
/** Slice out each <li …quick-move-in-card… data-template="QuickMoveInCommerceBlockCard">
* … </li> block by balancing nested <li>/<\/li>. */
function sliceCards(html: string): string[] {
const cards: string[] = [];
const marker = 'data-template="QuickMoveInCommerceBlockCard"';
let from = 0;
let at: number;
while ((at = html.indexOf(marker, from)) >= 0) {
const start = html.lastIndexOf("<li", at);
if (start < 0) {
from = at + marker.length;
continue;
}
let depth = 0;
let j = start;
let end = -1;
while (j < html.length) {
if (html.startsWith("<li", j)) {
depth++;
j += 3;
continue;
}
if (html.startsWith("</li>", j)) {
depth--;
if (depth === 0) {
end = j + 5;
break;
}
j += 5;
continue;
}
j++;
}
if (end < 0) {
from = at + marker.length;
continue;
}
cards.push(html.slice(start, end));
from = end;
}
return cards;
}
/** "2641 Gibraltar Drive | Lot 0187" → { address, lotNumber }. */
function splitAddress(raw: string | null): { address: string | null; lotNumber: string | null } {
const s = str(raw);
if (!s) return { address: null, lotNumber: null };
const m = s.match(/^(.*?)\s*\|\s*Lot\s*([A-Za-z0-9-]+)\s*$/i);
if (m) return { address: str(m[1]), lotNumber: str(m[2]) };
return { address: s, lotNumber: null };
}
export function parseQmiCards(html: string): QmiHome[] {
const g = (card: string, re: RegExp): string | null => {
const m = card.match(re);
return m ? str(m[1]!.replace(/\s+/g, " ")) : null;
};
return sliceCards(html).map((card) => {
const clean = card.replace(/<svg[\s\S]*?<\/svg>/g, "");
const addrRaw = g(clean, /street-number-text"[^>]*>([^<]+)</);
const { address, lotNumber } = splitAddress(addrRaw);
const bedbath = (label: string): number | null => {
const m = clean.match(new RegExp(`alt="${label}"[\\s\\S]{0,220}?<span>\\s*([0-9.,]+)`));
return m ? nonNegNum(m[1]) : null;
};
// status: the est-complete-date span may carry "Move-in Ready!" or "Est.
// Completion: <date>" (possibly after a <br/> and label). Take its trailing text.
let status = g(clean, /est-complete-date"[^>]*>[\s\S]*?<br\s*\/?>\s*([^<]+)</);
if (!status) status = g(clean, /est-complete-date"[^>]*>\s*([^<]+?)\s*<\/span>/);
if (status) status = status.replace(/Est\.\s*Completion:\s*/i, "").trim() || null;
return {
price: posNum(g(clean, /data-price="([^"]+)"/)),
sqft: posNum(g(clean, /data-sqft="([^"]+)"/)),
address,
lotNumber,
lotUrl: g(clean, /<a href="([^"]*\/lots\/[^"]*)"/),
plan: g(clean, /<span class="title">([^<]+)</),
homeType: g(clean, /home-type"[^>]*>([^<]+)</),
status,
beds: posNum(bedbath("Bedrooms")),
baths: nonNegNum(bedbath("Bathrooms")),
};
});
}
/** Century status text → construction-status enum. "Move-in Ready!" = finished;
* a dated / "Under Construction" text = in progress. Blank/unknown → null. */
function constructionStatus(status: string | null): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = (str(status) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("move-in ready") || s.includes("move in ready") || s.includes("ready now")) return "MOVE_IN_READY";
// A month/date estimate ("Aug. Move In", "Oct '26") or explicit under-construction = in progress.
if (s.includes("under construction") || s.includes("coming") || /move[\s-]?in/.test(s) || /'?\d{2}\b|\d{4}/.test(s)) {
return "UNDER_CONSTRUCTION";
}
return null;
}
// ---------------------------------------------------------------------------
// Adapter
// ---------------------------------------------------------------------------
export const centuryAdapter: SourceAdapter = {
key: "century-communities-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// LAYER 1a — findcommunities (geo-enriched seed; also emits community records).
let apiCommunities: FindCommunity[] = [];
try {
const api = await fetcher.fetch(FINDCOMMUNITIES_URL);
yield api; // extract() emits a community record per API row
apiCommunities = parseFindCommunities(api.body.toString("utf8"));
} catch (error) {
console.warn(` century findcommunities: ${error instanceof Error ? error.message : String(error)}`);
// Non-fatal — the sitemap can still seed the crawl.
}
// LAYER 1b — sitemap (the full ~490-community universe; the API caps at 12).
let sitemapUrls: string[] = [];
try {
const sm = await fetcher.fetch(SITEMAP_URL);
sitemapUrls = parseSitemapCommunityUrls(sm.body.toString("utf8"));
} catch (error) {
console.warn(` century sitemap: ${error instanceof Error ? error.message : String(error)}`);
}
const { urls, metaByUrl } = buildCrawlList(apiCommunities, sitemapUrls);
// LAYER 2 — crawl community pages, parse per-home QMI cards.
const norm = (u: string) => u.replace(/\/+$/, "/").toLowerCase();
for (const url of urls.slice(0, Math.max(0, PAGE_LIMIT))) {
let page: RawPage;
try {
page = await fetcher.fetch(url);
} catch (error) {
console.warn(` century community ${url}: ${error instanceof Error ? error.message : String(error)}`);
continue; // one blocked/failed community page doesn't stop the rest
}
const meta = encodeMeta(metaByUrl.get(norm(url)));
const injected = `<!-- ${META_MARKER} ${meta} -->\n${page.body.toString("utf8")}`;
yield { ...page, body: Buffer.from(injected, "utf8") };
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
// ---- findcommunities response: emit a community record per API row -----
if (page.url.includes("/api/search/findcommunities")) {
const communities = parseFindCommunities(html);
const records: ExtractedRecord[] = [];
for (const c of communities) {
const name = str(c.name);
if (!name) continue;
const csz = str(c.cityStateZip); // "Port Charlotte, FL 33981"
let city: string | null = null;
let state: string | null = null;
if (csz) {
const m = csz.match(/^(.*?),\s*([A-Za-z]{2})\s+\d{5}/);
if (m) {
city = str(m[1]);
state = normalizeStateCode(m[2]!);
} else {
city = str(csz.split(",")[0] ?? null);
}
}
const zip = zip5(c.zipCode) ?? (csz ? zip5(csz) : null);
const lat = toCoord(c.latitude);
const lon = toCoord(c.longitude);
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: name },
fields: {
name: fv(name, name, page.url, "findcommunities communities[].name"),
street: fv<string>(null, null, page.url),
city: fv(city, city, page.url),
state: fv(state, state === null ? null : csz, page.url),
zip: fv(zip, zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(lat, lat === null ? null : String(c.latitude), page.url, lat === null ? null : "communities[].latitude"),
lon: fv(lon, lon === null ? null : String(c.longitude), page.url, lon === null ? null : "communities[].longitude"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
}
if (!records.length) {
return { records: [], errors: [{ url: page.url, reason: "findcommunities returned no named communities" }] };
}
return { records, errors: [] };
}
// ---- community page: emit community (from ld+json) + its QMI homes ------
const ld = parseCommunityLd(html);
const meta = readInjectedMeta(html);
const cards = parseQmiCards(html);
// Community name: ld+json first, API meta as fallback.
const community = ld.name ?? meta?.apiName ?? null;
const city = ld.city ?? null;
const state = ld.state ?? null;
const zip = ld.zip ?? meta?.apiZip ?? null;
const lat = ld.lat ?? meta?.apiLat ?? null;
const lon = ld.lon ?? meta?.apiLon ?? null;
const metro = ld.metro ?? null;
if (!community) {
return { records: [], errors: [{ url: page.url, reason: "community page had no ld+json name and no injected API meta" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
// Community FIRST — publish creates/refreshes the FK target the homes need.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, page.url, "community-page ld+json"),
street: fv(ld.street, ld.street, page.url),
city: fv(city, city, page.url),
state: fv(state, ld.state === null ? null : "ld+json addressRegion", page.url),
zip: fv(zip, zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv(metro, metro, page.url),
lat: fv(lat, lat === null ? null : String(lat), page.url, lat === null ? null : "ld+json geo.latitude"),
lon: fv(lon, lon === null ? null : String(lon), page.url, lon === null ? null : "ld+json geo.longitude"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
if (!cards.length) {
// A community page with no QMI inventory is normal (aggregate-only /
// sold-out community). We still emitted the community; note it and move on.
errors.push({ url: page.url, reason: "no quick-move-in home cards on community page (aggregate-only)" });
return { records, errors };
}
for (const card of cards) {
const homeUrl = card.lotUrl ? absUrl(card.lotUrl) : page.url;
if (!card.address) {
errors.push({ url: homeUrl, reason: `QMI home ${card.lotNumber ?? "?"} missing street address — skipped` });
continue;
}
const cStatus = constructionStatus(card.status);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address: card.address,
builderInventoryId: card.lotNumber ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: card.plan ?? undefined,
},
fields: {
street: fv(card.address, card.address, homeUrl, "QMI card street-number-text"),
city: fv(city, city, homeUrl),
state: fv(state, ld.state === null ? null : "community ld+json", homeUrl),
zip: fv(zip, zip, homeUrl),
price: fv(card.price, card.price === null ? null : String(card.price), homeUrl, card.price === null ? null : `QMI card data-price ${card.price}`),
beds: fv(card.beds, card.beds === null ? null : String(card.beds), homeUrl),
bathsTotal: fv(card.baths, card.baths === null ? null : String(card.baths), homeUrl),
sqft: fv(card.sqft, card.sqft === null ? null : String(card.sqft), homeUrl),
stories: fv<number>(null, null, homeUrl),
garageSpaces: fv<number>(null, null, homeUrl),
homeType: fv(
card.homeType && /town|condo|multi/i.test(card.homeType)
? (/town/i.test(card.homeType) ? ("TOWNHOME" as const) : ("CONDO" as const))
: ("SINGLE_FAMILY" as const),
card.homeType,
homeUrl,
card.homeType ?? "Century QMI spec home",
),
constructionStatus: fv(cStatus, card.status, homeUrl, cStatus === null ? null : `status: ${card.status}`),
estCompletionDate: fv<string>(null, null, homeUrl),
lotNumber: fv(card.lotNumber, card.lotNumber, homeUrl),
builderInventoryId: fv(card.lotNumber, card.lotNumber, homeUrl),
lat: fv(lat, lat === null ? null : String(lat), homeUrl, lat === null ? null : "community ld+json geo.latitude"),
lon: fv(lon, lon === null ? null : String(lon), homeUrl, lon === null ? null : "community ld+json geo.longitude"),
planName: fv(card.plan, card.plan, homeUrl),
// facts-only: card carousel images exist but are intentionally dropped.
images: fv<string[]>([], null, homeUrl),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};