← back to Homesonspec
collectors/gl-homes/src/index.ts
441 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* GL Homes adapter — SERVER-RENDERED, community-organized source (recon 2026-07-28).
*
* glhomes.com is a large Florida-only regional builder (Optimizely/Episerver CMS,
* "areas/glhomes" theme). It is organized entirely by COMMUNITY (Valencia *,
* Apex at Avenir, Lotus Edge, The Estates at Nomar). There is NO inventory JSON
* API — a Playwright networkidle capture of a quick-move-in page returned ONLY
* tracking/chat XHRs (HubSpot, TikTok, OpenAI pixel); the home facts are fully
* server-rendered HTML. So this is a two-layer HTML crawl, not a feed.
*
* Layer 1 — the homepage nav lists the active communities as top-level slugs
* (<a href="/apex-at-avenir/">, /lotus-edge/, /valencia-ridge/ …). Each active
* community exposes its quick-move-in ("Early Move-In") inventory at
* GET https://www.glhomes.com/<slug>/early-move-in/
* (Community landing /<slug>/ carries the sales-center address
* "12874 Soaring View, Palm Beach Gardens, FL 33412" → community city/state/zip.
* NB: the page's "latitude"/"longitude" tokens are a JS template config, NOT
* real coordinates — so lat/lon are left null rather than guessed.)
*
* Layer 2 — the /<slug>/early-move-in/ page renders one <div class="early-move__card">
* per FLOORPLAN, carrying PLAN-LEVEL attributes that every home of that plan
* shares:
* .early-move__subtitle → plan name ("Calypso")
* .early-move__attr → "3 Bedrooms, 3 Bathrooms, 1 Half Bath, …, 3-Car Garage"
* .early-move__sqs > li → "2,464 a/c sq. ft." (living area) + "3,446 total sq. ft."
* floorplan href → "/apex-at-avenir/pinnacle-collection/calypso-508/"
* and INSIDE each card a <ul class="early-move__items"> with one <li> per
* INDIVIDUAL HOME:
* <span>13099 FLORIDA CRANE DRIVE</span> (street)
* <span>Lot 0201</span> (lot number)
* .early-move__current-price ($1,230,900 — the real sell price)
* .early-move__original-price (pre-savings, evidence only)
* .early-move-closing-date ("Available Now" | "Available Apr 2027")
*
* GRAIN (the key risk for a community-organized builder): one card = one plan
* but MANY homes. This adapter emits ONE inventory_home PER <li>, joining the
* card's plan-level beds/baths/sqft/garage to each home's own address/price/lot/
* completion. Parsing a card as a single home would collapse (e.g.) 66 real
* Valencia-Parc homes into 14 hollow floorplan rows — the Dream-Finders trap.
* Recon counts (2026-07-28): 68 plan-cards → 168 individual QMI homes across the
* 9 active communities.
*
* robots.txt (glhomes.com) is `User-agent: *` / `Disallow:` (empty) → allow /.
* Facts-only: card photos exist in the HTML and are intentionally dropped.
*
* Batch control: GL_PAGE_LIMIT (community early-move-in pages to crawl, default 10).
*/
const ORIGIN = "https://www.glhomes.com";
const BUILDER_SLUG = "gl-homes";
const PAGE_LIMIT = Number(process.env.GL_PAGE_LIMIT ?? 10);
const META_MARKER = "gl-community:";
// GL Homes is Florida-only — the sales-center address always resolves state FL,
// but we still parse it from the address text rather than hard-coding.
const HOME_URL = (slug: string) => `${ORIGIN}/${slug}/early-move-in/`;
const COMMUNITY_URL = (slug: string) => `${ORIGIN}/${slug}/`;
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
/** Community metadata injected into an early-move-in RawPage so extract() stays a
* pure function of the bytes it receives (city/state/zip only exist on the
* community landing page, not the early-move-in page). */
interface InjectedMeta {
slug: string;
name: string | null;
city: string | null;
state: string | null;
zip: string | null;
}
/** Title-case a hyphenated community slug → "Apex At Avenir", "Valencia Ridge". */
function slugToName(slug: string): string {
return slug
.split("-")
.map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w))
.join(" ");
}
/** Extract the active-community slugs from the homepage nav. GL lists each
* community as a top-level <a href="/<slug>/">; we keep only slugs that look
* like real community landing pages and drop the site's utility pages. */
const NON_COMMUNITY = new Set([
"about-us", "brokers", "careers", "coming-soon", "free-brochure", "philanthropy",
"privacy-policy", "smart-planning", "special-offers", "stay-and-play", "terms-of-use",
"videos", "florida-homes", "valencia-lifestyle", "case-studies", "webinar",
"utm-generator", "new-homes-for-sale",
]);
export function parseCommunitySlugs(html: string): string[] {
const slugs = new Set<string>();
for (const m of html.matchAll(/href="\/([a-z0-9-]+)\/"/g)) {
const slug = m[1]!;
if (NON_COMMUNITY.has(slug)) continue;
if (slug.includes("email-preferences")) continue;
slugs.add(slug);
}
return [...slugs];
}
/** Pull the community's city/state/zip out of the sales-center address on the
* community landing page. The address appears in several publisher formats:
* "12874 Soaring View, Palm Beach Gardens, FL 33412"
* "12320 SW Calm Pointe Ct, Port St. Lucie, FL 34987" (period in city)
* "9150 Serene Heron Drive, Boynton Beach, Fl 33473" (lowercase state)
* We anchor on the "<city>, <ST> <zip>" tail and take the city as the
* comma-segment immediately before the 2-letter state + 5-digit zip. */
export function parseCommunityAddress(html: string): { city: string | null; state: string | null; zip: string | null } {
const m = html.match(/,\s*([A-Za-z][A-Za-z .'-]*?),\s*([A-Za-z]{2})\s+(\d{5})\b/);
if (!m) return { city: null, state: null, zip: null };
const state = normalizeStateCode(str(m[2])?.toUpperCase() ?? null);
return {
city: str(m[1]),
state,
zip: zip5(m[3]),
};
}
/** Read the injected `<!-- gl-community: {json} -->` comment, if any. */
function readInjectedMeta(html: string): InjectedMeta | null {
const m = html.match(/<!--\s*gl-community:\s*(\{[\s\S]*?\})\s*-->/);
if (!m) return null;
try {
return JSON.parse(m[1]!) as InjectedMeta;
} catch {
return null;
}
}
interface PlanAttrs {
beds: number | null;
baths: number | null; // full + half*0.5
garages: number | null;
sqft: number | null; // a/c (living) sq ft
}
/** Parse the plan-level ".early-move__attr" line + sqft list into shared attrs.
* "3 Bedrooms, 3 Bathrooms, 1 Half Bath, Den/Opt. 4th Bed, 3-Car Garage" */
export function parsePlanAttrs(attr: string | null, sqftAcRaw: string | null): PlanAttrs {
const a = attr ?? "";
const beds = posNum(a.match(/(\d+)\s*Bedrooms?/i)?.[1] ?? null);
const full = nonNegNum(a.match(/(\d+)\s*Bathrooms?/i)?.[1] ?? null);
const half = nonNegNum(a.match(/(\d+)\s*Half\s*Baths?/i)?.[1] ?? null);
const baths = full === null ? null : full + (half ?? 0) * 0.5;
const garages = nonNegNum(a.match(/(\d+)\s*-?\s*Car\s*Garage/i)?.[1] ?? null);
const sqft = posNum(sqftAcRaw);
return { beds, baths, garages, sqft };
}
export interface GlHome {
street: string | null;
lot: string | null;
price: number | null;
priceRaw: string | null;
originalPriceRaw: string | null;
closingRaw: string | null;
}
export interface GlCard {
planName: string | null;
attr: string | null;
sqftAcRaw: string | null;
floorplanHref: string | null;
homes: GlHome[];
}
/** Parse every <div class="early-move__card"> … block on an early-move-in page.
* Each card is a PLAN; each <li> inside its .early-move__items is a HOME. */
export function parseCards(html: string): GlCard[] {
const cards: GlCard[] = [];
// Split on the card opener; the segment runs until the next card or the section
// close. We over-capture then trim at the next card boundary defensively.
const segments = html.split(/<div id="emi-[^"]*"\s+class="early-move__card"/);
for (let i = 1; i < segments.length; i++) {
// Trim this segment so it doesn't bleed into the following card's items.
const seg = segments[i]!;
const planName = seg.match(/early-move__subtitle">([^<]+)</)?.[1]?.trim() ?? null;
const attr = seg.match(/early-move__attr">([^<]+)</)?.[1]?.trim() ?? null;
// First .early-move__sqs <li> is the a/c (living) sqft; second is total.
const sqsBlock = seg.match(/early-move__sqs">([\s\S]*?)<\/ul>/)?.[1] ?? "";
const sqftAcRaw = sqsBlock.match(/<li>\s*([\d,]+)\s*a\/c/i)?.[1] ?? null;
const floorplanHref = seg.match(/href="(\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/)"\s+class="btn-link/)?.[1] ?? null;
const homes: GlHome[] = [];
const itemsBlock = seg.match(/<ul class="early-move__items">([\s\S]*?)<\/ul>/)?.[1] ?? "";
for (const liMatch of itemsBlock.matchAll(/<li>([\s\S]*?)<\/li>/g)) {
const li = liMatch[1]!;
// First <span> is the street; a "Lot NNNN" span is the lot.
const street = li.match(/<span>([0-9][^<]*)<\/span>/)?.[1]?.trim() ?? null;
const lot = li.match(/<span>\s*(Lot[^<]*)<\/span>/i)?.[1]?.trim() ?? null;
// Two publisher price classes: ".early-move__current-price" (when the home
// shows a struck-through original + savings) and
// ".early-move-current-price-standalone" (when it lists a single price).
const priceRaw =
li.match(/early-move__current-price">\s*\$?([\d,]+)/)?.[1] ??
li.match(/early-move-current-price-standalone">\s*\$?([\d,]+)/)?.[1] ??
null;
const originalPriceRaw = li.match(/early-move__original-price\s*">\s*\$?([\d,]+)/)?.[1] ?? null;
const closingRaw = li.match(/early-move-closing-date">([^<]+)</)?.[1]?.trim() ?? null;
// Skip a stray <li> that carries no street (defensive — real home <li>s
// always lead with the address span).
if (!street) continue;
homes.push({ street, lot, price: posNum(priceRaw), priceRaw, originalPriceRaw, closingRaw });
}
cards.push({ planName, attr, sqftAcRaw, floorplanHref, homes });
}
return cards;
}
/** GL closing-date text → construction-status enum. "Available Now" = ready;
* "Available <Mon Year>" = still under construction. Blank → null. */
function constructionStatus(closing: string | null): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = (str(closing) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("now")) return "MOVE_IN_READY";
if (/available\s+[a-z]{3}\s+\d{4}/i.test(s)) return "UNDER_CONSTRUCTION";
return null;
}
const MONTHS: Record<string, string> = {
jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06",
jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12",
};
/** "Available Apr 2027" → "2027-04-01" (est. completion, day defaulted). "Available
* Now" and unparseable text → null (never guessed). */
function estCompletionDate(closing: string | null): string | null {
const s = str(closing);
if (!s) return null;
const m = s.match(/([A-Za-z]{3})[a-z]*\s+(\d{4})/);
if (!m) return null;
const mon = MONTHS[m[1]!.toLowerCase()];
const year = Number(m[2]);
if (!mon || !Number.isFinite(year) || year < 2000 || year > 2100) return null;
return `${year}-${mon}-01`;
}
/** Absolute URL from a site-relative href. */
function abs(href: string): string {
if (/^https?:\/\//.test(href)) return href;
return `${ORIGIN}${href.startsWith("/") ? "" : "/"}${href}`;
}
export const glHomesAdapter: SourceAdapter = {
key: "gl-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let homepage: RawPage;
try {
homepage = await fetcher.fetch(`${ORIGIN}/`);
} catch (error) {
console.warn(` gl homepage: ${error instanceof Error ? error.message : String(error)}`);
return; // a block/403 on the homepage stops collection (source degraded)
}
const slugs = parseCommunitySlugs(homepage.body.toString("utf8"));
let crawled = 0;
for (const slug of slugs) {
if (crawled >= Math.max(0, PAGE_LIMIT)) break;
// Community landing → city/state/zip (early-move-in page lacks them).
let meta: InjectedMeta = { slug, name: slugToName(slug), city: null, state: null, zip: null };
try {
const landing = await fetcher.fetch(COMMUNITY_URL(slug));
const addr = parseCommunityAddress(landing.body.toString("utf8"));
meta = { slug, name: slugToName(slug), ...addr };
} catch (error) {
console.warn(` gl community ${slug}: ${error instanceof Error ? error.message : String(error)}`);
// Continue without city/zip — the home records are still valid facts.
}
// Quick-move-in ("Early Move-In") inventory page.
let page: RawPage;
try {
page = await fetcher.fetch(HOME_URL(slug));
} catch (error) {
console.warn(` gl early-move-in ${slug}: ${error instanceof Error ? error.message : String(error)}`);
continue; // one blocked/failed community doesn't stop the rest
}
const body = page.body.toString("utf8");
// Communities with no inventory page soft-404 to the homepage — skip those.
if (!body.includes("early-move__card")) continue;
crawled++;
const injected = `<!-- ${META_MARKER} ${JSON.stringify(meta)} -->\n${body}`;
yield { ...page, body: Buffer.from(injected, "utf8") };
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const meta = readInjectedMeta(html);
const slug = meta?.slug ?? page.url.match(/glhomes\.com\/([a-z0-9-]+)\//)?.[1] ?? null;
const community = meta?.name ?? (slug ? slugToName(slug) : null);
if (!community) {
return { records: [], errors: [{ url: page.url, reason: "early-move-in page missing community meta/slug" }] };
}
const cards = parseCards(html);
if (!cards.length) {
return { records: [], errors: [{ url: page.url, reason: "no early-move__card blocks on early-move-in page" }] };
}
const state = meta?.state ?? normalizeStateCode("FL"); // GL Homes is Florida-only
const city = meta?.city ?? null;
const zip = meta?.zip ?? null;
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
let emittedCommunity = false;
for (const card of cards) {
const plan = parsePlanAttrs(card.attr, card.sqftAcRaw);
const planUrl = card.floorplanHref ? abs(card.floorplanHref) : page.url;
for (const home of card.homes) {
if (!home.street) {
errors.push({ url: page.url, reason: `home in plan ${card.planName ?? "?"} missing street — skipped` });
continue;
}
// Community FIRST — publish creates/refreshes the FK target the home
// needs. Emit it once per page (all homes share the one community).
if (!emittedCommunity) {
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, page.url, "community slug/landing"),
street: fv<string>(null, null, page.url),
city: fv(city, city, page.url),
state: fv(state, meta?.state ?? "FL", page.url),
zip: fv(zip, zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
emittedCommunity = true;
}
const cStatus = constructionStatus(home.closingRaw);
const est = estCompletionDate(home.closingRaw);
// Preserve the address exactly as published (upper-cased street).
const street = str(home.street);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address: street ?? undefined,
builderInventoryId: home.lot ?? undefined,
planName: card.planName ?? undefined,
},
fields: {
street: fv(street, street, page.url, "early-move__items home address"),
city: fv(city, city, page.url),
state: fv(state, meta?.state ?? "FL", page.url),
zip: fv(zip, zip, page.url),
price: fv(
home.price,
home.priceRaw,
page.url,
home.price === null ? null : `current price $${home.priceRaw}${home.originalPriceRaw ? ` (was $${home.originalPriceRaw})` : ""}`,
),
beds: fv(plan.beds, plan.beds === null ? null : String(plan.beds), page.url, plan.beds === null ? null : `plan ${card.planName}: ${card.attr}`),
bathsTotal: fv(plan.baths, plan.baths === null ? null : String(plan.baths), page.url, plan.baths === null ? null : `plan ${card.planName}: ${card.attr}`),
sqft: fv(plan.sqft, plan.sqft === null ? null : String(plan.sqft), page.url, plan.sqft === null ? null : `${card.sqftAcRaw} a/c sq. ft.`),
stories: fv<number>(null, null, page.url),
garageSpaces: fv(plan.garages, plan.garages === null ? null : String(plan.garages), page.url),
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "GL Homes single-family spec home"),
constructionStatus: fv(cStatus, home.closingRaw, page.url, cStatus === null ? null : `closing: ${home.closingRaw}`),
estCompletionDate: fv(est, home.closingRaw, page.url, est === null ? null : `from "${home.closingRaw}"`),
lotNumber: fv(home.lot, home.lot, page.url),
builderInventoryId: fv(home.lot, home.lot, page.url, home.lot === null ? null : "GL lot number"),
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
planName: fv(card.planName, card.planName, planUrl),
// facts-only: card photos exist in the HTML but are intentionally dropped.
images: fv<string[]>([], null, page.url),
},
});
}
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};