← back to Homesonspec
collectors/harris-doyle/src/index.ts
479 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Harris Doyle Homes adapter — AL/FL site-built spec-home builder, a Clayton
* Properties Group brand (slug "harris-doyle"; recon + built 2026-08-30,
* TK-10487). Same architecture class as the goodall-homes adapter: Clayton has
* no unified site-built feed, so each brand gets its own adapter on its own
* domain.
*
* harrisdoyle.com is a React/Alpine SSR site (ColdFusion origin — note the
* `.cfm` XHR endpoints) that is FULLY server-rendered — a plain honest-UA GET
* of a per-home page returns ~110–130 KB of complete HTML (gzip on the wire;
* the shared LiveFetcher auto-decompresses). NO browser, NO bot-wall, NO
* XHR/WS for the facts. robots.txt is FULLY OPEN ("User-agent: * Disallow:",
* no rules) and references the sitemap.
*
* Sitemap https://www.harrisdoyle.com/sitemap.xml (single urlset, ~286 URLs)
* -> per-home inventory DETAIL pages are the 6-segment
* /new-homes/{state}/{metro}/{city}/{community}/{plan}/{street-address}
* (~72 of them, e.g.
* /new-homes/alabama/birmingham/hoover/the-foothills-at-blackridge/
* beaumont-pinnacle/4168-blackridge-crest).
* Shorter 4-seg URLs (…/{community}) are community LISTING pages and
* 5-seg URLs (…/{plan}) are floor-plan pages (many sibling homes) — both
* filtered out. The final segment can be a real address slug OR a
* `tbd-so-*` placeholder for a to-be-built lot, BUT the page's <h1>
* still carries the REAL street address either way.
*
* UNLIKE Goodall, the per-home JSON-LD here carries ONLY Organization + WebSite
* (no SingleFamilyResidence/Product), so all per-home facts come from the DOM.
* Each per-home page has ONE primary hero block rendered before the
* "Quick Move-Ins" section (whose sibling homes reuse the SAME
* `listing-card__*` classes — so we anchor on the FIRST rendered instance of
* each, which is unambiguously the current home):
*
* - <h1 class="page-section__page-title"> = street address (HTML-entity
* encoded spaces, e.g. "4168 Blackridge Crest").
* - a "font-sans-serif" <p> under the h1: community link, then
* "City, ST ZIP", then "Lot: N".
* - price: the "Priced at " label followed by <span class='fs-4'>$759,021</span>.
* - the FIRST <ul class="listing-card__specs">: iconic beds / baths / sqft /
* garages via <span class="listing-card__specs-value">N</span> +
* <span class="listing-card__specs-label">Beds|Baths|Sq. Ft.|Garages</span>.
* - builderInventoryId: the FIRST `/…/inventory.cfm?id=N` (the primary home's
* stable numeric id; sibling QMI cards carry their own later ids).
* - plan name: the URL {plan} segment, resolved to the rendered
* `<span class="fst-italic">the</span> Beaumont` h-title.
*
* Honest nulls (genuinely absent from the page, NOT fabricated):
* - geo lat/lon: the page renders a Google map by mapId only — no per-home
* latitude/longitude anywhere in the HTML -> null.
* - constructionStatus: the first-party dataLayer publishes pageType
* (currently "Move In Ready"). Unknown/absent values remain null.
* - estCompletionDate: no per-home completion date on the page -> null.
* - stories: not published as a discrete field on the per-home page -> null.
*
* Facts-only: images OMITTED (imgix photos exist but v1 stays images-off /
* mediaRights=NONE per HomesOnSpec policy). Plain HTTP; one detail page ==
* one inventory_home.
*
* Batch control: HARRISDOYLE_PAGE_LIMIT caps per-home pages/run (default 40).
* Optional HARRISDOYLE_STATE filters the sitemap URL list by the first
* /new-homes/{state} slug (comma-list, e.g. "alabama,florida").
*/
const BUILDER_SLUG = "harris-doyle";
const ORIGIN = "https://www.harrisdoyle.com";
const SITEMAP = `${ORIGIN}/sitemap.xml`;
const STATE_FILTER = (process.env.HARRISDOYLE_STATE ?? "")
.toLowerCase()
.split(",")
.map((s) => s.trim())
.filter(Boolean);
const PAGE_LIMIT = Number(process.env.HARRISDOYLE_PAGE_LIMIT ?? "40");
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN -> null.
const posInt = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
// A positive decimal (baths may be 4.5), or null.
const posDec = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, "")
// decode the numeric/hex entities harrisdoyle.com uses for spaces + punct.
.replace(/&#x([0-9a-f]+);/gi, (_m, h) => String.fromCodePoint(parseInt(h, 16)))
.replace(/&#(\d+);/g, (_m, d) => String.fromCodePoint(parseInt(d, 10)))
.replace(/'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
// Slug ("beaumont-pinnacle", "the-foothills-at-blackridge") -> Title Case
// ("Beaumont Pinnacle", "The Foothills At Blackridge"). Deterministic; used
// only as a fallback when the rendered name isn't recoverable.
const SLUG_LOWER = new Set(["at", "of", "in", "the", "and", "on", "by", "for", "a", "an"]);
const titleFromSlug = (slug: string | null): string | null => {
if (!slug) return null;
const words = slug.split("-").filter(Boolean);
const s = words
.map((w, i) =>
// Keep small joining words lowercase (never at the start) so the slug
// fallback reads "The Foothills at Blackridge", not "…At…".
i > 0 && SLUG_LOWER.has(w.toLowerCase())
? w.toLowerCase()
: w.charAt(0).toUpperCase() + w.slice(1),
)
.join(" ")
.trim();
return s || null;
};
interface HarrisDoyleHome {
url: string;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
community: string | null;
planName: string | null;
lotNumber: string | null;
price: number | null;
beds: number | null;
bathsTotal: number | null; // decimal, half-baths folded (site publishes a single "4.5")
sqft: number | null;
stories: number | null; // not published per-home -> null (honest)
garages: number | null;
lat: number | null; // no per-home geo on the page -> null (honest)
lon: number | null;
phone: string | null;
status: "PLANNED" | "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
builderInventoryId: string | null;
}
/** Pull every parsed JSON-LD object out of the page (flattening arrays + @graph). */
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const top = Array.isArray(data) ? data : [data];
for (const it of top) {
if (it && typeof it === "object") {
const g = (it as Record<string, unknown>)["@graph"];
if (Array.isArray(g)) {
for (const node of g) if (node && typeof node === "object") out.push(node as Record<string, unknown>);
} else {
out.push(it as Record<string, unknown>);
}
}
}
}
return out;
}
function hasType(obj: Record<string, unknown>, token: string): boolean {
const t = obj["@type"];
const types = Array.isArray(t) ? t : [t];
return types.some((x) => x === token);
}
/**
* Harris Doyle pushes a Google-Tag `dataLayer` object on every page carrying the
* builder's OWN authoritative, correctly-cased fields — e.g.
* "pageType":"Move In Ready","city":"Hoover","state":"AL",
* "community":"113 | The Foothills at Blackridge","model":"854 | Beaumont"
* Prefer it over DOM scraping: the hero community anchor is entity-encoded
* (`/` not `/`) so a slash-based regex is dead, and the URL slug mis-cases
* prepositions. Returns honest nulls when the block is absent. (TK-10487 Cody catch)
*/
function parseDataLayer(html: string): {
community: string | null;
city: string | null;
state: string | null;
pageType: string | null;
} {
const pick = (key: string): string | null => {
const m = html.match(new RegExp(`"${key}"\\s*:\\s*"([^"]*)"`, "i"));
return m ? clean(m[1] ?? null) : null;
};
const rawCommunity = pick("community"); // "113 | The Foothills at Blackridge"
const community = rawCommunity ? clean(rawCommunity.split("|").pop() ?? rawCommunity) : null;
return { community, city: pick("city"), state: normalizeStateCode(pick("state")), pageType: pick("pageType") };
}
/** Map the dataLayer pageType label to the schema constructionStatus enum. */
function statusFromPageType(pageType: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
if (!pageType) return null;
const p = pageType.toLowerCase();
if (p.includes("move in ready") || p.includes("move-in ready") || p.includes("quick move")) return "MOVE_IN_READY";
if (p.includes("under construction")) return "UNDER_CONSTRUCTION";
if (p.includes("presale") || p.includes("coming soon") || p.includes("to be built")) return "PLANNED";
return null;
}
// Minimal ZIP3-range guard for Harris Doyle's operating states (AL, FL). The
// site itself sometimes publishes a wrong ZIP in the hero (e.g. an AL home
// showing FL's 32542); rather than propagate data we've DETECTED is wrong, drop
// the ZIP to an honest null when its ZIP3 can't belong to the parsed state.
// Extensible — add a state's ranges as coverage grows. (TK-10487 Cody catch)
const STATE_ZIP3: Record<string, [number, number][]> = {
AL: [[350, 369]],
FL: [[320, 349]],
};
function zipConsistentWithState(zip: string | null, state: string | null): boolean {
if (!zip || !state) return true; // nothing to contradict
const ranges = STATE_ZIP3[state.toUpperCase()];
if (!ranges) return true; // no table for this state -> can't disprove, keep it
const z3 = Number(zip.slice(0, 3));
return ranges.some(([lo, hi]) => z3 >= lo && z3 <= hi);
}
/**
* Parse ONE Harris Doyle per-home page into a home, or null if not a resolvable
* per-home detail page. Anchors on the FIRST rendered instance of each
* primary-hero element (sibling "Quick Move-Ins" cards reuse the same classes
* and always render AFTER the hero).
*/
export function parseHome(html: string, pageUrl: string): HarrisDoyleHome | null {
// ---- street address: the primary <h1 page-title> ----
const street = clean(html.match(/class="page-section__page-title[^"]*"[^>]*>([\s\S]*?)<\/h1>/i)?.[1] ?? null);
if (!street) return null; // not a resolvable per-home listing
// ---- authoritative fields from the builder's own dataLayer (preferred) ----
const dl = parseDataLayer(html);
// ---- location line under the h1: "<community link> | City, ST ZIP | Lot: N" ----
// City/ST/ZIP: the classic "City, ST 35244" token anywhere in the primary block.
const noComment = html.replace(/<!--[\s\S]*?-->/g, "");
const locMatch = noComment.match(/>\s*([A-Za-z][A-Za-z .'-]+?),\s*([A-Z]{2})\s*(\d{5})\s*</);
// Prefer the dataLayer city/state (authoritative, correctly-cased); DOM fallback.
const city = dl.city ?? clean(locMatch?.[1] ?? null);
const state = dl.state ?? normalizeStateCode(clean(locMatch?.[2] ?? null));
const zipRaw = (clean(locMatch?.[3] ?? null) ?? "").match(/\d{5}/)?.[0] ?? null;
// Drop a ZIP the site published that can't belong to the parsed state (honest
// null over known-wrong data) — e.g. an AL home showing FL's 32542.
const zip = zipConsistentWithState(zipRaw, state) ? zipRaw : null;
// ---- lot number: "Lot: N" (entity-encoded colon), first occurrence ----
const lotNumber = clean(noComment.match(/Lot(?::|:)\s*([0-9A-Za-z-]+)/i)?.[1] ?? null);
// ---- price: the primary "Priced at " label -> the fs-4 dollar span ----
// The hero renders it as: Priced at </span> <span class='fs-4'>$759,021</span>
const priceStr =
noComment.match(/Priced at[\s\S]{0,120}?<span class='fs-4'>\s*\$?([\d,]+)/i)?.[1] ??
noComment.match(/Priced at[\s\S]{0,120}?\$\s*([\d,]+)/i)?.[1] ??
null;
const price = posInt(priceStr);
// ---- primary specs: the FIRST <ul class="listing-card__specs"> ----
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
let garages: number | null = null;
// NB: the SSR HTML is inconsistently minified — some pages render
// `<ul class="...">` (spaces) and others `<ul\nclass="...">` (newlines), so
// every open-tag matcher here uses \s+ between the tag and its attributes.
const specsUl = noComment.match(/<ul\s+class="listing-card__specs[^"]*"[^>]*>([\s\S]*?)<\/ul>/i)?.[1] ?? null;
if (specsUl) {
// Each spec renders as: <value-tag class="listing-card__specs-value" ...>VALUE</value-tag>
// <span class="listing-card__specs-label ...">LABEL</span>. The value tag is a
// <span> for beds/sqft/garages but an <abbr> for baths ("4.5" with a
// title="4 Baths + 1 Half Bath"), and the value carries surrounding whitespace
// — so match ANY value tag + close tag, label-anchored.
const item = (label: string): string | null =>
specsUl.match(
new RegExp(
`listing-card__specs-value"[^>]*>\\s*([\\d.,]+)\\s*<\\/[a-z]+>\\s*<span class="listing-card__specs-label[^"]*">\\s*${label}`,
"i",
),
)?.[1] ?? null;
beds = posInt(item("Beds?"));
bathsTotal = posDec(item("Baths?"));
sqft = posInt(item("Sq\\.\\s*Ft\\.?"));
garages = posInt(item("Garages?"));
}
// ---- builderInventoryId: the FIRST inventory.cfm?id=N (the primary home) ----
const builderInventoryId = clean(html.match(/inventory\.cfm\?id=(\d+)/i)?.[1] ?? null);
// ---- community + plan: derived from the 6-seg URL, resolved to rendered text ----
const parts = pageUrl.split("/new-homes/")[1]?.split(/[?#]/)[0]?.split("/") ?? [];
const communitySlug = parts.length >= 6 ? parts[3]! : null;
const planSlug = parts.length >= 6 ? parts[4]! : null;
// Community name: the dataLayer carries it correctly-cased ("The Foothills at
// Blackridge"); the slug fallback only fires when the dataLayer is absent.
const community = dl.community ?? titleFromSlug(communitySlug);
// rendered plan name = "<span ...fst-italic>the</span> Beaumont" h-title.
const planRendered = clean(
noComment.match(/fst-italic">\s*the\s*<\/span>\s*([A-Za-z][A-Za-z0-9 &.'-]*?)\s*<\/h/i)?.[1] ?? null,
);
const planName = planRendered ?? titleFromSlug(planSlug);
// ---- sales phone: first tel: link in the "We Can Help" consultant card ----
const phone = clean(html.match(/href="tel:(\+?[\d]+)"/i)?.[1] ?? null);
// Status from the builder's first-party dataLayer pageType (honest null when
// absent — never inferred from page placement or URL shape).
const status: HarrisDoyleHome["status"] = statusFromPageType(dl.pageType);
return {
url: pageUrl,
street,
city,
state,
zip,
community,
planName,
lotNumber,
price,
beds,
bathsTotal,
sqft,
stories: null, // not published per-home (honest null)
garages,
lat: null, // no per-home geo on the page (honest null)
lon: null,
phone,
status,
builderInventoryId,
};
}
/**
* A per-home inventory DETAIL URL: /new-homes/{state}/{metro}/{city}/{community}/{plan}/{address}
* — exactly SIX path segments after /new-homes/, with a real final segment.
* Rejects the 4-seg community listing pages and 5-seg floor-plan pages (which
* render many sibling homes), and any /undefined/ partial.
*/
function isDetailUrl(u: string): boolean {
const tail = u.split("/new-homes/")[1];
if (!tail) return false;
const segs = tail.replace(/\/$/, "").split(/[?#]/)[0]!.split("/");
if (segs.length !== 6) return false;
return segs.every((s) => !!s && s !== "undefined");
}
export const harrisDoyleAdapter: SourceAdapter = {
key: "harris-doyle-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const index = await fetcher.fetch(SITEMAP);
let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
.map((m) => m[1]!)
.filter(isDetailUrl);
if (STATE_FILTER.length) {
homeUrls = homeUrls.filter((u) => {
const st = u.split("/new-homes/")[1]?.split("/")[0]?.toLowerCase();
return st ? STATE_FILTER.includes(st) : false;
});
}
for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
const cname = h.community ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv<number>(null, null, page.url), // no per-home geo on the page
lon: fv<number>(null, null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(h.phone, h.phone, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
// The REAL street address (never a "Lot X" fallback) — the primary
// <h1 page-title>. `?? h.url` only fires if street is null, but
// parseHome already returned null in that case, so this is the street.
address: h.street ?? h.url,
builderInventoryId: h.builderInventoryId ?? h.url,
planName: h.planName ?? undefined,
lotNumber: h.lotNumber ?? undefined,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price}` : null,
page.url,
h.price != null ? `Priced at $${h.price}` : null,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv(h.stories, h.stories != null ? String(h.stories) : null, page.url),
garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url),
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Harris Doyle single-family inventory home"),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
pageTypeEvidence(html),
page.url,
pageTypeEvidence(html),
),
estCompletionDate: fv<string>(null, null, page.url), // no per-home completion date on the page
lotNumber: fv(h.lotNumber, h.lotNumber, page.url),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv<number>(null, null, page.url), // honest null: no per-home geo on the page
lon: fv<number>(null, null, page.url),
planName: fv(h.planName, h.planName, page.url),
// Facts-only: images intentionally omitted (mediaRights=NONE).
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};
function pageTypeEvidence(html: string): string | null {
return clean(html.match(/["']pageType["']\s*:\s*["']([^"']+)["']/i)?.[1] ?? null);
}