← back to Homesonspec
collectors/holt-homes/src/index.ts
359 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Holt Homes adapter — Pacific-Northwest builder (OREGON + WASHINGTON),
* recon + build 2026-07-28. holthomes.com is WordPress (WP REST + Yoast),
* but the inventory custom-post-types (`homes`, `communities`, `floorplans`)
* are NOT exposed via the public WP REST API (show_in_rest is off — /wp-json/wp/v2/
* lists only core types) and the /available-homes listing is a FacetWP grid.
*
* Bulk source = the Yoast `homes-sitemap.xml`, which lists every per-home page
* under /available-homes/<address-slug>/. Each home page is FULLY server-rendered
* — the facts live in the page's `.info-box` markup, NOT in JSON-LD (Holt's
* ld+json is just Yoast RealEstateListing chrome with no price/beds). We parse:
*
* <h2 class="price">$454,960</h2>
* <p class="address"><i.../> 121 Valemont Dr Eagle Point, OR 97524</p>
* <div class="stats"><div><strong>4</strong><span>Beds</span></div>
* <div><strong>3</strong><span>Baths</span></div>
* <div><strong>1890</strong><span>Sq. Ft.</span></div></div>
* <span class="status"><i.../> Move In Ready</span> (or "Under Construction")
* <div class="location-info" data-map-info='{"position":{"lat":..,"lng":..}}'></div>
* <a href=".../communities/quail-run/" class="community-link">Read more about Quail Run</a>
* <title>121 Valemont Dr | Quail Run 34 - Holt Homes</title> (community fallback)
*
* lat/lng are the per-home map pin (US longitude is negative — sign preserved).
* Community name comes from the `community-link` (canonical slug + name), falling
* back to the title's " | <Community> <homeId>" segment.
*
* Plain HTTP GET; one page == one inventory_home. Facts-only — images are
* intentionally dropped (mediaRights=NONE). robots.txt allows everything for our
* UA (only /wp/wp-admin/ is disallowed; the Yoast block has an empty Disallow =
* allow-all). No Turnstile/CAPTCHA, no login, no browser required.
*
* Batch control: HOLT_PAGE_LIMIT (per-home pages, default 10)
*/
const SITEMAP = "https://holthomes.com/homes-sitemap.xml";
const BUILDER_SLUG = "holt-homes";
const PAGE_LIMIT = Number(process.env.HOLT_PAGE_LIMIT ?? 10);
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths can legitimately read 0-ish), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
// A signed decimal — geo lat/lon can be negative, so the minus MUST survive.
const coord = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.\-]/g, "")) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
// strip tags + collapse whitespace + decode a couple of common entities
const text = (html: string): string =>
html
.replace(/<[^>]+>/g, " ")
.replace(/&/g, "&")
.replace(/�?39;|'/g, "'")
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
interface ParsedHome {
url: string;
price: number | null;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
beds: number | null;
bathsTotal: number | null;
sqft: number | null;
lat: number | null;
lon: number | null;
community: string | null;
status: string | null; // raw status text ("Move In Ready" / "Under Construction")
constructionStatus: "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
}
/** Holt status text → our construction-status enum. Unknown/blank → null (never guessed). */
function constructionStatus(raw: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null {
const s = (raw ?? "").toLowerCase();
if (!s) return null;
if (s.includes("move in ready") || s.includes("move-in ready") || s.includes("ready")) return "MOVE_IN_READY";
if (s.includes("under construction") || s.includes("construction") || s.includes("coming soon")) {
return "UNDER_CONSTRUCTION";
}
return null;
}
/**
* Split the raw address markup into parts. Holt renders the address as
* <i class="fa..."></i> 121 Valemont Dr Eagle Point, OR 97524
* where a DOUBLE-space (2+ whitespace) reliably separates the street from the
* (possibly multi-word) city, and the comma anchors "<ST> <ZIP>". We therefore
* split on the whitespace-run BEFORE collapsing, so "Eagle Point" / "Bonney
* Lake" / "Happy Valley" survive intact. If a page ever lacks the gap we keep
* the whole pre-comma text as `street` and leave `city` null — never guessed.
*
* `rawInner` is the raw <p class="address"> inner HTML (tags still present).
*/
function parseAddressLine(rawInner: string | null): {
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
} {
if (rawInner == null) return { street: null, city: null, state: null, zip: null };
// Drop the leading location-dot <i> (and any other tags) but PRESERVE the
// multi-space street/city gap; only decode a couple of entities.
const s = rawInner
.replace(/<[^>]+>/g, "")
.replace(/&/g, "&")
.replace(/�?39;|'/g, "'")
.replace(/ /g, " ")
.replace(/^\s+|\s+$/g, "");
if (!s) return { street: null, city: null, state: null, zip: null };
// "<pre>, <ST> <ZIP>" — split the state/zip off first.
const m = s.match(/^([\s\S]*?),\s*([A-Za-z]{2})\s+(\d{5})(?:-\d{4})?\s*$/);
if (!m) return { street: str(s.replace(/\s+/g, " ")), city: null, state: null, zip: null };
const pre = m[1]!.replace(/^\s+|\s+$/g, "");
const state = m[2]!.toUpperCase();
const zip = m[3]!;
// The DOUBLE-space (or tab) between street and city is the delimiter.
const gap = pre.match(/^([\s\S]*?\S)\s{2,}(\S[\s\S]*)$/);
let street = pre.replace(/\s+/g, " ");
let city: string | null = null;
if (gap) {
street = gap[1]!.replace(/\s+/g, " ").trim();
city = gap[2]!.replace(/\s+/g, " ").trim();
}
return { street: str(street), city: str(city), state, zip };
}
/** parse ONE Holt Homes per-home page into a home, or null if it's not a home page */
export function parseHome(html: string, pageUrl: string): ParsedHome | null {
// The listing index page (/available-homes/) has no .info-box detail block.
const infoIdx = html.indexOf('class="info-box"');
if (infoIdx < 0) return null;
// price
const priceM = html.match(/<h2\s+class="price">([^<]+)<\/h2>/i);
const price = posNum(priceM?.[1] ?? null);
// address — pass the RAW inner HTML so parseAddressLine can use the double-space
// street/city delimiter (text() would collapse it away).
const addrM = html.match(/<p\s+class="address">([\s\S]*?)<\/p>/i);
const { street, city, state, zip } = parseAddressLine(addrM ? addrM[1]! : null);
// stats: <strong>N</strong><span>Beds|Baths|Sq. Ft.</span> pairs
const statsM = html.match(/<div\s+class="stats">([\s\S]*?)<\/div>\s*<a/i);
let beds: number | null = null;
let bathsTotal: number | null = null;
let sqft: number | null = null;
if (statsM) {
for (const p of statsM[1]!.matchAll(/<strong>([^<]*)<\/strong>\s*<span>([^<]*)<\/span>/gi)) {
const val = p[1]!.trim();
const label = p[2]!.toLowerCase();
if (label.includes("bed")) beds = posNum(val);
else if (label.includes("bath")) bathsTotal = nonNegNum(val);
else if (label.includes("sq")) sqft = posNum(val);
}
}
// status
const statusM = html.match(/<span\s+class="status">([\s\S]*?)<\/span>/i);
const statusRaw = statusM ? text(statusM[1]!) : null;
// lat/lng from data-map-info='{"position":{"lat":..,"lng":..}}'
let lat: number | null = null;
let lon: number | null = null;
const mapM = html.match(/data-map-info='([^']+)'/i) ?? html.match(/data-map-info="([^"]+)"/i);
if (mapM) {
try {
const info = JSON.parse(mapM[1]!) as { position?: { lat?: unknown; lng?: unknown } };
lat = coord(info.position?.lat);
lon = coord(info.position?.lng);
} catch {
/* leave null */
}
}
// community — prefer the canonical community-link, fall back to the title
let community: string | null = null;
const linkM = html.match(/class="community-link"[^>]*>([\s\S]*?)<\/a>/i);
if (linkM) {
const t = text(linkM[1]!);
community = str(t.replace(/^read more about\s*/i, "").trim());
}
if (!community) {
const titleM = html.match(/<title>([^<]+)<\/title>/i);
const titleTxt = titleM ? text(titleM[1]!) : "";
// "121 Valemont Dr | Quail Run 34 - Holt Homes" → "Quail Run 34" → "Quail Run"
const core = titleTxt.replace(/\s*-\s*Holt Homes\s*$/i, "");
const bar = core.split("|");
if (bar.length > 1) community = str(bar[1]!.replace(/\s+\d+$/, "").trim());
}
return {
url: pageUrl,
price,
street,
city,
state,
zip,
beds,
bathsTotal,
sqft,
lat,
lon,
community,
status: statusRaw,
constructionStatus: constructionStatus(statusRaw),
};
}
export const holtHomesAdapter: SourceAdapter = {
key: "holt-homes-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let sitemap: RawPage;
try {
sitemap = await fetcher.fetch(SITEMAP);
} catch (error) {
console.warn(` holt sitemap ${SITEMAP}: ${error instanceof Error ? error.message : String(error)}`);
return; // no sitemap → nothing to collect (source marked degraded upstream)
}
// Every <loc> under /available-homes/<slug>/ that is NOT the bare index page.
const homeUrls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
.map((m) => m[1]!)
.filter((u) => /\/available-homes\/[^/]+\/?$/.test(u) && !/\/available-homes\/?$/.test(u));
for (const url of homeUrls.slice(0, Math.max(1, PAGE_LIMIT))) {
try {
yield await fetcher.fetch(url);
} catch (error) {
// 403/429 → BlockedError stops the whole run (bot protection is never
// circumvented); a single-page 404/timeout just skips that home.
console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const h = parseHome(html, page.url);
if (!h) return { records: [], errors: [] }; // sitemap/index page or non-home
if (!h.street) {
return { records: [], errors: [{ url: page.url, reason: "home page missing address — skipped" }] };
}
const state = normalizeStateCode(h.state);
const community = h.community;
const records: ExtractedRecord[] = [];
// The publisher requires an inventory home to hang off a community (FK).
// A home the page leaves community-less can't be published — skip + log it
// honestly rather than stage a record that will crash at publish.
if (!community) {
return {
records: [],
errors: [{ url: page.url, reason: `home ${h.street} has no community — cannot attach to a community, skipped` }],
};
}
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, page.url, "community-link text"),
street: fv<string>(null, null, page.url),
city: fv(h.city, h.city, page.url),
state: fv(state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address: h.street,
builderInventoryId: h.url, // no numeric job id in the page — the URL is the stable per-home key
lat: h.lat ?? undefined,
lon: h.lon ?? undefined,
planName: undefined,
},
fields: {
street: fv(h.street, h.street, page.url, "info-box address"),
city: fv(h.city, h.city, page.url),
state: fv(state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(h.price, h.price != null ? `$${h.price}` : null, page.url, h.price != null ? `price $${h.price}` : null),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
stories: fv<number>(null, null, page.url),
garageSpaces: fv<number>(null, null, page.url),
homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Holt Homes available home"),
constructionStatus: fv(h.constructionStatus, h.status, page.url, h.constructionStatus === null ? null : `status: ${h.status}`),
estCompletionDate: fv<string>(null, null, page.url),
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv(h.url, h.url, page.url),
lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url, h.lat === null ? null : "data-map-info position"),
lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url, h.lon === null ? null : "data-map-info position"),
planName: fv<string>(null, null, page.url),
// facts-only: images exist on the page but are intentionally dropped.
images: fv<string[]>([], null, page.url),
},
});
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};