← back to Homesonspec
collectors/highland-homes/src/index.ts
426 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Highland Homes adapter — SERVER-RENDERED + EMBEDDED_JSON source (recon 2026-07-28).
*
* highlandhomes.com is a Vue.js storefront over a Yii/PHP backend. The
* for-sale landing page
*
* GET https://www.highlandhomes.com/homes-for-sale
*
* server-embeds the full community list as an inline JS array —
*
* var communities = [ { objectID, name, url, _geoloc:{lat,lng}, city, zip,
* region, status, quickMoveIn, lowPrice, highPrice … } ]
*
* (111 communities, all with lat/lng + city + zip; ~101 carry quick-move-in
* inventory). Per-HOME facts do NOT live on the landing page; each community's
* page (community.url, e.g. /austin/kyle/6-creeks-at-waterridge) SERVER-RENDERS
* one <a class="home-card"> per available spec home, carrying:
* data-price / .home-price ($), .home-plan, .home-address ("street, city, ST"),
* .home-bedrooms, .home-baths, .home-garages, .home-stories,
* .home-squarefootage, a status tag ("Complete & Move-in Ready!" /
* "Est. Completion - Nov '26"), href (detail URL) and data-algolia-object-id.
*
* There is also an Algolia index (app KOMTD97D6N, index "highland", type:"5home"
* = 904 homes) queryable with the public search key, but those records are
* SEARCH-LEAN (address + community + url only — no price/beds/baths/sqft), so
* the community-page HTML cards are the authoritative facts source and this
* adapter prefers them.
*
* Strategy: fetch the landing page → parse the `communities` blob → fetch each
* community page that has inventory (up to HIGHLAND_PAGE_LIMIT) → parse its
* home cards. Community lat/lng/zip (only on the landing blob) are injected into
* each community page's bytes as a `<!-- highland-community: {json} -->` comment
* so extract() stays a pure function of the RawPage it receives.
*
* robots.txt (highlandhomes.com) is `User-agent: *` with no Disallow → allow /.
* Facts-only: featuredImage/photos exist in the feed and are dropped.
*
* Batch control: HIGHLAND_PAGE_LIMIT (community pages to crawl, default 10).
*/
const ORIGIN = "https://www.highlandhomes.com";
const LANDING_URL = `${ORIGIN}/homes-for-sale`;
const BUILDER_SLUG = "highland-homes";
const PAGE_LIMIT = Number(process.env.HIGHLAND_PAGE_LIMIT ?? 10);
const META_MARKER = "highland-community:";
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
interface Community {
objectID?: string;
name?: string;
url?: string;
_geoloc?: { lat?: string | number; lng?: string | number };
city?: string;
zip?: string | number;
region?: string;
status?: string;
quickMoveIn?: number;
}
/** The community metadata injected into a community-page RawPage so extract()
* can attach lat/lng/zip that only exist on the landing-page blob. */
interface InjectedMeta {
name: string | null;
city: string | null;
zip: string | null;
lat: number | null;
lon: number | null;
region: string | null;
}
/** Preserved signs: US lon is negative; a stringified "-97.9…" must stay negative. */
function toCoord(v: unknown): number | null {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
}
/** Scan the balanced `var communities = [ … ]` array literal out of the landing
* page HTML (string/escape aware) and JSON.parse it. Returns [] if absent. */
export function parseCommunitiesBlob(html: string): Community[] {
const needle = "var communities = ";
const at = html.indexOf(needle);
if (at < 0) return [];
const start = at + needle.length;
if (html[start] !== "[") return [];
let depth = 0;
let inStr = false;
let esc = false;
let end = -1;
for (let i = start; i < html.length; i++) {
const c = html[i]!;
if (esc) {
esc = false;
} else if (c === "\\") {
esc = true;
} else if (c === '"') {
inStr = !inStr;
} else if (!inStr) {
if (c === "[") depth++;
else if (c === "]") {
depth--;
if (depth === 0) {
end = i;
break;
}
}
}
}
if (end < 0) return [];
try {
const arr = JSON.parse(html.slice(start, end + 1));
return Array.isArray(arr) ? (arr as Community[]) : [];
} catch {
return [];
}
}
function communityMeta(c: Community): InjectedMeta {
return {
name: str(c.name),
city: str(c.city),
zip: zip5(c.zip),
lat: toCoord(c._geoloc?.lat),
lon: toCoord(c._geoloc?.lng),
region: str(c.region),
};
}
/** Absolute community-page URL from a `communities[].url` (which is relative). */
function communityPageUrl(relUrl: string): string {
if (/^https?:\/\//.test(relUrl)) return relUrl;
return `${ORIGIN}${relUrl.startsWith("/") ? "" : "/"}${relUrl}`;
}
/** Read the injected `<!-- highland-community: {json} -->` comment, if any. */
function readInjectedMeta(html: string): InjectedMeta | null {
const m = html.match(/<!--\s*highland-community:\s*(\{[\s\S]*?\})\s*-->/);
if (!m) return null;
try {
return JSON.parse(m[1]!) as InjectedMeta;
} catch {
return null;
}
}
interface HomeCard {
href: string | null;
objectID: string | null;
price: number | null;
priceRaw: string | null;
plan: string | null;
address: string | null; // "244 Basket Flower Loop, Kyle, TX"
beds: number | null;
baths: number | null;
garages: number | null;
stories: number | null;
sqft: number | null;
statusTag: string | null;
}
function pick(re: RegExp, block: string): string | null {
const m = block.match(re);
return m ? str(m[1]) : null;
}
/** Parse every <a class="home-card"> … </a> block out of a community page. */
export function parseHomeCards(html: string): HomeCard[] {
const cards: HomeCard[] = [];
for (const m of html.matchAll(/<a class="home-card[^>]*>([\s\S]*?)<\/a>/g)) {
const outer = m[0];
const priceRaw = pick(/data-price="([^"]+)"/, outer) ?? pick(/home-price">\$?([0-9,]+)/, outer);
cards.push({
href: pick(/href="([^"]+)"/, outer),
objectID: pick(/data-algolia-object-id="([^"]+)"/, outer),
price: posNum(priceRaw),
priceRaw,
plan: pick(/home-plan">([^<]+)</, outer),
address: pick(/home-address">([^<]+)</, outer),
beds: posNum(pick(/home-bedrooms[^>]*>([0-9.]+)/, outer)),
baths: nonNegNum(pick(/home-baths[^>]*>([0-9.]+)/, outer)),
garages: nonNegNum(pick(/home-garages[^>]*>([0-9.]+)/, outer)),
stories: posNum(pick(/home-stories[^>]*>([0-9.]+)/, outer)),
sqft: posNum(pick(/home-squarefootage[^>]*>([0-9,]+)/, outer)),
statusTag: pick(/home-tag[^>]*>([^<]+)</, outer),
});
}
return cards;
}
/** "244 Basket Flower Loop, Kyle, TX" → { street, city, state }. */
function parseAddress(addr: string | null): { street: string | null; city: string | null; state: string | null } {
if (!addr) return { street: null, city: null, state: null };
const parts = addr.split(",").map((p) => p.trim()).filter(Boolean);
if (parts.length >= 3) {
const state = normalizeStateCode(parts[parts.length - 1] ?? null);
const city = str(parts[parts.length - 2]);
const street = str(parts.slice(0, parts.length - 2).join(", "));
return { street, city, state };
}
if (parts.length === 2) {
return { street: str(parts[0]), city: str(parts[1]), state: null };
}
return { street: str(parts[0] ?? addr), city: null, state: null };
}
/** Highland status tag → our construction-status enum. "Complete & Move-in
* Ready!" = finished; "Est. Completion - …" / "Under Construction" = in
* progress. Unrecognized/blank → null (never guessed). */
function constructionStatus(tag: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = (str(tag) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("move-in ready") || s.includes("move in ready") || s.includes("complete")) return "MOVE_IN_READY";
if (s.includes("est. completion") || s.includes("under construction") || s.includes("coming")) {
return "UNDER_CONSTRUCTION";
}
return null;
}
export const highlandHomesAdapter: SourceAdapter = {
key: "highland-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
let landing: RawPage;
try {
landing = await fetcher.fetch(LANDING_URL);
} catch (error) {
console.warn(` highland landing: ${error instanceof Error ? error.message : String(error)}`);
return; // a block/403 on the landing page stops collection (source degraded)
}
// Yield the landing page — extract() emits every community from its blob.
yield landing;
const communities = parseCommunitiesBlob(landing.body.toString("utf8"))
.filter((c) => str(c.url) && (c.quickMoveIn ?? 0) > 0); // only crawl communities with inventory
for (const community of communities.slice(0, Math.max(0, PAGE_LIMIT))) {
const url = communityPageUrl(String(community.url));
let page: RawPage;
try {
page = await fetcher.fetch(url);
} catch (error) {
console.warn(` highland community ${url}: ${error instanceof Error ? error.message : String(error)}`);
continue; // one blocked/failed community page doesn't stop the rest
}
// Inject the community's geo/zip (only on the landing blob) so extract()
// stays pure over the bytes it receives.
const meta = JSON.stringify(communityMeta(community));
const injected = `<!-- ${META_MARKER} ${meta} -->\n${page.body.toString("utf8")}`;
yield { ...page, body: Buffer.from(injected, "utf8") };
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
// ---- Landing page: emit communities from the embedded blob -----------
if (page.url === LANDING_URL || html.includes("var communities = [")) {
const communities = parseCommunitiesBlob(html);
if (communities.length) {
const records: ExtractedRecord[] = [];
for (const c of communities) {
const name = str(c.name);
if (!name) continue;
const meta = communityMeta(c);
const state = normalizeStateCode(null); // landing blob has no state; city/zip only
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: name },
fields: {
name: fv(name, name, page.url, "landing-page communities blob"),
street: fv<string>(null, null, page.url),
city: fv(meta.city, meta.city, page.url),
state: fv(state, null, page.url),
zip: fv(meta.zip, meta.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv(meta.region, meta.region, page.url),
lat: fv(meta.lat, meta.lat === null ? null : String(meta.lat), page.url, meta.lat === null ? null : "_geoloc.lat"),
lon: fv(meta.lon, meta.lon === null ? null : String(meta.lon), page.url, meta.lon === null ? null : "_geoloc.lng"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
}
return { records, errors: [] };
}
// Landing page but no blob — report honestly.
if (page.url === LANDING_URL) {
return { records: [], errors: [{ url: page.url, reason: "landing page had no `var communities` blob" }] };
}
}
// ---- Community page: emit inventory homes from the card grid ---------
const meta = readInjectedMeta(html);
const cards = parseHomeCards(html);
if (!cards.length) {
return { records: [], errors: [{ url: page.url, reason: "no home-card blocks on community page" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
for (const card of cards) {
const parsed = parseAddress(card.address);
const homeUrl = card.href ? communityPageUrl(card.href) : page.url;
const community = meta?.name ?? null;
// The community name is required — the publisher hangs each home off a
// community FK. Without it we can't attach the home, so skip + log.
if (!community) {
errors.push({ url: homeUrl, reason: `home ${card.objectID ?? card.address ?? "?"} has no community meta — skipped` });
continue;
}
if (!parsed.street) {
errors.push({ url: homeUrl, reason: `home ${card.objectID ?? "?"} missing address — skipped` });
continue;
}
const state = parsed.state ?? normalizeStateCode(null);
const city = parsed.city ?? meta?.city ?? null;
const zip = meta?.zip ?? null;
const cStatus = constructionStatus(card.statusTag);
// Community FIRST — publish creates/refreshes the FK target the home needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, page.url, "injected community meta"),
street: fv<string>(null, null, page.url),
city: fv(meta?.city ?? null, meta?.city ?? null, page.url),
state: fv(state, null, page.url),
zip: fv(zip, zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv(meta?.region ?? null, meta?.region ?? null, page.url),
lat: fv(meta?.lat ?? null, meta?.lat == null ? null : String(meta.lat), page.url, meta?.lat == null ? null : "_geoloc.lat"),
lon: fv(meta?.lon ?? null, meta?.lon == null ? null : String(meta.lon), page.url, meta?.lon == null ? null : "_geoloc.lng"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address: parsed.street,
builderInventoryId: card.objectID ?? undefined,
lat: meta?.lat ?? undefined,
lon: meta?.lon ?? undefined,
planName: card.plan ?? undefined,
},
fields: {
street: fv(parsed.street, parsed.street, homeUrl, "community-page home-address"),
city: fv(city, city, homeUrl),
state: fv(state, parsed.state, homeUrl),
zip: fv(zip, zip, homeUrl),
price: fv(card.price, card.priceRaw, homeUrl, card.price === null ? null : `home card data-price ${card.priceRaw}`),
beds: fv(card.beds, card.beds === null ? null : String(card.beds), homeUrl),
bathsTotal: fv(card.baths, card.baths === null ? null : String(card.baths), homeUrl),
sqft: fv(card.sqft, card.sqft === null ? null : String(card.sqft), homeUrl),
stories: fv(card.stories, card.stories === null ? null : String(card.stories), homeUrl),
garageSpaces: fv(card.garages, card.garages === null ? null : String(card.garages), homeUrl),
homeType: fv("SINGLE_FAMILY" as const, null, homeUrl, "Highland single-family spec home"),
constructionStatus: fv(cStatus, card.statusTag, homeUrl, cStatus === null ? null : `tag: ${card.statusTag}`),
estCompletionDate: fv<string>(null, null, homeUrl),
lotNumber: fv<string>(null, null, homeUrl),
builderInventoryId: fv(card.objectID, card.objectID, homeUrl),
lat: fv(meta?.lat ?? null, meta?.lat == null ? null : String(meta.lat), homeUrl, meta?.lat == null ? null : "community _geoloc.lat"),
lon: fv(meta?.lon ?? null, meta?.lon == null ? null : String(meta.lon), homeUrl, meta?.lon == null ? null : "community _geoloc.lng"),
planName: fv(card.plan, card.plan, homeUrl),
// facts-only: card images exist in the feed but are intentionally dropped.
images: fv<string[]>([], null, homeUrl),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};