← back to Homesonspec
collectors/landsea-homes/src/index.ts
317 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Landsea Homes adapter — JSON-API source (recon 2026-07-28).
*
* Landsea rebranded to Risewell (risewellhomes.com). The site is WordPress
* fronting an Algolia index, proxied through a same-origin endpoint so no
* Algolia app-id/key is needed client-side:
*
* GET https://risewellhomes.com/api/algolia/search?query=<BASE64>
* where BASE64 = base64( JSON.stringify({
* "index": "wp_posts_homesites", "query": "", "hitsPerPage": 1000, "page": N }) )
*
* The response is a standard Algolia payload:
* { hits:[ homesite… ], nbHits, nbPages, page, hitsPerPage, … }
*
* PAGING CAVEAT (verified 2026-07-28): the WordPress proxy CAPS hitsPerPage at
* 120 AND *ignores the requested `page`* — every request echoes `page:0` and
* returns the SAME first 120 homesites. So although Algolia's metadata reports
* nbHits≈503 / nbPages≈5, only the first 120 homesites are actually reachable
* through this endpoint. We therefore fetch page 0, and if the server's echoed
* `page` does not match the page we asked for, we STOP (paging is a no-op — pulling
* more would only re-ingest the same 120 homes as duplicates). If Landsea ever
* fixes real paging, the same loop transparently walks the extra pages.
*
* Each hit carries the facts we keep (facts-only — main_image_* URLs are in the
* feed but intentionally DROPPED):
* address:{ address1, city, state("Texas"), zip, county }
* bedrooms, baths, sq_feet_total, stories, cars_spaces, _geoloc:{ lat, lng }
* price, base_price, moveInWindow("1-3 mo."), status("Move-In Ready"),
* neighborhood:{ name } (the community), floorplan:{ name } (the plan),
* region:{ name } (full state name), url, objectID.
*
* Access: the honest bot UA gets HTTP 200 from /api/algolia (verified). The raw
* domain (and /robots.txt) is Cloudflare-challenged (403) on a plain bot fetch;
* LiveFetcher treats a non-2xx robots.txt as "no robots.txt → allow all"
* (fail-open, standard convention), and the /api/algolia path itself answers 200,
* so collection proceeds without circumventing any protection. If Cloudflare ever
* starts challenging the API endpoint too (403), LiveFetcher STOPs (BlockedError)
* and the source is marked degraded upstream — we never bypass a challenge.
*
* Batch control: LANDSEA_PAGE_LIMIT (pages of ~120 homesites, default 10).
*/
const API_BASE = "https://risewellhomes.com/api/algolia/search";
const ALGOLIA_INDEX = "wp_posts_homesites";
const BUILDER_SLUG = "landsea-homes";
const PAGE_LIMIT = Number(process.env.LANDSEA_PAGE_LIMIT ?? 10);
/** Build the base64 `query` param for a given 0-indexed Algolia page. */
function queryUrl(page: number): string {
const payload = { index: ALGOLIA_INDEX, query: "", hitsPerPage: 1000, page };
const b64 = Buffer.from(JSON.stringify(payload), "utf8").toString("base64");
return `${API_BASE}?query=${encodeURIComponent(b64)}`;
}
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages/stories may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
interface Geoloc {
lat?: number;
lng?: number;
}
interface HomesiteAddress {
address1?: string;
city?: string;
state?: string; // full name, e.g. "Texas"
zip?: string | number;
county?: string;
}
interface Named {
id?: number;
name?: string;
}
interface Homesite {
objectID?: string;
name?: string;
status?: string; // "Move-In Ready", "Under Construction", …
moveInWindow?: string; // "1-3 mo.", "Ready Now", …
url?: string;
address?: HomesiteAddress;
neighborhood?: Named; // community
floorplan?: Named; // plan
region?: Named; // full state name
bedrooms?: number;
baths?: number;
sq_feet_total?: number;
stories?: number;
cars_spaces?: number;
price?: number;
base_price?: number;
_geoloc?: Geoloc;
}
interface AlgoliaResponse {
hits?: Homesite[];
nbHits?: number;
nbPages?: number;
page?: number; // the server echoes the page it actually served
hitsPerPage?: number;
}
/** Resolve a 2-letter state code from a homesite, preferring the full region
* name but falling back to address.state. region.name carries marketing regions
* like "Southern California" / "Northern California" that don't normalize; in
* those rows address.state is the real "California", so the fallback recovers
* them instead of dropping the state. Returns null only when neither resolves. */
function resolveState(h: Homesite): { code: string | null; raw: string | null } {
const addr = h.address ?? {};
const regionRaw = str(h.region?.name);
const addrRaw = str(addr.state);
const fromRegion = normalizeStateCode(regionRaw);
if (fromRegion) return { code: fromRegion, raw: regionRaw };
const fromAddr = normalizeStateCode(addrRaw);
if (fromAddr) return { code: fromAddr, raw: addrRaw };
return { code: null, raw: regionRaw ?? addrRaw };
}
function parseResponse(body: string): AlgoliaResponse | null {
try {
const parsed = JSON.parse(body) as AlgoliaResponse;
return parsed && Array.isArray(parsed.hits) ? parsed : null;
} catch {
return null;
}
}
/** { lat, lng } → { lat, lon } (US lon is negative — signs preserved; 0/NaN → null). */
function parseGeoloc(geo: Geoloc | undefined): { lat: number | null; lon: number | null } {
const lat = typeof geo?.lat === "number" ? geo.lat : NaN;
const lon = typeof geo?.lng === "number" ? geo.lng : NaN;
return {
lat: Number.isFinite(lat) && lat !== 0 ? lat : null,
lon: Number.isFinite(lon) && lon !== 0 ? lon : null,
};
}
/** Map Landsea status / moveInWindow → our construction-status enum.
* "Move-In Ready" / "Ready Now" = finished; "Under Construction" = in progress.
* Unrecognized/blank → null (never guessed). */
function constructionStatus(status: unknown, moveIn: unknown): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = `${str(status) ?? ""} ${str(moveIn) ?? ""}`.toLowerCase();
if (!s.trim()) return null;
if (s.includes("move-in ready") || s.includes("move in ready") || s.includes("ready now")) return "MOVE_IN_READY";
if (s.includes("under construction") || s.includes("construction")) return "UNDER_CONSTRUCTION";
return null;
}
export const landseaAdapter: SourceAdapter = {
key: "landsea-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
for (let pageNo = 0; pageNo < Math.max(1, PAGE_LIMIT); pageNo++) {
let page: RawPage;
try {
page = await fetcher.fetch(queryUrl(pageNo));
} catch (error) {
console.warn(` landsea page=${pageNo}: ${error instanceof Error ? error.message : String(error)}`);
return; // a block/403 stops collection (source marked degraded upstream)
}
const resp = parseResponse(page.body.toString("utf8"));
const servedPage = Number(resp?.page ?? 0);
// Paging guard (checked BEFORE yield so the duplicate is never ingested):
// this proxy ignores the requested page and always serves page 0. If we
// asked for page N>0 but the server echoes a different page, paging is a
// no-op — stop without yielding (would re-ingest the same 120 homes).
if (pageNo > 0 && servedPage !== pageNo) return;
yield page;
const got = resp?.hits?.length ?? 0;
const nbPages = Number(resp?.nbPages ?? 0);
// Normal termination: empty page, unparseable body, or last Algolia page.
if (!resp || got === 0 || (nbPages > 0 && pageNo + 1 >= nbPages)) return;
}
},
extract(page: RawPage): ExtractionOutput {
try {
const resp = parseResponse(page.body.toString("utf8"));
if (!resp) {
return { records: [], errors: [{ url: page.url, reason: "no hits array in Algolia response" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
for (const h of resp.hits ?? []) {
const addr = h.address ?? {};
// region.name first, falling back to address.state (handles marketing
// regions like "Southern California" whose real state is address.state).
const { code: state, raw: stateRaw } = resolveState(h);
const city = str(addr.city);
const zip = zip5(addr.zip);
const county = str(addr.county);
const address = str(addr.address1);
const community = str(h.neighborhood?.name);
const plan = str(h.floorplan?.name);
const { lat, lon } = parseGeoloc(h._geoloc);
const price = posNum(h.price);
const beds = posNum(h.bedrooms);
const bathsTotal = nonNegNum(h.baths);
const sqft = posNum(h.sq_feet_total);
const stories = posNum(h.stories);
const garages = nonNegNum(h.cars_spaces);
const homeId = str(h.objectID);
const url = str(h.url) ?? page.url;
const cStatus = constructionStatus(h.status, h.moveInWindow);
if (!address) {
errors.push({ url, reason: `homesite ${homeId ?? "?"} missing address1 — skipped` });
continue;
}
// The publisher requires an inventory home to hang off a community (FK).
// A homesite the feed leaves community-less can't be published — skip it
// and log it honestly rather than stage a record that crashes at publish.
if (!community) {
errors.push({ url, reason: `homesite ${address} has no neighborhood.name — cannot attach to a community, skipped` });
continue;
}
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "Algolia neighborhood.name"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, stateRaw, url),
zip: fv(zip, str(addr.zip), url),
county: fv(county, county, url),
metro: fv<string>(null, null, url),
lat: fv(lat, null, url),
lon: fv(lon, null, url),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
builderInventoryId: homeId ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, url, "Algolia address.address1"),
city: fv(city, city, url),
state: fv(state, stateRaw, url),
zip: fv(zip, str(addr.zip), url),
price: fv(price, price === null ? null : String(h.price), url, price === null ? null : `Algolia price ${h.price}`),
beds: fv(beds, beds === null ? null : String(h.bedrooms), url),
bathsTotal: fv(bathsTotal, bathsTotal === null ? null : String(h.baths), url),
sqft: fv(sqft, sqft === null ? null : String(h.sq_feet_total), url),
stories: fv(stories, stories === null ? null : String(h.stories), url),
garageSpaces: fv(garages, garages === null ? null : String(h.cars_spaces), url),
homeType: fv("SINGLE_FAMILY" as const, null, url, "Landsea single-family homesite"),
constructionStatus: fv(cStatus, str(h.status), url, cStatus === null ? null : `status: ${str(h.status)} / moveIn: ${str(h.moveInWindow)}`),
estCompletionDate: fv<string>(null, null, url),
lotNumber: fv<string>(null, null, url),
builderInventoryId: fv(homeId, homeId, url),
lat: fv(lat, null, url, lat === null ? null : "Algolia _geoloc"),
lon: fv(lon, null, url, lon === null ? null : "Algolia _geoloc"),
planName: fv(plan, plan, url),
// facts-only: main_image_* exist in the feed but are intentionally dropped.
images: fv<string[]>([], null, url),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};