← back to Homesonspec
collectors/fischer-homes/src/index.ts
359 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Fischer Homes adapter — JSON-API + detail-page source (recon 2026-07-28).
*
* fischerhomes.com (OH/KY/IN/GA/MO regional builder) is a Cloudflare-fronted SPA
* over a custom Laravel/"region-revamped" API. The move-in-ready inventory is a
* BULK, PAGINATED JSON GET keyed by numeric region id:
*
* GET https://www.fischerhomes.com/api/region-revamped/homes/{regionId}?page=N
* -> { "move-in-ready": { data:[ home… ], current_page, last_page, total } }
*
* The region ids come from a companion GET:
* GET https://www.fischerhomes.com/api/region-dropdown
* -> { regions:[ { id, name, seo_name, state } … ] }
*
* Each list home carries the CUSTOMER-FACING facts:
* { id, name(=plan), formattedAddress, formattedBeds, formattedBaths,
* formattedSqft, formattedFloors, formattedPrice(HTML), url(detail-page) }.
* The list is the truth for beds/baths/sqft/price (its formattedBaths includes
* the half-bath). It does NOT carry lat/lon or the community name.
*
* The per-home DETAIL PAGE (the `url` field) server-renders a schema.org "House"
* JSON-LD block that adds the two missing facts:
* - geo.latitude / geo.longitude (US lon negative — signs preserved), and
* - the community, parsed from the description ("built in <community> located in")
* with a <title> "... | <community> by Fischer Homes" fallback.
* (JSON-LD's numberOfPartialBathrooms is unreliable — it reports 0 when a half-bath
* exists — so the list's formattedBaths is authoritative for baths; JSON-LD is a
* fallback only.)
*
* To keep extract() pure and one-page-per-home, fetch() joins each list home with
* its detail-page facts and yields ONE synthetic combined-JSON RawPage per home.
*
* Verified: robots.txt (fischerhomes.com) disallows only an aggregate path, utm
* query params, .pdf/.swf, y_source= and calendar/create — the region-revamped API
* and find-new-homes/ready-now detail paths we use are all allowed. No auth/cookies
* (200 with the honest bot UA). Facts-only: image urls exist in the feed but are
* intentionally dropped.
*
* Batch control: FISCHER_PAGE_LIMIT (list pages per region, default 10)
* Optional scope: FISCHER_REGIONS (comma ids, e.g. "11,35" — else all regions)
*/
const BASE = "https://www.fischerhomes.com";
const BUILDER_SLUG = "fischer-homes";
const REGION_DROPDOWN = `${BASE}/api/region-dropdown`;
const homesUrl = (regionId: string | number, page: number) =>
`${BASE}/api/region-revamped/homes/${regionId}?page=${page}`;
const PAGE_LIMIT = Number(process.env.FISCHER_PAGE_LIMIT ?? 10);
const REGION_FILTER = (process.env.FISCHER_REGIONS ?? "")
.split(",")
.map((s) => s.trim())
.filter(Boolean);
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// Positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
const zip5 = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/\b(\d{5})\b/);
return m ? m[1]! : null;
};
/**
* Fischer's list `formattedBaths` uses "3½", "2 + ½ + ½", "3" — whole numbers plus
* a "½" glyph per half-bath. Sum the whole numbers and add 0.5 per ½. This is the
* authoritative bath source (JSON-LD's partial count is unreliable). >0 or null.
*/
function parseBaths(v: unknown): number | null {
const t = str(v);
if (!t) return null;
const halves = (t.match(/½/g) || []).length;
const wholes = t.match(/\d+/g);
const whole = wholes ? wholes.reduce((a, b) => a + Number(b), 0) : 0;
const total = whole + halves * 0.5;
return total > 0 ? total : null;
}
/** Pull a plain dollar amount out of the list's formattedPrice HTML span. */
function priceFromFormatted(html: unknown): number | null {
const s = str(html);
if (!s) return null;
const m = s.replace(/<[^>]*>/g, " ").match(/\$?\s*([\d,]{4,})/);
return m ? posNum(m[1]) : null;
}
// ---- detail-page (JSON-LD) facts: lat/lon + community (+ clean address) ----
interface DetailFacts {
lat: number | null;
lon: number | null;
community: string | null;
plan: string | null;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
}
function parseDetail(html: string): DetailFacts {
const out: DetailFacts = { lat: null, lon: null, community: null, plan: null, street: null, city: null, state: null, zip: null };
const m = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i);
if (m) {
try {
const d = JSON.parse(m[1]!) as Record<string, any>;
const addr = (d.address ?? {}) as Record<string, any>;
out.street = str(addr.streetAddress);
out.city = str(addr.addressLocality);
out.state = str(addr.addressRegion);
out.zip = zip5(addr.postalCode);
const lat = Number(d.geo?.latitude ?? d.latitude);
const lon = Number(d.geo?.longitude ?? d.longitude);
out.lat = Number.isFinite(lat) && lat !== 0 ? lat : null;
out.lon = Number.isFinite(lon) && lon !== 0 ? lon : null; // negative US lon preserved
out.plan = str(d.accommodationFloorPlan?.name);
const desc = String(d.description ?? "");
const cm = desc.match(/built in (.+?) located/i);
if (cm) out.community = str(cm[1]);
} catch {
/* fall through to title fallback */
}
}
if (!out.community) {
const tm = html.match(/<title>[^|]*\|\s*(.+?)\s+by Fischer Homes/i);
if (tm) out.community = str(tm[1]);
}
return out;
}
// ---- the synthetic combined page shape yielded by fetch() ----
interface CombinedHome {
__fischer: 1;
id?: string | number;
name?: string; // plan name (list)
formattedAddress?: string;
formattedBeds?: string;
formattedBaths?: string;
formattedSqft?: string;
formattedFloors?: string;
formattedPrice?: string;
url?: string; // detail-page path
region?: string;
detail: DetailFacts;
}
interface DropdownRegion { id: number; seo_name?: string; state?: string }
export const fischerHomesAdapter: SourceAdapter = {
key: "fischer-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// 1) resolve region ids.
let regionIds: string[] = REGION_FILTER;
if (regionIds.length === 0) {
try {
const rp = await fetcher.fetch(REGION_DROPDOWN);
const body = JSON.parse(rp.body.toString("utf8")) as { regions?: DropdownRegion[] };
regionIds = (body.regions ?? []).map((r) => String(r.id)).filter(Boolean);
} catch (error) {
console.warn(` fischer region-dropdown: ${error instanceof Error ? error.message : String(error)}`);
return; // can't discover regions → nothing to collect
}
}
// 2) per region: page the list API, then enrich each home via its detail page.
for (const regionId of regionIds) {
let page = 1;
let lastPage = 1;
let listPagesFetched = 0;
while (page <= lastPage && listPagesFetched < Math.max(1, PAGE_LIMIT)) {
let listPage: RawPage;
try {
listPage = await fetcher.fetch(homesUrl(regionId, page));
} catch (error) {
console.warn(` fischer region ${regionId} page ${page}: ${error instanceof Error ? error.message : String(error)}`);
break; // a block/403 stops this region (source marked degraded upstream)
}
listPagesFetched++;
let section: { data?: any[]; last_page?: number } = {};
try {
section = (JSON.parse(listPage.body.toString("utf8")) as any)?.["move-in-ready"] ?? {};
} catch {
break;
}
lastPage = Number(section.last_page ?? 1);
for (const h of section.data ?? []) {
const detailPath = str(h.url);
let detail: DetailFacts = { lat: null, lon: null, community: null, plan: null, street: null, city: null, state: null, zip: null };
if (detailPath) {
try {
const dp = await fetcher.fetch(BASE + detailPath);
detail = parseDetail(dp.body.toString("utf8"));
} catch (error) {
// Detail unreachable (403/404/net) — keep the list facts; community may
// be missing → extract() will skip-and-log honestly rather than guess.
console.warn(` fischer detail ${detailPath}: ${error instanceof Error ? error.message : String(error)}`);
}
}
const combined: CombinedHome = {
__fischer: 1,
id: h.id,
name: h.name,
formattedAddress: h.formattedAddress,
formattedBeds: h.formattedBeds,
formattedBaths: h.formattedBaths,
formattedSqft: h.formattedSqft,
formattedFloors: h.formattedFloors,
formattedPrice: h.formattedPrice,
url: h.url,
region: regionId,
detail,
};
const bytes = Buffer.from(JSON.stringify(combined), "utf8");
yield {
url: BASE + (detailPath ?? `/api/region-revamped/homes/${regionId}#${h.id}`),
retrievedAt: listPage.retrievedAt,
contentType: "application/json",
body: bytes,
contentHash: listPage.contentHash, // per-home hash source is the synthetic body
};
}
page++;
}
}
},
extract(page: RawPage): ExtractionOutput {
let h: CombinedHome;
try {
h = JSON.parse(page.body.toString("utf8")) as CombinedHome;
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: `unparseable combined page: ${String(error)}` }] };
}
if (!h || h.__fischer !== 1) {
return { records: [], errors: [{ url: page.url, reason: "not a fischer combined-home page" }] };
}
const url = page.url;
const d = h.detail ?? ({} as DetailFacts);
// Address: prefer the JSON-LD structured street; else the list formatted address head.
const listAddrHead = str(h.formattedAddress)?.split(",")[0] ?? null;
const address = d.street ?? listAddrHead;
const city = d.city ?? null;
const state = normalizeStateCode(d.state ?? null);
const zip = d.zip ?? zip5(h.formattedAddress);
const community = str(d.community);
const plan = str(h.name) ?? str(d.plan);
const lat = d.lat ?? null;
const lon = d.lon ?? null;
// Facts (list API authoritative; formattedBaths carries the half-bath).
const price = priceFromFormatted(h.formattedPrice);
const beds = posNum(h.formattedBeds);
const bathsTotal = parseBaths(h.formattedBaths);
const sqft = posNum(h.formattedSqft);
const stories = posNum(h.formattedFloors);
const homeId = str(h.id);
if (!address) {
return { records: [], errors: [{ url, reason: `home ${homeId ?? "?"} missing address — skipped` }] };
}
// The publisher requires an inventory home to hang off a community (FK). A home
// whose detail page yielded no community can't be attached — skip + log honestly
// rather than stage a record that will crash at publish.
if (!community) {
return { records: [], errors: [{ url, reason: `home ${address} has no community (detail page missing/unparsed) — skipped` }] };
}
const records: ExtractedRecord[] = [];
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "detail JSON-LD community"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, d.state, url),
zip: fv(zip, zip, url),
county: fv<string>(null, null, url),
metro: fv<string>(null, null, url),
lat: fv(lat, lat === null ? null : String(lat), url),
lon: fv(lon, lon === null ? null : String(lon), url),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
builderInventoryId: homeId ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, url, "home address"),
city: fv(city, city, url),
state: fv(state, d.state, url),
zip: fv(zip, zip, url),
price: fv(price, price === null ? null : String(price), url, price === null ? null : `formattedPrice ${str(h.formattedPrice)}`),
beds: fv(beds, beds === null ? null : str(h.formattedBeds), url),
bathsTotal: fv(bathsTotal, bathsTotal === null ? null : str(h.formattedBaths), url, bathsTotal === null ? null : `formattedBaths ${str(h.formattedBaths)}`),
sqft: fv(sqft, sqft === null ? null : str(h.formattedSqft), url),
stories: fv(stories, stories === null ? null : str(h.formattedFloors), url),
garageSpaces: fv<number>(null, null, url),
homeType: fv("SINGLE_FAMILY" as const, null, url, "Fischer single-family inventory home"),
constructionStatus: fv("MOVE_IN_READY" as const, null, url, "Fischer move-in-ready (ready-now) inventory"),
estCompletionDate: fv<string>(null, null, url),
lotNumber: fv<string>(null, null, url),
builderInventoryId: fv(homeId, homeId, url),
lat: fv(lat, lat === null ? null : String(lat), url, lat === null ? null : "detail JSON-LD geo"),
lon: fv(lon, lon === null ? null : String(lon), url, lon === null ? null : "detail JSON-LD geo"),
planName: fv(plan, plan, url),
// facts-only: image urls exist in the feed but are intentionally dropped.
images: fv<string[]>([], null, url),
},
});
return { records, errors: [] };
},
};