← back to Homesonspec
collectors/pulte/src/index.ts
194 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Pulte adapter — JSON_API source (recon 2026-07-23). Cleanest of the set.
* `GET /api/community/getcommunities?state=…` enumerates communities; each
* community's inventory is `GET /api/plan/qmiplans?communityId=<id>` — a JSON
* array of quick-move-in homes carrying price/beds/baths/sqft/address/status +
* communityLatitude/Longitude + salesPhoneNumber (so homes get real geo).
* Plain HTTP, no anti-bot, no auth.
*/
const API = "https://www.pulte.com";
const BUILDER_SLUG = "pultegroup";
const STATE = process.env.PULTE_STATE ?? "California";
const BRAND = process.env.PULTE_BRAND ?? "Pulte";
const PAGE_LIMIT = Number(process.env.PULTE_PAGE_LIMIT ?? 200);
// Full state name → 2-letter code, so a multi-state drain labels homes with
// their REAL state instead of a hardcoded "CA". stateAbbreviation from the API
// is already 2-letter; this is the fallback when it's missing.
const US_STATE: Record<string, string> = {
california: "CA", texas: "TX", arizona: "AZ", colorado: "CO", florida: "FL",
nevada: "NV", "new mexico": "NM", washington: "WA", oregon: "OR", georgia: "GA",
"north carolina": "NC", "south carolina": "SC", tennessee: "TN", indiana: "IN",
illinois: "IL", minnesota: "MN", michigan: "MI", ohio: "OH", "new york": "NY",
massachusetts: "MA", maryland: "MD", virginia: "VA", pennsylvania: "PA",
"new jersey": "NJ", connecticut: "CT", missouri: "MO", kentucky: "KY",
};
function stateCode(abbr: unknown, fallbackFullName: string): string | null {
const a = typeof abbr === "string" ? abbr.trim() : "";
if (/^[A-Za-z]{2}$/.test(a)) return a.toUpperCase();
return US_STATE[fallbackFullName.toLowerCase()] ?? null;
}
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const num = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// lat/lon may be negative — num()'s >0 guard would wrongly drop western longitudes.
const geo = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(v) : NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
const str = (v: unknown): string | null => {
const s = typeof v === "string" ? v.trim() : null;
return s ? s : null;
};
function titleCase(slug: string): string {
return slug.replace(/-\d+$/, "").split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
}
function statusOf(dateAvailable: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | "PLANNED" {
const s = (dateAvailable ?? "").toLowerCase();
if (/available now|move.?in|ready|complete/.test(s)) return "MOVE_IN_READY";
if (/coming soon|planned/.test(s)) return "PLANNED";
return "UNDER_CONSTRUCTION";
}
export const pulteAdapter: SourceAdapter = {
key: "pultegroup-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const commsPage = await fetcher.fetch(
`${API}/api/community/getcommunities?brand=${encodeURIComponent(BRAND)}&state=${encodeURIComponent(STATE)}®ion=&cityNames=&pageSize=1000000&pageNumber=0`,
);
let comms: Record<string, unknown>[] = [];
try {
const parsed = JSON.parse(commsPage.body.toString("utf8"));
comms = Array.isArray(parsed) ? parsed : (parsed.communities ?? parsed.data ?? []);
} catch { /* leave empty */ }
const withInventory = comms.filter((c) => c.hasActiveQMI || num(c.inventoryCount));
for (const c of withInventory.slice(0, PAGE_LIMIT)) {
const id = c.id ?? c.communityId;
if (id == null) continue;
try {
yield await fetcher.fetch(`${API}/api/plan/qmiplans?communityId=${id}`);
} catch (error) {
console.warn(` skip community ${id}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
const errors: { url: string; reason: string }[] = [];
try {
const parsed = JSON.parse(page.body.toString("utf8"));
const homes: Record<string, unknown>[] = Array.isArray(parsed) ? parsed : (parsed.plans ?? parsed.data ?? []);
if (!Array.isArray(homes) || homes.length === 0) {
return { records: [], errors: [] }; // community with no current inventory
}
const first = homes[0]!;
const firstUrl = str(first.inventoryPageURL) ?? "";
const slug = firstUrl.split("/").filter(Boolean)[4] ?? "";
const communityName = str(first.community) ?? (slug ? titleCase(slug) : null);
if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community name" }] };
const addr = (first.address as Record<string, unknown>) ?? {};
const lat = geo(first.communityLatitude);
const lon = geo(first.communityLongitude);
const phoneRaw = str(first.salesPhoneNumber);
const phone = phoneRaw ? phoneRaw.replace(/[^0-9+]/g, "") : null;
const records: ExtractedRecord[] = [];
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
fields: {
name: fv(communityName, communityName, page.url),
street: fv<string>(null, null, page.url),
city: fv(str(addr.city), str(addr.city), page.url),
state: fv(stateCode(addr.stateAbbreviation, STATE), str(addr.stateAbbreviation) ?? STATE, page.url),
zip: fv(str(addr.zipCode), str(addr.zipCode), page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(lat, lat === null ? null : String(lat), page.url),
lon: fv(lon, lon === null ? null : String(lon), page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(phone && phone.length >= 10 ? phone : null, phoneRaw, page.url, "community sales phone"),
},
});
for (const h of homes) {
const a = (h.address as Record<string, unknown>) ?? {};
const address = str(a.street1);
if (!address) continue;
const price = num(h.finalPrice) ?? num(h.price);
// Pulte's image objects carry the URL under `path` (a PicturePark CDN link),
// not url/src — recon 2026-07-28. Rank order is already meaningful (imageRank 1
// first), so keep source order. Missing `path` was why Pulte had 0 photos.
const images = Array.isArray(h.images) ? (h.images as unknown[]) : [];
const img = images.map((im) => (typeof im === "string" ? im : str((im as Record<string, unknown>)?.url) ?? str((im as Record<string, unknown>)?.src) ?? str((im as Record<string, unknown>)?.path)))
.filter((u): u is string => !!u).map((u) => (u.startsWith("http") ? u : API + u));
const invId = str(h.inventoryHomeID) ?? str(h.lotBlock);
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName,
address,
builderInventoryId: invId,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: str(h.planName),
},
fields: {
street: fv(address, address, page.url),
city: fv(str(a.city), str(a.city), page.url),
state: fv(stateCode(a.stateAbbreviation, STATE), str(a.stateAbbreviation) ?? STATE, page.url),
zip: fv(str(a.zipCode), str(a.zipCode), page.url),
price: fv(price, price === null ? null : String(price), page.url, price ? `finalPrice ${price}` : null),
beds: fv(num(h.bedrooms), null, page.url),
bathsTotal: fv(num(h.totalBaths) ?? num(h.bathrooms), null, page.url),
sqft: fv(num(h.squareFeet), null, page.url),
stories: fv(num(h.floors), null, page.url),
garageSpaces: fv(num(h.garages), null, page.url),
homeType: fv((h.isSingleFamily === false ? "TOWNHOME" : "SINGLE_FAMILY") as never, null, page.url, "Pulte inventory home"),
constructionStatus: fv(statusOf(str(h.dateAvailable)), str(h.dateAvailable), page.url),
estCompletionDate: fv<string>(null, null, page.url),
lotNumber: fv(str(h.lotBlock), str(h.lotBlock), page.url),
builderInventoryId: fv(invId, invId, page.url),
planName: fv(str(h.planName), str(h.planName), page.url),
availabilityStatus: fv(str(h.dateAvailable), str(h.dateAvailable), page.url),
lat: fv(lat, null, page.url),
lon: fv(lon, null, page.url),
images: fv<string[]>(img.slice(0, 6), null, page.url, img.length ? "builder listing photo" : null),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};