← back to Homesonspec
collectors/fulton-homes/src/index.ts
344 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Fulton Homes adapter — legacy ASP.NET AJAX HTML source (recon 2026-07-28).
*
* fultonhomes.com is a single-metro ARIZONA builder on an old jQuery /
* Bootstrap-3 / ASP.NET WebForms stack (NOT a Next.js/Algolia SPA). The
* "Find Your Home" search page (/find-your-home) client-injects its results via
* a jQuery AJAX GET against an ASMX-style web service; the SPEC-HOME (quick-move-
* in) inventory feed is a single GET that returns the whole company inventory in
* one shot (no pagination, no cookies, no auth):
*
* GET https://www.fultonhomes.com/ws.svc/GetSpecs
* ?search="Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|NB"
* &clicked_element=""
* &sort_field="SqFt"&sort_direction="ASC"
*
* The `search` string is the site's own default all-filters-Any selector
* (city|hometype|beds|baths|garage|sqft|price… all "Any", trailing "NB"); it
* returns EVERY spec home statewide. The response is JSON-wrapped HTML:
*
* { "d": { "__type":"HomesSearchResponse:#FultonHomes",
* "count": 284, "html": "<table class='table results table-spec'>…" } }
*
* Each spec home is a <tr> carrying a `?ih=<projectCode>|<lot>` deep-link and
* columns [thumb, Lot, Floorplan, Elevation, Neighborhood, City, SqFt, Price,
* Beds, Bath]. `ih` (project|lot) is the stable per-home id. The Price cell is
* one of: a plain "$###,###" (available), a "$was$now" markdown pair (take the
* LAST = current price), or a struck-through price + "Sold"/"Pending" — those
* are NOT sellable inventory and are skipped (not a live listable home).
*
* Per-HOME, not aggregate — verified: 284 spec rows, 284 distinct `ih` ids, 98
* distinct sqft, 22 neighborhoods across 6 AZ cities; 202 available w/ real
* prices, 79 Sold + 3 Pending excluded. This is the individual-physical-home
* table (GetSpecs, lot-numbered), NOT the sibling GetHomes "New Built Homes"
* floorplan-aggregate table (count 155, no lot) which would be the hollow trap.
*
* Fields the bulk list does NOT carry (garage bays, plan stories, street number,
* lat/lon, est. completion date) live only on each home's detail page and are
* left null here (facts-only; never guessed). There is no street number in the
* list — the home's identity is Lot + Neighborhood + City, so `street` is the
* lot-qualified label "Lot <lot>, <neighborhood>".
*
* Auth: none — plain honest-UA GET returns 200 with no cookie/login. robots.txt
* (fultonhomes.com) Disallows only /bin/ /envision_sso/ /mfh/ /survey/
* /warranty/ — the /ws.svc/ and /find-your-home paths we use are allowed (and
* enforced in-code by LiveFetcher). STOP-on-403/401/429 is inherited.
*
* Facts-only: elevation thumbnails exist in the feed; they are intentionally
* dropped (mediaRights=NONE).
*
* Batch control: FULTON_PAGE_LIMIT (default 10). The feed is single-call, so
* this is a safety cap only — one page returns the whole statewide inventory.
*/
const ORIGIN = "https://www.fultonhomes.com";
// The site's own default "all filters = Any" selector; NB = New Built delivery set.
const SEARCH =
'"Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|NB"';
const GETSPECS_URL =
`${ORIGIN}/ws.svc/GetSpecs` +
`?search=${encodeURIComponent(SEARCH)}` +
`&clicked_element=${encodeURIComponent('""')}` +
`&sort_field=${encodeURIComponent('"SqFt"')}` +
`&sort_direction=${encodeURIComponent('"ASC"')}`;
const BUILDER_SLUG = "fulton-homes";
const PAGE_LIMIT = Number(process.env.FULTON_PAGE_LIMIT ?? 10);
// Fulton is a single-metro ARIZONA builder; the list carries no state column.
const STATE_RAW = "AZ";
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
// First positive number in a string, or null. Handles bed/bath RANGES ("3-4", "5 - 6")
// by taking the base (first) value — posNum() would strip the dash and read "3-4" as 34,
// producing bogus 34-bedroom homes the validator rejects (contrarian fix, verified 2026-07-28).
const firstNum = (v: unknown): number | null => {
const m = String(v ?? "").match(/\d+(?:\.\d+)?/);
if (!m) return null;
const n = Number(m[0]);
return Number.isFinite(n) && n > 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
/** Strip HTML tags and collapse whitespace → plain text. */
const text = (htmlFragment: string): string =>
decodeEntities(htmlFragment.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
/** Minimal HTML-entity decode for the handful the feed emits (& ' " etc.). */
function decodeEntities(s: string): string {
return s
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/"/g, '"')
.replace(/�?39;/g, "'")
.replace(/'/g, "'")
.replace(/ /g, " ")
.replace(/&#(\d+);/g, (_, d) => {
const code = Number(d);
return Number.isFinite(code) ? String.fromCharCode(code) : _;
});
}
/** Pull the JSON-wrapped HTML table out of a GetSpecs response body. */
function specHtml(body: string): { count: number; html: string } | null {
let parsed: unknown;
try {
parsed = JSON.parse(body);
} catch {
return null;
}
const d = (parsed as { d?: { count?: number; html?: string } })?.d;
if (!d || typeof d.html !== "string") return null;
return { count: Number(d.count ?? 0), html: decodeEntities(d.html) };
}
interface SpecRow {
ih: string; // "<projectCode>|<lot>" — stable per-home id
lot: string | null;
plan: string | null;
elevation: string | null;
neighborhood: string | null;
city: string | null;
sqft: number | null;
/** null when the home is Sold/Pending (not sellable) or has no listed price. */
price: number | null;
/** true only for a home that shows a live, current asking price. */
available: boolean;
status: string | null; // "Sold" | "Pending" | null (available)
beds: number | null;
baths: number | null;
}
/**
* Parse the current (last) dollar figure out of a Price cell. The cell may be:
* - "$412,900" → 412900 (available)
* - "$447,449$429,449" → 429449 (was/now markdown; take the LAST)
* - struck-through "$…" + "Sold" → null, status "Sold"
* - struck-through "$…" + "Pending"→ null, status "Pending"
*/
function parsePriceCell(cellHtml: string): { price: number | null; status: string | null } {
const plain = text(cellHtml);
if (/\bSold\b/i.test(plain)) return { price: null, status: "Sold" };
if (/\bPending\b/i.test(plain)) return { price: null, status: "Pending" };
const matches = plain.match(/\$[\d,]+/g);
if (!matches || matches.length === 0) return { price: null, status: null };
// was/now pairs list the was-price first and the current price last.
const price = posNum(matches[matches.length - 1]);
return { price, status: null };
}
/** Parse the GetSpecs HTML table into one SpecRow per physical spec home. */
function parseSpecRows(html: string): SpecRow[] {
const rows: SpecRow[] = [];
const seen = new Set<string>();
const trRe = /<tr\b[^>]*>([\s\S]*?)<\/tr>/gi;
let m: RegExpExecArray | null;
while ((m = trRe.exec(html)) !== null) {
const rowHtml = m[0];
if (/table-header/i.test(rowHtml)) continue; // header row
const ihMatch = rowHtml.match(/\?ih=([a-z0-9]+)\|(\d+)/i);
if (!ihMatch) continue; // detail/expand rows (no ?ih=) — skip
const ih = `${ihMatch[1]}|${ihMatch[2]}`;
if (seen.has(ih)) continue; // guard against the paired detail row re-matching
const cellsRaw = [...rowHtml.matchAll(/<td\b[^>]*>([\s\S]*?)<\/td>/gi)].map((c) => c[1]);
if (cellsRaw.length < 9) continue; // not a full data row
// Column order: [thumb, Lot, Floorplan, Elv, Neighborhood, City, SqFt, Price, Beds, Bath]
const [_, lotC, planC, elvC, nbhdC, cityC, sqftC, priceC, bedsC, bathC] = cellsRaw;
const { price, status } = parsePriceCell(priceC ?? "");
seen.add(ih);
rows.push({
ih,
lot: str(ihMatch[2]) ?? str(text(lotC ?? "")),
plan: str(text(planC ?? "")),
elevation: str(text(elvC ?? "")),
neighborhood: str(text(nbhdC ?? "")),
city: str(text(cityC ?? "")),
sqft: posNum(text(sqftC ?? "")),
price,
available: price !== null,
status,
beds: firstNum(text(bedsC ?? "")),
baths: firstNum(text(bathC ?? "")),
});
}
return rows;
}
/** The home-detail path for a spec home (evidence URL). Prefer the row's own
* anchor href when present, else synthesize the ?ih= deep-link. */
function homeUrl(rowHtml: string, ih: string): string {
const href = rowHtml.match(/href="((?:our-communities\/)?[^"]*\?ih=[^"]+)"/i)?.[1];
if (href) return href.startsWith("http") ? href : `${ORIGIN}/${href.replace(/^\//, "")}`;
return `${ORIGIN}/find-your-home?ih=${encodeURIComponent(ih)}`;
}
export const fultonHomesAdapter: SourceAdapter = {
key: "fulton-homes-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// The GetSpecs feed is single-call (returns the whole statewide inventory in one response),
// so there is exactly one page regardless of FULTON_PAGE_LIMIT (which stays for interface
// parity with the other adapters). Fetch it once. [was Math.min(1,PAGE_LIMIT) dead code]
for (let pageNo = 0; pageNo < 1; pageNo++) {
let page: RawPage;
try {
page = await fetcher.fetch(GETSPECS_URL);
} catch (error) {
console.warn(` fulton GetSpecs: ${error instanceof Error ? error.message : String(error)}`);
return; // a block/403 stops collection (source marked degraded upstream)
}
yield page;
}
},
extract(page: RawPage): ExtractionOutput {
try {
const parsed = specHtml(page.body.toString("utf8"));
if (!parsed) {
return { records: [], errors: [{ url: page.url, reason: "no GetSpecs { d: { html } } payload in response" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
// Re-walk the <tr> blocks so each row keeps its own anchor href for evidence.
const trRe = /<tr\b[^>]*>([\s\S]*?)<\/tr>/gi;
const rowHtmlByIh = new Map<string, string>();
let mm: RegExpExecArray | null;
while ((mm = trRe.exec(parsed.html)) !== null) {
const im = mm[0].match(/\?ih=([a-z0-9]+)\|(\d+)/i);
if (im && !rowHtmlByIh.has(`${im[1]}|${im[2]}`)) rowHtmlByIh.set(`${im[1]}|${im[2]}`, mm[0]);
}
const rows = parseSpecRows(parsed.html);
const state = normalizeStateCode(STATE_RAW);
for (const r of rows) {
// Only live, sellable inventory — Sold/Pending homes are not listable.
if (!r.available || r.price === null) {
errors.push({ url: page.url, reason: `spec ${r.ih} not available (status: ${r.status ?? "no price"}) — skipped` });
continue;
}
const community = r.neighborhood;
// The publisher requires an inventory home to hang off a community (FK).
if (!community) {
errors.push({ url: page.url, reason: `spec ${r.ih} has no neighborhood — cannot attach to a community, skipped` });
continue;
}
const url = homeUrl(rowHtmlByIh.get(r.ih) ?? "", r.ih);
const city = r.city;
// No street number in the list; identity is Lot + Neighborhood + City.
const street = r.lot ? `Lot ${r.lot}, ${community}` : community;
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "GetSpecs Neighborhood column"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, STATE_RAW, url, "Fulton Homes is an Arizona-only builder"),
zip: fv<string>(null, null, url),
county: fv<string>(null, null, url),
metro: fv<string>("Phoenix", null, url, "Fulton Homes serves the Phoenix / Valley metro"),
lat: fv<number>(null, null, url),
lon: fv<number>(null, null, url),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address: street,
builderInventoryId: r.ih,
planName: r.plan ?? undefined,
},
fields: {
street: fv(street, r.lot, url, r.lot ? `Lot ${r.lot} in ${community}` : "GetSpecs Neighborhood"),
city: fv(city, city, url),
state: fv(state, STATE_RAW, url, "Fulton Homes is an Arizona-only builder"),
zip: fv<string>(null, null, url),
price: fv(r.price, r.price === null ? null : String(r.price), url, r.price === null ? null : `GetSpecs price $${r.price}`),
beds: fv(r.beds, r.beds === null ? null : String(r.beds), url),
bathsTotal: fv(r.baths, r.baths === null ? null : String(r.baths), url, r.baths === null ? null : "GetSpecs Bath column"),
sqft: fv(r.sqft, r.sqft === null ? null : String(r.sqft), url),
// Garage bays & plan stories are not in the bulk list (detail-page only) — not guessed.
stories: fv<number>(null, null, url),
garageSpaces: fv<number>(null, null, url),
homeType: fv("SINGLE_FAMILY" as const, null, url, "Fulton Homes single-family spec home"),
// GetSpecs are quick-move-in spec homes; without a per-home stage we don't
// assert MOVE_IN_READY vs UNDER_CONSTRUCTION (that lives on the detail page).
constructionStatus: fv<"UNDER_CONSTRUCTION" | "MOVE_IN_READY">(null, null, url),
estCompletionDate: fv<string>(null, null, url),
lotNumber: fv(r.lot, r.lot, url, r.lot ? `Lot ${r.lot}` : null),
builderInventoryId: fv(r.ih, r.ih, url, "GetSpecs ?ih=<project>|<lot> id"),
// No lat/lon in the GetSpecs feed (both null); not guessed.
lat: fv<number>(null, null, url),
lon: fv<number>(null, null, url),
planName: fv(r.plan, r.plan, url),
// facts-only: elevation thumbnails exist in the feed but are intentionally dropped.
images: fv<string[]>([], null, url),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};