← back to Homesonspec
collectors/mattamy/src/index.ts
338 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Mattamy Homes adapter — Sitecore JSS server-rendered JSON feed (recon 2026-07-29).
*
* mattamyhomes.com is a Sitecore JSS (React) SPA. The public site was previously
* bucketed D-hard, but the inventory ("Quick Move-In" / QMI) homes are exposed as
* a clean, bulk, per-home JSON feed via the JSS Layout Service:
*
* GET https://mattamyhomes.com/sitecore/api/layout/render/jss
* ?item=/search-data
* &sc_apikey={8C3D041E-BB12-4CC6-908A-4CF43E542E5B}
* &market=<State or Metro>
* &IsState=1 (1 = whole state; 0 = single metro)
*
* The response is the JSS layout for the search page's data route. The home list
* lives at:
* .sitecore.route.placeholders["jss-main"][0].fields.qmiCards.value = [ home… ]
* where each QMI card carries:
* title (street address), city, community, state, stateAbbreviation,
* latitudeCommunity / longitudeCommunity (community-level geo — Mattamy does not
* publish per-home lat/lon), price.price ("$420,990"), price.previousPriceQMI,
* attributes[] (free-text labels keyed by icon: bed/bath/half-baths/ruler(sqft)/
* stories/car(garage)), homeType, isCondo, den, planName, community_sheet via
* `community`, `date` (YYYYMMDDT000000 completion), availableDateMessage
* ("Ready Now" | "Ready October 2026"), url, id, countryQMI ("United States").
*
* Auth: the sc_apikey is the site's ANONYMOUS Sitecore JSS key (embedded in every
* page of mattamyhomes.com; a request WITHOUT it 400s, but it is not a credential —
* no login, no cookie, no token). We send it because the layout service requires
* the key to resolve the app, exactly as the public site does. Honest UA
* throughout; no anti-bot bypass — the feed serves our HomesOnSpecBot UA directly.
*
* robots.txt (mattamyhomes.com): `User-agent: * Allow: /` — only AspiegelBot /
* PetalBot are blocked, so our honest UA is allowed on /sitecore/api/**.
*
* US inventory is confined to 4 states (Arizona, Florida, North Carolina, Texas —
* from the feed's own metrosFilter, country=="USA"). We iterate those 4 states,
* one GET each (IsState=1 pulls the whole state in a single response), and filter
* to countryQMI=="United States" so Canadian inventory never leaks in. Facts-only:
* `image.src` exists in the feed but is intentionally dropped (mediaRights=NONE).
*
* Batch control: MATTAMY_PAGE_LIMIT (max states to fetch, default 10 → all 4).
*/
const FEED_BASE = "https://mattamyhomes.com/sitecore/api/layout/render/jss";
const SC_APIKEY = "{8C3D041E-BB12-4CC6-908A-4CF43E542E5B}";
const BUILDER_SLUG = "mattamy";
const PAGE_LIMIT = Number(process.env.MATTAMY_PAGE_LIMIT ?? 10);
// US states that carry Mattamy inventory (from the feed's metrosFilter, country=USA).
const US_STATES = ["Arizona", "Florida", "North Carolina", "Texas"];
/** Feed URL for one state (IsState=1 returns the whole state's inventory in one call). */
const stateFeedUrl = (state: string): string => {
const p = new URLSearchParams({
item: "/search-data",
sc_apikey: SC_APIKEY,
market: state,
IsState: "1",
});
return `${FEED_BASE}?${p.toString()}`;
};
function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
// Parse the FIRST numeric run out of a value. Labels like "1,565 Sq. Ft." carry a
// trailing "." (from "Sq. Ft.") and thousands commas, so we strip commas, then
// grab the first integer/decimal token — a blind [^0-9.] strip would fold the
// "Sq. Ft." dots into the number and yield NaN.
const firstNumber = (v: unknown): number => {
if (typeof v === "number") return v;
if (typeof v !== "string") return NaN;
const m = v.replace(/,/g, "").match(/-?\d+(?:\.\d+)?/);
return m ? Number(m[0]) : NaN;
};
// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
const posNum = (v: unknown): number | null => {
const n = firstNumber(v);
return Number.isFinite(n) && n > 0 ? n : null;
};
// A non-negative number (baths/garages may legitimately be 0), or null.
const nonNegNum = (v: unknown): number | null => {
const n = firstNumber(v);
return Number.isFinite(n) && n >= 0 ? n : null;
};
const str = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v).trim();
return s ? s : null;
};
interface QmiAttribute {
icon?: string;
label?: string; // e.g. "3 Bed", "1,565 Sq. Ft.", "2 Car Garage"
attribute?: string | null;
}
interface QmiPrice {
price?: string; // "$420,990"
previousPriceQMI?: string;
}
interface QmiCard {
id?: string;
title?: string; // street address
city?: string;
community?: string;
state?: string;
stateAbbreviation?: string;
latitudeCommunity?: string;
longitudeCommunity?: string;
price?: QmiPrice;
attributes?: QmiAttribute[];
homeType?: string;
isCondo?: boolean;
den?: boolean;
planName?: string;
url?: string;
date?: string; // "YYYYMMDDT000000"
availableDateMessage?: string; // "Ready Now" | "Ready October 2026"
countryQMI?: string; // "United States" | "Canada"
}
/** Pull the qmiCards array out of a JSS /search-data layout response. */
function qmiCards(body: string): QmiCard[] | null {
let parsed: unknown;
try {
parsed = JSON.parse(body);
} catch {
return null;
}
const main = (parsed as any)?.sitecore?.route?.placeholders?.["jss-main"];
if (!Array.isArray(main)) return null;
for (const comp of main) {
const cards = comp?.fields?.qmiCards?.value;
if (Array.isArray(cards)) return cards as QmiCard[];
}
return null;
}
/** Read one metric out of the free-text attributes[] list by icon key. */
function attr(attrs: QmiAttribute[] | undefined, icon: string): string | null {
if (!Array.isArray(attrs)) return null;
const a = attrs.find((x) => x?.icon === icon);
return a ? str(a.label) : null;
}
/** "$420,990" → 420990 (positive) or null. */
const parsePrice = (p: QmiPrice | undefined): { value: number | null; raw: string | null } => {
const raw = str(p?.price);
return { value: posNum(raw), raw };
};
/** community-level "33.46" string → number, US lon negative — sign preserved; 0/NaN → null. */
const geo = (v: unknown): number | null => {
const s = str(v);
if (!s) return null;
const n = Number(s);
return Number.isFinite(n) && n !== 0 ? n : null;
};
/** "YYYYMMDDT000000" → "YYYY-MM-DD", clamped to a sane window; else null. */
const isoFromCompact = (v: unknown): string | null => {
const s = str(v);
if (!s) return null;
const m = s.match(/^(\d{4})(\d{2})(\d{2})/);
if (!m) return null;
const [, y, mo, d] = m;
const year = Number(y);
if (year < 2000 || year > 2100) return null;
const mm = Number(mo);
const dd = Number(d);
if (mm < 1 || mm > 12 || dd < 1 || dd > 31) return null;
return `${y}-${mo}-${d}`;
};
/** "Ready Now" → MOVE_IN_READY; a future ready-date → UNDER_CONSTRUCTION; blank → null. */
function constructionStatus(msg: unknown): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
const s = (str(msg) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("ready now") || s.includes("move-in ready") || s.includes("move in ready")) return "MOVE_IN_READY";
if (s.includes("ready")) return "UNDER_CONSTRUCTION"; // "Ready <future month>"
return null;
}
/** Mattamy homeType label + flags → our enum. */
function homeType(card: QmiCard): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" | "OTHER" | null {
if (card.isCondo === true) return "CONDO";
const s = (str(card.homeType) ?? "").toLowerCase();
if (!s) return null;
if (s.includes("single family") || s.includes("single-family")) return "SINGLE_FAMILY";
if (s.includes("town")) return "TOWNHOME";
if (s.includes("condo")) return "CONDO";
return "OTHER";
}
export const mattamyAdapter: SourceAdapter = {
key: "mattamy-site",
version: "1.0.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
const states = US_STATES.slice(0, Math.max(1, PAGE_LIMIT));
for (const state of states) {
try {
yield await fetcher.fetch(stateFeedUrl(state));
} catch (error) {
// A block/403/429 or transient error on one state stops that state only;
// other states still collect. (source marked degraded upstream)
console.warn(` mattamy ${state}: ${error instanceof Error ? error.message : String(error)}`);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const cards = qmiCards(page.body.toString("utf8"));
if (!cards) {
return { records: [], errors: [{ url: page.url, reason: "no qmiCards in Mattamy /search-data layout response" }] };
}
const records: ExtractedRecord[] = [];
const errors: { url: string; reason: string }[] = [];
for (const h of cards) {
// Facts-only + US-only: never stage Canadian inventory.
if (str(h.countryQMI) && str(h.countryQMI) !== "United States") continue;
const state = normalizeStateCode(str(h.stateAbbreviation) ?? str(h.state));
const city = str(h.city);
const address = str(h.title);
const community = str(h.community);
const plan = str(h.planName);
const lat = geo(h.latitudeCommunity);
const lon = geo(h.longitudeCommunity);
const { value: price, raw: priceRaw } = parsePrice(h.price);
const beds = posNum(attr(h.attributes, "bed"));
const full = nonNegNum(attr(h.attributes, "bath"));
const half = nonNegNum(attr(h.attributes, "half-baths"));
const bathsTotal = full === null ? null : full + (half ?? 0) * 0.5;
const sqft = posNum(attr(h.attributes, "ruler"));
const stories = posNum(attr(h.attributes, "stories"));
const garages = nonNegNum(attr(h.attributes, "car"));
const homeId = str(h.id);
const url = str(h.url) ? `https://mattamyhomes.com${str(h.url)}` : page.url;
const cStatus = constructionStatus(h.availableDateMessage);
const estCompletion = isoFromCompact(h.date);
const hType = homeType(h);
if (!address) {
errors.push({ url, reason: `home ${homeId ?? "?"} missing address — skipped` });
continue;
}
// The publisher requires an inventory home to hang off a community (FK).
// A home the feed leaves community-less can't be published — skip + log it
// honestly rather than stage a record that will crash at publish.
if (!community) {
errors.push({ url, reason: `home ${address} has no community — cannot attach to a community, skipped` });
continue;
}
// Community FIRST — publish creates the FK target the home record needs.
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
fields: {
name: fv(community, community, url, "QMI card community"),
street: fv<string>(null, null, url),
city: fv(city, city, url),
state: fv(state, str(h.stateAbbreviation) ?? str(h.state), url),
zip: fv<string>(null, null, url), // Mattamy feed carries no zip
county: fv<string>(null, null, url),
metro: fv<string>(null, null, url),
lat: fv(lat, str(h.latitudeCommunity), url),
lon: fv(lon, str(h.longitudeCommunity), url),
hoaFeeMonthly: fv<number>(null, null, url),
schoolDistrict: fv<string>(null, null, url),
ageRestricted: fv<boolean>(null, null, url),
},
});
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: community,
address,
builderInventoryId: homeId ?? undefined,
lat: lat ?? undefined,
lon: lon ?? undefined,
planName: plan ?? undefined,
},
fields: {
street: fv(address, address, url, "QMI card address (title)"),
city: fv(city, city, url),
state: fv(state, str(h.stateAbbreviation) ?? str(h.state), url),
zip: fv<string>(null, null, url), // Mattamy feed carries no zip
price: fv(price, priceRaw, url, price === null ? null : `QMI price ${priceRaw}`),
beds: fv(beds, beds === null ? null : String(beds), url, attr(h.attributes, "bed")),
bathsTotal: fv(bathsTotal, bathsTotal === null ? null : String(bathsTotal), url, bathsTotal === null ? null : `${full} full + ${half ?? 0} half`),
sqft: fv(sqft, sqft === null ? null : String(sqft), url, attr(h.attributes, "ruler")),
stories: fv(stories, stories === null ? null : String(stories), url, attr(h.attributes, "stories")),
garageSpaces: fv(garages, garages === null ? null : String(garages), url, attr(h.attributes, "car")),
homeType: fv(hType, str(h.homeType), url, hType === null ? null : `homeType: ${str(h.homeType)}`),
constructionStatus: fv(cStatus, str(h.availableDateMessage), url, cStatus === null ? null : `availableDateMessage: ${str(h.availableDateMessage)}`),
estCompletionDate: fv(estCompletion, str(h.date), url),
lotNumber: fv<string>(null, null, url),
builderInventoryId: fv(homeId, homeId, url),
lat: fv(lat, str(h.latitudeCommunity), url, lat === null ? null : "QMI community latitude"),
lon: fv(lon, str(h.longitudeCommunity), url, lon === null ? null : "QMI community longitude"),
planName: fv(plan, plan, url),
// facts-only: image.src exists in the feed but is intentionally dropped.
images: fv<string[]>([], null, url),
},
});
}
return { records, errors };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};