← back to Homesonspec
auto-save: 2026-07-29T09:07:36 (1 files) — collectors/mattamy/
718fc16bc1c47da0cef34fe8add427b9025531f5 · 2026-07-29 09:07:43 -0700 · Steve Abrams
Files touched
A collectors/mattamy/package.jsonA collectors/mattamy/src/index.tsA collectors/mattamy/tsconfig.json
Diff
commit 718fc16bc1c47da0cef34fe8add427b9025531f5
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 29 09:07:43 2026 -0700
auto-save: 2026-07-29T09:07:36 (1 files) — collectors/mattamy/
---
collectors/mattamy/package.json | 15 ++
collectors/mattamy/src/index.ts | 337 +++++++++++++++++++++++++++++++++++++++
collectors/mattamy/tsconfig.json | 1 +
3 files changed, 353 insertions(+)
diff --git a/collectors/mattamy/package.json b/collectors/mattamy/package.json
new file mode 100644
index 00000000..9478b4f6
--- /dev/null
+++ b/collectors/mattamy/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-mattamy",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/mattamy/src/index.ts b/collectors/mattamy/src/index.ts
new file mode 100644
index 00000000..c9b7a2d7
--- /dev/null
+++ b/collectors/mattamy/src/index.ts
@@ -0,0 +1,337 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Mattamy Homes adapter — Sitecore JSS server-rendered JSON feed (recon 2026-07-29).
+ *
+ * mattamyhomes.com is a Sitecore JSS (React) SPA. The public site was previously
+ * bucketed D-hard, but the inventory ("Quick Move-In" / QMI) homes are exposed as
+ * a clean, bulk, per-home JSON feed via the JSS Layout Service:
+ *
+ * GET https://mattamyhomes.com/sitecore/api/layout/render/jss
+ * ?item=/search-data
+ * &sc_apikey={8C3D041E-BB12-4CC6-908A-4CF43E542E5B}
+ * &market=<State or Metro>
+ * &IsState=1 (1 = whole state; 0 = single metro)
+ *
+ * The response is the JSS layout for the search page's data route. The home list
+ * lives at:
+ * .sitecore.route.placeholders["jss-main"][0].fields.qmiCards.value = [ home… ]
+ * where each QMI card carries:
+ * title (street address), city, community, state, stateAbbreviation,
+ * latitudeCommunity / longitudeCommunity (community-level geo — Mattamy does not
+ * publish per-home lat/lon), price.price ("$420,990"), price.previousPriceQMI,
+ * attributes[] (free-text labels keyed by icon: bed/bath/half-baths/ruler(sqft)/
+ * stories/car(garage)), homeType, isCondo, den, planName, community_sheet via
+ * `community`, `date` (YYYYMMDDT000000 completion), availableDateMessage
+ * ("Ready Now" | "Ready October 2026"), url, id, countryQMI ("United States").
+ *
+ * Auth: the sc_apikey is the site's ANONYMOUS Sitecore JSS key (embedded in every
+ * page of mattamyhomes.com; a request WITHOUT it 400s, but it is not a credential —
+ * no login, no cookie, no token). We send it because the layout service requires
+ * the key to resolve the app, exactly as the public site does. Honest UA
+ * throughout; no anti-bot bypass — the feed serves our HomesOnSpecBot UA directly.
+ *
+ * robots.txt (mattamyhomes.com): `User-agent: * Allow: /` — only AspiegelBot /
+ * PetalBot are blocked, so our honest UA is allowed on /sitecore/api/**.
+ *
+ * US inventory is confined to 4 states (Arizona, Florida, North Carolina, Texas —
+ * from the feed's own metrosFilter, country=="USA"). We iterate those 4 states,
+ * one GET each (IsState=1 pulls the whole state in a single response), and filter
+ * to countryQMI=="United States" so Canadian inventory never leaks in. Facts-only:
+ * `image.src` exists in the feed but is intentionally dropped (mediaRights=NONE).
+ *
+ * Batch control: MATTAMY_PAGE_LIMIT (max states to fetch, default 10 → all 4).
+ */
+
+const FEED_BASE = "https://mattamyhomes.com/sitecore/api/layout/render/jss";
+const SC_APIKEY = "{8C3D041E-BB12-4CC6-908A-4CF43E542E5B}";
+const BUILDER_SLUG = "mattamy";
+const PAGE_LIMIT = Number(process.env.MATTAMY_PAGE_LIMIT ?? 10);
+
+// US states that carry Mattamy inventory (from the feed's metrosFilter, country=USA).
+const US_STATES = ["Arizona", "Florida", "North Carolina", "Texas"];
+
+/** Feed URL for one state (IsState=1 returns the whole state's inventory in one call). */
+const stateFeedUrl = (state: string): string => {
+ const p = new URLSearchParams({
+ item: "/search-data",
+ sc_apikey: SC_APIKEY,
+ market: state,
+ IsState: "1",
+ });
+ return `${FEED_BASE}?${p.toString()}`;
+};
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// Parse the FIRST numeric run out of a value. Labels like "1,565 Sq. Ft." carry a
+// trailing "." (from "Sq. Ft.") and thousands commas, so we strip commas, then
+// grab the first integer/decimal token — a blind [^0-9.] strip would fold the
+// "Sq. Ft." dots into the number and yield NaN.
+const firstNumber = (v: unknown): number => {
+ if (typeof v === "number") return v;
+ if (typeof v !== "string") return NaN;
+ const m = v.replace(/,/g, "").match(/-?\d+(?:\.\d+)?/);
+ return m ? Number(m[0]) : NaN;
+};
+// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = firstNumber(v);
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+// A non-negative number (baths/garages may legitimately be 0), or null.
+const nonNegNum = (v: unknown): number | null => {
+ const n = firstNumber(v);
+ return Number.isFinite(n) && n >= 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+
+interface QmiAttribute {
+ icon?: string;
+ label?: string; // e.g. "3 Bed", "1,565 Sq. Ft.", "2 Car Garage"
+ attribute?: string | null;
+}
+interface QmiPrice {
+ price?: string; // "$420,990"
+ previousPriceQMI?: string;
+}
+interface QmiCard {
+ id?: string;
+ title?: string; // street address
+ city?: string;
+ community?: string;
+ state?: string;
+ stateAbbreviation?: string;
+ latitudeCommunity?: string;
+ longitudeCommunity?: string;
+ price?: QmiPrice;
+ attributes?: QmiAttribute[];
+ homeType?: string;
+ isCondo?: boolean;
+ den?: boolean;
+ planName?: string;
+ url?: string;
+ date?: string; // "YYYYMMDDT000000"
+ availableDateMessage?: string; // "Ready Now" | "Ready October 2026"
+ countryQMI?: string; // "United States" | "Canada"
+}
+
+/** Pull the qmiCards array out of a JSS /search-data layout response. */
+function qmiCards(body: string): QmiCard[] | null {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(body);
+ } catch {
+ return null;
+ }
+ const main = (parsed as any)?.sitecore?.route?.placeholders?.["jss-main"];
+ if (!Array.isArray(main)) return null;
+ for (const comp of main) {
+ const cards = comp?.fields?.qmiCards?.value;
+ if (Array.isArray(cards)) return cards as QmiCard[];
+ }
+ return null;
+}
+
+/** Read one metric out of the free-text attributes[] list by icon key. */
+function attr(attrs: QmiAttribute[] | undefined, icon: string): string | null {
+ if (!Array.isArray(attrs)) return null;
+ const a = attrs.find((x) => x?.icon === icon);
+ return a ? str(a.label) : null;
+}
+
+/** "$420,990" → 420990 (positive) or null. */
+const parsePrice = (p: QmiPrice | undefined): { value: number | null; raw: string | null } => {
+ const raw = str(p?.price);
+ return { value: posNum(raw), raw };
+};
+
+/** community-level "33.46" string → number, US lon negative — sign preserved; 0/NaN → null. */
+const geo = (v: unknown): number | null => {
+ const s = str(v);
+ if (!s) return null;
+ const n = Number(s);
+ return Number.isFinite(n) && n !== 0 ? n : null;
+};
+
+/** "YYYYMMDDT000000" → "YYYY-MM-DD", clamped to a sane window; else null. */
+const isoFromCompact = (v: unknown): string | null => {
+ const s = str(v);
+ if (!s) return null;
+ const m = s.match(/^(\d{4})(\d{2})(\d{2})/);
+ if (!m) return null;
+ const [, y, mo, d] = m;
+ const year = Number(y);
+ if (year < 2000 || year > 2100) return null;
+ const mm = Number(mo);
+ const dd = Number(d);
+ if (mm < 1 || mm > 12 || dd < 1 || dd > 31) return null;
+ return `${y}-${mo}-${d}`;
+};
+
+/** "Ready Now" → MOVE_IN_READY; a future ready-date → UNDER_CONSTRUCTION; blank → null. */
+function constructionStatus(msg: unknown): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
+ const s = (str(msg) ?? "").toLowerCase();
+ if (!s) return null;
+ if (s.includes("ready now") || s.includes("move-in ready") || s.includes("move in ready")) return "MOVE_IN_READY";
+ if (s.includes("ready")) return "UNDER_CONSTRUCTION"; // "Ready <future month>"
+ return null;
+}
+
+/** Mattamy homeType label + flags → our enum. */
+function homeType(card: QmiCard): "SINGLE_FAMILY" | "TOWNHOME" | "CONDO" | "OTHER" | null {
+ if (card.isCondo === true) return "CONDO";
+ const s = (str(card.homeType) ?? "").toLowerCase();
+ if (!s) return null;
+ if (s.includes("single family") || s.includes("single-family")) return "SINGLE_FAMILY";
+ if (s.includes("town")) return "TOWNHOME";
+ if (s.includes("condo")) return "CONDO";
+ return "OTHER";
+}
+
+export const mattamyAdapter: SourceAdapter = {
+ key: "mattamy-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ const states = US_STATES.slice(0, Math.max(1, PAGE_LIMIT));
+ for (const state of states) {
+ try {
+ yield await fetcher.fetch(stateFeedUrl(state));
+ } catch (error) {
+ // A block/403/429 or transient error on one state stops that state only;
+ // other states still collect. (source marked degraded upstream)
+ console.warn(` mattamy ${state}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const cards = qmiCards(page.body.toString("utf8"));
+ if (!cards) {
+ return { records: [], errors: [{ url: page.url, reason: "no qmiCards in Mattamy /search-data layout response" }] };
+ }
+ const records: ExtractedRecord[] = [];
+ const errors: { url: string; reason: string }[] = [];
+
+ for (const h of cards) {
+ // Facts-only + US-only: never stage Canadian inventory.
+ if (str(h.countryQMI) && str(h.countryQMI) !== "United States") continue;
+
+ const state = normalizeStateCode(str(h.stateAbbreviation) ?? str(h.state));
+ const city = str(h.city);
+ const address = str(h.title);
+ const community = str(h.community);
+ const plan = str(h.planName);
+ const lat = geo(h.latitudeCommunity);
+ const lon = geo(h.longitudeCommunity);
+ const { value: price, raw: priceRaw } = parsePrice(h.price);
+
+ const beds = posNum(attr(h.attributes, "bed"));
+ const full = nonNegNum(attr(h.attributes, "bath"));
+ const half = nonNegNum(attr(h.attributes, "half-baths"));
+ const bathsTotal = full === null ? null : full + (half ?? 0) * 0.5;
+ const sqft = posNum(attr(h.attributes, "ruler"));
+ const stories = posNum(attr(h.attributes, "stories"));
+ const garages = nonNegNum(attr(h.attributes, "car"));
+
+ const homeId = str(h.id);
+ const url = str(h.url) ? `https://mattamyhomes.com${str(h.url)}` : page.url;
+ const cStatus = constructionStatus(h.availableDateMessage);
+ const estCompletion = isoFromCompact(h.date);
+ const hType = homeType(h);
+
+ if (!address) {
+ errors.push({ url, reason: `home ${homeId ?? "?"} missing address — skipped` });
+ continue;
+ }
+ // The publisher requires an inventory home to hang off a community (FK).
+ // A home the feed leaves community-less can't be published — skip + log it
+ // honestly rather than stage a record that will crash at publish.
+ if (!community) {
+ errors.push({ url, reason: `home ${address} has no community — cannot attach to a community, skipped` });
+ continue;
+ }
+
+ // Community FIRST — publish creates the FK target the home record needs.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, url, "QMI card community"),
+ street: fv<string>(null, null, url),
+ city: fv(city, city, url),
+ state: fv(state, str(h.stateAbbreviation) ?? str(h.state), url),
+ zip: fv<string>(null, null, url), // Mattamy feed carries no zip
+ county: fv<string>(null, null, url),
+ metro: fv<string>(null, null, url),
+ lat: fv(lat, str(h.latitudeCommunity), url),
+ lon: fv(lon, str(h.longitudeCommunity), url),
+ hoaFeeMonthly: fv<number>(null, null, url),
+ schoolDistrict: fv<string>(null, null, url),
+ ageRestricted: fv<boolean>(null, null, url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address,
+ builderInventoryId: homeId ?? undefined,
+ lat: lat ?? undefined,
+ lon: lon ?? undefined,
+ planName: plan ?? undefined,
+ },
+ fields: {
+ street: fv(address, address, url, "QMI card address (title)"),
+ city: fv(city, city, url),
+ state: fv(state, str(h.stateAbbreviation) ?? str(h.state), url),
+ zip: fv<string>(null, null, url), // Mattamy feed carries no zip
+ price: fv(price, priceRaw, url, price === null ? null : `QMI price ${priceRaw}`),
+ beds: fv(beds, beds === null ? null : String(beds), url, attr(h.attributes, "bed")),
+ bathsTotal: fv(bathsTotal, bathsTotal === null ? null : String(bathsTotal), url, bathsTotal === null ? null : `${full} full + ${half ?? 0} half`),
+ sqft: fv(sqft, sqft === null ? null : String(sqft), url, attr(h.attributes, "ruler")),
+ stories: fv(stories, stories === null ? null : String(stories), url, attr(h.attributes, "stories")),
+ garageSpaces: fv(garages, garages === null ? null : String(garages), url, attr(h.attributes, "car")),
+ homeType: fv(hType, str(h.homeType), url, hType === null ? null : `homeType: ${str(h.homeType)}`),
+ constructionStatus: fv(cStatus, str(h.availableDateMessage), url, cStatus === null ? null : `availableDateMessage: ${str(h.availableDateMessage)}`),
+ estCompletionDate: fv(estCompletion, str(h.date), url),
+ lotNumber: fv<string>(null, null, url),
+ builderInventoryId: fv(homeId, homeId, url),
+ lat: fv(lat, str(h.latitudeCommunity), url, lat === null ? null : "QMI community latitude"),
+ lon: fv(lon, str(h.longitudeCommunity), url, lon === null ? null : "QMI community longitude"),
+ planName: fv(plan, plan, url),
+ // facts-only: image.src exists in the feed but is intentionally dropped.
+ images: fv<string[]>([], null, url),
+ },
+ });
+ }
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/mattamy/tsconfig.json b/collectors/mattamy/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/mattamy/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
← e70a69b5 lennar: durable Mac-side refresh runner (keepalive tunnel +
·
back to Homesonspec
·
collectors: add Mattamy Homes — cracked the 'bucket-D' natio 5c09c690 →