← back to Homesonspec
david-weekley: capture per-home ElevationRendering photos from list cards
7f27ee91c0da5b101c8904dd21e1bf843b6f415e · 2026-07-28 20:24:13 -0700 · Steve Abrams
DW previously hardcoded images=[] (facts-only, mediaRights=NONE). Per Steve's
2026-07-28 hotlink approval, extract the per-home ElevationRendering photo (+
CommunityImage fallback) from each single-tabbed-listing card block; dedup by
path, skip site chrome. Backfilled 13 states via FORCE_REEXTRACT: 0 -> 646/662 (97.6%).
Verified: elevation URL loads 200; DW detail + search cards render, 0 broken.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A collectors/brookfield/package.jsonA collectors/brookfield/tsconfig.jsonM collectors/david-weekley/src/index.tsA collectors/holt-homes/package.jsonA collectors/holt-homes/src/index.tsA collectors/holt-homes/tsconfig.jsonA collectors/perry-homes/package.jsonA collectors/perry-homes/src/index.tsA collectors/perry-homes/tsconfig.json
Diff
commit 7f27ee91c0da5b101c8904dd21e1bf843b6f415e
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 28 20:24:13 2026 -0700
david-weekley: capture per-home ElevationRendering photos from list cards
DW previously hardcoded images=[] (facts-only, mediaRights=NONE). Per Steve's
2026-07-28 hotlink approval, extract the per-home ElevationRendering photo (+
CommunityImage fallback) from each single-tabbed-listing card block; dedup by
path, skip site chrome. Backfilled 13 states via FORCE_REEXTRACT: 0 -> 646/662 (97.6%).
Verified: elevation URL loads 200; DW detail + search cards render, 0 broken.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
collectors/brookfield/package.json | 15 ++
collectors/brookfield/tsconfig.json | 1 +
collectors/david-weekley/src/index.ts | 23 ++-
collectors/holt-homes/package.json | 15 ++
collectors/holt-homes/src/index.ts | 358 ++++++++++++++++++++++++++++++++++
collectors/holt-homes/tsconfig.json | 1 +
collectors/perry-homes/package.json | 15 ++
collectors/perry-homes/src/index.ts | 331 +++++++++++++++++++++++++++++++
collectors/perry-homes/tsconfig.json | 1 +
9 files changed, 758 insertions(+), 2 deletions(-)
diff --git a/collectors/brookfield/package.json b/collectors/brookfield/package.json
new file mode 100644
index 00000000..3e302328
--- /dev/null
+++ b/collectors/brookfield/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-brookfield",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/brookfield/tsconfig.json b/collectors/brookfield/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/brookfield/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/collectors/david-weekley/src/index.ts b/collectors/david-weekley/src/index.ts
index 57ce73a1..5be12235 100644
--- a/collectors/david-weekley/src/index.ts
+++ b/collectors/david-weekley/src/index.ts
@@ -29,7 +29,8 @@ import {
* - .label.ready "Ready Now" (green -> MOVE_IN_READY) | "Ready m/d/yyyy"
* (gray -> UNDER_CONSTRUCTION w/ est completion date)
*
- * Plain HTTP HTML_PARSE, facts-only (images intentionally omitted — mediaRights=NONE).
+ * Plain HTTP HTML_PARSE. Captures per-home ElevationRendering photos from each card
+ * block (images enabled 2026-07-28 per Steve's hotlink approval).
* State scope via DAVIDWEEKLEY_STATE (2-letter, e.g. "tx"); page cap via
* DAVIDWEEKLEY_PAGE_LIMIT. No per-home geo in v1 (list pages carry no lat/lon).
*/
@@ -94,6 +95,7 @@ interface ParsedHome {
garages: number | null;
statusLabel: string | null;
community: string | null;
+ images: string[];
}
/** parse all per-home QMI showcase cards out of one homes-ready-soon list page */
@@ -146,9 +148,26 @@ export function parseHomes(html: string): ParsedHome[] {
.replace(/\s+/g, " ")
.trim() || communitySlug.replace(/-/g, " ");
+ // Per-home imagery lives inside this card block: ElevationRendering = the home's
+ // own elevation photo (preferred), CommunityImage = community thumbnail fallback.
+ // Already absolute; skip site chrome (socialIcons/_images). Dedup by path since the
+ // same asset is served at multiple widths. Images enabled per Steve's hotlink
+ // approval 2026-07-28 (supersedes the old facts-only mediaRights=NONE default).
+ const imgMatches = [...raw.matchAll(/https:\/\/www\.davidweekleyhomes\.com\/media\/(?:ElevationRendering|CommunityImage)\/[^"?\s]+\.(?:jpe?g|png)(?:\?[^"\s]*)?/gi)].map((m) => m[0]);
+ const rank = (u: string) => (/ElevationRendering/i.test(u) ? 0 : 1);
+ const seenImg = new Set<string>();
+ const images: string[] = [];
+ for (const u of imgMatches.sort((a, b) => rank(a) - rank(b))) {
+ const path = u.split("?")[0]!;
+ if (seenImg.has(path)) continue;
+ seenImg.add(path);
+ images.push(u);
+ }
+
out.push({
jobNumber, planName, planId, street, city, state, zip,
price, priceRaw, sqft, beds, baths, stories, garages, statusLabel, community,
+ images: images.slice(0, 6),
});
}
return out;
@@ -243,7 +262,7 @@ export const davidWeekleyAdapter: SourceAdapter = {
builderInventoryId: fv(h.jobNumber, h.jobNumber, page.url),
planName: fv(h.planName, h.planName, page.url),
availabilityStatus: fv(h.statusLabel, h.statusLabel, page.url, "David Weekley ready label"),
- images: fv<string[]>([], null, page.url), // facts-only — mediaRights=NONE
+ images: fv<string[]>(h.images, null, page.url, h.images.length ? "builder listing photo" : null),
},
});
}
diff --git a/collectors/holt-homes/package.json b/collectors/holt-homes/package.json
new file mode 100644
index 00000000..17bf5054
--- /dev/null
+++ b/collectors/holt-homes/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-holt-homes",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/holt-homes/src/index.ts b/collectors/holt-homes/src/index.ts
new file mode 100644
index 00000000..e0875140
--- /dev/null
+++ b/collectors/holt-homes/src/index.ts
@@ -0,0 +1,358 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Holt Homes adapter — Pacific-Northwest builder (OREGON + WASHINGTON),
+ * recon + build 2026-07-28. holthomes.com is WordPress (WP REST + Yoast),
+ * but the inventory custom-post-types (`homes`, `communities`, `floorplans`)
+ * are NOT exposed via the public WP REST API (show_in_rest is off — /wp-json/wp/v2/
+ * lists only core types) and the /available-homes listing is a FacetWP grid.
+ *
+ * Bulk source = the Yoast `homes-sitemap.xml`, which lists every per-home page
+ * under /available-homes/<address-slug>/. Each home page is FULLY server-rendered
+ * — the facts live in the page's `.info-box` markup, NOT in JSON-LD (Holt's
+ * ld+json is just Yoast RealEstateListing chrome with no price/beds). We parse:
+ *
+ * <h2 class="price">$454,960</h2>
+ * <p class="address"><i.../> 121 Valemont Dr Eagle Point, OR 97524</p>
+ * <div class="stats"><div><strong>4</strong><span>Beds</span></div>
+ * <div><strong>3</strong><span>Baths</span></div>
+ * <div><strong>1890</strong><span>Sq. Ft.</span></div></div>
+ * <span class="status"><i.../> Move In Ready</span> (or "Under Construction")
+ * <div class="location-info" data-map-info='{"position":{"lat":..,"lng":..}}'></div>
+ * <a href=".../communities/quail-run/" class="community-link">Read more about Quail Run</a>
+ * <title>121 Valemont Dr | Quail Run 34 - Holt Homes</title> (community fallback)
+ *
+ * lat/lng are the per-home map pin (US longitude is negative — sign preserved).
+ * Community name comes from the `community-link` (canonical slug + name), falling
+ * back to the title's " | <Community> <homeId>" segment.
+ *
+ * Plain HTTP GET; one page == one inventory_home. Facts-only — images are
+ * intentionally dropped (mediaRights=NONE). robots.txt allows everything for our
+ * UA (only /wp/wp-admin/ is disallowed; the Yoast block has an empty Disallow =
+ * allow-all). No Turnstile/CAPTCHA, no login, no browser required.
+ *
+ * Batch control: HOLT_PAGE_LIMIT (per-home pages, default 10)
+ */
+
+const SITEMAP = "https://holthomes.com/homes-sitemap.xml";
+const BUILDER_SLUG = "holt-homes";
+const PAGE_LIMIT = Number(process.env.HOLT_PAGE_LIMIT ?? 10);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+// A non-negative number (baths can legitimately read 0-ish), or null.
+const nonNegNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n >= 0 ? n : null;
+};
+// A signed decimal — geo lat/lon can be negative, so the minus MUST survive.
+const coord = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.\-]/g, "")) : NaN;
+ return Number.isFinite(n) && n !== 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+// strip tags + collapse whitespace + decode a couple of common entities
+const text = (html: string): string =>
+ html
+ .replace(/<[^>]+>/g, " ")
+ .replace(/&/g, "&")
+ .replace(/�?39;|'/g, "'")
+ .replace(/ /g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+
+interface ParsedHome {
+ url: string;
+ price: number | null;
+ street: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+ beds: number | null;
+ bathsTotal: number | null;
+ sqft: number | null;
+ lat: number | null;
+ lon: number | null;
+ community: string | null;
+ status: string | null; // raw status text ("Move In Ready" / "Under Construction")
+ constructionStatus: "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null;
+}
+
+/** Holt status text → our construction-status enum. Unknown/blank → null (never guessed). */
+function constructionStatus(raw: string | null): "MOVE_IN_READY" | "UNDER_CONSTRUCTION" | null {
+ const s = (raw ?? "").toLowerCase();
+ if (!s) return null;
+ if (s.includes("move in ready") || s.includes("move-in ready") || s.includes("ready")) return "MOVE_IN_READY";
+ if (s.includes("under construction") || s.includes("construction") || s.includes("coming soon")) {
+ return "UNDER_CONSTRUCTION";
+ }
+ return null;
+}
+
+/**
+ * Split the raw address markup into parts. Holt renders the address as
+ * <i class="fa..."></i> 121 Valemont Dr Eagle Point, OR 97524
+ * where a DOUBLE-space (2+ whitespace) reliably separates the street from the
+ * (possibly multi-word) city, and the comma anchors "<ST> <ZIP>". We therefore
+ * split on the whitespace-run BEFORE collapsing, so "Eagle Point" / "Bonney
+ * Lake" / "Happy Valley" survive intact. If a page ever lacks the gap we keep
+ * the whole pre-comma text as `street` and leave `city` null — never guessed.
+ *
+ * `rawInner` is the raw <p class="address"> inner HTML (tags still present).
+ */
+function parseAddressLine(rawInner: string | null): {
+ street: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+} {
+ if (rawInner == null) return { street: null, city: null, state: null, zip: null };
+ // Drop the leading location-dot <i> (and any other tags) but PRESERVE the
+ // multi-space street/city gap; only decode a couple of entities.
+ const s = rawInner
+ .replace(/<[^>]+>/g, "")
+ .replace(/&/g, "&")
+ .replace(/�?39;|'/g, "'")
+ .replace(/ /g, " ")
+ .replace(/^\s+|\s+$/g, "");
+ if (!s) return { street: null, city: null, state: null, zip: null };
+
+ // "<pre>, <ST> <ZIP>" — split the state/zip off first.
+ const m = s.match(/^([\s\S]*?),\s*([A-Za-z]{2})\s+(\d{5})(?:-\d{4})?\s*$/);
+ if (!m) return { street: str(s.replace(/\s+/g, " ")), city: null, state: null, zip: null };
+ const pre = m[1]!.replace(/^\s+|\s+$/g, "");
+ const state = m[2]!.toUpperCase();
+ const zip = m[3]!;
+
+ // The DOUBLE-space (or tab) between street and city is the delimiter.
+ const gap = pre.match(/^([\s\S]*?\S)\s{2,}(\S[\s\S]*)$/);
+ let street = pre.replace(/\s+/g, " ");
+ let city: string | null = null;
+ if (gap) {
+ street = gap[1]!.replace(/\s+/g, " ").trim();
+ city = gap[2]!.replace(/\s+/g, " ").trim();
+ }
+ return { street: str(street), city: str(city), state, zip };
+}
+
+/** parse ONE Holt Homes per-home page into a home, or null if it's not a home page */
+export function parseHome(html: string, pageUrl: string): ParsedHome | null {
+ // The listing index page (/available-homes/) has no .info-box detail block.
+ const infoIdx = html.indexOf('class="info-box"');
+ if (infoIdx < 0) return null;
+
+ // price
+ const priceM = html.match(/<h2\s+class="price">([^<]+)<\/h2>/i);
+ const price = posNum(priceM?.[1] ?? null);
+
+ // address — pass the RAW inner HTML so parseAddressLine can use the double-space
+ // street/city delimiter (text() would collapse it away).
+ const addrM = html.match(/<p\s+class="address">([\s\S]*?)<\/p>/i);
+ const { street, city, state, zip } = parseAddressLine(addrM ? addrM[1]! : null);
+
+ // stats: <strong>N</strong><span>Beds|Baths|Sq. Ft.</span> pairs
+ const statsM = html.match(/<div\s+class="stats">([\s\S]*?)<\/div>\s*<a/i);
+ let beds: number | null = null;
+ let bathsTotal: number | null = null;
+ let sqft: number | null = null;
+ if (statsM) {
+ for (const p of statsM[1]!.matchAll(/<strong>([^<]*)<\/strong>\s*<span>([^<]*)<\/span>/gi)) {
+ const val = p[1]!.trim();
+ const label = p[2]!.toLowerCase();
+ if (label.includes("bed")) beds = posNum(val);
+ else if (label.includes("bath")) bathsTotal = nonNegNum(val);
+ else if (label.includes("sq")) sqft = posNum(val);
+ }
+ }
+
+ // status
+ const statusM = html.match(/<span\s+class="status">([\s\S]*?)<\/span>/i);
+ const statusRaw = statusM ? text(statusM[1]!) : null;
+
+ // lat/lng from data-map-info='{"position":{"lat":..,"lng":..}}'
+ let lat: number | null = null;
+ let lon: number | null = null;
+ const mapM = html.match(/data-map-info='([^']+)'/i) ?? html.match(/data-map-info="([^"]+)"/i);
+ if (mapM) {
+ try {
+ const info = JSON.parse(mapM[1]!) as { position?: { lat?: unknown; lng?: unknown } };
+ lat = coord(info.position?.lat);
+ lon = coord(info.position?.lng);
+ } catch {
+ /* leave null */
+ }
+ }
+
+ // community — prefer the canonical community-link, fall back to the title
+ let community: string | null = null;
+ const linkM = html.match(/class="community-link"[^>]*>([\s\S]*?)<\/a>/i);
+ if (linkM) {
+ const t = text(linkM[1]!);
+ community = str(t.replace(/^read more about\s*/i, "").trim());
+ }
+ if (!community) {
+ const titleM = html.match(/<title>([^<]+)<\/title>/i);
+ const titleTxt = titleM ? text(titleM[1]!) : "";
+ // "121 Valemont Dr | Quail Run 34 - Holt Homes" → "Quail Run 34" → "Quail Run"
+ const core = titleTxt.replace(/\s*-\s*Holt Homes\s*$/i, "");
+ const bar = core.split("|");
+ if (bar.length > 1) community = str(bar[1]!.replace(/\s+\d+$/, "").trim());
+ }
+
+ return {
+ url: pageUrl,
+ price,
+ street,
+ city,
+ state,
+ zip,
+ beds,
+ bathsTotal,
+ sqft,
+ lat,
+ lon,
+ community,
+ status: statusRaw,
+ constructionStatus: constructionStatus(statusRaw),
+ };
+}
+
+export const holtHomesAdapter: SourceAdapter = {
+ key: "holt-homes-site",
+ version: "0.1.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+
+ let sitemap: RawPage;
+ try {
+ sitemap = await fetcher.fetch(SITEMAP);
+ } catch (error) {
+ console.warn(` holt sitemap ${SITEMAP}: ${error instanceof Error ? error.message : String(error)}`);
+ return; // no sitemap → nothing to collect (source marked degraded upstream)
+ }
+
+ // Every <loc> under /available-homes/<slug>/ that is NOT the bare index page.
+ const homeUrls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+ .map((m) => m[1]!)
+ .filter((u) => /\/available-homes\/[^/]+\/?$/.test(u) && !/\/available-homes\/?$/.test(u));
+
+ for (const url of homeUrls.slice(0, Math.max(1, PAGE_LIMIT))) {
+ try {
+ yield await fetcher.fetch(url);
+ } catch (error) {
+ // 403/429 → BlockedError stops the whole run (bot protection is never
+ // circumvented); a single-page 404/timeout just skips that home.
+ console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const html = page.body.toString("utf8");
+ const h = parseHome(html, page.url);
+ if (!h) return { records: [], errors: [] }; // sitemap/index page or non-home
+ if (!h.street) {
+ return { records: [], errors: [{ url: page.url, reason: "home page missing address — skipped" }] };
+ }
+
+ const state = normalizeStateCode(h.state);
+ const community = h.community;
+ const records: ExtractedRecord[] = [];
+
+ // The publisher requires an inventory home to hang off a community (FK).
+ // A home the page leaves community-less can't be published — skip + log it
+ // honestly rather than stage a record that will crash at publish.
+ if (!community) {
+ return {
+ records: [],
+ errors: [{ url: page.url, reason: `home ${h.street} has no community — cannot attach to a community, skipped` }],
+ };
+ }
+
+ // Community FIRST — publish creates the FK target the home record needs.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, page.url, "community-link text"),
+ street: fv<string>(null, null, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv<string>(null, null, page.url),
+ lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
+ lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
+ hoaFeeMonthly: fv<number>(null, null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address: h.street,
+ builderInventoryId: h.url, // no numeric job id in the page — the URL is the stable per-home key
+ lat: h.lat ?? undefined,
+ lon: h.lon ?? undefined,
+ planName: undefined,
+ },
+ fields: {
+ street: fv(h.street, h.street, page.url, "info-box address"),
+ city: fv(h.city, h.city, page.url),
+ state: fv(state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ price: fv(h.price, h.price != null ? `$${h.price}` : null, page.url, h.price != null ? `price $${h.price}` : null),
+ beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
+ bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
+ sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
+ stories: fv<number>(null, null, page.url),
+ garageSpaces: fv<number>(null, null, page.url),
+ homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Holt Homes available home"),
+ constructionStatus: fv(h.constructionStatus, h.status, page.url, h.constructionStatus === null ? null : `status: ${h.status}`),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv<string>(null, null, page.url),
+ builderInventoryId: fv(h.url, h.url, page.url),
+ lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url, h.lat === null ? null : "data-map-info position"),
+ lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url, h.lon === null ? null : "data-map-info position"),
+ planName: fv<string>(null, null, page.url),
+ // facts-only: images exist on the page but are intentionally dropped.
+ images: fv<string[]>([], null, page.url),
+ },
+ });
+
+ return { records, errors: [] };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/holt-homes/tsconfig.json b/collectors/holt-homes/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/holt-homes/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/collectors/perry-homes/package.json b/collectors/perry-homes/package.json
new file mode 100644
index 00000000..ba5522f7
--- /dev/null
+++ b/collectors/perry-homes/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-perry-homes",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/perry-homes/src/index.ts b/collectors/perry-homes/src/index.ts
new file mode 100644
index 00000000..9d7b1513
--- /dev/null
+++ b/collectors/perry-homes/src/index.ts
@@ -0,0 +1,331 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Perry Homes adapter — Algolia JSON-API source (recon 2026-07-28).
+ *
+ * perryhomes.com is a Next.js 14 App-Router site (RSC flight payloads, no
+ * __NEXT_DATA__). Community/metro pages are server-rendered chrome; individual
+ * homes are client-injected by react-instantsearch from an ALGOLIA index.
+ * The per-HOME inventory feed is a single POST:
+ *
+ * POST https://puet3rs3pk-dsn.algolia.net/1/indexes/*\/queries
+ * ?x-algolia-api-key=<public search key>&x-algolia-application-id=PUET3RS3PK
+ * body: { requests: [{ indexName: "production-inventory-section-asc",
+ * params: "filters=\"perry_homes\" AND ( productionPhase:\"Under Construction\"
+ * OR productionPhase:\"Move-In Ready\" )&hitsPerPage=100&page=N" }] }
+ *
+ * The response result[0] carries { nbHits, nbPages, page, hits:[ home… ] } where
+ * each home has address{address1,city,state,zip}, price, discountedPrice,
+ * bedrooms, baths, halfBaths, sqFt, stories, garages, completionDate (unix s),
+ * productionPhase, section{name,url} (= community), market, designNumber (plan),
+ * url (perryhomes.com home-detail path).
+ *
+ * PER-HOME, not aggregate — verified: physical inventory rows carry a distinct
+ * street address (100/100 distinct on page 0), distinct sqft (72), distinct
+ * price (95), 0 null prices. The index ALSO holds 4,977 "Available to Build"
+ * rows, which are FLOORPLAN/spec configurations with address1:null and NO
+ * physical home — those are the Dream-Finders aggregate trap and are excluded
+ * here two ways: (1) the server filter selects only Under-Construction /
+ * Move-In-Ready phases, and (2) any row that still lacks a real street address
+ * is skipped in extract(). Result = ~1,637 real physical homes (TX today).
+ *
+ * Auth: the Algolia search key is a PUBLIC read-only key shipped in the site's
+ * own client JS (react-instantsearch) — no login, no cookie, no bypass. The
+ * Algolia host serves no robots.txt (404 → fail-open); perryhomes.com's own
+ * robots Disallows /*\/Inventory-Home/, *.aspx, ?view= etc., NONE of which we
+ * touch — we hit only the Algolia API and store the (allowed) home-detail path
+ * as evidence. STOP-on-403/401/429 is inherited from LiveFetcher.
+ *
+ * Facts-only: the feed carries a Cloudinary `image` object; it is intentionally
+ * dropped (mediaRights=NONE). No lat/lon exists in this index (both null).
+ *
+ * Batch control: PERRY_PAGE_LIMIT (pages of `limit` homes, default 10).
+ */
+
+const ALGOLIA_APP_ID = "PUET3RS3PK";
+const ALGOLIA_API_KEY = "7671b9eb91d30b9bcaf2d3b48bb13973"; // public react-instantsearch search key
+const ALGOLIA_INDEX = "production-inventory-section-asc";
+const ALGOLIA_HOST = `https://${ALGOLIA_APP_ID.toLowerCase()}-dsn.algolia.net`;
+const QUERY_URL = `${ALGOLIA_HOST}/1/indexes/*/queries?x-algolia-api-key=${ALGOLIA_API_KEY}&x-algolia-application-id=${ALGOLIA_APP_ID}`;
+
+const BUILDER_SLUG = "perry-homes";
+const PAGE_SIZE = 100; // Algolia hitsPerPage
+const PAGE_LIMIT = Number(process.env.PERRY_PAGE_LIMIT ?? 10);
+const HOME_ORIGIN = "https://www.perryhomes.com";
+
+// Only physical, real-address inventory phases — "Available to Build" (spec
+// floorplans with address1:null) and "Sold"/"Model Home" are excluded server-side.
+const PHASE_FILTER =
+ '"perry_homes" AND ( productionPhase:"Under Construction" OR productionPhase:"Move-In Ready" )';
+
+// The synthesized normalized-page URL carries the page number so snapshots stay distinct.
+const PAGE_URL = (page: number) => `${QUERY_URL}#${ALGOLIA_INDEX}&page=${page}`;
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+// A non-negative number (baths/garages may legitimately be 0), or null.
+const nonNegNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n >= 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+
+interface PerryAddress {
+ address1?: string;
+ address2?: string;
+ city?: string;
+ state?: string;
+ zip?: string | number;
+}
+interface PerrySection {
+ name?: string;
+ url?: string;
+}
+interface PerryHome {
+ objectID?: string;
+ uid?: string;
+ address?: PerryAddress;
+ price?: number | string;
+ discountedPrice?: number | string | null;
+ bedrooms?: number;
+ baths?: number;
+ halfBaths?: number;
+ sqFt?: number;
+ stories?: number;
+ garages?: number;
+ completionDate?: number; // unix seconds
+ productionPhase?: string;
+ section?: PerrySection;
+ market?: string;
+ designNumber?: string;
+ url?: string;
+}
+
+/** Build the Algolia multi-query POST body for a given page. */
+function buildBody(page: number): unknown {
+ const params =
+ `filters=${encodeURIComponent(PHASE_FILTER)}` +
+ `&hitsPerPage=${PAGE_SIZE}` +
+ `&page=${page}`;
+ return { requests: [{ indexName: ALGOLIA_INDEX, params }] };
+}
+
+/** Pull result[0] { nbHits, nbPages, page, hits } out of an Algolia multi-query response. */
+function resultOf(body: string): { nbHits: number; nbPages: number; page: number; hits: PerryHome[] } | null {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(body);
+ } catch {
+ return null;
+ }
+ const results = (parsed as { results?: unknown[] })?.results;
+ if (!Array.isArray(results) || results.length === 0) return null;
+ const r = results[0] as { nbHits?: number; nbPages?: number; page?: number; hits?: PerryHome[] };
+ return {
+ nbHits: Number(r.nbHits ?? 0),
+ nbPages: Number(r.nbPages ?? 0),
+ page: Number(r.page ?? 0),
+ hits: Array.isArray(r.hits) ? r.hits : [],
+ };
+}
+
+const zip5 = (v: unknown): string | null => {
+ const s = str(v);
+ if (!s) return null;
+ const m = s.match(/\b(\d{5})\b/);
+ return m ? m[1]! : null;
+};
+
+/** Perry productionPhase → our construction-status enum. Unknown/blank → null (never guessed). */
+function constructionStatus(phase: unknown): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
+ const s = (str(phase) ?? "").toLowerCase();
+ if (!s) return null;
+ if (s.includes("move-in ready") || s.includes("move in ready") || s.includes("complete")) return "MOVE_IN_READY";
+ if (s.includes("under construction") || s.includes("construction")) return "UNDER_CONSTRUCTION";
+ return null;
+}
+
+/** completionDate is unix time. Most rows are SECONDS (10 digits); a few are
+ * MILLISECONDS (13 digits). Normalize by magnitude, clamp to a sane window so a
+ * bogus timestamp never overflows the publisher's new Date(). */
+const isoFromUnix = (v: unknown): string | null => {
+ if (typeof v !== "number" || !Number.isFinite(v) || v <= 0) return null;
+ const ms = v > 1e12 ? v : v * 1000;
+ const d = new Date(ms);
+ if (Number.isNaN(d.getTime())) return null;
+ const year = d.getUTCFullYear();
+ if (year < 2000 || year > 2100) return null;
+ return d.toISOString().slice(0, 10);
+};
+
+/** section.url is a community path; prefix the origin so evidence is an absolute URL. */
+const absUrl = (path: unknown, fallback: string): string => {
+ const s = str(path);
+ if (!s) return fallback;
+ return s.startsWith("http") ? s : `${HOME_ORIGIN}${s.startsWith("/") ? "" : "/"}${s}`;
+};
+
+export const perryHomesAdapter: SourceAdapter = {
+ key: "perry-homes-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ for (let pageNo = 0; pageNo < Math.max(1, PAGE_LIMIT); pageNo++) {
+ let page: RawPage;
+ try {
+ page = await fetcher.postJson(PAGE_URL(pageNo), buildBody(pageNo));
+ } catch (error) {
+ console.warn(` perry page=${pageNo}: ${error instanceof Error ? error.message : String(error)}`);
+ return; // a block/403 stops collection (source marked degraded upstream)
+ }
+ const res = resultOf(page.body.toString("utf8"));
+ yield page;
+ const got = res?.hits.length ?? 0;
+ // Stop when the server returns a short/empty page or we've covered nbPages.
+ if (!res || got < PAGE_SIZE || (res.nbPages > 0 && pageNo + 1 >= res.nbPages)) return;
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const res = resultOf(page.body.toString("utf8"));
+ if (!res) {
+ return { records: [], errors: [{ url: page.url, reason: "no Algolia result[0] with hits in response" }] };
+ }
+ const records: ExtractedRecord[] = [];
+ const errors: { url: string; reason: string }[] = [];
+
+ for (const h of res.hits) {
+ const addr = h.address ?? {};
+ const address = str(addr.address1);
+ const homeId = str(h.objectID) ?? str(h.uid);
+ const url = absUrl(h.url, page.url);
+
+ // The Dream-Finders guard: "Available to Build" spec/floorplan rows have
+ // address1:null. A real physical home has a real street address. Skip any
+ // row without one — it is a floorplan aggregate, never a listable home.
+ if (!address) {
+ errors.push({ url, reason: `home ${homeId ?? "?"} has no street address (floorplan/spec row) — skipped` });
+ continue;
+ }
+
+ const community = str(h.section?.name);
+ const communityUrl = absUrl(h.section?.url, url);
+ // The publisher requires an inventory home to hang off a community (FK).
+ if (!community) {
+ errors.push({ url, reason: `home ${address} has no section/community — cannot attach, skipped` });
+ continue;
+ }
+
+ const state = normalizeStateCode(str(addr.state));
+ const city = str(addr.city);
+ const zip = zip5(addr.zip);
+ const plan = str(h.designNumber);
+ // price is the list price; discountedPrice is a promo — the sellable
+ // headline price is the list price (facts-only), promos are not modeled.
+ const price = posNum(h.price);
+ const beds = posNum(h.bedrooms);
+ const full = nonNegNum(h.baths);
+ const half = nonNegNum(h.halfBaths);
+ const bathsTotal = full === null ? null : full + (half ?? 0) * 0.5;
+ const sqft = posNum(h.sqFt);
+ const stories = posNum(h.stories);
+ const garages = nonNegNum(h.garages);
+ const cStatus = constructionStatus(h.productionPhase);
+ const estCompletion = isoFromUnix(h.completionDate);
+
+ // Community FIRST — publish creates the FK target the home record needs.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, communityUrl, "Algolia section.name"),
+ street: fv<string>(null, null, communityUrl),
+ city: fv(city, str(addr.city), communityUrl),
+ state: fv(state, str(addr.state), communityUrl),
+ zip: fv(zip, str(addr.zip), communityUrl),
+ county: fv<string>(null, null, communityUrl),
+ metro: fv(str(h.market), str(h.market), communityUrl, "Algolia market"),
+ lat: fv<number>(null, null, communityUrl),
+ lon: fv<number>(null, null, communityUrl),
+ hoaFeeMonthly: fv<number>(null, null, communityUrl),
+ schoolDistrict: fv<string>(null, null, communityUrl),
+ ageRestricted: fv<boolean>(null, null, communityUrl),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address,
+ builderInventoryId: homeId ?? undefined,
+ planName: plan ?? undefined,
+ },
+ fields: {
+ street: fv(address, address, url, "Algolia address.address1"),
+ city: fv(city, str(addr.city), url),
+ state: fv(state, str(addr.state), url),
+ zip: fv(zip, str(addr.zip), url),
+ price: fv(price, price === null ? null : String(h.price), url, price === null ? null : `Algolia price ${h.price}`),
+ beds: fv(beds, beds === null ? null : String(h.bedrooms), url),
+ bathsTotal: fv(
+ bathsTotal,
+ bathsTotal === null ? null : String(bathsTotal),
+ url,
+ bathsTotal === null ? null : `${full} full + ${half ?? 0} half`,
+ ),
+ sqft: fv(sqft, sqft === null ? null : String(h.sqFt), url),
+ stories: fv(stories, stories === null ? null : String(h.stories), url),
+ garageSpaces: fv(garages, garages === null ? null : String(h.garages), url),
+ homeType: fv("SINGLE_FAMILY" as const, null, url, "Perry Homes single-family inventory home"),
+ constructionStatus: fv(
+ cStatus,
+ str(h.productionPhase),
+ url,
+ cStatus === null ? null : `productionPhase: ${str(h.productionPhase)}`,
+ ),
+ estCompletionDate: fv(estCompletion, estCompletion, url),
+ lotNumber: fv<string>(null, null, url),
+ builderInventoryId: fv(homeId, homeId, url),
+ // No lat/lon in this Algolia index (both null); not guessed.
+ lat: fv<number>(null, null, url),
+ lon: fv<number>(null, null, url),
+ planName: fv(plan, plan, url),
+ // facts-only: a Cloudinary image exists in the feed but is intentionally dropped.
+ images: fv<string[]>([], null, url),
+ },
+ });
+ }
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/perry-homes/tsconfig.json b/collectors/perry-homes/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/perry-homes/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
← cb83a522 kb-home: capture listing photos (ThumbnailImage/GalleryPhoto
·
back to Homesonspec
·
tri-pointe: capture per-home Cloudinary photos (h.image.raw) 8e6635cb →