← back to Homesonspec
Add Summit Homes KC collector adapter (TK-10487 batch 2)
e1d685cb321df4d14032ae42c3cf20a1aeed721f · 2026-08-17 04:24:34 -0700 · steve@designerwallcoverings.com
- New adapter collectors/summit-homes-kc: React SSR site-scraper using
JSON-LD (SingleFamilyResidence/Product) + HomeOverview_content text block
for beds/baths/sqft/garage/plan/status; gzip-decompressed HTTP GET
- Wired into workers (cli.ts + package.json dependency)
- Seed script: seed-cpg-builders.ts registers Builder+SourceRegistry for
goodall-homes, brohn-homes, oakwood-homes, summit-homes-kc (batch 1+2)
- Pipeline test: 40 pages, 80 records published, 0 errors
Files touched
M apps/workers/package.jsonM apps/workers/src/cli.tsA collectors/summit-homes-kc/src/index.tsA packages/database/prisma/seed-cpg-builders.tsM pnpm-lock.yaml
Diff
commit e1d685cb321df4d14032ae42c3cf20a1aeed721f
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Mon Aug 17 04:24:34 2026 -0700
Add Summit Homes KC collector adapter (TK-10487 batch 2)
- New adapter collectors/summit-homes-kc: React SSR site-scraper using
JSON-LD (SingleFamilyResidence/Product) + HomeOverview_content text block
for beds/baths/sqft/garage/plan/status; gzip-decompressed HTTP GET
- Wired into workers (cli.ts + package.json dependency)
- Seed script: seed-cpg-builders.ts registers Builder+SourceRegistry for
goodall-homes, brohn-homes, oakwood-homes, summit-homes-kc (batch 1+2)
- Pipeline test: 40 pages, 80 records published, 0 errors
---
apps/workers/package.json | 3 +-
apps/workers/src/cli.ts | 2 +
collectors/summit-homes-kc/src/index.ts | 341 ++++++++++++++++++++++++++
packages/database/prisma/seed-cpg-builders.ts | 96 ++++++++
pnpm-lock.yaml | 5 +
5 files changed, 446 insertions(+), 1 deletion(-)
diff --git a/apps/workers/package.json b/apps/workers/package.json
index b94a7f47..eb68bdd9 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -52,7 +52,8 @@
"@homesonspec/collector-smith-douglas": "workspace:*",
"@homesonspec/collector-goodall-homes": "workspace:*",
"@homesonspec/collector-brohn-homes": "workspace:*",
- "@homesonspec/collector-oakwood-homes": "workspace:*"
+ "@homesonspec/collector-oakwood-homes": "workspace:*",
+ "@homesonspec/collector-summit-homes-kc": "workspace:*"
},
"devDependencies": {
"tsx": "^4.19.2",
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index f5dc864c..cbf58b6a 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -33,6 +33,7 @@ import { smithDouglasAdapter } from "@homesonspec/collector-smith-douglas";
import { goodallHomesAdapter } from "@homesonspec/collector-goodall-homes";
import { brohnHomesAdapter } from "@homesonspec/collector-brohn-homes";
import { oakwoodHomesAdapter } from "@homesonspec/collector-oakwood-homes";
+import { summitHomesKcAdapter } from "@homesonspec/collector-summit-homes-kc";
import { runPipeline } from "./pipeline";
import { recordSourceRun } from "./verify";
@@ -75,6 +76,7 @@ const ADAPTERS = {
[goodallHomesAdapter.key]: goodallHomesAdapter,
[brohnHomesAdapter.key]: brohnHomesAdapter,
[oakwoodHomesAdapter.key]: oakwoodHomesAdapter,
+ [summitHomesKcAdapter.key]: summitHomesKcAdapter,
};
async function main() {
diff --git a/collectors/summit-homes-kc/src/index.ts b/collectors/summit-homes-kc/src/index.ts
new file mode 100644
index 00000000..cf97ee0d
--- /dev/null
+++ b/collectors/summit-homes-kc/src/index.ts
@@ -0,0 +1,341 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Summit Homes KC adapter — Kansas City area site-built spec-home builder,
+ * a Clayton Properties Group brand (slug "summit-homes-kc"; recon 2026-08-12,
+ * built 2026-08-17, TK-10487).
+ *
+ * www.summithomeskc.com is a React SSR site (data-reactid markers, react-text
+ * comment split) that is FULLY server-rendered — a plain honest-UA GET with
+ * Accept-Encoding gzip of a per-home page returns complete HTML. robots.txt is
+ * open (Allow: /). NO browser, NO bot-wall, NO XHR for the facts.
+ *
+ * Sitemap https://www.summithomeskc.com/sitemap.xml (~92 per-home URLs)
+ * -> per-home detail pages: /homes/{area}/{city}/{community}/{street-slug}
+ * (4 path segments after /homes/, all non-undefined = detail pages).
+ * Aggregate /homes/{area} and /homes/{area}/{city} pages are rejected.
+ * /homes/undefined/* stubs are rejected.
+ *
+ * Each per-home page carries two complementary sources:
+ *
+ * (1) JSON-LD block, @type["SingleFamilyResidence","Product"]:
+ * name (street + city-state-zip), address{streetAddress, addressLocality,
+ * addressRegion, postalCode}, geo{latitude, longitude} (per-home),
+ * offers[{price}], containedIn.name (community/subdivision),
+ * sku / productId (24-hex MongoDB ObjectId = builderInventoryId).
+ * An Organization/LocalBusiness/HomeAndConstructionBusiness block carries
+ * the builder-wide telephone "Summit Homes KC" and builder-wide geo
+ * (office address, NOT per-home).
+ *
+ * (2) HomeOverview_content section rendered as React SSR text. After stripping
+ * CSS (the React server inlines component CSS before the markup), the text
+ * node exposes the facts in a fixed order:
+ * "Beds N Baths N SQFT N,NNN Garage N -Car Lot # N {plan} Price $N
+ * Community {community} Floor Plan {plan-full} Status {Active|Move In Ready}"
+ * Beds / Baths / SQFT / Garage parse via simple regex on this text block.
+ * Status: "Active" -> UNDER_CONSTRUCTION; "Move In Ready" -> MOVE_IN_READY;
+ * "Model Home Not For Sale" -> treat as UNDER_CONSTRUCTION (not spec inventory).
+ * Plan name: text immediately after "Floor Plan" in the text block.
+ *
+ * Facts-only: images intentionally omitted (mediaRights=NONE per HomesOnSpec policy).
+ * Geo IS captured (per-home lat/lon in JSON-LD). Plain HTTP GET with gzip decompression.
+ *
+ * Batch control: SUMMIT_PAGE_LIMIT (default 40).
+ */
+const BUILDER_SLUG = "summit-homes-kc";
+const ORIGIN = "https://www.summithomeskc.com";
+const SITEMAP = `${ORIGIN}/sitemap.xml`;
+const PAGE_LIMIT = Number(process.env.SUMMIT_PAGE_LIMIT ?? "40");
+
+type SummitStatus = "MOVE_IN_READY" | "UNDER_CONSTRUCTION";
+
+interface SummitHome {
+ url: string;
+ street: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+ community: string | null;
+ planName: string | null;
+ price: number | null;
+ beds: number | null;
+ bathsTotal: number | null;
+ sqft: number | null;
+ garages: number | null;
+ lat: number | null;
+ lon: number | null;
+ phone: string | null;
+ status: SummitStatus;
+ builderInventoryId: string | null;
+}
+
+function fv<T>(
+ value: T | null,
+ raw: string | null,
+ sourceUrl: string,
+ evidenceText?: string | null,
+): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+function clean(v: unknown): string | null {
+ if (v == null) return null;
+ const s = String(v).replace(/<!--.*?-->/gs, "").trim();
+ return s || null;
+}
+
+function posInt(v: unknown): number | null {
+ if (v == null) return null;
+ const n = parseInt(String(v).replace(/[,\s]/g, ""), 10);
+ return isFinite(n) && n > 0 ? n : null;
+}
+
+function posDec(v: unknown): number | null {
+ if (v == null) return null;
+ const n = parseFloat(String(v).replace(/[,\s]/g, ""));
+ return isFinite(n) && n > 0 ? n : null;
+}
+
+function coord(v: unknown): number | null {
+ if (v == null) return null;
+ const n = parseFloat(String(v));
+ return isFinite(n) && n !== 0 ? n : null;
+}
+
+function hasType(obj: Record<string, unknown>, t: string): boolean {
+ const tp = obj["@type"];
+ return typeof tp === "string" ? tp === t : Array.isArray(tp) && tp.includes(t);
+}
+
+/** Strip <style> blocks (React inline CSS) from HTML before text extraction. */
+function stripStyles(html: string): string {
+ return html.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
+}
+
+/** Extract plain text from a region of stripped HTML. */
+function plainText(html: string): string {
+ return html
+ .replace(/<!--[\s\S]*?-->/g, " ")
+ .replace(/<[^>]+>/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+function parseHome(html: string, pageUrl: string): SummitHome | null {
+ // --- JSON-LD ---
+ const ldBlocks = [...html.matchAll(/<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)]
+ .map((m) => {
+ try {
+ // Clean control characters that occasionally appear in React SSR JSON
+ return JSON.parse(m[1]!.replace(/[\x00-\x1f\x7f]/g, " ")) as Record<string, unknown>;
+ } catch {
+ return null;
+ }
+ })
+ .filter((o): o is Record<string, unknown> => o != null);
+
+ const prod = ldBlocks.find(
+ (o) => hasType(o, "SingleFamilyResidence") || hasType(o, "Product"),
+ );
+ if (!prod) return null;
+
+ const addr = ((prod.address ?? {}) as Record<string, unknown>);
+ const geo = ((prod.geo ?? {}) as Record<string, unknown>);
+ const offers = Array.isArray(prod.offers)
+ ? (prod.offers[0] as Record<string, unknown>)
+ : ((prod.offers ?? {}) as Record<string, unknown>);
+ const containedIn = ((prod.containedIn ?? {}) as Record<string, unknown>);
+
+ const org = ldBlocks.find(
+ (o) => hasType(o, "Organization") || hasType(o, "LocalBusiness") || hasType(o, "HomeAndConstructionBusiness"),
+ );
+ const phone = clean(org?.telephone);
+
+ const street = clean(addr.streetAddress);
+ const city = clean(addr.addressLocality);
+ const stateRaw = clean(addr.addressRegion);
+ const zip = (clean(addr.postalCode) ?? "").match(/\d{5}/)?.[0] ?? null;
+ const community = clean(containedIn.name);
+ const price = posInt(offers.price);
+ const builderInventoryId = clean(prod.sku) ?? clean(prod.productId);
+
+ // --- HomeOverview text block ---
+ // After stripping styles, the HomeOverview_content section renders plain text
+ // in a fixed order: "Beds N Baths N SQFT N,NNN Garage N -Car Lot # N ..."
+ const stripped = stripStyles(html);
+ const ovIdx = stripped.indexOf("HomeOverview_content");
+ let overviewText = "";
+ if (ovIdx >= 0) {
+ // Take up to 3000 chars from the section start
+ overviewText = plainText(stripped.slice(ovIdx, ovIdx + 3000));
+ }
+
+ // Extract numeric facts from overview text
+ const bedsM = overviewText.match(/\bBeds\s+([\d,]+)/i);
+ const bathsM = overviewText.match(/\bBaths\s+([\d,.]+)/i);
+ const sqftM = overviewText.match(/\bSQFT\s+([\d,]+)/i) ?? overviewText.match(/\bSq\.\s*Ft\.\s+([\d,]+)/i);
+ const garageM = overviewText.match(/\bGarage\s+([\d]+)/i);
+
+ const beds = posInt(bedsM?.[1] ?? null);
+ const bathsTotal = posDec(bathsM?.[1] ?? null);
+ const sqft = posInt(sqftM?.[1] ?? null);
+ const garages = posInt(garageM?.[1] ?? null);
+
+ // Plan name: text after "Floor Plan" up to the next major label
+ const planM = overviewText.match(/Floor Plan\s+(.+?)(?:\s+Status\s|\s+Community\s|\s*$)/i);
+ const planName = clean(planM?.[1]?.trim() ?? null);
+
+ // Status: "Move In Ready" -> MOVE_IN_READY; "Active" or anything else -> UNDER_CONSTRUCTION
+ const statusM = overviewText.match(/Status\s+(\S.+?)(?:\s{2,}|\s+Have a|\s+About\s|\s*$)/i);
+ const statusText = statusM?.[1]?.trim() ?? "";
+ const status: SummitStatus = /move\s*in\s*ready/i.test(statusText)
+ ? "MOVE_IN_READY"
+ : "UNDER_CONSTRUCTION";
+
+ return {
+ url: clean(prod.url) ?? pageUrl,
+ street,
+ city,
+ state: normalizeStateCode(stateRaw),
+ zip,
+ community,
+ planName,
+ price,
+ beds,
+ bathsTotal,
+ sqft,
+ garages,
+ lat: coord(geo.latitude),
+ lon: coord(geo.longitude),
+ phone,
+ status,
+ builderInventoryId,
+ };
+}
+
+/** A per-home detail URL: /homes/{area}/{city}/{community}/{addr} (4 non-undefined segments) */
+function isDetailUrl(u: string): boolean {
+ const m = u.match(/^https?:\/\/[^/]+\/homes\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/?$/);
+ if (!m) return false;
+ const [, area, city, community, addr] = m;
+ return (
+ !!addr &&
+ addr !== "undefined" &&
+ area !== "undefined" &&
+ city !== "undefined" &&
+ community !== "undefined"
+ );
+}
+
+export const summitHomesKcAdapter: SourceAdapter = {
+ key: "summit-homes-kc-site",
+ version: "0.1.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ const index = await fetcher.fetch(SITEMAP);
+ const homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
+ .map((m) => m[1]!)
+ .filter(isDetailUrl);
+
+ for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
+ try {
+ yield await fetcher.fetch(url);
+ } catch (error) {
+ console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const html = page.body.toString("utf8");
+ const h = parseHome(html, page.url);
+ if (!h || !h.street) return { records: [], errors: [] };
+
+ const cname = h.community ?? "Unknown";
+ const records: ExtractedRecord[] = [];
+
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
+ fields: {
+ name: fv(cname, cname, page.url),
+ street: fv<string>(null, null, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(h.state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv<string>(null, null, page.url),
+ lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
+ lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
+ hoaFeeMonthly: fv<number>(null, null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ salesPhone: fv(h.phone, h.phone, page.url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: cname,
+ address: h.street,
+ builderInventoryId: h.builderInventoryId ?? h.url,
+ lat: h.lat ?? undefined,
+ lon: h.lon ?? undefined,
+ planName: h.planName ?? undefined,
+ },
+ fields: {
+ street: fv(h.street, h.street, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(h.state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ price: fv(
+ h.price,
+ h.price != null ? `$${h.price}` : null,
+ page.url,
+ h.price != null ? `Product offer price $${h.price}` : null,
+ ),
+ beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
+ bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
+ sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
+ stories: fv<number>(null, null, page.url), // not published per-home on summit
+ garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url),
+ homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Summit Homes KC single-family inventory"),
+ constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
+ h.status,
+ null,
+ page.url,
+ h.status === "MOVE_IN_READY" ? "Status: Move In Ready" : "Status: Active (under construction)",
+ ),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv<string>(null, null, page.url),
+ builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
+ lat: fv(h.lat, h.lat != null ? String(h.lat) : null, page.url),
+ lon: fv(h.lon, h.lon != null ? String(h.lon) : null, page.url),
+ planName: fv(h.planName, h.planName, page.url),
+ images: fv<string[]>([], null, page.url),
+ },
+ });
+
+ return { records, errors: [] };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/packages/database/prisma/seed-cpg-builders.ts b/packages/database/prisma/seed-cpg-builders.ts
new file mode 100644
index 00000000..11e44d0e
--- /dev/null
+++ b/packages/database/prisma/seed-cpg-builders.ts
@@ -0,0 +1,96 @@
+import { prisma } from "../src/index.js";
+
+/**
+ * Clayton Properties Group sub-brand adapters — batch 1 + 2 (TK-10487).
+ * Built 2026-08-12 (batch 1: goodall/brohn/oakwood) and 2026-08-17 (batch 2: summit-homes-kc).
+ * Parent builder slug: "clayton-properties" (already seeded).
+ */
+
+const CPG_BRANDS: {
+ slug: string;
+ name: string;
+ legalName: string;
+ website: string;
+ termsUrl: string;
+ coverage: string;
+}[] = [
+ {
+ slug: "goodall-homes",
+ name: "Goodall Homes",
+ legalName: "Goodall Homes, LLC",
+ website: "https://www.goodallhomes.com",
+ termsUrl: "https://www.goodallhomes.com/terms-of-use",
+ coverage: "Tennessee — Nashville metro",
+ },
+ {
+ slug: "brohn-homes",
+ name: "Brohn Homes",
+ legalName: "Brohn Homes, LLC",
+ website: "https://www.brohnhomes.com",
+ termsUrl: "https://www.brohnhomes.com/terms-of-use",
+ coverage: "Texas — Austin/San Antonio metro",
+ },
+ {
+ slug: "oakwood-homes",
+ name: "Oakwood Homes",
+ legalName: "Oakwood Homes, LLC",
+ website: "https://www.oakwoodhomesnc.com",
+ termsUrl: "https://www.oakwoodhomesnc.com/terms-of-use",
+ coverage: "North Carolina — Charlotte/Triad/Triangle markets",
+ },
+ {
+ slug: "summit-homes-kc",
+ name: "Summit Homes KC",
+ legalName: "Summit Homes KC, LLC",
+ website: "https://www.summithomeskc.com",
+ termsUrl: "https://www.summithomeskc.com/terms-of-use",
+ coverage: "Kansas — Kansas City metro",
+ },
+];
+
+async function main() {
+ for (const spec of CPG_BRANDS) {
+ const builder = await prisma.builder.upsert({
+ where: { slug: spec.slug },
+ update: { name: spec.name, legalName: spec.legalName, websiteUrl: spec.website },
+ create: {
+ slug: spec.slug,
+ name: spec.name,
+ legalName: spec.legalName,
+ websiteUrl: spec.website,
+ coverageArea: spec.coverage,
+ logoRightsStatus: "none",
+ partnershipStatus: null,
+ isDemo: false,
+ },
+ });
+ const regKey = `${spec.slug}-site`;
+ await prisma.sourceRegistry.upsert({
+ where: { key: regKey },
+ update: { termsUrl: spec.termsUrl },
+ create: {
+ key: regKey,
+ name: `${spec.name} (builder website)`,
+ builderId: builder.id,
+ baseUrl: spec.website,
+ collectionMethod: "CRAWL",
+ termsUrl: spec.termsUrl,
+ mediaRights: "NONE",
+ rateLimitRpm: 10,
+ freshIntervalHours: 24,
+ agingIntervalHours: 72,
+ health: "HEALTHY",
+ active: true,
+ notes: "Clayton Properties Group sub-brand. robots.txt open. Recon 2026-08-12. Adapter live.",
+ },
+ });
+ console.log(`registered ${spec.name} → ${regKey} (builder.id=${builder.id})`);
+ }
+}
+
+main()
+ .catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ })
+ .finally(() => prisma.$disconnect());
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 09cd5f4d..1c94c248 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -302,6 +302,9 @@ importers:
'@homesonspec/collector-smith-douglas':
specifier: workspace:*
version: link:../../collectors/smith-douglas
+ '@homesonspec/collector-summit-homes-kc':
+ specifier: workspace:*
+ version: link:../../collectors/summit-homes-kc
'@homesonspec/collector-taylor-morrison':
specifier: workspace:*
version: link:../../collectors/taylor-morrison
@@ -2973,10 +2976,12 @@ packages:
'@xmldom/xmldom@0.8.13':
resolution: {integrity: sha512-KRYzxepc14G/CEpEGc3Yn+JKaAeT63smlDr+vjB8jRfgTBBI9wRj/nkQEO+ucV8p8I9bfKLWp37uHgFrbntPvw==}
engines: {node: '>=10.0.0'}
+ deprecated: this version has critical issues, please update to the latest version
'@xmldom/xmldom@0.9.10':
resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==}
engines: {node: '>=14.6'}
+ deprecated: this version has critical issues, please update to the latest version
abort-controller@3.0.0:
resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
← d5513153 HomesOnSpec search: make Table columns bidirectionally sorta
·
back to Homesonspec
·
auto-data-snapshot: 2026-08-24T04:45:01 (6 data files) — col 67563c20 →