← back to Homesonspec
collectors: add Smith Douglas Homes inventory adapter (facts-only)
ec4d72cfa5cbbf6dd08187c1ea51ad72d5bffcde · 2026-08-10 21:51:14 -0700 · Steve
Southeast builder (SDHC). smithdouglas.com is React SSR with fully open
robots.txt; per-home detail pages (/homes/{metro}/{city}/{community}/{addr})
are server-rendered, so this is a plain-fetch adapter (no browser, no bot-wall).
Extracts facts-only from the SSR DOM: address, city/state/zip, price
(null on Contact-For-Pricing homes), beds, bathsTotal (decimal, half folded),
sqft, garage, lot#, plan name, community. Per-home geo + stories absent on the
page -> null. Images OMITTED, mediaRights=NONE. builderInventoryId = detail URL.
Representative self-test (25 homes random across all 752 detail URLs, all real
distinct addresses): address 100%, beds 100%, baths 100%, sqft 96%, price 92%.
Typecheck clean. Scoped to collectors/smith-douglas/ only.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A collectors/smith-douglas/src/index.ts
Diff
commit ec4d72cfa5cbbf6dd08187c1ea51ad72d5bffcde
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Aug 10 21:51:14 2026 -0700
collectors: add Smith Douglas Homes inventory adapter (facts-only)
Southeast builder (SDHC). smithdouglas.com is React SSR with fully open
robots.txt; per-home detail pages (/homes/{metro}/{city}/{community}/{addr})
are server-rendered, so this is a plain-fetch adapter (no browser, no bot-wall).
Extracts facts-only from the SSR DOM: address, city/state/zip, price
(null on Contact-For-Pricing homes), beds, bathsTotal (decimal, half folded),
sqft, garage, lot#, plan name, community. Per-home geo + stories absent on the
page -> null. Images OMITTED, mediaRights=NONE. builderInventoryId = detail URL.
Representative self-test (25 homes random across all 752 detail URLs, all real
distinct addresses): address 100%, beds 100%, baths 100%, sqft 96%, price 92%.
Typecheck clean. Scoped to collectors/smith-douglas/ only.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
collectors/smith-douglas/src/index.ts | 305 ++++++++++++++++++++++++++++++++++
1 file changed, 305 insertions(+)
diff --git a/collectors/smith-douglas/src/index.ts b/collectors/smith-douglas/src/index.ts
new file mode 100644
index 00000000..c37c8a52
--- /dev/null
+++ b/collectors/smith-douglas/src/index.ts
@@ -0,0 +1,305 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Smith Douglas Homes adapter — Southeast builder (public builder SDHC; slug
+ * "smith-douglas", recon + built 2026-08-10 TK-10001).
+ *
+ * smithdouglas.com is a React SSR site (react-helmet + `data-reactid` markers).
+ * robots.txt is fully open ("User-agent: * Allow: /") and references the
+ * sitemap. Every per-home detail page is FULLY server-rendered — the facts we
+ * keep live in the delivered HTML, not behind an XHR/WebSocket — so this is a
+ * plain-fetch adapter (no browser, no bot-wall). Verified 16/16 homes across
+ * GA/TX/NC/TN/AL for address + beds + baths + garage + sqft.
+ *
+ * Sitemap https://www.smithdouglas.com/sitemap.xml (single urlset, ~1688 URLs)
+ * -> per-home inventory detail pages of the form
+ * /homes/{metro}/{city}/{community}/{street-address-slug}
+ * (~627 fully-slugged + ~125 with `undefined` metro/city/community
+ * segments but a real street-address slug — both render valid facts).
+ * Community/landing pages (/communities/…, /homes/{metro}, /plan/…) and
+ * the "undefined/undefined" partials are filtered out — we keep only the
+ * 4-real-segment /homes/… detail URLs (a detail page has the full home
+ * card; a listing page renders many HomeCards which are OTHER homes).
+ *
+ * Each detail page carries (facts-only, from the SSR DOM):
+ * <h1 class="HomeOverview_name"> first text node = street address, then a
+ * <span> "City, ST ZIP", an optional "| Lot# NN" span, an optional
+ * "Floor Plan: <a>The Plan</a>", and "Community: <a>Name</a>".
+ * <h4 class="HomeOverview_priceFrom"> "Priced at $NNN,NNN" | "Contact For
+ * Pricing" (many active spec homes withhold price -> price null, correct).
+ * <ul class="HomeOverview_list"> icon-labelled Beds / Baths / Garage /
+ * SQ.FT. — baths is a single decimal ("2.5") with half-baths already
+ * folded in (schema bathsTotal is one number).
+ *
+ * NO per-home geo (only the corporate Organization GeoCoordinates lives in the
+ * JSON-LD — that is the HQ, not the home; per-home lat/lon is absent, left null,
+ * exactly like richmond-american). NO stories field on the page -> null. Plain
+ * HTTP; one detail page == one inventory_home. Facts-only, images OMITTED,
+ * mediaRights=NONE.
+ *
+ * Metro scope via SMITHDOUGLAS_METRO (comma-list of metro slugs, e.g.
+ * "houston-tx,atlanta-ga"; default = all metros). Per-run home cap via
+ * SMITHDOUGLAS_PAGE_LIMIT (default 40).
+ */
+const BUILDER_SLUG = "smith-douglas";
+const ORIGIN = "https://www.smithdouglas.com";
+const SITEMAP = `${ORIGIN}/sitemap.xml`;
+const METRO_FILTER = (process.env.SMITHDOUGLAS_METRO ?? "")
+ .toLowerCase()
+ .split(",")
+ .map((s) => s.trim())
+ .filter(Boolean);
+const PAGE_LIMIT = Number(process.env.SMITHDOUGLAS_PAGE_LIMIT ?? "40");
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+const intNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const decNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const clean = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v)
+ .replace(/'|'|'/g, "'")
+ .replace(/&/g, "&")
+ .replace(/<[^>]+>/g, "")
+ .replace(/\s+/g, " ")
+ .trim();
+ return s || null;
+};
+
+/** strip HTML comments + tags from a fragment, collapse whitespace */
+function stripTags(fragment: string): string {
+ return fragment
+ .replace(/<!--[\s\S]*?-->/g, "")
+ .replace(/<[^>]+>/g, "")
+ .replace(/'|'|'/g, "'")
+ .replace(/&/g, "&")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+interface ParsedHome {
+ street: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+ community: string | null;
+ planName: string | null;
+ lotNumber: string | null;
+ price: number | null;
+ priceRaw: string | null;
+ beds: number | null;
+ bathsTotal: number | null; // decimal, half-baths folded
+ garages: number | null;
+ sqft: number | null;
+}
+
+/**
+ * Parse ONE Smith Douglas detail page (SSR react HTML) into a home, or null if
+ * the page has no HomeOverview block (a non-detail page slipped through).
+ * Exported for the self-test / fixtures.
+ */
+export function parseHome(html: string): ParsedHome | null {
+ const h1m = html.match(/<h1[^>]*class="[^"]*HomeOverview_name[^"]*"[^>]*>([\s\S]*?)<\/h1>/i);
+ if (!h1m) return null;
+ const h1 = h1m[1]!;
+
+ // The first text node of the <h1> (before the first child <span>/<a>) is the
+ // true street address — even on model homes where <title> shows the plan name.
+ const firstSpan = h1.search(/<span\b/i);
+ const addrFragment = firstSpan >= 0 ? h1.slice(0, firstSpan) : h1;
+ const street = clean(stripTags(addrFragment));
+
+ // The leading <span> carries "City, ST ZIP".
+ const spans = [...h1.matchAll(/<span[^>]*>([\s\S]*?)<\/span>/gi)].map((m) => stripTags(m[1]!));
+ const locLine = spans.find((s) => /,\s*[A-Za-z]{2}\.?\s*\d{5}/.test(s)) ?? null;
+ const locM = locLine?.match(/^(.*?),\s*([A-Za-z]{2})\.?\s*(\d{5})/);
+ const city = clean(locM?.[1]);
+ const state = normalizeStateCode(locM?.[2] ?? null);
+ const zip = (locM?.[3] ?? "").match(/\d{5}/)?.[0] ?? null;
+
+ // "| Lot# 13" (text-nowrap span).
+ const lotNumber = clean(h1.match(/Lot#\s*([0-9A-Za-z\-]+)/i)?.[1]);
+
+ // Anchored labels: "Floor Plan: <a>The James</a>" / "Community: <a>Name</a>".
+ const planName = clean(
+ h1.match(/Floor Plan:[\s\S]*?<a[^>]*>([\s\S]*?)<\/a>/i)?.[1],
+ );
+ const community = clean(
+ h1.match(/Community:[\s\S]*?<a[^>]*>([\s\S]*?)<\/a>/i)?.[1],
+ );
+
+ // Price: <h4 class="HomeOverview_priceFrom">…</h4>. Text is either a
+ // "$NNN,NNN" amount or "Contact For Pricing" (-> price null, correct).
+ const priceBlock = html.match(/<h4[^>]*class="[^"]*HomeOverview_priceFrom[^"]*"[^>]*>([\s\S]*?)<\/h4>/i);
+ const priceText = priceBlock ? stripTags(priceBlock[1]!) : null;
+ const priceMatch = priceText?.match(/\$\s*([0-9][0-9,]{2,})/);
+ const price = priceMatch ? intNum(priceMatch[1]) : null;
+ const priceRaw = priceMatch ? priceMatch[0] : priceText;
+
+ // Overview list: icon-labelled Beds / Baths / Garage / SQ.FT.
+ // <img ... alt="icon beds"/> <h4>value</h4>. Scope to the HomeOverview_list so
+ // we never read a sibling HomeCard's numbers.
+ const listBlock = html.match(/<ul[^>]*class="[^"]*HomeOverview_list[^"]*"[^>]*>([\s\S]*?)<\/ul>/i);
+ let beds: number | null = null;
+ let bathsTotal: number | null = null;
+ let garages: number | null = null;
+ let sqft: number | null = null;
+ if (listBlock) {
+ for (const m of listBlock[1]!.matchAll(/alt="icon\s+(\w+)"[^>]*\/?>\s*<h4[^>]*>([\s\S]*?)<\/h4>/gi)) {
+ const label = m[1]!.toLowerCase();
+ const val = stripTags(m[2]!);
+ if (label === "beds") beds = intNum(val);
+ else if (label === "baths") bathsTotal = decNum(val);
+ else if (label === "garage") garages = intNum(val);
+ else if (label === "sqft") sqft = intNum(val);
+ }
+ }
+
+ return {
+ street,
+ city,
+ state,
+ zip,
+ community,
+ planName,
+ lotNumber,
+ price,
+ priceRaw,
+ beds,
+ bathsTotal,
+ garages,
+ sqft,
+ };
+}
+
+/**
+ * A per-home inventory DETAIL URL: /homes/{metro}/{city}/{community}/{addr}
+ * — exactly four path segments after /homes/, with a REAL final address segment.
+ * The metro/city/community segments are sometimes literally "undefined" in the
+ * sitemap (a builder-CMS quirk) yet the page still renders full per-home facts,
+ * so we accept those too and recover their community from the page's H1
+ * "Community:" link. What we reject: aggregate listing pages
+ * (/homes/{metro}, /homes/{metro}/{city}, /homes/undefined/undefined) which
+ * render many sibling HomeCards, and any URL whose final segment is "undefined".
+ */
+function isDetailUrl(u: string): boolean {
+ const m = u.match(/^https?:\/\/[^/]+\/homes\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/([^/?#]+)\/?$/);
+ if (!m) return false;
+ const addr = m[4]!;
+ return !!addr && addr !== "undefined"; // real address slug required; upstream segments may be "undefined"
+}
+
+export const smithDouglasAdapter: SourceAdapter = {
+ key: "smith-douglas-site",
+ version: "0.1.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ const index = await fetcher.fetch(SITEMAP);
+ let homeUrls = [...index.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g)]
+ .map((m) => m[1]!)
+ .filter(isDetailUrl);
+ if (METRO_FILTER.length) {
+ homeUrls = homeUrls.filter((u) => {
+ const metro = u.match(/\/homes\/([^/]+)\//)?.[1]?.toLowerCase();
+ return metro ? METRO_FILTER.includes(metro) : false;
+ });
+ }
+ for (const url of homeUrls.slice(0, PAGE_LIMIT)) {
+ try {
+ yield await fetcher.fetch(url);
+ } catch (error) {
+ console.warn(` skip ${url}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const html = page.body.toString("utf8");
+ const h = parseHome(html);
+ if (!h || !h.street) return { records: [], errors: [] }; // not a resolvable detail page
+
+ const cname = h.community ?? "Unknown";
+ const records: ExtractedRecord[] = [];
+
+ // community record (no per-home geo available; sales phone is builder-wide
+ // and lives outside the home block, so left null on the community too)
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
+ fields: {
+ name: fv(cname, cname, page.url),
+ street: fv<string>(null, null, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(h.state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv<string>(null, null, page.url),
+ lat: fv<number>(null, null, page.url),
+ lon: fv<number>(null, null, page.url),
+ hoaFeeMonthly: fv<number>(null, null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ salesPhone: fv<string>(null, null, page.url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: cname,
+ address: h.street,
+ builderInventoryId: page.url, // no numeric job id on the page — detail URL is the stable per-home key
+ planName: h.planName,
+ },
+ fields: {
+ street: fv(h.street, h.street, page.url),
+ city: fv(h.city, h.city, page.url),
+ state: fv(h.state, h.state, page.url),
+ zip: fv(h.zip, h.zip, page.url),
+ // per-home geo NOT published by Smith Douglas — left null (like richmond-american)
+ lat: fv<number>(null, null, page.url),
+ lon: fv<number>(null, null, page.url),
+ price: fv(h.price, h.priceRaw, page.url, h.priceRaw ? `price ${h.priceRaw}` : null),
+ beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
+ bathsTotal: fv(h.bathsTotal, h.bathsTotal != null ? String(h.bathsTotal) : null, page.url),
+ sqft: fv(h.sqft, h.sqft != null ? String(h.sqft) : null, page.url),
+ stories: fv<number>(null, null, page.url), // not stated on the page
+ garageSpaces: fv(h.garages, h.garages != null ? String(h.garages) : null, page.url),
+ homeType: fv("SINGLE_FAMILY" as never, null, page.url, "Smith Douglas inventory home"),
+ constructionStatus: fv("UNDER_CONSTRUCTION" as never, null, page.url, "Smith Douglas active inventory (no status label on page)"),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv(h.lotNumber, h.lotNumber, page.url),
+ builderInventoryId: fv(page.url, page.url, page.url),
+ planName: fv(h.planName, h.planName, page.url),
+ },
+ });
+
+ return { records, errors: [] };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
← 03e75060 auto-data-snapshot: 2026-08-10T21:49:05 (6 data files) — col
·
back to Homesonspec
·
clayton-properties: DEFER recon — scope mismatch (manufactur 8d0fbaf1 →