← back to Homesonspec
collectors/chafin-communities/src/index.ts
514 lines
import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
import { normalizeStateCode } from "@homesonspec/shared";
import {
fetchFixtures,
LiveFetcher,
type ExtractionOutput,
type FetchContext,
type RawPage,
type SourceAdapter,
} from "@homesonspec/collectors-common";
/**
* Chafin Communities adapter — Metro Atlanta (GA) site-built spec-home
* builder, a Clayton Properties Group brand (slug "chafin-communities";
* recon + built 2026-08-24, TK-10487, batch 2).
*
* chafincommunities.com is a WordPress site (Yoast SEO, `rvadv` builder
* plugin) that is FULLY server-rendered — a plain GET returns complete HTML
* with all available-home data inline. NO browser, NO bot-wall, NO XHR.
*
* robots.txt: `User-agent: * Disallow:` (everything allowed) + Yoast sitemap
* index at https://www.chafincommunities.com/sitemap_index.xml.
*
* Sitemap index https://www.chafincommunities.com/sitemap_index.xml
* -> rvadv_communities-sitemap.xml (the per-COMMUNITY inventory, the
* data source for this adapter; currently ~34 communities in GA).
* URL pattern:
* /communities/{state}/{county}/{city}/{community-slug}/
* The plan sitemap (rvadv_floorplans_adv-sitemap.xml) lists model
* floor-plan pages, NOT per-home spec inventory — skip.
*
* VERIFIED DOM STRUCTURE (2026-08-24, live page sampling):
*
* <div class="single_community_available_homes_wrapper">
* <div class="...single_community_available_homes_listing"
* data-lot="86C" data-id="7733108" data-status="Active">
* <a>
* <div class="...single_community_available_homes_listing_info">
* <h2>$356,440</h2> <- price (formatted USD)
* <div class="grid-60...">
* <p>3739 Ivy Cottage Drive<br /> <- per-home street address
* Snellville GA 30039</p> <- city state zip (no comma)
* </div>
* </div>
* </a>
* <div class="...single_community_available_homes_listing_details">
* <p><strong>3</strong>BEDS</p> <- beds
* <p><strong><strong>2</strong> .5</strong>BATHS</p> <- baths (nested strong)
* <span data-price="356440" ...> <- payment calc span (int USD)
* </div>
* </div>
* </div>
*
* NOTE: data-price is NOT on the outer listing div; it is on a payment-
* calculator <span> inside _listing_details. The <h2> price (with $ comma
* formatting) is the primary price signal; the span data-price is a clean
* integer backup. Per-home street addresses ARE published (contrary to initial
* recon — the address is embedded in the listing info block).
*
* JSON-LD (community page — nested in @graph array):
* @type: LocalBusiness
* name -> community name
* address.streetAddress / addressLocality / addressRegion / postalCode
* geo.latitude / geo.longitude
* telephone
*
* Status mapping: "Active" / "Open" -> MOVE_IN_READY;
* "Sold" -> skip; "Model Home" -> skip;
* any other status -> UNDER_CONSTRUCTION.
*
* Batch control:
* CHAFIN_COMMUNITY_LIMIT max community pages per run (default 20)
* CHAFIN_HOME_LIMIT max homes per community (default 50)
*
* Cost: $0 (plain HTTP fetch).
*/
const BUILDER_SLUG = "chafin-communities";
const ORIGIN = "https://www.chafincommunities.com";
const COMMUNITIES_SITEMAP = `${ORIGIN}/rvadv_communities-sitemap.xml`;
const COMMUNITY_LIMIT = Number(process.env.CHAFIN_COMMUNITY_LIMIT ?? "20");
const HOME_LIMIT = Number(process.env.CHAFIN_HOME_LIMIT ?? "50");
/** 4 path segments after /communities/ (state/county/city/community-slug). */
const COMMUNITY_URL_RE =
/^https?:\/\/[^/]+\/communities\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/?$/i;
// ── helpers ──────────────────────────────────────────────────────────────────
function fv<T>(
value: T | null,
raw: string | null,
sourceUrl: string,
evidenceText?: string | null,
): FieldValue<T> {
return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
}
const posInt = (v: unknown): number | null => {
const n =
typeof v === "number"
? v
: typeof v === "string"
? Number(String(v).replace(/[^0-9]/g, ""))
: NaN;
return Number.isFinite(n) && n > 0 ? Math.trunc(n) : null;
};
const posDec = (v: unknown): number | null => {
const n =
typeof v === "number"
? v
: typeof v === "string"
? Number(String(v).replace(/[^0-9.]/g, ""))
: NaN;
return Number.isFinite(n) && n > 0 ? n : null;
};
const coord = (v: unknown): number | null => {
const n =
typeof v === "number"
? v
: typeof v === "string"
? Number(v.trim())
: NaN;
return Number.isFinite(n) && n !== 0 ? n : null;
};
const clean = (v: unknown): string | null => {
if (v == null) return null;
const s = String(v)
.replace(/<[^>]+>/g, " ")
.replace(/'|'|'/g, "'")
.replace(/&/g, "&")
.replace(/"/g, '"')
.replace(/ /g, " ")
.replace(/\s+/g, " ")
.trim();
return s || null;
};
/**
* Pull all JSON-LD objects from the page, unwrapping @graph arrays.
* Chafin uses `{ "@context": ..., "@graph": [...] }` wrappers.
*/
function jsonLdObjects(html: string): Record<string, unknown>[] {
const out: Record<string, unknown>[] = [];
for (const b of html.matchAll(
/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi,
)) {
let data: unknown;
try {
data = JSON.parse(b[1]!.trim());
} catch {
continue;
}
const items = Array.isArray(data) ? data : [data];
for (const it of items) {
if (!it || typeof it !== "object") continue;
const obj = it as Record<string, unknown>;
// Unwrap @graph arrays (Yoast SEO pattern)
if (Array.isArray(obj["@graph"])) {
for (const g of obj["@graph"] as unknown[]) {
if (g && typeof g === "object") out.push(g as Record<string, unknown>);
}
} else {
out.push(obj);
}
}
}
return out;
}
// ── per-home DOM types ────────────────────────────────────────────────────────
type HomeStatus = "MOVE_IN_READY" | "UNDER_CONSTRUCTION";
interface ChafinHome {
lot: string | null;
builderInventoryId: string | null;
statusRaw: string | null;
status: HomeStatus;
price: number | null;
beds: number | null;
baths: number | null;
/** Per-home street address embedded in the listing_info block. */
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
}
interface ChafinCommunity {
name: string | null;
street: string | null;
city: string | null;
state: string | null;
zip: string | null;
lat: number | null;
lon: number | null;
phone: string | null;
}
// ── page parser ───────────────────────────────────────────────────────────────
/**
* Parse a Chafin community page into community info + available homes.
*
* Splits on `data-lot="` (unique to the outer listing divs, not child divs)
* to carve up the home blocks without fighting nested-div regex hell.
*/
export function parseCommunityPage(
html: string,
_pageUrl: string,
): { community: ChafinCommunity; homes: ChafinHome[] } | null {
if (!html.includes("single_community_available_homes")) return null;
// ── community info from JSON-LD LocalBusiness (@graph unwrapped) ──
const objs = jsonLdObjects(html);
const lb = objs.find(
(o) => o["@type"] === "LocalBusiness" || o["@type"] === "RealEstateAgent",
) as Record<string, unknown> | undefined;
const addr = (lb?.address ?? {}) as Record<string, unknown>;
const geo = (lb?.geo ?? {}) as Record<string, unknown>;
const lbName = clean(lb?.name);
const titleName = clean(html.match(/<title[^>]*>([^<|]+)/i)?.[1]);
const name = lbName ?? titleName ?? null;
const community: ChafinCommunity = {
name,
street: clean(addr.streetAddress),
city: clean(addr.addressLocality),
state: normalizeStateCode(clean(addr.addressRegion)),
zip: (clean(addr.postalCode) ?? "").match(/\d{5}/)?.[0] ?? null,
lat: coord(geo.latitude),
lon: coord(geo.longitude),
phone: clean(lb?.telephone),
};
// ── homes — split on data-lot= attribute (only on outer listing divs) ──
//
// Strategy: find each block that starts with the outer listing div opening tag
// (which has data-lot= on it) and ends just before the NEXT outer listing div.
// We identify outer listing blocks by locating their opening tags via
// data-lot= attr, then carve the segment up to the next such tag (or EOF).
const homes: ChafinHome[] = [];
// Find all positions of `data-lot="` in the listing section
const wrapperStart = html.indexOf("single_community_available_homes_wrapper");
if (wrapperStart < 0) return { community, homes };
// Find position of wrapper end (approx)
const wrapperEnd = html.indexOf("</section>", wrapperStart);
const section = wrapperEnd > wrapperStart ? html.slice(wrapperStart, wrapperEnd) : html.slice(wrapperStart);
// Split the section by outer-listing block openings.
//
// Anchor on the OUTER listing div opening tag itself — a `<div>` whose class
// list contains the whole token `single_community_available_homes_listing`
// (word-boundary: NOT the `_info` / `_details` / `_wrapper` child variants),
// and which carries `data-lot=`. In practice ~131 `data-lot=` attrs appear on
// the page (child payment-calc divs carry them too), but only the ~10 outer
// listing divs bear the listing class, so we split on those.
//
// NB: the previous implementation used `(?<!"[^"]*)\bdata-lot="` — a
// variable-length negative lookbehind that ALWAYS fails in a large HTML doc
// (there is always a `"` somewhere earlier), so it matched 0 times and every
// community yielded 0 homes. Fixed 2026-08-30 (TK-10487 verification).
const blockStarts: number[] = [];
const listingOpenRe =
/<div\b[^>]*\bclass="[^"]*\bsingle_community_available_homes_listing\b(?!_)[^"]*"[^>]*\bdata-lot="/gi;
let m: RegExpExecArray | null;
while ((m = listingOpenRe.exec(section)) !== null) {
blockStarts.push(m.index); // m.index is the `<div` position
}
blockStarts.push(section.length); // sentinel
for (let i = 0; i < blockStarts.length - 1; i++) {
const block = section.slice(blockStarts[i]!, blockStarts[i + 1]!);
// Extract data-lot, data-id, data-status from the opening tag
const openTag = block.slice(0, Math.min(600, block.length));
const lot = clean(openTag.match(/data-lot="([^"]*)"/)?.[1]);
const id = clean(openTag.match(/data-id="([^"]*)"/)?.[1]);
const statusRaw = clean(openTag.match(/data-status="([^"]*)"/)?.[1]);
// Skip sold / model homes
if (statusRaw && /sold|model/i.test(statusRaw)) continue;
const status: HomeStatus =
statusRaw && /active|open|available/i.test(statusRaw)
? "MOVE_IN_READY"
: "UNDER_CONSTRUCTION";
// Price from <h2> (primary — formatted "$356,440") or payment-span data-price (clean int)
const h2Price = clean(block.match(/<h2[^>]*>([\s\S]*?)<\/h2>/i)?.[1]);
const spanPrice = block.match(/\bdata-price="(\d+)"/)?.[1];
const price = posInt(spanPrice) ?? posInt(h2Price?.replace(/[^0-9]/g, "") ?? null);
// Address from grid-60 <p>: street<br/>city state zip
const addressPara = block.match(
/class="[^"]*grid-60[^"]*"[^>]*>[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/i,
)?.[1];
let street: string | null = null;
let city: string | null = null;
let state: string | null = null;
let zip: string | null = null;
if (addressPara) {
// "3739 Ivy Cottage Drive<br />Snellville GA 30039"
const lines = addressPara
.replace(/<br\s*\/?>/gi, "\n")
.replace(/<[^>]+>/g, "")
.split(/\n/)
.map((l) => l.trim())
.filter(Boolean);
if (lines.length >= 2) {
street = lines[0]!;
// "Snellville GA 30039" -> city="Snellville" state="GA" zip="30039"
const cityStateLine = lines[1]!;
const csm = cityStateLine.match(/^(.+?)\s+([A-Z]{2})\s+(\d{5})/);
if (csm) {
city = csm[1]!.trim();
state = normalizeStateCode(csm[2]!);
zip = csm[3]!;
} else {
city = cityStateLine;
}
} else if (lines.length === 1) {
street = lines[0]!;
}
}
// Fall back to community address for city/state/zip
city = city ?? community.city;
state = state ?? community.state;
zip = zip ?? community.zip;
// Beds / baths each live in their OWN `<p>…BEDS</p>` / `<p>…BATHS</p>`
// element, e.g. `<p><strong>3</strong>BEDS</p>` and, for half-baths,
// `<p><strong><strong>2</strong> .5</strong>BATHS</p>`.
//
// We scope the value extraction to the SINGLE <p> that carries the label,
// then take digits from that <p> only. This avoids the earlier bug where a
// non-greedy `<strong>…</strong>\s*BATHS` swallowed the preceding BEDS <p>
// (…</strong>BEDS</p><p><strong><strong>2…) and fabricated a "32.5"-bath
// home. The `[^<]*` before the label keeps the match anchored to the label's
// own paragraph. Fixed 2026-08-30 (TK-10487 verification).
const bedsP = block.match(/<p\b[^>]*>((?:(?!<\/p>)[\s\S])*?)BEDS<\/p>/i)?.[1];
const beds = posInt(bedsP?.replace(/<[^>]+>/g, " ").replace(/[^0-9]/g, ""));
const bathsP = block.match(/<p\b[^>]*>((?:(?!<\/p>)[\s\S])*?)BATHS<\/p>/i)?.[1];
const bathsText = bathsP?.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim() ?? "";
const baths = posDec(bathsText.replace(/[^0-9.]/g, ""));
homes.push({
lot,
builderInventoryId: id,
statusRaw,
status,
price,
beds,
baths,
street,
city,
state,
zip,
});
}
return { community, homes };
}
// ── adapter ───────────────────────────────────────────────────────────────────
export const chafinCommunitiesAdapter: SourceAdapter = {
key: "chafin-communities-site",
version: "0.1.0",
async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
if (ctx.mode === "fixture") {
yield* fetchFixtures(ctx);
return;
}
const fetcher = new LiveFetcher(ctx.registry);
// 1. Fetch communities sitemap
let communityUrls: string[] = [];
try {
const sm = await fetcher.fetch(COMMUNITIES_SITEMAP);
communityUrls = [
...sm.body.toString("utf8").matchAll(/<loc>\s*([^<\s]+)\s*<\/loc>/g),
]
.map((m) => m[1]!)
.filter((u) => COMMUNITY_URL_RE.test(u));
} catch (error) {
console.warn(
` chafin: failed to fetch communities sitemap: ${error instanceof Error ? error.message : String(error)}`,
);
return;
}
// 2. Fetch each community page (bounded)
for (const url of communityUrls.slice(0, COMMUNITY_LIMIT)) {
try {
yield await fetcher.fetch(url);
} catch (error) {
console.warn(
` chafin: skip ${url}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
},
extract(page: RawPage): ExtractionOutput {
try {
const html = page.body.toString("utf8");
const parsed = parseCommunityPage(html, page.url);
if (!parsed) return { records: [], errors: [] };
const { community: c, homes } = parsed;
if (!homes.length) return { records: [], errors: [] };
const cname = c.name ?? "Unknown";
const records: ExtractedRecord[] = [];
// Community record
records.push({
entityType: "community",
canonicalHints: { builderSlug: BUILDER_SLUG, communityName: cname },
fields: {
name: fv(cname, cname, page.url),
street: fv(c.street, c.street, page.url),
city: fv(c.city, c.city, page.url),
state: fv(c.state, c.state, page.url),
zip: fv(c.zip, c.zip, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
lat: fv(c.lat, c.lat != null ? String(c.lat) : null, page.url),
lon: fv(c.lon, c.lon != null ? String(c.lon) : null, page.url),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
salesPhone: fv(c.phone, c.phone, page.url),
},
});
// Per-home records
for (const h of homes.slice(0, HOME_LIMIT)) {
const addrStr = h.street
? `${h.street}, ${h.city ?? ""} ${h.state ?? ""} ${h.zip ?? ""}`.trim()
: null;
records.push({
entityType: "inventory_home",
canonicalHints: {
builderSlug: BUILDER_SLUG,
communityName: cname,
// Precedence: ?? binds looser than ?:, so the ternary MUST be
// parenthesized — otherwise `addrStr ?? h.lot ? …` parses as
// `(addrStr ?? h.lot) ? …` and every real street address emits
// "Lot X" as the canonical dedupe key (TK-10487 contrarian catch).
address:
addrStr ?? (h.lot ? `Lot ${h.lot}` : (h.builderInventoryId ?? page.url)),
builderInventoryId: h.builderInventoryId ?? page.url,
},
fields: {
street: fv(h.street, h.street, page.url),
city: fv(h.city, h.city, page.url),
state: fv(h.state, h.state, page.url),
zip: fv(h.zip, h.zip, page.url),
price: fv(
h.price,
h.price != null ? `$${h.price.toLocaleString()}` : null,
page.url,
),
beds: fv(h.beds, h.beds != null ? String(h.beds) : null, page.url),
bathsTotal: fv(
h.baths,
h.baths != null ? String(h.baths) : null,
page.url,
),
// sqft not published in the available-homes DOM section.
sqft: fv<number>(null, null, page.url),
stories: fv<number>(null, null, page.url),
garageSpaces: fv<number>(null, null, page.url),
homeType: fv(
"SINGLE_FAMILY" as const,
null,
page.url,
"Chafin Communities single-family inventory home",
),
constructionStatus: fv<"PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY">(
h.status,
h.statusRaw,
page.url,
h.statusRaw ? `data-status="${h.statusRaw}"` : null,
),
estCompletionDate: fv<string>(null, null, page.url),
lotNumber: fv(h.lot, h.lot, page.url, h.lot ? `Lot ${h.lot}` : null),
builderInventoryId: fv(h.builderInventoryId, h.builderInventoryId, page.url),
lat: fv<number>(null, null, page.url),
lon: fv<number>(null, null, page.url),
planName: fv<string>(null, null, page.url),
images: fv<string[]>([], null, page.url),
},
});
}
return { records, errors: [] };
} catch (error) {
return { records: [], errors: [{ url: page.url, reason: String(error) }] };
}
},
};