← back to Homesonspec
collectors: add Fischer, Fulton, GL Homes adapters (+3 builders, yolo iter-2)
840c359e4c89fca5e1858860c52e9677d37fe6d4 · 2026-07-28 22:02:14 -0700 · Steve Abrams
DTD-panel-approved (5/5 verdict A) increment. All facts-only/honest-UA/robots-respected,
built via 3 parallel isolated recon+build subagents + serial gated integration:
- Fischer (fischer-homes-site): region API /api/region-revamped/homes/{id} + per-home detail
JSON-LD for geo; OH/KY/IN/GA/MO; 100% price/beds/baths/sqft/lat-lon. region-not-area avoids
the view-filter trap.
- Fulton (fulton-homes-site): ASP.NET GetSpecs web service, AZ, 166 homes 100% price/beds/sqft.
GetSpecs-not-GetHomes avoids the floorplan aggregate.
- GL Homes (gl-homes-site): two-layer HTML crawl (community -> per-QMI-<li>), FL, 168 homes 100%
price/beds/sqft. Per-<li> emission avoids collapsing 168 homes into 68 floorplan cards.
Fulton/GL every-sweep; Fischer throttled (per-home geo GETs). All 3 pre-seeded w/ Builder+SourceRegistry.
Files touched
M apps/workers/package.jsonM apps/workers/src/cli.tsA collectors/fischer-homes/package.jsonA collectors/fischer-homes/selftest.mjsA collectors/fischer-homes/src/index.tsA collectors/fischer-homes/tsconfig.jsonA collectors/fulton-homes/package.jsonA collectors/fulton-homes/src/index.tsA collectors/fulton-homes/tsconfig.jsonA collectors/gl-homes/package.jsonA collectors/gl-homes/src/index.tsA collectors/gl-homes/tsconfig.jsonM pnpm-lock.yamlM scripts/build-loop.sh
Diff
commit 840c359e4c89fca5e1858860c52e9677d37fe6d4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 28 22:02:14 2026 -0700
collectors: add Fischer, Fulton, GL Homes adapters (+3 builders, yolo iter-2)
DTD-panel-approved (5/5 verdict A) increment. All facts-only/honest-UA/robots-respected,
built via 3 parallel isolated recon+build subagents + serial gated integration:
- Fischer (fischer-homes-site): region API /api/region-revamped/homes/{id} + per-home detail
JSON-LD for geo; OH/KY/IN/GA/MO; 100% price/beds/baths/sqft/lat-lon. region-not-area avoids
the view-filter trap.
- Fulton (fulton-homes-site): ASP.NET GetSpecs web service, AZ, 166 homes 100% price/beds/sqft.
GetSpecs-not-GetHomes avoids the floorplan aggregate.
- GL Homes (gl-homes-site): two-layer HTML crawl (community -> per-QMI-<li>), FL, 168 homes 100%
price/beds/sqft. Per-<li> emission avoids collapsing 168 homes into 68 floorplan cards.
Fulton/GL every-sweep; Fischer throttled (per-home geo GETs). All 3 pre-seeded w/ Builder+SourceRegistry.
---
apps/workers/package.json | 5 +-
apps/workers/src/cli.ts | 6 +
collectors/fischer-homes/package.json | 15 ++
collectors/fischer-homes/selftest.mjs | 156 ++++++++++++
collectors/fischer-homes/src/index.ts | 358 +++++++++++++++++++++++++++
collectors/fischer-homes/tsconfig.json | 1 +
collectors/fulton-homes/package.json | 15 ++
collectors/fulton-homes/src/index.ts | 333 +++++++++++++++++++++++++
collectors/fulton-homes/tsconfig.json | 1 +
collectors/gl-homes/package.json | 15 ++
collectors/gl-homes/src/index.ts | 440 +++++++++++++++++++++++++++++++++
collectors/gl-homes/tsconfig.json | 1 +
pnpm-lock.yaml | 75 ++++++
scripts/build-loop.sh | 13 +-
14 files changed, 1431 insertions(+), 3 deletions(-)
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 7ecbc0cf..4a1300fc 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -42,7 +42,10 @@
"@homesonspec/collector-drees-homes": "workspace:*",
"@homesonspec/collector-perry-homes": "workspace:*",
"@homesonspec/collector-brookfield": "workspace:*",
- "@homesonspec/collector-holt-homes": "workspace:*"
+ "@homesonspec/collector-holt-homes": "workspace:*",
+ "@homesonspec/collector-fischer-homes": "workspace:*",
+ "@homesonspec/collector-fulton-homes": "workspace:*",
+ "@homesonspec/collector-gl-homes": "workspace:*"
},
"devDependencies": {
"tsx": "^4.19.2",
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index e41c5eec..582ac70b 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -23,6 +23,9 @@ import { dreesHomesAdapter } from "@homesonspec/collector-drees-homes";
import { perryHomesAdapter } from "@homesonspec/collector-perry-homes";
import { brookfieldAdapter } from "@homesonspec/collector-brookfield";
import { holtHomesAdapter } from "@homesonspec/collector-holt-homes";
+import { fischerHomesAdapter } from "@homesonspec/collector-fischer-homes";
+import { fultonHomesAdapter } from "@homesonspec/collector-fulton-homes";
+import { glHomesAdapter } from "@homesonspec/collector-gl-homes";
import { runPipeline } from "./pipeline";
import { recordSourceRun } from "./verify";
@@ -55,6 +58,9 @@ const ADAPTERS = {
[perryHomesAdapter.key]: perryHomesAdapter,
[brookfieldAdapter.key]: brookfieldAdapter,
[holtHomesAdapter.key]: holtHomesAdapter,
+ [fischerHomesAdapter.key]: fischerHomesAdapter,
+ [fultonHomesAdapter.key]: fultonHomesAdapter,
+ [glHomesAdapter.key]: glHomesAdapter,
};
async function main() {
diff --git a/collectors/fischer-homes/package.json b/collectors/fischer-homes/package.json
new file mode 100644
index 00000000..f71e9c29
--- /dev/null
+++ b/collectors/fischer-homes/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-fischer-homes",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/fischer-homes/selftest.mjs b/collectors/fischer-homes/selftest.mjs
new file mode 100644
index 00000000..10b3f2f0
--- /dev/null
+++ b/collectors/fischer-homes/selftest.mjs
@@ -0,0 +1,156 @@
+// Self-contained data-quality proof for the Fischer Homes adapter.
+// Hits the LIVE source with the honest bot UA, applies the same extraction the
+// adapter will use, and reports coverage / distinct-address / distinct-sqft / samples.
+// $0 (local). No DB, no writes.
+
+const UA = "HomesOnSpecBot/0.1 (+https://homesonspec.com/bot; contact: data@homesonspec.com)";
+const BASE = "https://www.fischerhomes.com";
+const PAGE_LIMIT = Number(process.env.FISCHER_PAGE_LIMIT ?? 10);
+// Region ids proven in recon (subset of /api/region-dropdown covering OH/KY/IN/GA/MO):
+// 11 Cincinnati OH, 16 Dayton OH, 22 Columbus OH, 35 Indianapolis IN, 47 Atlanta GA,
+// 48 Louisville KY, 13 Northern KY, 51 St Louis MO. Self-test uses a couple to keep it quick.
+const REGIONS = (process.env.FISCHER_REGIONS ?? "11,35,47").split(",").map((s) => s.trim()).filter(Boolean);
+
+async function getJson(url) {
+ const r = await fetch(url, { headers: { "User-Agent": UA, Accept: "application/json,*/*;q=0.8" }, redirect: "follow", signal: AbortSignal.timeout(25000) });
+ if (!r.ok) throw new Error(`GET ${url} -> ${r.status}`);
+ return r.json();
+}
+async function getText(url) {
+ const r = await fetch(url, { headers: { "User-Agent": UA, Accept: "text/html,*/*;q=0.8" }, redirect: "follow", signal: AbortSignal.timeout(25000) });
+ if (!r.ok) throw new Error(`GET ${url} -> ${r.status}`);
+ return r.text();
+}
+
+const num = (v) => { const n = Number(String(v ?? "").replace(/[^0-9.]/g, "")); return Number.isFinite(n) && n > 0 ? n : null; };
+const stripTags = (s) => String(s ?? "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+
+// Baths like "3½", "2 + ½ + ½", "3" -> total number (½ = 0.5)
+function parseBaths(s) {
+ const t = String(s ?? "");
+ if (!t.trim()) return null;
+ const halves = (t.match(/½/g) || []).length;
+ const wholeMatch = t.match(/\d+/g);
+ const whole = wholeMatch ? wholeMatch.reduce((a, b) => a + Number(b), 0) : 0;
+ const total = whole + halves * 0.5;
+ return total > 0 ? total : null;
+}
+function priceFromFormatted(html) {
+ const m = String(html ?? "").match(/\$?\s*([\d,]{4,})/);
+ return m ? num(m[1]) : null;
+}
+
+// --- List API: all homes for a region (bulk, paginated) ---
+async function fetchRegionHomes(regionId) {
+ const out = [];
+ let page = 1, lastPage = 1;
+ while (page <= lastPage && page <= 20) {
+ const url = `${BASE}/api/region-revamped/homes/${regionId}?page=${page}`;
+ const body = await getJson(url);
+ const sec = body["move-in-ready"] || {};
+ lastPage = Number(sec.last_page ?? 1);
+ for (const h of sec.data ?? []) out.push(h);
+ page++;
+ await new Promise((r) => setTimeout(r, 300));
+ }
+ return out;
+}
+
+// --- Detail page JSON-LD enrichment: lat/lon + community + confirm sqft/beds/baths + exact price ---
+function extractDetail(html) {
+ const out = { lat: null, lon: null, community: null, plan: null, price: null, sqft: null, beds: null, bathsFull: null, bathsHalf: null, city: null, state: null, zip: null, street: null };
+ const m = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i);
+ if (m) {
+ try {
+ const d = JSON.parse(m[1]);
+ const addr = d.address || {};
+ out.street = addr.streetAddress ?? null;
+ out.city = addr.addressLocality ?? null;
+ out.state = addr.addressRegion ?? null;
+ out.zip = addr.postalCode ?? null;
+ const lat = Number(d.latitude ?? d.geo?.latitude);
+ const lon = Number(d.longitude ?? d.geo?.longitude);
+ out.lat = Number.isFinite(lat) && lat !== 0 ? lat : null;
+ out.lon = Number.isFinite(lon) && lon !== 0 ? lon : null;
+ out.sqft = num(d.floorSize?.value);
+ out.beds = num(d.numberOfBedrooms);
+ out.bathsFull = d.numberOfFullBathrooms != null ? Number(d.numberOfFullBathrooms) : null;
+ out.bathsHalf = d.numberOfPartialBathrooms != null ? Number(d.numberOfPartialBathrooms) : null;
+ out.plan = d.accommodationFloorPlan?.name ?? null;
+ // community: JSON-LD description "built in <community> located in"
+ const desc = String(d.description ?? "");
+ const cm = desc.match(/built in ([^,]+?) located/i);
+ if (cm) out.community = cm[1].trim();
+ } catch {}
+ }
+ // community fallback from <title> "... | <community> by Fischer Homes"
+ if (!out.community) {
+ const tm = html.match(/<title>[^|]*\|\s*(.+?)\s+by Fischer Homes/i);
+ if (tm) out.community = tm[1].trim();
+ }
+ // exact price from sale-price element
+ const pm = html.match(/class="[^"]*sale-price-bold[^"]*"[^>]*>\s*\$?\s*([\d,]+)/i);
+ if (pm) out.price = num(pm[1]);
+ return out;
+}
+
+// ---- run ----
+console.log(`Fischer self-test | UA=${UA}\nRegions=${REGIONS.join(",")} PAGE_LIMIT(list-pages/region cap)=20\n`);
+let summaries = [];
+for (const r of REGIONS) {
+ try {
+ const hs = await fetchRegionHomes(r);
+ console.log(` region ${r}: ${hs.length} homes from list API`);
+ summaries = summaries.concat(hs.map((h) => ({ ...h, _region: r })));
+ } catch (e) { console.log(` region ${r}: LIST FAIL ${e.message}`); }
+}
+console.log(`\nTotal list-API homes: ${summaries.length}`);
+
+// Enrich each home via its detail page (cap for self-test speed via PAGE_LIMIT*3 homes)
+const cap = Math.min(summaries.length, Math.max(30, PAGE_LIMIT * 6));
+const rows = [];
+let enriched = 0;
+for (const s of summaries.slice(0, cap)) {
+ const detailUrl = BASE + s.url;
+ let det = {};
+ try { det = extractDetail(await getText(detailUrl)); enriched++; } catch (e) { /* keep list data */ }
+ // Facts policy: the LIST API's formatted fields are the customer-facing truth
+ // for beds/baths/sqft/price (complete, incl. half-baths). JSON-LD's
+ // numberOfPartialBathrooms is unreliable (reports 0 when a half-bath exists),
+ // so it's used ONLY as a fallback. Detail page is authoritative for lat/lon +
+ // community, which the list API lacks.
+ const price = priceFromFormatted(s.formattedPrice) ?? det.price;
+ const beds = num(s.formattedBeds) ?? det.beds;
+ const baths = parseBaths(s.formattedBaths) ?? ((det.bathsFull != null) ? det.bathsFull + (det.bathsHalf ?? 0) * 0.5 : null);
+ const sqft = num(s.formattedSqft) ?? det.sqft;
+ rows.push({
+ street: det.street ?? (s.formattedAddress || "").split(",")[0]?.trim() ?? null,
+ fullAddress: s.formattedAddress ?? null,
+ city: det.city, state: det.state, zip: det.zip,
+ community: det.community, plan: det.plan ?? s.name ?? null,
+ price, beds, baths, sqft, lat: det.lat, lon: det.lon,
+ region: s._region,
+ });
+ await new Promise((r) => setTimeout(r, 250));
+}
+
+const N = rows.length;
+const pct = (f) => N ? ((rows.filter(f).length / N) * 100).toFixed(1) + "%" : "0%";
+const distinct = (f) => new Set(rows.map(f).filter((x) => x != null)).size;
+
+console.log(`\n===== DATA QUALITY (${N} enriched homes; ${enriched} detail pages fetched) =====`);
+console.log(`price coverage: ${pct((r) => r.price != null)}`);
+console.log(`beds coverage: ${pct((r) => r.beds != null)}`);
+console.log(`baths coverage: ${pct((r) => r.baths != null)}`);
+console.log(`sqft coverage: ${pct((r) => r.sqft != null)}`);
+console.log(`lat/lon coverage: ${pct((r) => r.lat != null && r.lon != null)}`);
+console.log(`community coverage: ${pct((r) => r.community != null)}`);
+console.log(`plan coverage: ${pct((r) => r.plan != null)}`);
+console.log(`distinct addresses: ${distinct((r) => r.fullAddress)} / ${N}`);
+console.log(`distinct sqft: ${distinct((r) => r.sqft)}`);
+console.log(`distinct prices: ${distinct((r) => r.price)}`);
+console.log(`distinct communities:${distinct((r) => r.community)}`);
+console.log(`negative-lon (US): ${rows.filter((r) => r.lon != null && r.lon < 0).length}/${rows.filter((r) => r.lon != null).length}`);
+
+console.log(`\n===== 6 SAMPLE ROWS =====`);
+for (const r of rows.slice(0, 6)) console.log(JSON.stringify(r));
diff --git a/collectors/fischer-homes/src/index.ts b/collectors/fischer-homes/src/index.ts
new file mode 100644
index 00000000..48490762
--- /dev/null
+++ b/collectors/fischer-homes/src/index.ts
@@ -0,0 +1,358 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Fischer Homes adapter — JSON-API + detail-page source (recon 2026-07-28).
+ *
+ * fischerhomes.com (OH/KY/IN/GA/MO regional builder) is a Cloudflare-fronted SPA
+ * over a custom Laravel/"region-revamped" API. The move-in-ready inventory is a
+ * BULK, PAGINATED JSON GET keyed by numeric region id:
+ *
+ * GET https://www.fischerhomes.com/api/region-revamped/homes/{regionId}?page=N
+ * -> { "move-in-ready": { data:[ home… ], current_page, last_page, total } }
+ *
+ * The region ids come from a companion GET:
+ * GET https://www.fischerhomes.com/api/region-dropdown
+ * -> { regions:[ { id, name, seo_name, state } … ] }
+ *
+ * Each list home carries the CUSTOMER-FACING facts:
+ * { id, name(=plan), formattedAddress, formattedBeds, formattedBaths,
+ * formattedSqft, formattedFloors, formattedPrice(HTML), url(detail-page) }.
+ * The list is the truth for beds/baths/sqft/price (its formattedBaths includes
+ * the half-bath). It does NOT carry lat/lon or the community name.
+ *
+ * The per-home DETAIL PAGE (the `url` field) server-renders a schema.org "House"
+ * JSON-LD block that adds the two missing facts:
+ * - geo.latitude / geo.longitude (US lon negative — signs preserved), and
+ * - the community, parsed from the description ("built in <community> located in")
+ * with a <title> "... | <community> by Fischer Homes" fallback.
+ * (JSON-LD's numberOfPartialBathrooms is unreliable — it reports 0 when a half-bath
+ * exists — so the list's formattedBaths is authoritative for baths; JSON-LD is a
+ * fallback only.)
+ *
+ * To keep extract() pure and one-page-per-home, fetch() joins each list home with
+ * its detail-page facts and yields ONE synthetic combined-JSON RawPage per home.
+ *
+ * Verified: robots.txt (fischerhomes.com) disallows only an aggregate path, utm
+ * query params, .pdf/.swf, y_source= and calendar/create — the region-revamped API
+ * and find-new-homes/ready-now detail paths we use are all allowed. No auth/cookies
+ * (200 with the honest bot UA). Facts-only: image urls exist in the feed but are
+ * intentionally dropped.
+ *
+ * Batch control: FISCHER_PAGE_LIMIT (list pages per region, default 10)
+ * Optional scope: FISCHER_REGIONS (comma ids, e.g. "11,35" — else all regions)
+ */
+
+const BASE = "https://www.fischerhomes.com";
+const BUILDER_SLUG = "fischer-homes";
+const REGION_DROPDOWN = `${BASE}/api/region-dropdown`;
+const homesUrl = (regionId: string | number, page: number) =>
+ `${BASE}/api/region-revamped/homes/${regionId}?page=${page}`;
+const PAGE_LIMIT = Number(process.env.FISCHER_PAGE_LIMIT ?? 10);
+const REGION_FILTER = (process.env.FISCHER_REGIONS ?? "")
+ .split(",")
+ .map((s) => s.trim())
+ .filter(Boolean);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// Positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+const zip5 = (v: unknown): string | null => {
+ const s = str(v);
+ if (!s) return null;
+ const m = s.match(/\b(\d{5})\b/);
+ return m ? m[1]! : null;
+};
+
+/**
+ * Fischer's list `formattedBaths` uses "3½", "2 + ½ + ½", "3" — whole numbers plus
+ * a "½" glyph per half-bath. Sum the whole numbers and add 0.5 per ½. This is the
+ * authoritative bath source (JSON-LD's partial count is unreliable). >0 or null.
+ */
+function parseBaths(v: unknown): number | null {
+ const t = str(v);
+ if (!t) return null;
+ const halves = (t.match(/½/g) || []).length;
+ const wholes = t.match(/\d+/g);
+ const whole = wholes ? wholes.reduce((a, b) => a + Number(b), 0) : 0;
+ const total = whole + halves * 0.5;
+ return total > 0 ? total : null;
+}
+
+/** Pull a plain dollar amount out of the list's formattedPrice HTML span. */
+function priceFromFormatted(html: unknown): number | null {
+ const s = str(html);
+ if (!s) return null;
+ const m = s.replace(/<[^>]*>/g, " ").match(/\$?\s*([\d,]{4,})/);
+ return m ? posNum(m[1]) : null;
+}
+
+// ---- detail-page (JSON-LD) facts: lat/lon + community (+ clean address) ----
+interface DetailFacts {
+ lat: number | null;
+ lon: number | null;
+ community: string | null;
+ plan: string | null;
+ street: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+}
+
+function parseDetail(html: string): DetailFacts {
+ const out: DetailFacts = { lat: null, lon: null, community: null, plan: null, street: null, city: null, state: null, zip: null };
+ const m = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i);
+ if (m) {
+ try {
+ const d = JSON.parse(m[1]!) as Record<string, any>;
+ const addr = (d.address ?? {}) as Record<string, any>;
+ out.street = str(addr.streetAddress);
+ out.city = str(addr.addressLocality);
+ out.state = str(addr.addressRegion);
+ out.zip = zip5(addr.postalCode);
+ const lat = Number(d.geo?.latitude ?? d.latitude);
+ const lon = Number(d.geo?.longitude ?? d.longitude);
+ out.lat = Number.isFinite(lat) && lat !== 0 ? lat : null;
+ out.lon = Number.isFinite(lon) && lon !== 0 ? lon : null; // negative US lon preserved
+ out.plan = str(d.accommodationFloorPlan?.name);
+ const desc = String(d.description ?? "");
+ const cm = desc.match(/built in (.+?) located/i);
+ if (cm) out.community = str(cm[1]);
+ } catch {
+ /* fall through to title fallback */
+ }
+ }
+ if (!out.community) {
+ const tm = html.match(/<title>[^|]*\|\s*(.+?)\s+by Fischer Homes/i);
+ if (tm) out.community = str(tm[1]);
+ }
+ return out;
+}
+
+// ---- the synthetic combined page shape yielded by fetch() ----
+interface CombinedHome {
+ __fischer: 1;
+ id?: string | number;
+ name?: string; // plan name (list)
+ formattedAddress?: string;
+ formattedBeds?: string;
+ formattedBaths?: string;
+ formattedSqft?: string;
+ formattedFloors?: string;
+ formattedPrice?: string;
+ url?: string; // detail-page path
+ region?: string;
+ detail: DetailFacts;
+}
+
+interface DropdownRegion { id: number; seo_name?: string; state?: string }
+
+export const fischerHomesAdapter: SourceAdapter = {
+ key: "fischer-homes-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+
+ // 1) resolve region ids.
+ let regionIds: string[] = REGION_FILTER;
+ if (regionIds.length === 0) {
+ try {
+ const rp = await fetcher.fetch(REGION_DROPDOWN);
+ const body = JSON.parse(rp.body.toString("utf8")) as { regions?: DropdownRegion[] };
+ regionIds = (body.regions ?? []).map((r) => String(r.id)).filter(Boolean);
+ } catch (error) {
+ console.warn(` fischer region-dropdown: ${error instanceof Error ? error.message : String(error)}`);
+ return; // can't discover regions → nothing to collect
+ }
+ }
+
+ // 2) per region: page the list API, then enrich each home via its detail page.
+ for (const regionId of regionIds) {
+ let page = 1;
+ let lastPage = 1;
+ let listPagesFetched = 0;
+ while (page <= lastPage && listPagesFetched < Math.max(1, PAGE_LIMIT)) {
+ let listPage: RawPage;
+ try {
+ listPage = await fetcher.fetch(homesUrl(regionId, page));
+ } catch (error) {
+ console.warn(` fischer region ${regionId} page ${page}: ${error instanceof Error ? error.message : String(error)}`);
+ break; // a block/403 stops this region (source marked degraded upstream)
+ }
+ listPagesFetched++;
+ let section: { data?: any[]; last_page?: number } = {};
+ try {
+ section = (JSON.parse(listPage.body.toString("utf8")) as any)?.["move-in-ready"] ?? {};
+ } catch {
+ break;
+ }
+ lastPage = Number(section.last_page ?? 1);
+
+ for (const h of section.data ?? []) {
+ const detailPath = str(h.url);
+ let detail: DetailFacts = { lat: null, lon: null, community: null, plan: null, street: null, city: null, state: null, zip: null };
+ if (detailPath) {
+ try {
+ const dp = await fetcher.fetch(BASE + detailPath);
+ detail = parseDetail(dp.body.toString("utf8"));
+ } catch (error) {
+ // Detail unreachable (403/404/net) — keep the list facts; community may
+ // be missing → extract() will skip-and-log honestly rather than guess.
+ console.warn(` fischer detail ${detailPath}: ${error instanceof Error ? error.message : String(error)}`);
+ }
+ }
+ const combined: CombinedHome = {
+ __fischer: 1,
+ id: h.id,
+ name: h.name,
+ formattedAddress: h.formattedAddress,
+ formattedBeds: h.formattedBeds,
+ formattedBaths: h.formattedBaths,
+ formattedSqft: h.formattedSqft,
+ formattedFloors: h.formattedFloors,
+ formattedPrice: h.formattedPrice,
+ url: h.url,
+ region: regionId,
+ detail,
+ };
+ const bytes = Buffer.from(JSON.stringify(combined), "utf8");
+ yield {
+ url: BASE + (detailPath ?? `/api/region-revamped/homes/${regionId}#${h.id}`),
+ retrievedAt: listPage.retrievedAt,
+ contentType: "application/json",
+ body: bytes,
+ contentHash: listPage.contentHash, // per-home hash source is the synthetic body
+ };
+ }
+ page++;
+ }
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ let h: CombinedHome;
+ try {
+ h = JSON.parse(page.body.toString("utf8")) as CombinedHome;
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: `unparseable combined page: ${String(error)}` }] };
+ }
+ if (!h || h.__fischer !== 1) {
+ return { records: [], errors: [{ url: page.url, reason: "not a fischer combined-home page" }] };
+ }
+
+ const url = page.url;
+ const d = h.detail ?? ({} as DetailFacts);
+
+ // Address: prefer the JSON-LD structured street; else the list formatted address head.
+ const listAddrHead = str(h.formattedAddress)?.split(",")[0] ?? null;
+ const address = d.street ?? listAddrHead;
+ const city = d.city ?? null;
+ const state = normalizeStateCode(d.state ?? null);
+ const zip = d.zip ?? zip5(h.formattedAddress);
+ const community = str(d.community);
+ const plan = str(h.name) ?? str(d.plan);
+ const lat = d.lat ?? null;
+ const lon = d.lon ?? null;
+
+ // Facts (list API authoritative; formattedBaths carries the half-bath).
+ const price = priceFromFormatted(h.formattedPrice);
+ const beds = posNum(h.formattedBeds);
+ const bathsTotal = parseBaths(h.formattedBaths);
+ const sqft = posNum(h.formattedSqft);
+ const stories = posNum(h.formattedFloors);
+ const homeId = str(h.id);
+
+ if (!address) {
+ return { records: [], errors: [{ url, reason: `home ${homeId ?? "?"} missing address — skipped` }] };
+ }
+ // The publisher requires an inventory home to hang off a community (FK). A home
+ // whose detail page yielded no community can't be attached — skip + log honestly
+ // rather than stage a record that will crash at publish.
+ if (!community) {
+ return { records: [], errors: [{ url, reason: `home ${address} has no community (detail page missing/unparsed) — skipped` }] };
+ }
+
+ const records: ExtractedRecord[] = [];
+
+ // Community FIRST — publish creates the FK target the home record needs.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, url, "detail JSON-LD community"),
+ street: fv<string>(null, null, url),
+ city: fv(city, city, url),
+ state: fv(state, d.state, url),
+ zip: fv(zip, zip, url),
+ county: fv<string>(null, null, url),
+ metro: fv<string>(null, null, url),
+ lat: fv(lat, lat === null ? null : String(lat), url),
+ lon: fv(lon, lon === null ? null : String(lon), url),
+ hoaFeeMonthly: fv<number>(null, null, url),
+ schoolDistrict: fv<string>(null, null, url),
+ ageRestricted: fv<boolean>(null, null, url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address,
+ builderInventoryId: homeId ?? undefined,
+ lat: lat ?? undefined,
+ lon: lon ?? undefined,
+ planName: plan ?? undefined,
+ },
+ fields: {
+ street: fv(address, address, url, "home address"),
+ city: fv(city, city, url),
+ state: fv(state, d.state, url),
+ zip: fv(zip, zip, url),
+ price: fv(price, price === null ? null : String(price), url, price === null ? null : `formattedPrice ${str(h.formattedPrice)}`),
+ beds: fv(beds, beds === null ? null : str(h.formattedBeds), url),
+ bathsTotal: fv(bathsTotal, bathsTotal === null ? null : str(h.formattedBaths), url, bathsTotal === null ? null : `formattedBaths ${str(h.formattedBaths)}`),
+ sqft: fv(sqft, sqft === null ? null : str(h.formattedSqft), url),
+ stories: fv(stories, stories === null ? null : str(h.formattedFloors), url),
+ garageSpaces: fv<number>(null, null, url),
+ homeType: fv("SINGLE_FAMILY" as const, null, url, "Fischer single-family inventory home"),
+ constructionStatus: fv("MOVE_IN_READY" as const, null, url, "Fischer move-in-ready (ready-now) inventory"),
+ estCompletionDate: fv<string>(null, null, url),
+ lotNumber: fv<string>(null, null, url),
+ builderInventoryId: fv(homeId, homeId, url),
+ lat: fv(lat, lat === null ? null : String(lat), url, lat === null ? null : "detail JSON-LD geo"),
+ lon: fv(lon, lon === null ? null : String(lon), url, lon === null ? null : "detail JSON-LD geo"),
+ planName: fv(plan, plan, url),
+ // facts-only: image urls exist in the feed but are intentionally dropped.
+ images: fv<string[]>([], null, url),
+ },
+ });
+
+ return { records, errors: [] };
+ },
+};
diff --git a/collectors/fischer-homes/tsconfig.json b/collectors/fischer-homes/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/fischer-homes/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/collectors/fulton-homes/package.json b/collectors/fulton-homes/package.json
new file mode 100644
index 00000000..3447fd2c
--- /dev/null
+++ b/collectors/fulton-homes/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-fulton-homes",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/fulton-homes/src/index.ts b/collectors/fulton-homes/src/index.ts
new file mode 100644
index 00000000..af071582
--- /dev/null
+++ b/collectors/fulton-homes/src/index.ts
@@ -0,0 +1,333 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * Fulton Homes adapter — legacy ASP.NET AJAX HTML source (recon 2026-07-28).
+ *
+ * fultonhomes.com is a single-metro ARIZONA builder on an old jQuery /
+ * Bootstrap-3 / ASP.NET WebForms stack (NOT a Next.js/Algolia SPA). The
+ * "Find Your Home" search page (/find-your-home) client-injects its results via
+ * a jQuery AJAX GET against an ASMX-style web service; the SPEC-HOME (quick-move-
+ * in) inventory feed is a single GET that returns the whole company inventory in
+ * one shot (no pagination, no cookies, no auth):
+ *
+ * GET https://www.fultonhomes.com/ws.svc/GetSpecs
+ * ?search="Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|NB"
+ * &clicked_element=""
+ * &sort_field="SqFt"&sort_direction="ASC"
+ *
+ * The `search` string is the site's own default all-filters-Any selector
+ * (city|hometype|beds|baths|garage|sqft|price… all "Any", trailing "NB"); it
+ * returns EVERY spec home statewide. The response is JSON-wrapped HTML:
+ *
+ * { "d": { "__type":"HomesSearchResponse:#FultonHomes",
+ * "count": 284, "html": "<table class='table results table-spec'>…" } }
+ *
+ * Each spec home is a <tr> carrying a `?ih=<projectCode>|<lot>` deep-link and
+ * columns [thumb, Lot, Floorplan, Elevation, Neighborhood, City, SqFt, Price,
+ * Beds, Bath]. `ih` (project|lot) is the stable per-home id. The Price cell is
+ * one of: a plain "$###,###" (available), a "$was$now" markdown pair (take the
+ * LAST = current price), or a struck-through price + "Sold"/"Pending" — those
+ * are NOT sellable inventory and are skipped (not a live listable home).
+ *
+ * Per-HOME, not aggregate — verified: 284 spec rows, 284 distinct `ih` ids, 98
+ * distinct sqft, 22 neighborhoods across 6 AZ cities; 202 available w/ real
+ * prices, 79 Sold + 3 Pending excluded. This is the individual-physical-home
+ * table (GetSpecs, lot-numbered), NOT the sibling GetHomes "New Built Homes"
+ * floorplan-aggregate table (count 155, no lot) which would be the hollow trap.
+ *
+ * Fields the bulk list does NOT carry (garage bays, plan stories, street number,
+ * lat/lon, est. completion date) live only on each home's detail page and are
+ * left null here (facts-only; never guessed). There is no street number in the
+ * list — the home's identity is Lot + Neighborhood + City, so `street` is the
+ * lot-qualified label "Lot <lot>, <neighborhood>".
+ *
+ * Auth: none — plain honest-UA GET returns 200 with no cookie/login. robots.txt
+ * (fultonhomes.com) Disallows only /bin/ /envision_sso/ /mfh/ /survey/
+ * /warranty/ — the /ws.svc/ and /find-your-home paths we use are allowed (and
+ * enforced in-code by LiveFetcher). STOP-on-403/401/429 is inherited.
+ *
+ * Facts-only: elevation thumbnails exist in the feed; they are intentionally
+ * dropped (mediaRights=NONE).
+ *
+ * Batch control: FULTON_PAGE_LIMIT (default 10). The feed is single-call, so
+ * this is a safety cap only — one page returns the whole statewide inventory.
+ */
+
+const ORIGIN = "https://www.fultonhomes.com";
+// The site's own default "all filters = Any" selector; NB = New Built delivery set.
+const SEARCH =
+ '"Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|Any|NB"';
+const GETSPECS_URL =
+ `${ORIGIN}/ws.svc/GetSpecs` +
+ `?search=${encodeURIComponent(SEARCH)}` +
+ `&clicked_element=${encodeURIComponent('""')}` +
+ `&sort_field=${encodeURIComponent('"SqFt"')}` +
+ `&sort_direction=${encodeURIComponent('"ASC"')}`;
+
+const BUILDER_SLUG = "fulton-homes";
+const PAGE_LIMIT = Number(process.env.FULTON_PAGE_LIMIT ?? 10);
+// Fulton is a single-metro ARIZONA builder; the list carries no state column.
+const STATE_RAW = "AZ";
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+
+/** Strip HTML tags and collapse whitespace → plain text. */
+const text = (htmlFragment: string): string =>
+ decodeEntities(htmlFragment.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
+
+/** Minimal HTML-entity decode for the handful the feed emits (& ' " etc.). */
+function decodeEntities(s: string): string {
+ return s
+ .replace(/&/g, "&")
+ .replace(/</g, "<")
+ .replace(/>/g, ">")
+ .replace(/"/g, '"')
+ .replace(/�?39;/g, "'")
+ .replace(/'/g, "'")
+ .replace(/ /g, " ")
+ .replace(/&#(\d+);/g, (_, d) => {
+ const code = Number(d);
+ return Number.isFinite(code) ? String.fromCharCode(code) : _;
+ });
+}
+
+/** Pull the JSON-wrapped HTML table out of a GetSpecs response body. */
+function specHtml(body: string): { count: number; html: string } | null {
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(body);
+ } catch {
+ return null;
+ }
+ const d = (parsed as { d?: { count?: number; html?: string } })?.d;
+ if (!d || typeof d.html !== "string") return null;
+ return { count: Number(d.count ?? 0), html: decodeEntities(d.html) };
+}
+
+interface SpecRow {
+ ih: string; // "<projectCode>|<lot>" — stable per-home id
+ lot: string | null;
+ plan: string | null;
+ elevation: string | null;
+ neighborhood: string | null;
+ city: string | null;
+ sqft: number | null;
+ /** null when the home is Sold/Pending (not sellable) or has no listed price. */
+ price: number | null;
+ /** true only for a home that shows a live, current asking price. */
+ available: boolean;
+ status: string | null; // "Sold" | "Pending" | null (available)
+ beds: number | null;
+ baths: number | null;
+}
+
+/**
+ * Parse the current (last) dollar figure out of a Price cell. The cell may be:
+ * - "$412,900" → 412900 (available)
+ * - "$447,449$429,449" → 429449 (was/now markdown; take the LAST)
+ * - struck-through "$…" + "Sold" → null, status "Sold"
+ * - struck-through "$…" + "Pending"→ null, status "Pending"
+ */
+function parsePriceCell(cellHtml: string): { price: number | null; status: string | null } {
+ const plain = text(cellHtml);
+ if (/\bSold\b/i.test(plain)) return { price: null, status: "Sold" };
+ if (/\bPending\b/i.test(plain)) return { price: null, status: "Pending" };
+ const matches = plain.match(/\$[\d,]+/g);
+ if (!matches || matches.length === 0) return { price: null, status: null };
+ // was/now pairs list the was-price first and the current price last.
+ const price = posNum(matches[matches.length - 1]);
+ return { price, status: null };
+}
+
+/** Parse the GetSpecs HTML table into one SpecRow per physical spec home. */
+function parseSpecRows(html: string): SpecRow[] {
+ const rows: SpecRow[] = [];
+ const seen = new Set<string>();
+ const trRe = /<tr\b[^>]*>([\s\S]*?)<\/tr>/gi;
+ let m: RegExpExecArray | null;
+ while ((m = trRe.exec(html)) !== null) {
+ const rowHtml = m[0];
+ if (/table-header/i.test(rowHtml)) continue; // header row
+ const ihMatch = rowHtml.match(/\?ih=([a-z0-9]+)\|(\d+)/i);
+ if (!ihMatch) continue; // detail/expand rows (no ?ih=) — skip
+ const ih = `${ihMatch[1]}|${ihMatch[2]}`;
+ if (seen.has(ih)) continue; // guard against the paired detail row re-matching
+ const cellsRaw = [...rowHtml.matchAll(/<td\b[^>]*>([\s\S]*?)<\/td>/gi)].map((c) => c[1]);
+ if (cellsRaw.length < 9) continue; // not a full data row
+ // Column order: [thumb, Lot, Floorplan, Elv, Neighborhood, City, SqFt, Price, Beds, Bath]
+ const [_, lotC, planC, elvC, nbhdC, cityC, sqftC, priceC, bedsC, bathC] = cellsRaw;
+ const { price, status } = parsePriceCell(priceC ?? "");
+ seen.add(ih);
+ rows.push({
+ ih,
+ lot: str(ihMatch[2]) ?? str(text(lotC ?? "")),
+ plan: str(text(planC ?? "")),
+ elevation: str(text(elvC ?? "")),
+ neighborhood: str(text(nbhdC ?? "")),
+ city: str(text(cityC ?? "")),
+ sqft: posNum(text(sqftC ?? "")),
+ price,
+ available: price !== null,
+ status,
+ beds: posNum(text(bedsC ?? "")),
+ baths: posNum(text(bathC ?? "")),
+ });
+ }
+ return rows;
+}
+
+/** The home-detail path for a spec home (evidence URL). Prefer the row's own
+ * anchor href when present, else synthesize the ?ih= deep-link. */
+function homeUrl(rowHtml: string, ih: string): string {
+ const href = rowHtml.match(/href="((?:our-communities\/)?[^"]*\?ih=[^"]+)"/i)?.[1];
+ if (href) return href.startsWith("http") ? href : `${ORIGIN}/${href.replace(/^\//, "")}`;
+ return `${ORIGIN}/find-your-home?ih=${encodeURIComponent(ih)}`;
+}
+
+export const fultonHomesAdapter: SourceAdapter = {
+ key: "fulton-homes-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ // The GetSpecs feed is single-call (returns the whole statewide inventory);
+ // FULTON_PAGE_LIMIT is a safety cap, so we fetch at most one page.
+ for (let pageNo = 0; pageNo < Math.max(1, Math.min(1, PAGE_LIMIT)); pageNo++) {
+ let page: RawPage;
+ try {
+ page = await fetcher.fetch(GETSPECS_URL);
+ } catch (error) {
+ console.warn(` fulton GetSpecs: ${error instanceof Error ? error.message : String(error)}`);
+ return; // a block/403 stops collection (source marked degraded upstream)
+ }
+ yield page;
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const parsed = specHtml(page.body.toString("utf8"));
+ if (!parsed) {
+ return { records: [], errors: [{ url: page.url, reason: "no GetSpecs { d: { html } } payload in response" }] };
+ }
+ const records: ExtractedRecord[] = [];
+ const errors: { url: string; reason: string }[] = [];
+
+ // Re-walk the <tr> blocks so each row keeps its own anchor href for evidence.
+ const trRe = /<tr\b[^>]*>([\s\S]*?)<\/tr>/gi;
+ const rowHtmlByIh = new Map<string, string>();
+ let mm: RegExpExecArray | null;
+ while ((mm = trRe.exec(parsed.html)) !== null) {
+ const im = mm[0].match(/\?ih=([a-z0-9]+)\|(\d+)/i);
+ if (im && !rowHtmlByIh.has(`${im[1]}|${im[2]}`)) rowHtmlByIh.set(`${im[1]}|${im[2]}`, mm[0]);
+ }
+
+ const rows = parseSpecRows(parsed.html);
+ const state = normalizeStateCode(STATE_RAW);
+
+ for (const r of rows) {
+ // Only live, sellable inventory — Sold/Pending homes are not listable.
+ if (!r.available || r.price === null) {
+ errors.push({ url: page.url, reason: `spec ${r.ih} not available (status: ${r.status ?? "no price"}) — skipped` });
+ continue;
+ }
+ const community = r.neighborhood;
+ // The publisher requires an inventory home to hang off a community (FK).
+ if (!community) {
+ errors.push({ url: page.url, reason: `spec ${r.ih} has no neighborhood — cannot attach to a community, skipped` });
+ continue;
+ }
+ const url = homeUrl(rowHtmlByIh.get(r.ih) ?? "", r.ih);
+ const city = r.city;
+ // No street number in the list; identity is Lot + Neighborhood + City.
+ const street = r.lot ? `Lot ${r.lot}, ${community}` : community;
+
+ // Community FIRST — publish creates the FK target the home record needs.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, url, "GetSpecs Neighborhood column"),
+ street: fv<string>(null, null, url),
+ city: fv(city, city, url),
+ state: fv(state, STATE_RAW, url, "Fulton Homes is an Arizona-only builder"),
+ zip: fv<string>(null, null, url),
+ county: fv<string>(null, null, url),
+ metro: fv<string>("Phoenix", null, url, "Fulton Homes serves the Phoenix / Valley metro"),
+ lat: fv<number>(null, null, url),
+ lon: fv<number>(null, null, url),
+ hoaFeeMonthly: fv<number>(null, null, url),
+ schoolDistrict: fv<string>(null, null, url),
+ ageRestricted: fv<boolean>(null, null, url),
+ },
+ });
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address: street,
+ builderInventoryId: r.ih,
+ planName: r.plan ?? undefined,
+ },
+ fields: {
+ street: fv(street, r.lot, url, r.lot ? `Lot ${r.lot} in ${community}` : "GetSpecs Neighborhood"),
+ city: fv(city, city, url),
+ state: fv(state, STATE_RAW, url, "Fulton Homes is an Arizona-only builder"),
+ zip: fv<string>(null, null, url),
+ price: fv(r.price, r.price === null ? null : String(r.price), url, r.price === null ? null : `GetSpecs price $${r.price}`),
+ beds: fv(r.beds, r.beds === null ? null : String(r.beds), url),
+ bathsTotal: fv(r.baths, r.baths === null ? null : String(r.baths), url, r.baths === null ? null : "GetSpecs Bath column"),
+ sqft: fv(r.sqft, r.sqft === null ? null : String(r.sqft), url),
+ // Garage bays & plan stories are not in the bulk list (detail-page only) — not guessed.
+ stories: fv<number>(null, null, url),
+ garageSpaces: fv<number>(null, null, url),
+ homeType: fv("SINGLE_FAMILY" as const, null, url, "Fulton Homes single-family spec home"),
+ // GetSpecs are quick-move-in spec homes; without a per-home stage we don't
+ // assert MOVE_IN_READY vs UNDER_CONSTRUCTION (that lives on the detail page).
+ constructionStatus: fv<"UNDER_CONSTRUCTION" | "MOVE_IN_READY">(null, null, url),
+ estCompletionDate: fv<string>(null, null, url),
+ lotNumber: fv(r.lot, r.lot, url, r.lot ? `Lot ${r.lot}` : null),
+ builderInventoryId: fv(r.ih, r.ih, url, "GetSpecs ?ih=<project>|<lot> id"),
+ // No lat/lon in the GetSpecs feed (both null); not guessed.
+ lat: fv<number>(null, null, url),
+ lon: fv<number>(null, null, url),
+ planName: fv(r.plan, r.plan, url),
+ // facts-only: elevation thumbnails exist in the feed but are intentionally dropped.
+ images: fv<string[]>([], null, url),
+ },
+ });
+ }
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/fulton-homes/tsconfig.json b/collectors/fulton-homes/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/fulton-homes/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/collectors/gl-homes/package.json b/collectors/gl-homes/package.json
new file mode 100644
index 00000000..33874549
--- /dev/null
+++ b/collectors/gl-homes/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@homesonspec/collector-gl-homes",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@homesonspec/collectors-common": "workspace:*",
+ "@homesonspec/schemas": "workspace:*",
+ "@homesonspec/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/gl-homes/src/index.ts b/collectors/gl-homes/src/index.ts
new file mode 100644
index 00000000..8d813e93
--- /dev/null
+++ b/collectors/gl-homes/src/index.ts
@@ -0,0 +1,440 @@
+import type { ExtractedRecord, FieldValue } from "@homesonspec/schemas";
+import { normalizeStateCode } from "@homesonspec/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@homesonspec/collectors-common";
+
+/**
+ * GL Homes adapter — SERVER-RENDERED, community-organized source (recon 2026-07-28).
+ *
+ * glhomes.com is a large Florida-only regional builder (Optimizely/Episerver CMS,
+ * "areas/glhomes" theme). It is organized entirely by COMMUNITY (Valencia *,
+ * Apex at Avenir, Lotus Edge, The Estates at Nomar). There is NO inventory JSON
+ * API — a Playwright networkidle capture of a quick-move-in page returned ONLY
+ * tracking/chat XHRs (HubSpot, TikTok, OpenAI pixel); the home facts are fully
+ * server-rendered HTML. So this is a two-layer HTML crawl, not a feed.
+ *
+ * Layer 1 — the homepage nav lists the active communities as top-level slugs
+ * (<a href="/apex-at-avenir/">, /lotus-edge/, /valencia-ridge/ …). Each active
+ * community exposes its quick-move-in ("Early Move-In") inventory at
+ * GET https://www.glhomes.com/<slug>/early-move-in/
+ * (Community landing /<slug>/ carries the sales-center address
+ * "12874 Soaring View, Palm Beach Gardens, FL 33412" → community city/state/zip.
+ * NB: the page's "latitude"/"longitude" tokens are a JS template config, NOT
+ * real coordinates — so lat/lon are left null rather than guessed.)
+ *
+ * Layer 2 — the /<slug>/early-move-in/ page renders one <div class="early-move__card">
+ * per FLOORPLAN, carrying PLAN-LEVEL attributes that every home of that plan
+ * shares:
+ * .early-move__subtitle → plan name ("Calypso")
+ * .early-move__attr → "3 Bedrooms, 3 Bathrooms, 1 Half Bath, …, 3-Car Garage"
+ * .early-move__sqs > li → "2,464 a/c sq. ft." (living area) + "3,446 total sq. ft."
+ * floorplan href → "/apex-at-avenir/pinnacle-collection/calypso-508/"
+ * and INSIDE each card a <ul class="early-move__items"> with one <li> per
+ * INDIVIDUAL HOME:
+ * <span>13099 FLORIDA CRANE DRIVE</span> (street)
+ * <span>Lot 0201</span> (lot number)
+ * .early-move__current-price ($1,230,900 — the real sell price)
+ * .early-move__original-price (pre-savings, evidence only)
+ * .early-move-closing-date ("Available Now" | "Available Apr 2027")
+ *
+ * GRAIN (the key risk for a community-organized builder): one card = one plan
+ * but MANY homes. This adapter emits ONE inventory_home PER <li>, joining the
+ * card's plan-level beds/baths/sqft/garage to each home's own address/price/lot/
+ * completion. Parsing a card as a single home would collapse (e.g.) 66 real
+ * Valencia-Parc homes into 14 hollow floorplan rows — the Dream-Finders trap.
+ * Recon counts (2026-07-28): 68 plan-cards → 168 individual QMI homes across the
+ * 9 active communities.
+ *
+ * robots.txt (glhomes.com) is `User-agent: *` / `Disallow:` (empty) → allow /.
+ * Facts-only: card photos exist in the HTML and are intentionally dropped.
+ *
+ * Batch control: GL_PAGE_LIMIT (community early-move-in pages to crawl, default 10).
+ */
+
+const ORIGIN = "https://www.glhomes.com";
+const BUILDER_SLUG = "gl-homes";
+const PAGE_LIMIT = Number(process.env.GL_PAGE_LIMIT ?? 10);
+const META_MARKER = "gl-community:";
+// GL Homes is Florida-only — the sales-center address always resolves state FL,
+// but we still parse it from the address text rather than hard-coding.
+const HOME_URL = (slug: string) => `${ORIGIN}/${slug}/early-move-in/`;
+const COMMUNITY_URL = (slug: string) => `${ORIGIN}/${slug}/`;
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+// A positive finite number, or null. Never guesses; 0 / negative / NaN → null.
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
+// A non-negative number (baths/garages may legitimately be 0), or null.
+const nonNegNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n >= 0 ? n : null;
+};
+const str = (v: unknown): string | null => {
+ if (v == null) return null;
+ const s = String(v).trim();
+ return s ? s : null;
+};
+const zip5 = (v: unknown): string | null => {
+ const s = str(v);
+ if (!s) return null;
+ const m = s.match(/\b(\d{5})\b/);
+ return m ? m[1]! : null;
+};
+
+/** Community metadata injected into an early-move-in RawPage so extract() stays a
+ * pure function of the bytes it receives (city/state/zip only exist on the
+ * community landing page, not the early-move-in page). */
+interface InjectedMeta {
+ slug: string;
+ name: string | null;
+ city: string | null;
+ state: string | null;
+ zip: string | null;
+}
+
+/** Title-case a hyphenated community slug → "Apex At Avenir", "Valencia Ridge". */
+function slugToName(slug: string): string {
+ return slug
+ .split("-")
+ .map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w))
+ .join(" ");
+}
+
+/** Extract the active-community slugs from the homepage nav. GL lists each
+ * community as a top-level <a href="/<slug>/">; we keep only slugs that look
+ * like real community landing pages and drop the site's utility pages. */
+const NON_COMMUNITY = new Set([
+ "about-us", "brokers", "careers", "coming-soon", "free-brochure", "philanthropy",
+ "privacy-policy", "smart-planning", "special-offers", "stay-and-play", "terms-of-use",
+ "videos", "florida-homes", "valencia-lifestyle", "case-studies", "webinar",
+ "utm-generator", "new-homes-for-sale",
+]);
+
+export function parseCommunitySlugs(html: string): string[] {
+ const slugs = new Set<string>();
+ for (const m of html.matchAll(/href="\/([a-z0-9-]+)\/"/g)) {
+ const slug = m[1]!;
+ if (NON_COMMUNITY.has(slug)) continue;
+ if (slug.includes("email-preferences")) continue;
+ slugs.add(slug);
+ }
+ return [...slugs];
+}
+
+/** Pull the community's city/state/zip out of the sales-center address on the
+ * community landing page. The address appears in several publisher formats:
+ * "12874 Soaring View, Palm Beach Gardens, FL 33412"
+ * "12320 SW Calm Pointe Ct, Port St. Lucie, FL 34987" (period in city)
+ * "9150 Serene Heron Drive, Boynton Beach, Fl 33473" (lowercase state)
+ * We anchor on the "<city>, <ST> <zip>" tail and take the city as the
+ * comma-segment immediately before the 2-letter state + 5-digit zip. */
+export function parseCommunityAddress(html: string): { city: string | null; state: string | null; zip: string | null } {
+ const m = html.match(/,\s*([A-Za-z][A-Za-z .'-]*?),\s*([A-Za-z]{2})\s+(\d{5})\b/);
+ if (!m) return { city: null, state: null, zip: null };
+ const state = normalizeStateCode(str(m[2])?.toUpperCase() ?? null);
+ return {
+ city: str(m[1]),
+ state,
+ zip: zip5(m[3]),
+ };
+}
+
+/** Read the injected `<!-- gl-community: {json} -->` comment, if any. */
+function readInjectedMeta(html: string): InjectedMeta | null {
+ const m = html.match(/<!--\s*gl-community:\s*(\{[\s\S]*?\})\s*-->/);
+ if (!m) return null;
+ try {
+ return JSON.parse(m[1]!) as InjectedMeta;
+ } catch {
+ return null;
+ }
+}
+
+interface PlanAttrs {
+ beds: number | null;
+ baths: number | null; // full + half*0.5
+ garages: number | null;
+ sqft: number | null; // a/c (living) sq ft
+}
+
+/** Parse the plan-level ".early-move__attr" line + sqft list into shared attrs.
+ * "3 Bedrooms, 3 Bathrooms, 1 Half Bath, Den/Opt. 4th Bed, 3-Car Garage" */
+export function parsePlanAttrs(attr: string | null, sqftAcRaw: string | null): PlanAttrs {
+ const a = attr ?? "";
+ const beds = posNum(a.match(/(\d+)\s*Bedrooms?/i)?.[1] ?? null);
+ const full = nonNegNum(a.match(/(\d+)\s*Bathrooms?/i)?.[1] ?? null);
+ const half = nonNegNum(a.match(/(\d+)\s*Half\s*Baths?/i)?.[1] ?? null);
+ const baths = full === null ? null : full + (half ?? 0) * 0.5;
+ const garages = nonNegNum(a.match(/(\d+)\s*-?\s*Car\s*Garage/i)?.[1] ?? null);
+ const sqft = posNum(sqftAcRaw);
+ return { beds, baths, garages, sqft };
+}
+
+export interface GlHome {
+ street: string | null;
+ lot: string | null;
+ price: number | null;
+ priceRaw: string | null;
+ originalPriceRaw: string | null;
+ closingRaw: string | null;
+}
+
+export interface GlCard {
+ planName: string | null;
+ attr: string | null;
+ sqftAcRaw: string | null;
+ floorplanHref: string | null;
+ homes: GlHome[];
+}
+
+/** Parse every <div class="early-move__card"> … block on an early-move-in page.
+ * Each card is a PLAN; each <li> inside its .early-move__items is a HOME. */
+export function parseCards(html: string): GlCard[] {
+ const cards: GlCard[] = [];
+ // Split on the card opener; the segment runs until the next card or the section
+ // close. We over-capture then trim at the next card boundary defensively.
+ const segments = html.split(/<div id="emi-[^"]*"\s+class="early-move__card"/);
+ for (let i = 1; i < segments.length; i++) {
+ // Trim this segment so it doesn't bleed into the following card's items.
+ const seg = segments[i]!;
+ const planName = seg.match(/early-move__subtitle">([^<]+)</)?.[1]?.trim() ?? null;
+ const attr = seg.match(/early-move__attr">([^<]+)</)?.[1]?.trim() ?? null;
+ // First .early-move__sqs <li> is the a/c (living) sqft; second is total.
+ const sqsBlock = seg.match(/early-move__sqs">([\s\S]*?)<\/ul>/)?.[1] ?? "";
+ const sqftAcRaw = sqsBlock.match(/<li>\s*([\d,]+)\s*a\/c/i)?.[1] ?? null;
+ const floorplanHref = seg.match(/href="(\/[a-z0-9-]+\/[a-z0-9-]+\/[a-z0-9-]+\/)"\s+class="btn-link/)?.[1] ?? null;
+
+ const homes: GlHome[] = [];
+ const itemsBlock = seg.match(/<ul class="early-move__items">([\s\S]*?)<\/ul>/)?.[1] ?? "";
+ for (const liMatch of itemsBlock.matchAll(/<li>([\s\S]*?)<\/li>/g)) {
+ const li = liMatch[1]!;
+ // First <span> is the street; a "Lot NNNN" span is the lot.
+ const street = li.match(/<span>([0-9][^<]*)<\/span>/)?.[1]?.trim() ?? null;
+ const lot = li.match(/<span>\s*(Lot[^<]*)<\/span>/i)?.[1]?.trim() ?? null;
+ // Two publisher price classes: ".early-move__current-price" (when the home
+ // shows a struck-through original + savings) and
+ // ".early-move-current-price-standalone" (when it lists a single price).
+ const priceRaw =
+ li.match(/early-move__current-price">\s*\$?([\d,]+)/)?.[1] ??
+ li.match(/early-move-current-price-standalone">\s*\$?([\d,]+)/)?.[1] ??
+ null;
+ const originalPriceRaw = li.match(/early-move__original-price\s*">\s*\$?([\d,]+)/)?.[1] ?? null;
+ const closingRaw = li.match(/early-move-closing-date">([^<]+)</)?.[1]?.trim() ?? null;
+ // Skip a stray <li> that carries no street (defensive — real home <li>s
+ // always lead with the address span).
+ if (!street) continue;
+ homes.push({ street, lot, price: posNum(priceRaw), priceRaw, originalPriceRaw, closingRaw });
+ }
+
+ cards.push({ planName, attr, sqftAcRaw, floorplanHref, homes });
+ }
+ return cards;
+}
+
+/** GL closing-date text → construction-status enum. "Available Now" = ready;
+ * "Available <Mon Year>" = still under construction. Blank → null. */
+function constructionStatus(closing: string | null): "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
+ const s = (str(closing) ?? "").toLowerCase();
+ if (!s) return null;
+ if (s.includes("now")) return "MOVE_IN_READY";
+ if (/available\s+[a-z]{3}\s+\d{4}/i.test(s)) return "UNDER_CONSTRUCTION";
+ return null;
+}
+
+const MONTHS: Record<string, string> = {
+ jan: "01", feb: "02", mar: "03", apr: "04", may: "05", jun: "06",
+ jul: "07", aug: "08", sep: "09", oct: "10", nov: "11", dec: "12",
+};
+
+/** "Available Apr 2027" → "2027-04-01" (est. completion, day defaulted). "Available
+ * Now" and unparseable text → null (never guessed). */
+function estCompletionDate(closing: string | null): string | null {
+ const s = str(closing);
+ if (!s) return null;
+ const m = s.match(/([A-Za-z]{3})[a-z]*\s+(\d{4})/);
+ if (!m) return null;
+ const mon = MONTHS[m[1]!.toLowerCase()];
+ const year = Number(m[2]);
+ if (!mon || !Number.isFinite(year) || year < 2000 || year > 2100) return null;
+ return `${year}-${mon}-01`;
+}
+
+/** Absolute URL from a site-relative href. */
+function abs(href: string): string {
+ if (/^https?:\/\//.test(href)) return href;
+ return `${ORIGIN}${href.startsWith("/") ? "" : "/"}${href}`;
+}
+
+export const glHomesAdapter: SourceAdapter = {
+ key: "gl-homes-site",
+ version: "1.0.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+
+ let homepage: RawPage;
+ try {
+ homepage = await fetcher.fetch(`${ORIGIN}/`);
+ } catch (error) {
+ console.warn(` gl homepage: ${error instanceof Error ? error.message : String(error)}`);
+ return; // a block/403 on the homepage stops collection (source degraded)
+ }
+
+ const slugs = parseCommunitySlugs(homepage.body.toString("utf8"));
+ let crawled = 0;
+ for (const slug of slugs) {
+ if (crawled >= Math.max(0, PAGE_LIMIT)) break;
+
+ // Community landing → city/state/zip (early-move-in page lacks them).
+ let meta: InjectedMeta = { slug, name: slugToName(slug), city: null, state: null, zip: null };
+ try {
+ const landing = await fetcher.fetch(COMMUNITY_URL(slug));
+ const addr = parseCommunityAddress(landing.body.toString("utf8"));
+ meta = { slug, name: slugToName(slug), ...addr };
+ } catch (error) {
+ console.warn(` gl community ${slug}: ${error instanceof Error ? error.message : String(error)}`);
+ // Continue without city/zip — the home records are still valid facts.
+ }
+
+ // Quick-move-in ("Early Move-In") inventory page.
+ let page: RawPage;
+ try {
+ page = await fetcher.fetch(HOME_URL(slug));
+ } catch (error) {
+ console.warn(` gl early-move-in ${slug}: ${error instanceof Error ? error.message : String(error)}`);
+ continue; // one blocked/failed community doesn't stop the rest
+ }
+ const body = page.body.toString("utf8");
+ // Communities with no inventory page soft-404 to the homepage — skip those.
+ if (!body.includes("early-move__card")) continue;
+ crawled++;
+
+ const injected = `<!-- ${META_MARKER} ${JSON.stringify(meta)} -->\n${body}`;
+ yield { ...page, body: Buffer.from(injected, "utf8") };
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ try {
+ const html = page.body.toString("utf8");
+ const meta = readInjectedMeta(html);
+ const slug = meta?.slug ?? page.url.match(/glhomes\.com\/([a-z0-9-]+)\//)?.[1] ?? null;
+ const community = meta?.name ?? (slug ? slugToName(slug) : null);
+
+ if (!community) {
+ return { records: [], errors: [{ url: page.url, reason: "early-move-in page missing community meta/slug" }] };
+ }
+
+ const cards = parseCards(html);
+ if (!cards.length) {
+ return { records: [], errors: [{ url: page.url, reason: "no early-move__card blocks on early-move-in page" }] };
+ }
+
+ const state = meta?.state ?? normalizeStateCode("FL"); // GL Homes is Florida-only
+ const city = meta?.city ?? null;
+ const zip = meta?.zip ?? null;
+
+ const records: ExtractedRecord[] = [];
+ const errors: { url: string; reason: string }[] = [];
+ let emittedCommunity = false;
+
+ for (const card of cards) {
+ const plan = parsePlanAttrs(card.attr, card.sqftAcRaw);
+ const planUrl = card.floorplanHref ? abs(card.floorplanHref) : page.url;
+
+ for (const home of card.homes) {
+ if (!home.street) {
+ errors.push({ url: page.url, reason: `home in plan ${card.planName ?? "?"} missing street — skipped` });
+ continue;
+ }
+
+ // Community FIRST — publish creates/refreshes the FK target the home
+ // needs. Emit it once per page (all homes share the one community).
+ if (!emittedCommunity) {
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName: community },
+ fields: {
+ name: fv(community, community, page.url, "community slug/landing"),
+ street: fv<string>(null, null, page.url),
+ city: fv(city, city, page.url),
+ state: fv(state, meta?.state ?? "FL", page.url),
+ zip: fv(zip, zip, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv<string>(null, null, page.url),
+ lat: fv<number>(null, null, page.url),
+ lon: fv<number>(null, null, page.url),
+ hoaFeeMonthly: fv<number>(null, null, page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ },
+ });
+ emittedCommunity = true;
+ }
+
+ const cStatus = constructionStatus(home.closingRaw);
+ const est = estCompletionDate(home.closingRaw);
+ // Preserve the address exactly as published (upper-cased street).
+ const street = str(home.street);
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName: community,
+ address: street ?? undefined,
+ builderInventoryId: home.lot ?? undefined,
+ planName: card.planName ?? undefined,
+ },
+ fields: {
+ street: fv(street, street, page.url, "early-move__items home address"),
+ city: fv(city, city, page.url),
+ state: fv(state, meta?.state ?? "FL", page.url),
+ zip: fv(zip, zip, page.url),
+ price: fv(
+ home.price,
+ home.priceRaw,
+ page.url,
+ home.price === null ? null : `current price $${home.priceRaw}${home.originalPriceRaw ? ` (was $${home.originalPriceRaw})` : ""}`,
+ ),
+ beds: fv(plan.beds, plan.beds === null ? null : String(plan.beds), page.url, plan.beds === null ? null : `plan ${card.planName}: ${card.attr}`),
+ bathsTotal: fv(plan.baths, plan.baths === null ? null : String(plan.baths), page.url, plan.baths === null ? null : `plan ${card.planName}: ${card.attr}`),
+ sqft: fv(plan.sqft, plan.sqft === null ? null : String(plan.sqft), page.url, plan.sqft === null ? null : `${card.sqftAcRaw} a/c sq. ft.`),
+ stories: fv<number>(null, null, page.url),
+ garageSpaces: fv(plan.garages, plan.garages === null ? null : String(plan.garages), page.url),
+ homeType: fv("SINGLE_FAMILY" as const, null, page.url, "GL Homes single-family spec home"),
+ constructionStatus: fv(cStatus, home.closingRaw, page.url, cStatus === null ? null : `closing: ${home.closingRaw}`),
+ estCompletionDate: fv(est, home.closingRaw, page.url, est === null ? null : `from "${home.closingRaw}"`),
+ lotNumber: fv(home.lot, home.lot, page.url),
+ builderInventoryId: fv(home.lot, home.lot, page.url, home.lot === null ? null : "GL lot number"),
+ lat: fv<number>(null, null, page.url),
+ lon: fv<number>(null, null, page.url),
+ planName: fv(card.planName, card.planName, planUrl),
+ // facts-only: card photos exist in the HTML but are intentionally dropped.
+ images: fv<string[]>([], null, page.url),
+ },
+ });
+ }
+ }
+
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/gl-homes/tsconfig.json b/collectors/gl-homes/tsconfig.json
new file mode 100644
index 00000000..e9bfc482
--- /dev/null
+++ b/collectors/gl-homes/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index addbc674..f7808598 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -169,6 +169,15 @@ importers:
'@homesonspec/collector-drees-homes':
specifier: workspace:*
version: link:../../collectors/drees-homes
+ '@homesonspec/collector-fischer-homes':
+ specifier: workspace:*
+ version: link:../../collectors/fischer-homes
+ '@homesonspec/collector-fulton-homes':
+ specifier: workspace:*
+ version: link:../../collectors/fulton-homes
+ '@homesonspec/collector-gl-homes':
+ specifier: workspace:*
+ version: link:../../collectors/gl-homes
'@homesonspec/collector-highland-homes':
specifier: workspace:*
version: link:../../collectors/highland-homes
@@ -469,6 +478,72 @@ importers:
specifier: ^4.0.0
version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+ collectors/fischer-homes:
+ dependencies:
+ '@homesonspec/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@homesonspec/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@homesonspec/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
+ collectors/fulton-homes:
+ dependencies:
+ '@homesonspec/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@homesonspec/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@homesonspec/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
+ collectors/gl-homes:
+ dependencies:
+ '@homesonspec/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@homesonspec/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@homesonspec/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
collectors/highland-homes:
dependencies:
'@homesonspec/collectors-common':
diff --git a/scripts/build-loop.sh b/scripts/build-loop.sh
index 98a54ebb..80f9f501 100644
--- a/scripts/build-loop.sh
+++ b/scripts/build-loop.sh
@@ -123,6 +123,13 @@ sweep_brookfield() { BROOKFIELD_PAGE_LIMIT=10 CLI brookfield-site | sed "s/^/
# Holt = per-home sitemap crawl (OR/WA, ~128 homes = 128 rate-limited GETs) — throttle w/ heavy crawlers.
sweep_holt() { HOLT_PAGE_LIMIT=140 CLI holt-homes-site | sed "s/^/ hlt REGIONAL: /"; }
+# Fulton = single ASP.NET GetSpecs web-service call (AZ, ~202 homes), no pagination, fast — every sweep.
+sweep_fulton() { FULTON_PAGE_LIMIT=5 CLI fulton-homes-site | sed "s/^/ ful REGIONAL: /"; }
+# GL Homes = two-layer HTML crawl (~10 FL community pages, ~168 homes), light — every sweep.
+sweep_gl() { GL_PAGE_LIMIT=60 CLI gl-homes-site | sed "s/^/ gl REGIONAL: /"; }
+# Fischer = region API + one detail-page GET PER HOME (OH/KY/IN/GA/MO, many requests) — throttle w/ heavy crawlers.
+sweep_fischer() { FISCHER_PAGE_LIMIT=20 CLI fischer-homes-site | sed "s/^/ fis REGIONAL: /"; }
+
# metro-iterating adapters (footprint = METROS, not states) — scoped to served metros only
# (Cody WAF-lesson from the start). Ashton Woods publishes per-metro quick-move-in homes.
sweep_metros() {
@@ -141,7 +148,7 @@ while pgrep -f "cli.ts --adapter" 2>/dev/null | grep -qv "^$$\$"; do sleep 20; d
# a transient failure shouldn't permanently sideline a chosen builder, and genuine breakage is
# surfaced by the dashboard's per-source failed_runs + the freshness canary (not silent). To
# retire a builder, remove it from THIS list (don't rely on auto-pause surviving a restart).
-for k in dr-horton-site kb-home-site pultegroup-site tri-pointe-site lennar-site toll-brothers-site david-weekley-site taylor-morrison-site ashton-woods-site discovery-homes-site meritage-homes-site ryan-homes-site highland-homes-site landsea-homes-site shea-homes-site lgi-homes-site century-communities-site beazer-site drees-homes-site perry-homes-site brookfield-site holt-homes-site; do
+for k in dr-horton-site kb-home-site pultegroup-site tri-pointe-site lennar-site toll-brothers-site david-weekley-site taylor-morrison-site ashton-woods-site discovery-homes-site meritage-homes-site ryan-homes-site highland-homes-site landsea-homes-site shea-homes-site lgi-homes-site century-communities-site beazer-site drees-homes-site perry-homes-site brookfield-site holt-homes-site fulton-homes-site gl-homes-site fischer-homes-site; do
PSQL "update \"SourceRegistry\" set active=true, health='HEALTHY', \"consecutiveFailures\"=0 where key='$k';" >/dev/null
done
@@ -161,8 +168,10 @@ while [ ! -f "$STOP" ] && [ "$sweep" -lt "$MAX_SWEEPS" ]; do
sweep_shea &
sweep_perry &
sweep_brookfield &
+ sweep_fulton &
+ sweep_gl &
# heavy crawlers: throttled to every 4th sweep (hundreds of GETs each, slow-changing data)
- if [ $((sweep % 4)) -eq 1 ]; then sweep_lgi & sweep_century & sweep_beazer & sweep_drees & sweep_holt & fi
+ if [ $((sweep % 4)) -eq 1 ]; then sweep_lgi & sweep_century & sweep_beazer & sweep_drees & sweep_holt & sweep_fischer & fi
sweep_adapter drh &
sweep_adapter plt &
sweep_adapter tri &
← 3cf38266 cody-gate fixes: AW gallery pollution, DW community-photo he
·
back to Homesonspec
·
ashton-woods: tighten image blocklist + purge residual pollu d9a0af8c →