← back to Homesonspec
auto-save: 2026-07-29T07:06:42 (3 files) — collectors/dream-finders/src/index.ts apps/workers/apply-lennar-images.sh apps/workers/src/_lennar-mac-fetch.mts
0b79470c2b7e699b82ea43c9d2082bbe20b46b89 · 2026-07-29 07:06:55 -0700 · Steve Abrams
Files touched
A apps/workers/apply-lennar-images.shA apps/workers/src/_lennar-mac-fetch.mtsM collectors/dream-finders/src/index.ts
Diff
commit 0b79470c2b7e699b82ea43c9d2082bbe20b46b89
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 29 07:06:55 2026 -0700
auto-save: 2026-07-29T07:06:42 (3 files) — collectors/dream-finders/src/index.ts apps/workers/apply-lennar-images.sh apps/workers/src/_lennar-mac-fetch.mts
---
apps/workers/apply-lennar-images.sh | 26 ++++++++
apps/workers/src/_lennar-mac-fetch.mts | 41 ++++++++++++
collectors/dream-finders/src/index.ts | 118 ++++++++++++++++++---------------
3 files changed, 131 insertions(+), 54 deletions(-)
diff --git a/apps/workers/apply-lennar-images.sh b/apps/workers/apply-lennar-images.sh
new file mode 100644
index 00000000..06e7e172
--- /dev/null
+++ b/apps/workers/apply-lennar-images.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Apply Mac-fetched Lennar images (/tmp/lennar-images.jsonl) to the Kamatera DB.
+# Matches on (builderInventoryId, street) with an images='{}' guard so it ONLY fills
+# currently-empty Lennar homes and never overwrites existing good images. [TK-10005]
+set -euo pipefail
+JSONL=/tmp/lennar-images.jsonl
+[ -s "$JSONL" ] || { echo "no JSONL to apply"; exit 1; }
+echo "shipping $(wc -l < "$JSONL") rows to Kamatera…"
+gzip -c "$JSONL" | ssh root@45.61.58.125 'gunzip > /tmp/lennar-images.jsonl; cd /root/Projects/homesonspec && set -a; . .env; set +a; \
+psql "$DATABASE_URL" -v ON_ERROR_STOP=1 <<SQL
+CREATE TEMP TABLE _limg (j jsonb);
+\copy _limg (j) FROM '"'"'/tmp/lennar-images.jsonl'"'"'
+CREATE TEMP TABLE _lstage AS
+ SELECT j->>'"'"'invId'"'"' AS inv, j->>'"'"'street'"'"' AS street,
+ ARRAY(SELECT jsonb_array_elements_text(j->'"'"'images'"'"')) AS images
+ FROM _limg;
+-- de-dup: keep one image-set per (inv,street)
+CREATE TEMP TABLE _lstage1 AS SELECT DISTINCT ON (inv,street) inv,street,images FROM _lstage;
+WITH b AS (SELECT id FROM "Builder" WHERE name='"'"'Lennar'"'"')
+UPDATE "InventoryHome" h SET images = s.images, "updatedAt"=now()
+FROM _lstage1 s, b
+WHERE h."builderId"=b.id AND h."builderInventoryId"=s.inv AND h.street=s.street
+ AND h.images='"'"'{}'"'"' AND h.status='"'"'PUBLISHED'"'"';
+SELECT '"'"'Lennar imaged now: '"'"'||count(*) filter (where images<>'"'"'{}'"'"')||'"'"'/'"'"'||count(*)||'"'"' ('"'"'||round(100.0*count(*) filter (where images<>'"'"'{}'"'"')/count(*),1)||'"'"'%)'"'"'
+FROM "InventoryHome" h JOIN "Builder" b ON b.id=h."builderId" WHERE b.name='"'"'Lennar'"'"' AND h.status='"'"'PUBLISHED'"'"';
+SQL'
diff --git a/apps/workers/src/_lennar-mac-fetch.mts b/apps/workers/src/_lennar-mac-fetch.mts
new file mode 100644
index 00000000..e4139965
--- /dev/null
+++ b/apps/workers/src/_lennar-mac-fetch.mts
@@ -0,0 +1,41 @@
+// Mac-side Lennar fetch (Mac IP is NOT 403-blocked by lennar.com, unlike Kamatera).
+// Drives the real lennarAdapter.fetch()+extract() (full robots/rate-limit politeness via
+// LiveFetcher), collects per-home {invId, street, city, state, images}, writes JSONL for
+// application to the Kamatera DB. NO DB writes here — pure fetch→extract→file. [TK-10005]
+import { lennarAdapter } from "@homesonspec/collector-lennar";
+import { writeFileSync, appendFileSync } from "fs";
+
+const STATES = ["california", "colorado", "florida", "arizona", "alabama", "arkansas", "delaware"];
+const OUT = "/tmp/lennar-images.jsonl";
+writeFileSync(OUT, "");
+const registry = { key: "lennar-site", collectionMethod: "CRAWL", rateLimitRpm: 60, mediaRights: "LICENSED" } as any;
+
+let grandTotal = 0, grandImg = 0;
+for (const st of STATES) {
+ process.env.LENNAR_STATE_FILTER = st;
+ let sTotal = 0, sImg = 0;
+ try {
+ for await (const page of lennarAdapter.fetch({ mode: "live", fixtureDir: "/tmp/none", registry })) {
+ const out = lennarAdapter.extract(page);
+ for (const r of out.records) {
+ if (r.entityType !== "inventory_home") continue;
+ sTotal++;
+ const f: any = r.fields;
+ const imgs: string[] = f.images?.value ?? [];
+ const invId: string | null = f.builderInventoryId?.value ?? null;
+ const street: string | null = f.street?.value ?? null;
+ const city: string | null = f.city?.value ?? null;
+ const state: string | null = f.state?.value ?? null;
+ if (imgs.length && invId && street) {
+ sImg++;
+ appendFileSync(OUT, JSON.stringify({ invId, street, city, state, images: imgs }) + "\n");
+ }
+ }
+ }
+ } catch (e) {
+ console.error(` ${st} ERROR:`, (e as Error).message);
+ }
+ grandTotal += sTotal; grandImg += sImg;
+ console.log(`${st}: ${sImg}/${sTotal} homes with images`);
+}
+console.log(`TOTAL: ${grandImg}/${grandTotal} written to ${OUT}`);
diff --git a/collectors/dream-finders/src/index.ts b/collectors/dream-finders/src/index.ts
index f0615101..6884f4c2 100644
--- a/collectors/dream-finders/src/index.ts
+++ b/collectors/dream-finders/src/index.ts
@@ -55,7 +55,10 @@ function parseSpecUrl(url: string): UrlParts | null {
function extractLdJsonBlocks(html: string): unknown[] {
const blocks: unknown[] = [];
- for (const match of html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/g)) {
+ // NB: Dream Finders emits type='application/ld+json' with SINGLE quotes — the original regex
+ // required double quotes and matched ZERO blocks, which was the true root of the hollow-data
+ // failure (no ld+json ever parsed). Accept either quote style. [2026-07-29]
+ for (const match of html.matchAll(/<script[^>]*type=['"]application\/ld\+json['"][^>]*>([\s\S]*?)<\/script>/g)) {
try {
blocks.push(JSON.parse(match[1]!));
} catch {
@@ -65,52 +68,59 @@ function extractLdJsonBlocks(html: string): unknown[] {
return blocks;
}
-/** Find the Product whose Offer.url matches THIS page (pages also list sibling homes). */
-function findOwnOffer(blocks: unknown[], pageUrl: string): { price: number | null; raw: string | null } {
- const normalized = pageUrl.replace(/\/$/, "");
- const walk = (node: unknown): { price: number | null; raw: string | null } | null => {
- if (!node || typeof node !== "object") return null;
- if (Array.isArray(node)) {
- for (const item of node) {
- const found = walk(item);
- if (found) return found;
- }
- return null;
- }
- const obj = node as Record<string, unknown>;
- const offers = obj.offers as Record<string, unknown> | undefined;
- if (offers && typeof offers === "object") {
- const offerUrl = String(offers.url ?? "").replace(/\/$/, "");
- if (offerUrl === normalized) {
- const price = Number(offers.price);
- return Number.isFinite(price) && price > 0
- ? { price, raw: String(offers.price) }
- : { price: null, raw: offers.price === undefined ? null : String(offers.price) };
- }
- }
- for (const value of Object.values(obj)) {
- const found = walk(value);
- if (found) return found;
- }
- return null;
- };
- for (const block of blocks) {
- const found = walk(block);
- if (found) return found;
- }
- return { price: null, raw: null };
+interface OwnHome {
+ price: number | null; priceRaw: string | null;
+ beds: number | null; baths: number | null; sqft: number | null;
+ lat: number | null; lon: number | null;
}
+const posNum = (v: unknown): number | null => {
+ const n = typeof v === "number" ? v : typeof v === "string" ? Number(String(v).replace(/[^0-9.]/g, "")) : NaN;
+ return Number.isFinite(n) && n > 0 ? n : null;
+};
-/** "3,214 sq. ft. ... 5 bedrooms, 4 baths" — regex facts out of description text. */
-function factsFromText(html: string): { beds: number | null; baths: number | null; sqft: number | null } {
- const bedsMatch = html.match(/(\d+)\s*(?:bedrooms|beds|bed\b)/i);
- const bathsMatch = html.match(/(\d+(?:\.\d+)?)\s*(?:bathrooms|baths|bath\b)/i);
- const sqftMatch = html.match(/([\d,]{3,6})\s*(?:sq\.?\s?ft|square feet)/i);
- const sqft = sqftMatch ? Number(sqftMatch[1]!.replace(/,/g, "")) : null;
+/**
+ * Find THIS page's own home in the ld+json. A DF spec page carries a Product/SingleFamilyResidence
+ * block PER home (its own + siblings in the community); the page's own home is the block whose `url`
+ * ends with the page's address slug. The original findOwnOffer matched offers.url === pageUrl exactly
+ * and got 0 (siblings' urls differ, trailing-slash drift), producing hollow 0%-price rows — the fix is
+ * a slug match, and the same block also yields beds/baths/sqft + GeoCoordinates ($0 geo). [2026-07-29]
+ */
+function findOwnHome(blocks: unknown[], addressSlug: string): OwnHome {
+ const empty: OwnHome = { price: null, priceRaw: null, beds: null, baths: null, sqft: null, lat: null, lon: null };
+ const products: Record<string, unknown>[] = [];
+ const walk = (node: unknown) => {
+ if (!node || typeof node !== "object") return;
+ if (Array.isArray(node)) { node.forEach(walk); return; }
+ const obj = node as Record<string, unknown>;
+ const t = JSON.stringify(obj["@type"] ?? "");
+ if (/Product|SingleFamilyResidence|House/.test(t) && (obj.url || obj.offers)) products.push(obj);
+ for (const v of Object.values(obj)) walk(v);
+ };
+ blocks.forEach(walk);
+ const slug = addressSlug.replace(/\/$/, "").toLowerCase();
+ // own home = the product whose url ends with the page's address slug. CRITICAL: no products[0]
+ // fallback — some DF pages carry only SIBLING homes in ld+json (not their own), and blindly taking
+ // the first product assigns one home's price/beds to every sibling page = the hollow-data trap that
+ // originally sank this adapter. If the page's own home isn't in ld+json, return null (honest) rather
+ // than a wrong home's facts. [2026-07-29]
+ const own = products.find((p) => String((p.url ?? (p.offers as Record<string, unknown>)?.url) ?? "").replace(/\/$/, "").toLowerCase().endsWith(slug));
+ if (!own) return empty;
+ const offers = (own.offers as Record<string, unknown>) ?? {};
+ const geo = (own.geo as Record<string, unknown>) ?? {};
+ const bathsTotal = posNum(own.numberOfBathroomsTotal)
+ ?? (posNum(own.numberOfFullBathrooms) !== null
+ ? posNum(own.numberOfFullBathrooms)! + (posNum(own.numberOfPartialBathrooms) ?? 0) * 0.5
+ : null);
+ const sqftNode = own.floorSize as Record<string, unknown> | undefined;
+ const lat = posNum(geo.latitude), lonRaw = geo.longitude;
+ const lon = lonRaw != null && Number.isFinite(Number(lonRaw)) && Number(lonRaw) !== 0 ? Number(lonRaw) : null;
return {
- beds: bedsMatch ? Number(bedsMatch[1]) : null,
- baths: bathsMatch ? Number(bathsMatch[1]) : null,
- sqft: sqft !== null && Number.isFinite(sqft) ? sqft : null,
+ price: posNum(offers.price), priceRaw: offers.price != null ? String(offers.price) : null,
+ beds: posNum(own.numberOfBedrooms),
+ baths: bathsTotal,
+ sqft: posNum(sqftNode?.value ?? own.floorSize),
+ lat: lat, // US bbox-gated below via coord()
+ lon: lon !== null && lon < 0 && lon > -180 ? lon : null,
};
}
@@ -146,8 +156,8 @@ export const dreamFindersAdapter: SourceAdapter = {
}
const html = page.body.toString("utf8");
const blocks = extractLdJsonBlocks(html);
- const offer = findOwnOffer(blocks, page.url);
- const facts = factsFromText(html);
+ const addressSlug = page.url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "").split("/").filter(Boolean).pop() ?? "";
+ const own = findOwnHome(blocks, addressSlug);
const records: ExtractedRecord[] = [
// Community first — publish creates/refreshes the FK target.
@@ -162,8 +172,8 @@ export const dreamFindersAdapter: SourceAdapter = {
zip: fv<string>(null, null, page.url),
county: fv<string>(null, null, page.url),
metro: fv<string>(null, null, page.url),
- lat: fv<number>(null, null, page.url),
- lon: fv<number>(null, null, page.url),
+ lat: fv(own.lat, own.lat === null ? null : String(own.lat), page.url, own.lat === null ? null : "ld+json GeoCoordinates"),
+ lon: fv(own.lon, own.lon === null ? null : String(own.lon), page.url, own.lon === null ? null : "ld+json GeoCoordinates"),
hoaFeeMonthly: fv<number>(null, null, page.url),
schoolDistrict: fv<string>(null, null, page.url),
ageRestricted: fv<boolean>(null, null, page.url),
@@ -182,10 +192,10 @@ export const dreamFindersAdapter: SourceAdapter = {
city: fv(parts.city, parts.city, page.url),
state: fv(parts.state, parts.state, page.url),
zip: fv<string>(null, null, page.url),
- price: fv(offer.price, offer.raw, page.url, offer.raw ? `ld+json Offer price ${offer.raw}` : null),
- beds: fv(facts.beds, facts.beds === null ? null : String(facts.beds), page.url),
- bathsTotal: fv(facts.baths, facts.baths === null ? null : String(facts.baths), page.url),
- sqft: fv(facts.sqft, facts.sqft === null ? null : String(facts.sqft), page.url),
+ price: fv(own.price, own.priceRaw, page.url, own.priceRaw ? `ld+json Offer price ${own.priceRaw}` : null),
+ beds: fv(own.beds, own.beds === null ? null : String(own.beds), page.url, own.beds === null ? null : "ld+json numberOfBedrooms"),
+ bathsTotal: fv(own.baths, own.baths === null ? null : String(own.baths), page.url, own.baths === null ? null : "ld+json numberOfBathrooms"),
+ sqft: fv(own.sqft, own.sqft === null ? null : String(own.sqft), page.url, own.sqft === null ? null : "ld+json floorSize"),
stories: fv<number>(null, null, page.url),
garageSpaces: fv<number>(null, null, page.url),
homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Dream Finders spec home listing"),
@@ -193,8 +203,8 @@ export const dreamFindersAdapter: SourceAdapter = {
estCompletionDate: fv<string>(null, null, page.url),
lotNumber: fv<string>(null, null, page.url),
builderInventoryId: fv<string>(null, null, page.url),
- lat: fv<number>(null, null, page.url),
- lon: fv<number>(null, null, page.url),
+ lat: fv(own.lat, own.lat === null ? null : String(own.lat), page.url, own.lat === null ? null : "ld+json GeoCoordinates"),
+ lon: fv(own.lon, own.lon === null ? null : String(own.lon), page.url, own.lon === null ? null : "ld+json GeoCoordinates"),
planName: fv(parts.plan, parts.plan, page.url),
},
},
← de09b351 david-weekley: capture city-gated community MapPoint coord (
·
back to Homesonspec
·
lennar: Mac-side fetch->Kamatera image apply (routes around 6a68cd77 →