← back to Homesonspec
lennar live adapter (172 real homes from 8 pages, Apollo state extraction), streaming runPipeline for large sweeps, honest early-preview banner
45f9ee299d0713a8257f3caa81664f45413578bc · 2026-07-22 11:55:44 -0700 · Steve Abrams
Files touched
M apps/web/src/app/layout.tsxM apps/workers/package.jsonM apps/workers/src/cli.tsM apps/workers/src/pipeline.tsA collectors/lennar/package.jsonA collectors/lennar/src/index.tsA collectors/lennar/tsconfig.jsonM pnpm-lock.yaml
Diff
commit 45f9ee299d0713a8257f3caa81664f45413578bc
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 22 11:55:44 2026 -0700
lennar live adapter (172 real homes from 8 pages, Apollo state extraction), streaming runPipeline for large sweeps, honest early-preview banner
---
apps/web/src/app/layout.tsx | 7 +-
apps/workers/package.json | 3 +-
apps/workers/src/cli.ts | 2 +
apps/workers/src/pipeline.ts | 44 +++++++--
collectors/lennar/package.json | 15 +++
collectors/lennar/src/index.ts | 199 ++++++++++++++++++++++++++++++++++++++++
collectors/lennar/tsconfig.json | 1 +
pnpm-lock.yaml | 25 +++++
8 files changed, 286 insertions(+), 10 deletions(-)
diff --git a/apps/web/src/app/layout.tsx b/apps/web/src/app/layout.tsx
index 2cd57b5..9184828 100644
--- a/apps/web/src/app/layout.tsx
+++ b/apps/web/src/app/layout.tsx
@@ -12,10 +12,11 @@ export default function RootLayout({ children }: { children: React.ReactNode })
return (
<html lang="en">
<body className="min-h-screen bg-white text-neutral-900 antialiased">
- {/* All inventory in this build is synthetic — never present demo data as real. */}
+ {/* Early-access honesty: real builder-website data + labeled synthetic demo data coexist. */}
<div className="bg-violet-700 px-4 py-1.5 text-center text-xs font-medium text-white">
- Demonstration inventory — all homes, builders, communities, and offers shown are synthetic sample
- data, clearly labeled. No real listings.
+ Early preview — listings labeled “Verified from builder website” come from live builder sites;
+ listings labeled “Demonstration data” are synthetic samples. Every listing shows its verification
+ label and last-verified time.
</div>
<header className="border-b border-neutral-200">
<nav className="mx-auto flex max-w-7xl items-center justify-between px-4 py-3">
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 88e0c02..b361a75 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -19,7 +19,8 @@
"@spechomes/collector-meridian-homes": "workspace:*",
"pg-boss": "^10.1.5",
"@spechomes/publisher": "workspace:*",
- "@spechomes/collector-toll-brothers": "workspace:*"
+ "@spechomes/collector-toll-brothers": "workspace:*",
+ "@spechomes/collector-lennar": "workspace:*"
},
"devDependencies": {
"tsx": "^4.19.2",
diff --git a/apps/workers/src/cli.ts b/apps/workers/src/cli.ts
index b7b6a42..623debe 100644
--- a/apps/workers/src/cli.ts
+++ b/apps/workers/src/cli.ts
@@ -1,6 +1,7 @@
import { prisma } from "@spechomes/database";
import { meridianHomesAdapter } from "@spechomes/collector-meridian-homes";
import { tollBrothersAdapter } from "@spechomes/collector-toll-brothers";
+import { lennarAdapter } from "@spechomes/collector-lennar";
import { runPipeline } from "./pipeline";
/**
@@ -10,6 +11,7 @@ import { runPipeline } from "./pipeline";
const ADAPTERS = {
[meridianHomesAdapter.key]: meridianHomesAdapter,
[tollBrothersAdapter.key]: tollBrothersAdapter,
+ [lennarAdapter.key]: lennarAdapter,
};
async function main() {
diff --git a/apps/workers/src/pipeline.ts b/apps/workers/src/pipeline.ts
index cae9862..406ecb0 100644
--- a/apps/workers/src/pipeline.ts
+++ b/apps/workers/src/pipeline.ts
@@ -243,13 +243,45 @@ export async function runPipeline(adapter: SourceAdapter) {
errors: [] as { url: string; reason: string }[],
};
- const pages = await fetchStage(adapter, source);
- summary.pages = pages.length;
- summary.changed = pages.filter((p) => p.changed).length;
+ // STREAMING: each page is snapshotted, extracted, validated, and published
+ // before the next fetch — memory stays flat across multi-thousand-page
+ // sweeps, and a mid-run failure loses nothing already processed.
+ const dir = join(SNAPSHOT_DIR, source.key);
+ await mkdir(dir, { recursive: true });
+ const isFixture = source.collectionMethod === "FIXTURE" || source.collectionMethod === "SYNTHETIC";
+
+ for await (const page of adapter.fetch({
+ mode: isFixture ? "fixture" : "live",
+ fixtureDir: join(REPO_ROOT, "fixtures/raw-pages", adapter.key.replace(/-fixtures$/, "")),
+ registry: {
+ key: source.key,
+ collectionMethod: source.collectionMethod,
+ rateLimitRpm: source.rateLimitRpm,
+ mediaRights: source.mediaRights,
+ },
+ })) {
+ summary.pages += 1;
+ const existing = await prisma.rawSnapshot.findUnique({
+ where: { sourceId_contentHash: { sourceId: source.id, contentHash: page.contentHash } },
+ });
+ if (existing) continue; // unchanged bytes → nothing new to extract
+ summary.changed += 1;
+
+ const storagePath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
+ await writeFile(storagePath, page.body);
+ const snapshot = await prisma.rawSnapshot.create({
+ data: {
+ sourceId: source.id,
+ url: page.url,
+ retrievedAt: new Date(page.retrievedAt),
+ contentHash: page.contentHash,
+ storagePath,
+ contentType: page.contentType,
+ httpStatus: 200,
+ },
+ });
- for (const { page, snapshotId, changed } of pages) {
- if (!changed) continue; // unchanged bytes → nothing new to extract
- const { stagedIds, errors } = await extractStage(adapter, source, page, snapshotId);
+ const { stagedIds, errors } = await extractStage(adapter, source, page, snapshot.id);
summary.errors.push(...errors);
summary.staged += stagedIds.length;
diff --git a/collectors/lennar/package.json b/collectors/lennar/package.json
new file mode 100644
index 0000000..07f0c1a
--- /dev/null
+++ b/collectors/lennar/package.json
@@ -0,0 +1,15 @@
+{
+ "name": "@spechomes/collector-lennar",
+ "version": "0.1.0",
+ "private": true,
+ "type": "module",
+ "main": "./src/index.ts",
+ "types": "./src/index.ts",
+ "scripts": { "test": "vitest run --passWithNoTests", "typecheck": "tsc --noEmit" },
+ "dependencies": {
+ "@spechomes/collectors-common": "workspace:*",
+ "@spechomes/schemas": "workspace:*",
+ "@spechomes/shared": "workspace:*"
+ },
+ "devDependencies": { "typescript": "^5.7.2", "vitest": "^4.0.0", "@types/node": "^22.10.5" }
+}
diff --git a/collectors/lennar/src/index.ts b/collectors/lennar/src/index.ts
new file mode 100644
index 0000000..fe04203
--- /dev/null
+++ b/collectors/lennar/src/index.ts
@@ -0,0 +1,199 @@
+import type { ExtractedRecord, FieldValue } from "@spechomes/schemas";
+import { normalizeStateCode } from "@spechomes/shared";
+import {
+ fetchFixtures,
+ LiveFetcher,
+ type ExtractionOutput,
+ type FetchContext,
+ type RawPage,
+ type SourceAdapter,
+} from "@spechomes/collectors-common";
+
+/**
+ * Lennar adapter — EMBEDDED_JSON source (recon 2026-07-22, difficulty 2).
+ * Community pages under /new-homes/ ship a fully-hydrated Apollo state in
+ * __NEXT_DATA__: CommunityType (name, cityRef, stateCode, zipCode, hoa,
+ * lat/lon) + per-home HomesiteType records (price, beds, baths, sqft,
+ * status, address, lat/lon, planRef). robots.txt permits the /new-homes
+ * tree (only fragment paths disallowed).
+ *
+ * Controlled batching: LENNAR_PAGE_LIMIT community pages per run
+ * (default 8) at the registry rate limit; change detection makes
+ * re-visits cheap (unchanged bytes stage nothing).
+ */
+
+const SITEMAP_URL = "https://www.lennar.com/api/images/sitemapfe.xml";
+const BUILDER_SLUG = "lennar";
+const PAGE_LIMIT = Number(process.env.LENNAR_PAGE_LIMIT ?? 8);
+
+function fv<T>(value: T | null, raw: string | null, sourceUrl: string, evidenceText?: string | null): FieldValue<T> {
+ return { value, raw, evidenceText: evidenceText ?? raw, sourceUrl, confidence: value === null ? 0 : 1 };
+}
+
+/** Community URL = /new-homes/{state}/{metro}/{city}/{community} exactly. */
+function isCommunityUrl(url: string): boolean {
+ const path = url.replace(/^https?:\/\/[^/]+/, "").replace(/\/$/, "");
+ const segments = path.split("/").filter(Boolean);
+ return segments[0] === "new-homes" && segments.length === 5;
+}
+
+function titleCase(slug: string): string {
+ return slug.split("-").map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)).join(" ");
+}
+
+type Apollo = Record<string, Record<string, unknown> | undefined>;
+
+/** Resolve an Apollo `{__ref}` pointer (or return the value if plain). */
+function deref(apollo: Apollo, value: unknown): Record<string, unknown> | null {
+ if (value && typeof value === "object") {
+ const ref = (value as { __ref?: string }).__ref;
+ if (ref) return apollo[ref] ?? null;
+ return value as Record<string, unknown>;
+ }
+ return null;
+}
+
+function str(value: unknown): string | null {
+ return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function numOrNull(value: unknown): number | null {
+ const n = Number(value);
+ return Number.isFinite(n) && n !== 0 ? n : null; // Lennar uses 0 for "no value"
+}
+
+function statusToEnum(status: string | null): "PLANNED" | "UNDER_CONSTRUCTION" | "MOVE_IN_READY" | null {
+ if (!status) return null;
+ if (/under.?construction/i.test(status)) return "UNDER_CONSTRUCTION";
+ if (/move.?in.?ready|completed|ready/i.test(status)) return "MOVE_IN_READY";
+ if (/coming.?soon|planned|future/i.test(status)) return "PLANNED";
+ return null; // UNDEFINED etc — recorded raw in availabilityStatus, never guessed
+}
+
+export const lennarAdapter: SourceAdapter = {
+ key: "lennar-site",
+ version: "1.1.0",
+
+ async *fetch(ctx: FetchContext): AsyncIterable<RawPage> {
+ if (ctx.mode === "fixture") {
+ yield* fetchFixtures(ctx);
+ return;
+ }
+ const fetcher = new LiveFetcher(ctx.registry);
+ const sitemap = await fetcher.fetch(SITEMAP_URL);
+ const urls = [...sitemap.body.toString("utf8").matchAll(/<loc>([^<]+)<\/loc>/g)]
+ .map((m) => m[1]!)
+ .filter(isCommunityUrl);
+ urls.sort();
+ for (const url of urls.slice(0, PAGE_LIMIT)) {
+ yield await fetcher.fetch(url);
+ }
+ },
+
+ extract(page: RawPage): ExtractionOutput {
+ const errors: { url: string; reason: string }[] = [];
+ try {
+ const html = page.body.toString("utf8");
+ const match = html.match(/<script id="__NEXT_DATA__" type="application\/json"[^>]*>([\s\S]*?)<\/script>/);
+ if (!match) return { records: [], errors: [{ url: page.url, reason: "no __NEXT_DATA__" }] };
+ const nextData = JSON.parse(match[1]!) as {
+ props?: { pageProps?: { initialApolloState?: Apollo } };
+ };
+ const apollo: Apollo = nextData.props?.pageProps?.initialApolloState ?? {};
+
+ const segments = page.url.replace(/^https?:\/\/[^/]+/, "").split("/").filter(Boolean);
+ const urlState = normalizeStateCode(segments[1] ?? null);
+ const urlCity = segments[3] ? titleCase(segments[3]) : null;
+
+ const rollup = Object.entries(apollo).find(([key]) => key.startsWith("CommunityType:"))?.[1];
+ const communityName = str(rollup?.name) ?? (segments[4] ? titleCase(segments[4]) : null);
+ if (!communityName) return { records: [], errors: [{ url: page.url, reason: "no community name" }] };
+
+ const cityNode = deref(apollo, rollup?.city);
+ const cityName = str(cityNode?.name) ?? urlCity;
+ const stateCode = normalizeStateCode(str(rollup?.stateCode)) ?? urlState;
+ const zipRaw = str(rollup?.zipCode);
+ const zip = zipRaw && /^\d{5}$/.test(zipRaw) ? zipRaw : null;
+ const hoaMonthly = numOrNull(rollup?.monthlyHoaFee);
+
+ const records: ExtractedRecord[] = [];
+
+ // Community first — publish creates the FK target before homes validate.
+ records.push({
+ entityType: "community",
+ canonicalHints: { builderSlug: BUILDER_SLUG, communityName },
+ fields: {
+ name: fv(communityName, str(rollup?.name), page.url),
+ street: fv(str(rollup?.address), str(rollup?.address), page.url),
+ city: fv(cityName, str(cityNode?.name) ?? urlCity, page.url),
+ state: fv(stateCode, str(rollup?.stateCode) ?? segments[1] ?? null, page.url),
+ zip: fv(zip, zipRaw, page.url),
+ county: fv<string>(null, null, page.url),
+ metro: fv(segments[2] ? titleCase(segments[2]) : null, segments[2] ?? null, page.url),
+ lat: fv(numOrNull(rollup?.latitude), rollup?.latitude === undefined ? null : String(rollup?.latitude), page.url),
+ lon: fv(numOrNull(rollup?.longitude), rollup?.longitude === undefined ? null : String(rollup?.longitude), page.url),
+ hoaFeeMonthly: fv(hoaMonthly, rollup?.monthlyHoaFee === undefined ? null : String(rollup?.monthlyHoaFee), page.url),
+ schoolDistrict: fv<string>(null, null, page.url),
+ ageRestricted: fv<boolean>(null, null, page.url),
+ },
+ });
+
+ for (const [key, entry] of Object.entries(apollo)) {
+ if (!key.startsWith("HomesiteType:") || !entry) continue;
+ const home = entry;
+ const status = str(home.status);
+ if (status && /sold|closed/i.test(status)) continue; // not available inventory
+ const address = str(home.address);
+ if (!address) continue; // no address → no canonical identity for a per-home record
+
+ const bathsFull = numOrNull(home.baths) ?? (home.baths === 0 ? null : null);
+ const halfBaths = Number(home.halfBaths ?? 0);
+ const baths = bathsFull !== null ? bathsFull + (Number.isFinite(halfBaths) ? halfBaths * 0.5 : 0) : null;
+
+ const plan = deref(apollo, home.plan);
+ const planName = str(plan?.name);
+ const garages = numOrNull(plan?.garages);
+ const lotid = str(home.lotid) ?? str(home.id);
+
+ records.push({
+ entityType: "inventory_home",
+ canonicalHints: {
+ builderSlug: BUILDER_SLUG,
+ communityName,
+ address,
+ builderInventoryId: lotid,
+ lat: numOrNull(home.latitude),
+ lon: numOrNull(home.longitude),
+ planName,
+ },
+ fields: {
+ street: fv(address, str(home.address), page.url),
+ city: fv(cityName, str(cityNode?.name) ?? urlCity, page.url),
+ state: fv(stateCode, str(rollup?.stateCode) ?? segments[1] ?? null, page.url),
+ zip: fv(zip, zipRaw, page.url),
+ price: fv(numOrNull(home.price), home.price === undefined ? null : String(home.price), page.url,
+ home.price ? `price: ${String(home.price)} (${key})` : null),
+ beds: fv(numOrNull(home.beds), home.beds === undefined ? null : String(home.beds), page.url),
+ bathsTotal: fv(baths, home.baths === undefined ? null : `${String(home.baths)} + ${String(home.halfBaths ?? 0)} half`, page.url),
+ sqft: fv(numOrNull(home.sqft), home.sqft === undefined ? null : String(home.sqft), page.url),
+ stories: fv<number>(null, null, page.url),
+ garageSpaces: fv(garages, plan?.garages === undefined ? null : String(plan?.garages), page.url),
+ homeType: fv("SINGLE_FAMILY" as const, null, page.url, "Lennar homesite listing"),
+ constructionStatus: fv(statusToEnum(status), status, page.url),
+ estCompletionDate: fv<string>(null, null, page.url),
+ lotNumber: fv(str(home.number), str(home.number), page.url),
+ builderInventoryId: fv(lotid, lotid, page.url),
+ lat: fv(numOrNull(home.latitude), home.latitude === undefined ? null : String(home.latitude), page.url),
+ lon: fv(numOrNull(home.longitude), home.longitude === undefined ? null : String(home.longitude), page.url),
+ planName: fv(planName, planName, page.url),
+ availabilityStatus: fv(status, status, page.url),
+ },
+ });
+ }
+
+ return { records, errors };
+ } catch (error) {
+ return { records: [], errors: [{ url: page.url, reason: String(error) }] };
+ }
+ },
+};
diff --git a/collectors/lennar/tsconfig.json b/collectors/lennar/tsconfig.json
new file mode 100644
index 0000000..e9bfc48
--- /dev/null
+++ b/collectors/lennar/tsconfig.json
@@ -0,0 +1 @@
+{ "extends": "../../tsconfig.base.json", "compilerOptions": { "noEmit": true }, "include": ["src/**/*.ts"] }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7c99656..4d49b01 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -142,6 +142,9 @@ importers:
apps/workers:
dependencies:
+ '@spechomes/collector-lennar':
+ specifier: workspace:*
+ version: link:../../collectors/lennar
'@spechomes/collector-meridian-homes':
specifier: workspace:*
version: link:../../collectors/meridian-homes
@@ -205,6 +208,28 @@ importers:
specifier: ^4.0.0
version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+ collectors/lennar:
+ dependencies:
+ '@spechomes/collectors-common':
+ specifier: workspace:*
+ version: link:../common
+ '@spechomes/schemas':
+ specifier: workspace:*
+ version: link:../../packages/schemas
+ '@spechomes/shared':
+ specifier: workspace:*
+ version: link:../../packages/shared
+ devDependencies:
+ '@types/node':
+ specifier: ^22.10.5
+ version: 22.20.1
+ typescript:
+ specifier: ^5.7.2
+ version: 5.9.3
+ vitest:
+ specifier: ^4.0.0
+ version: 4.1.10(@types/node@22.20.1)(vite@8.1.5(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.1))
+
collectors/meridian-homes:
dependencies:
'@spechomes/collectors-common':
← f0e81ad top-30 builder registry + recon findings recorded; toll-brot
·
back to Homesonspec
·
source runbooks for toll-brothers + lennar live adapters 4ba331c →