`apps/workers/src/lennar-mac-fetch.mts`

42 lines
// Mac-side Lennar fetch (Mac IP is NOT 403-blocked by lennar.com, unlike Kamatera).
// Drives the real lennarAdapter.fetch()+extract() (full robots/rate-limit politeness via
// LiveFetcher), collects per-home {invId, street, city, state, images}, writes JSONL for
// application to the Kamatera DB. NO DB writes here — pure fetch→extract→file. [TK-10005]
import { lennarAdapter } from "@homesonspec/collector-lennar";
import { writeFileSync, appendFileSync } from "fs";

const STATES = ["california", "colorado", "florida", "arizona", "alabama", "arkansas", "delaware"];
const OUT = "/tmp/lennar-images.jsonl";
writeFileSync(OUT, "");
const registry = { key: "lennar-site", collectionMethod: "CRAWL", rateLimitRpm: 60, mediaRights: "LICENSED" } as any;

let grandTotal = 0, grandImg = 0;
for (const st of STATES) {
  process.env.LENNAR_STATE_FILTER = st;
  let sTotal = 0, sImg = 0;
  try {
    for await (const page of lennarAdapter.fetch({ mode: "live", fixtureDir: "/tmp/none", registry })) {
      const out = lennarAdapter.extract(page);
      for (const r of out.records) {
        if (r.entityType !== "inventory_home") continue;
        sTotal++;
        const f: any = r.fields;
        const imgs: string[] = f.images?.value ?? [];
        const invId: string | null = f.builderInventoryId?.value ?? null;
        const street: string | null = f.street?.value ?? null;
        const city: string | null = f.city?.value ?? null;
        const state: string | null = f.state?.value ?? null;
        if (imgs.length && invId && street) {
          sImg++;
          appendFileSync(OUT, JSON.stringify({ invId, street, city, state, images: imgs }) + "\n");
        }
      }
    }
  } catch (e) {
    console.error(`  ${st} ERROR:`, (e as Error).message);
  }
  grandTotal += sTotal; grandImg += sImg;
  console.log(`${st}: ${sImg}/${sTotal} homes with images`);
}
console.log(`TOTAL: ${grandImg}/${grandTotal} written to ${OUT}`);