`collectors/fischer-homes/selftest.mjs`

157 lines
// Self-contained data-quality proof for the Fischer Homes adapter.
// Hits the LIVE source with the honest bot UA, applies the same extraction the
// adapter will use, and reports coverage / distinct-address / distinct-sqft / samples.
// $0 (local). No DB, no writes.

const UA = "HomesOnSpecBot/0.1 (+https://homesonspec.com/bot; contact: data@homesonspec.com)";
const BASE = "https://www.fischerhomes.com";
const PAGE_LIMIT = Number(process.env.FISCHER_PAGE_LIMIT ?? 10);
// Region ids proven in recon (subset of /api/region-dropdown covering OH/KY/IN/GA/MO):
// 11 Cincinnati OH, 16 Dayton OH, 22 Columbus OH, 35 Indianapolis IN, 47 Atlanta GA,
// 48 Louisville KY, 13 Northern KY, 51 St Louis MO. Self-test uses a couple to keep it quick.
const REGIONS = (process.env.FISCHER_REGIONS ?? "11,35,47").split(",").map((s) => s.trim()).filter(Boolean);

async function getJson(url) {
  const r = await fetch(url, { headers: { "User-Agent": UA, Accept: "application/json,*/*;q=0.8" }, redirect: "follow", signal: AbortSignal.timeout(25000) });
  if (!r.ok) throw new Error(`GET ${url} -> ${r.status}`);
  return r.json();
}
async function getText(url) {
  const r = await fetch(url, { headers: { "User-Agent": UA, Accept: "text/html,*/*;q=0.8" }, redirect: "follow", signal: AbortSignal.timeout(25000) });
  if (!r.ok) throw new Error(`GET ${url} -> ${r.status}`);
  return r.text();
}

const num = (v) => { const n = Number(String(v ?? "").replace(/[^0-9.]/g, "")); return Number.isFinite(n) && n > 0 ? n : null; };
const stripTags = (s) => String(s ?? "").replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();

// Baths like "3½", "2 + ½ + ½", "3" -> total number (½ = 0.5)
function parseBaths(s) {
  const t = String(s ?? "");
  if (!t.trim()) return null;
  const halves = (t.match(/½/g) || []).length;
  const wholeMatch = t.match(/\d+/g);
  const whole = wholeMatch ? wholeMatch.reduce((a, b) => a + Number(b), 0) : 0;
  const total = whole + halves * 0.5;
  return total > 0 ? total : null;
}
function priceFromFormatted(html) {
  const m = String(html ?? "").match(/\$?\s*([\d,]{4,})/);
  return m ? num(m[1]) : null;
}

// --- List API: all homes for a region (bulk, paginated) ---
async function fetchRegionHomes(regionId) {
  const out = [];
  let page = 1, lastPage = 1;
  while (page <= lastPage && page <= 20) {
    const url = `${BASE}/api/region-revamped/homes/${regionId}?page=${page}`;
    const body = await getJson(url);
    const sec = body["move-in-ready"] || {};
    lastPage = Number(sec.last_page ?? 1);
    for (const h of sec.data ?? []) out.push(h);
    page++;
    await new Promise((r) => setTimeout(r, 300));
  }
  return out;
}

// --- Detail page JSON-LD enrichment: lat/lon + community + confirm sqft/beds/baths + exact price ---
function extractDetail(html) {
  const out = { lat: null, lon: null, community: null, plan: null, price: null, sqft: null, beds: null, bathsFull: null, bathsHalf: null, city: null, state: null, zip: null, street: null };
  const m = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i);
  if (m) {
    try {
      const d = JSON.parse(m[1]);
      const addr = d.address || {};
      out.street = addr.streetAddress ?? null;
      out.city = addr.addressLocality ?? null;
      out.state = addr.addressRegion ?? null;
      out.zip = addr.postalCode ?? null;
      const lat = Number(d.latitude ?? d.geo?.latitude);
      const lon = Number(d.longitude ?? d.geo?.longitude);
      out.lat = Number.isFinite(lat) && lat !== 0 ? lat : null;
      out.lon = Number.isFinite(lon) && lon !== 0 ? lon : null;
      out.sqft = num(d.floorSize?.value);
      out.beds = num(d.numberOfBedrooms);
      out.bathsFull = d.numberOfFullBathrooms != null ? Number(d.numberOfFullBathrooms) : null;
      out.bathsHalf = d.numberOfPartialBathrooms != null ? Number(d.numberOfPartialBathrooms) : null;
      out.plan = d.accommodationFloorPlan?.name ?? null;
      // community: JSON-LD description "built in <community> located in"
      const desc = String(d.description ?? "");
      const cm = desc.match(/built in ([^,]+?) located/i);
      if (cm) out.community = cm[1].trim();
    } catch {}
  }
  // community fallback from <title> "... | <community> by Fischer Homes"
  if (!out.community) {
    const tm = html.match(/<title>[^|]*\|\s*(.+?)\s+by Fischer Homes/i);
    if (tm) out.community = tm[1].trim();
  }
  // exact price from sale-price element
  const pm = html.match(/class="[^"]*sale-price-bold[^"]*"[^>]*>\s*\$?\s*([\d,]+)/i);
  if (pm) out.price = num(pm[1]);
  return out;
}

// ---- run ----
console.log(`Fischer self-test | UA=${UA}\nRegions=${REGIONS.join(",")} PAGE_LIMIT(list-pages/region cap)=20\n`);
let summaries = [];
for (const r of REGIONS) {
  try {
    const hs = await fetchRegionHomes(r);
    console.log(`  region ${r}: ${hs.length} homes from list API`);
    summaries = summaries.concat(hs.map((h) => ({ ...h, _region: r })));
  } catch (e) { console.log(`  region ${r}: LIST FAIL ${e.message}`); }
}
console.log(`\nTotal list-API homes: ${summaries.length}`);

// Enrich each home via its detail page (cap for self-test speed via PAGE_LIMIT*3 homes)
const cap = Math.min(summaries.length, Math.max(30, PAGE_LIMIT * 6));
const rows = [];
let enriched = 0;
for (const s of summaries.slice(0, cap)) {
  const detailUrl = BASE + s.url;
  let det = {};
  try { det = extractDetail(await getText(detailUrl)); enriched++; } catch (e) { /* keep list data */ }
  // Facts policy: the LIST API's formatted fields are the customer-facing truth
  // for beds/baths/sqft/price (complete, incl. half-baths). JSON-LD's
  // numberOfPartialBathrooms is unreliable (reports 0 when a half-bath exists),
  // so it's used ONLY as a fallback. Detail page is authoritative for lat/lon +
  // community, which the list API lacks.
  const price = priceFromFormatted(s.formattedPrice) ?? det.price;
  const beds = num(s.formattedBeds) ?? det.beds;
  const baths = parseBaths(s.formattedBaths) ?? ((det.bathsFull != null) ? det.bathsFull + (det.bathsHalf ?? 0) * 0.5 : null);
  const sqft = num(s.formattedSqft) ?? det.sqft;
  rows.push({
    street: det.street ?? (s.formattedAddress || "").split(",")[0]?.trim() ?? null,
    fullAddress: s.formattedAddress ?? null,
    city: det.city, state: det.state, zip: det.zip,
    community: det.community, plan: det.plan ?? s.name ?? null,
    price, beds, baths, sqft, lat: det.lat, lon: det.lon,
    region: s._region,
  });
  await new Promise((r) => setTimeout(r, 250));
}

const N = rows.length;
const pct = (f) => N ? ((rows.filter(f).length / N) * 100).toFixed(1) + "%" : "0%";
const distinct = (f) => new Set(rows.map(f).filter((x) => x != null)).size;

console.log(`\n===== DATA QUALITY (${N} enriched homes; ${enriched} detail pages fetched) =====`);
console.log(`price coverage:      ${pct((r) => r.price != null)}`);
console.log(`beds coverage:       ${pct((r) => r.beds != null)}`);
console.log(`baths coverage:      ${pct((r) => r.baths != null)}`);
console.log(`sqft coverage:       ${pct((r) => r.sqft != null)}`);
console.log(`lat/lon coverage:    ${pct((r) => r.lat != null && r.lon != null)}`);
console.log(`community coverage:  ${pct((r) => r.community != null)}`);
console.log(`plan coverage:       ${pct((r) => r.plan != null)}`);
console.log(`distinct addresses:  ${distinct((r) => r.fullAddress)} / ${N}`);
console.log(`distinct sqft:       ${distinct((r) => r.sqft)}`);
console.log(`distinct prices:     ${distinct((r) => r.price)}`);
console.log(`distinct communities:${distinct((r) => r.community)}`);
console.log(`negative-lon (US):   ${rows.filter((r) => r.lon != null && r.lon < 0).length}/${rows.filter((r) => r.lon != null).length}`);

console.log(`\n===== 6 SAMPLE ROWS =====`);
for (const r of rows.slice(0, 6)) console.log(JSON.stringify(r));