`src/ingest/listings/weichert.ts`

78 lines
/**
 * Weichert adapter. Discovery: paginated listings sitemap
 * `sitemaplistings.ashx?page=N&justlisted=False&weichert=True` → PDP URLs
 * `https://www.weichert.com/{id}/`. Facts from JSON-LD Residence + Product/Offer +
 * Place.geo (address, price, lat/lng; Weichert does not expose beds/baths/sqft in
 * JSON-LD, left null). source_id = the numeric id in the URL. Robots: no listing
 * disallows (verified 2026-07-30); the engine re-checks each path before fetch.
 */
import type { ListingAdapter, ListingFacts } from './types.ts';
import { extractJsonLd, typeMatches, num, rawFetch } from './shared.ts';

const SITEMAP = (page: number) =>
  `https://www.weichert.com/sitemaplistings.ashx?page=${page}&justlisted=False&weichert=True`;

/** Flatten JSON-LD: expand @graph, keep plain nodes. */
function flatten(nodes: any[]): any[] {
  return nodes.flatMap(n => (Array.isArray(n?.['@graph']) ? n['@graph'] : [n])).filter(Boolean);
}

export const weichert: ListingAdapter = {
  source: 'weichert',
  // FULL: discovery paginates the complete `sitemaplistings.ashx` (justlisted=False) set, i.e.
  // every currently-listed property, so a fresh crawl re-surfaces still-active listings and a
  // missing one is genuinely gone → withdrawal detection is valid for this source.
  mode: 'full',
  host: 'www.weichert.com',
  listingsPathHint: '/137101627/', // representative PDP path for the robots honor check

  async discover(cap) {
    const urls: string[] = [];
    for (let page = 1; page <= 30 && urls.length < cap; page++) {
      const { ok, body } = await rawFetch(SITEMAP(page));
      if (!ok) break;
      const locs = [...body.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)]
        .map(m => m[1].replace(/&amp;/g, '&'))
        .filter(u => /^https?:\/\/www\.weichert\.com\/\d+\/?$/.test(u));
      if (!locs.length) break; // ran past the last populated page
      urls.push(...locs);
    }
    return urls.slice(0, cap);
  },

  parse(url, html): ListingFacts | null {
    const idm = url.match(/\/(\d+)\/?$/);
    const sourceId = idm?.[1];
    if (!sourceId) return null;

    const flat = flatten(extractJsonLd(html));
    // Weichert splits facts across nodes: Residence(address) · Product(.offers.price) ·
    // (no geo in JSON-LD — region resolves off addressRegion/state instead).
    const residence = flat.find(n => typeMatches(n, 'Residence', 'SingleFamilyResidence', 'House'));
    const product = flat.find(n => typeMatches(n, 'Product'));
    const offer = product?.offers || flat.find(n => typeMatches(n, 'Offer')) || {};
    const addr = residence?.address || flat.find(n => typeMatches(n, 'PostalAddress')) || {};
    const geo = (flat.find(n => n?.geo)?.geo) || {};

    const streetParts = [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode]
      .filter(Boolean).join(', ');
    const facts: ListingFacts = {
      sourceId,
      url,
      address: streetParts || residence?.name || product?.name || null,
      city: addr.addressLocality ?? null,
      state: addr.addressRegion ?? null,
      zip: addr.postalCode ?? null,
      price: num(offer?.price),
      beds: num(residence?.numberOfBedrooms),
      baths: num(residence?.numberOfBathroomsTotal),
      sqft: null,
      lat: geo.latitude != null ? Number(geo.latitude) : null,
      lng: geo.longitude != null ? Number(geo.longitude) : null,
      status: 'active',
    };
    if (!facts.address && facts.price == null) return null;
    return facts;
  },
};