← back to Nationalrealestate
Add Weichert listings adapter (3rd homes source) + fix extractJsonLd for unquoted ld+json (helps all adapters) + resolveRegionByZip fallback so geo-less listings get county-attributed via postalCode
1bf4cc45d70b0aae4d51038c49fa9909f82d6a47 · 2026-07-30 19:30:57 -0700 · steve@designerwallcoverings.com
Files touched
M src/ingest/listings/engine.tsM src/ingest/listings/shared.tsM src/ingest/listings/types.tsA src/ingest/listings/weichert.ts
Diff
commit 1bf4cc45d70b0aae4d51038c49fa9909f82d6a47
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Thu Jul 30 19:30:57 2026 -0700
Add Weichert listings adapter (3rd homes source) + fix extractJsonLd for unquoted ld+json (helps all adapters) + resolveRegionByZip fallback so geo-less listings get county-attributed via postalCode
---
src/ingest/listings/engine.ts | 8 +++--
src/ingest/listings/shared.ts | 23 ++++++++++++-
src/ingest/listings/types.ts | 1 +
src/ingest/listings/weichert.ts | 73 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 101 insertions(+), 4 deletions(-)
diff --git a/src/ingest/listings/engine.ts b/src/ingest/listings/engine.ts
index 007655a..b87f59e 100644
--- a/src/ingest/listings/engine.ts
+++ b/src/ingest/listings/engine.ts
@@ -18,9 +18,10 @@ import { openRun, closeRun } from '../run.ts';
import type { ListingAdapter } from './types.ts';
import { coldwellbanker } from './coldwellbanker.ts';
import { realtytexas } from './realtytexas.ts';
-import { politeFetch, robotsGate, resolveFirmByHost, resolveRegion } from './shared.ts';
+import { weichert } from './weichert.ts';
+import { politeFetch, robotsGate, resolveFirmByHost, resolveRegion, resolveRegionByZip } from './shared.ts';
-const ADAPTERS: ListingAdapter[] = [coldwellbanker, realtytexas];
+const ADAPTERS: ListingAdapter[] = [coldwellbanker, realtytexas, weichert];
const CAP = Number(process.env.USRE_LISTINGS_CAP || 100); // per-site cap (env-tunable; was pilot 50)
async function upsertListing(
@@ -77,7 +78,8 @@ async function runAdapter(a: ListingAdapter): Promise<{ upserted: number; skippe
if (!ok) { console.log(`[${a.source}] fetch ${status} ${url}`); skipped++; continue; }
const facts = a.parse(url, body);
if (!facts) { skipped++; continue; }
- const regionId = await resolveRegion(facts.state, facts.lat, facts.lng);
+ const regionId = (await resolveRegion(facts.state, facts.lat, facts.lng))
+ ?? (await resolveRegionByZip(facts.zip ?? null));
await upsertListing(a.source, facts, firmId, regionId);
upserted++;
if (upserted % 5 === 0) console.log(`[${a.source}] upserted ${upserted}…`);
diff --git a/src/ingest/listings/shared.ts b/src/ingest/listings/shared.ts
index 86bf6a0..3ec81c4 100644
--- a/src/ingest/listings/shared.ts
+++ b/src/ingest/listings/shared.ts
@@ -69,7 +69,8 @@ export async function robotsGate(host: string): Promise<(path: string) => boolea
/** Extract all parseable JSON-LD objects (flattening @graph + arrays) from a page. */
export function extractJsonLd(html: string): any[] {
const out: any[] = [];
- const re = /<script[^>]*type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
+ // quotes optional: some sites (e.g. Weichert) emit `<script type=application/ld+json>` unquoted
+ const re = /<script[^>]*type=["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(html))) {
let parsed: any;
@@ -147,6 +148,26 @@ export async function resolveRegion(
return best;
}
+/**
+ * Fallback region resolver by ZIP → county_fips → county region, for adapters whose
+ * listings carry a postalCode but no lat/lng (e.g. Weichert). Chain:
+ * zip_county.zip → zip_county.county_fips = region.fips (region_type='county').
+ */
+const zipRegionCache = new Map<string, number | null>();
+export async function resolveRegionByZip(zip: string | null): Promise<number | null> {
+ if (!zip) return null;
+ const z = zip.trim().slice(0, 5);
+ if (!/^\d{5}$/.test(z)) return null;
+ if (zipRegionCache.has(z)) return zipRegionCache.get(z)!;
+ const r = await query<{ id: number }>(
+ `SELECT r.id FROM zip_county zc
+ JOIN region r ON r.fips = zc.county_fips AND r.region_type = 'county'
+ WHERE zc.zip = $1 LIMIT 1`, [z]);
+ const id = r.rows[0]?.id ?? null;
+ zipRegionCache.set(z, id);
+ return id;
+}
+
/** Pull all leaf <loc> URLs from a sitemap or sitemap-index (one level of recursion). */
export async function sitemapLocs(url: string, depth = 1): Promise<string[]> {
const { ok, body } = await rawFetch(url);
diff --git a/src/ingest/listings/types.ts b/src/ingest/listings/types.ts
index 90e9a13..c29f47f 100644
--- a/src/ingest/listings/types.ts
+++ b/src/ingest/listings/types.ts
@@ -13,6 +13,7 @@ export interface ListingFacts {
address: string | null;
city: string | null;
state: string | null; // 2-letter, used for firm + region crosswalk
+ zip?: string | null; // postal code — ZIP→county region fallback when lat/lng absent
price: number | null;
beds: number | null;
baths: number | null;
diff --git a/src/ingest/listings/weichert.ts b/src/ingest/listings/weichert.ts
new file mode 100644
index 0000000..3414293
--- /dev/null
+++ b/src/ingest/listings/weichert.ts
@@ -0,0 +1,73 @@
+/**
+ * Weichert adapter. Discovery: paginated listings sitemap
+ * `sitemaplistings.ashx?page=N&justlisted=False&weichert=True` → PDP URLs
+ * `https://www.weichert.com/{id}/`. Facts from JSON-LD Residence + Product/Offer +
+ * Place.geo (address, price, lat/lng; Weichert does not expose beds/baths/sqft in
+ * JSON-LD, left null). source_id = the numeric id in the URL. Robots: no listing
+ * disallows (verified 2026-07-30); the engine re-checks each path before fetch.
+ */
+import type { ListingAdapter, ListingFacts } from './types.ts';
+import { extractJsonLd, typeMatches, num, rawFetch } from './shared.ts';
+
+const SITEMAP = (page: number) =>
+ `https://www.weichert.com/sitemaplistings.ashx?page=${page}&justlisted=False&weichert=True`;
+
+/** Flatten JSON-LD: expand @graph, keep plain nodes. */
+function flatten(nodes: any[]): any[] {
+ return nodes.flatMap(n => (Array.isArray(n?.['@graph']) ? n['@graph'] : [n])).filter(Boolean);
+}
+
+export const weichert: ListingAdapter = {
+ source: 'weichert',
+ host: 'www.weichert.com',
+ listingsPathHint: '/137101627/', // representative PDP path for the robots honor check
+
+ async discover(cap) {
+ const urls: string[] = [];
+ for (let page = 1; page <= 30 && urls.length < cap; page++) {
+ const { ok, body } = await rawFetch(SITEMAP(page));
+ if (!ok) break;
+ const locs = [...body.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)]
+ .map(m => m[1].replace(/&/g, '&'))
+ .filter(u => /^https?:\/\/www\.weichert\.com\/\d+\/?$/.test(u));
+ if (!locs.length) break; // ran past the last populated page
+ urls.push(...locs);
+ }
+ return urls.slice(0, cap);
+ },
+
+ parse(url, html): ListingFacts | null {
+ const idm = url.match(/\/(\d+)\/?$/);
+ const sourceId = idm?.[1];
+ if (!sourceId) return null;
+
+ const flat = flatten(extractJsonLd(html));
+ // Weichert splits facts across nodes: Residence(address) · Product(.offers.price) ·
+ // (no geo in JSON-LD — region resolves off addressRegion/state instead).
+ const residence = flat.find(n => typeMatches(n, 'Residence', 'SingleFamilyResidence', 'House'));
+ const product = flat.find(n => typeMatches(n, 'Product'));
+ const offer = product?.offers || flat.find(n => typeMatches(n, 'Offer')) || {};
+ const addr = residence?.address || flat.find(n => typeMatches(n, 'PostalAddress')) || {};
+ const geo = (flat.find(n => n?.geo)?.geo) || {};
+
+ const streetParts = [addr.streetAddress, addr.addressLocality, addr.addressRegion, addr.postalCode]
+ .filter(Boolean).join(', ');
+ const facts: ListingFacts = {
+ sourceId,
+ url,
+ address: streetParts || residence?.name || product?.name || null,
+ city: addr.addressLocality ?? null,
+ state: addr.addressRegion ?? null,
+ zip: addr.postalCode ?? null,
+ price: num(offer?.price),
+ beds: num(residence?.numberOfBedrooms),
+ baths: num(residence?.numberOfBathroomsTotal),
+ sqft: null,
+ lat: geo.latitude != null ? Number(geo.latitude) : null,
+ lng: geo.longitude != null ? Number(geo.longitude) : null,
+ status: 'active',
+ };
+ if (!facts.address && facts.price == null) return null;
+ return facts;
+ },
+};
← 7241fde resolver: confidence-tier matches — only set firm.website wh
·
back to Nationalrealestate
·
Fix parcel upsert crash on within-batch duplicate (county_fi 77583bc →