← back to Homesonspec
polite LiveFetcher (rate-limited, never circumvents blocks) + top-10 national builder registry seed (sources PAUSED until recon + activation)
14131617cdbe0750fc52991f98188a383c2a5284 · 2026-07-22 11:43:50 -0700 · Steve Abrams
Files touched
M collectors/common/src/index.tsA collectors/common/src/live-fetch.tsA packages/database/prisma/seed-national-builders.ts
Diff
commit 14131617cdbe0750fc52991f98188a383c2a5284
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 22 11:43:50 2026 -0700
polite LiveFetcher (rate-limited, never circumvents blocks) + top-10 national builder registry seed (sources PAUSED until recon + activation)
---
collectors/common/src/index.ts | 1 +
collectors/common/src/live-fetch.ts | 63 ++++++++++++++++++
packages/database/prisma/seed-national-builders.ts | 75 ++++++++++++++++++++++
3 files changed, 139 insertions(+)
diff --git a/collectors/common/src/index.ts b/collectors/common/src/index.ts
index 631bd0f..8a9e66d 100644
--- a/collectors/common/src/index.ts
+++ b/collectors/common/src/index.ts
@@ -1 +1,2 @@
export * from "./adapter";
+export * from "./live-fetch";
diff --git a/collectors/common/src/live-fetch.ts b/collectors/common/src/live-fetch.ts
new file mode 100644
index 0000000..325c2f9
--- /dev/null
+++ b/collectors/common/src/live-fetch.ts
@@ -0,0 +1,63 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { sha256, type RawPage, type SourceRegistryEntry } from "./adapter";
+
+/**
+ * Polite live fetcher for permissioned/limited factual collection.
+ * - Honest User-Agent with a contact address (never disguised).
+ * - Rate-limited per source (registry rateLimitRpm, default 12/min).
+ * - Never retries a 403/401/429 challenge — bot protection means STOP;
+ * the caller records the source as blocked/degraded.
+ * - GET only. No cookies, no login, no media downloads.
+ */
+
+const USER_AGENT = "SpecHomesBot/0.1 (+https://spechomes.com/bot; contact: data@spechomes.com)";
+
+export class LiveFetcher {
+ private lastRequestAt = 0;
+ private readonly minIntervalMs: number;
+
+ constructor(private readonly registry: SourceRegistryEntry) {
+ const rpm = registry.rateLimitRpm ?? 12;
+ this.minIntervalMs = Math.ceil(60_000 / Math.max(1, rpm));
+ }
+
+ async fetch(url: string): Promise<RawPage> {
+ const wait = this.lastRequestAt + this.minIntervalMs - Date.now();
+ if (wait > 0) await sleep(wait);
+ this.lastRequestAt = Date.now();
+
+ const response = await fetch(url, {
+ method: "GET",
+ headers: { "User-Agent": USER_AGENT, Accept: "text/html,application/json" },
+ redirect: "follow",
+ signal: AbortSignal.timeout(25_000),
+ });
+
+ if (response.status === 401 || response.status === 403 || response.status === 429) {
+ // Access control / rate limiting — never circumvented.
+ throw new BlockedError(url, response.status);
+ }
+ if (!response.ok) {
+ throw new Error(`fetch ${url} → HTTP ${response.status}`);
+ }
+
+ const body = Buffer.from(await response.arrayBuffer());
+ return {
+ url,
+ retrievedAt: new Date().toISOString(),
+ contentType: response.headers.get("content-type") ?? "text/html",
+ body,
+ contentHash: sha256(body),
+ };
+ }
+}
+
+export class BlockedError extends Error {
+ constructor(
+ public readonly url: string,
+ public readonly status: number,
+ ) {
+ super(`blocked by access control (HTTP ${status}) at ${url} — collection stops, source marked degraded`);
+ this.name = "BlockedError";
+ }
+}
diff --git a/packages/database/prisma/seed-national-builders.ts b/packages/database/prisma/seed-national-builders.ts
new file mode 100644
index 0000000..16b6907
--- /dev/null
+++ b/packages/database/prisma/seed-national-builders.ts
@@ -0,0 +1,75 @@
+import { prisma } from "../src/index.js";
+
+/**
+ * Top-10 national builders (by closings) — REAL companies. This seed creates
+ * Builder rows + SourceRegistry entries only. Sources start INACTIVE with
+ * health PAUSED: nothing collects until recon findings (terms, robots,
+ * feasibility) are recorded and the source is explicitly activated.
+ * No media rights are assumed (mediaRights NONE — facts only, link back).
+ */
+
+const BUILDERS: {
+ slug: string;
+ name: string;
+ legalName: string;
+ website: string;
+ termsUrl: string;
+ coverage: string;
+}[] = [
+ { slug: "dr-horton", name: "D.R. Horton", legalName: "D.R. Horton, Inc.", website: "https://www.drhorton.com", termsUrl: "https://www.drhorton.com/terms-of-use", coverage: "National — 33+ states" },
+ { slug: "lennar", name: "Lennar", legalName: "Lennar Corporation", website: "https://www.lennar.com", termsUrl: "https://www.lennar.com/legal/terms-of-use", coverage: "National — 26+ states" },
+ { slug: "pultegroup", name: "PulteGroup", legalName: "PulteGroup, Inc.", website: "https://www.pulte.com", termsUrl: "https://www.pulte.com/terms-of-use", coverage: "National — Pulte / Del Webb / Centex brands" },
+ { slug: "ryan-homes", name: "Ryan Homes (NVR)", legalName: "NVR, Inc.", website: "https://www.ryanhomes.com", termsUrl: "https://www.ryanhomes.com/terms-and-conditions", coverage: "East/Midwest — 15+ states" },
+ { slug: "meritage-homes", name: "Meritage Homes", legalName: "Meritage Homes Corporation", website: "https://www.meritagehomes.com", termsUrl: "https://www.meritagehomes.com/terms-of-use", coverage: "Sun Belt — 10+ states" },
+ { slug: "kb-home", name: "KB Home", legalName: "KB Home", website: "https://www.kbhome.com", termsUrl: "https://www.kbhome.com/terms-of-use", coverage: "West/South — 8+ states" },
+ { slug: "taylor-morrison", name: "Taylor Morrison", legalName: "Taylor Morrison Home Corporation", website: "https://www.taylormorrison.com", termsUrl: "https://www.taylormorrison.com/terms-of-use", coverage: "Sun Belt — 11+ states" },
+ { slug: "century-communities", name: "Century Communities", legalName: "Century Communities, Inc.", website: "https://www.centurycommunities.com", termsUrl: "https://www.centurycommunities.com/terms-of-use", coverage: "National — 17+ states" },
+ { slug: "lgi-homes", name: "LGI Homes", legalName: "LGI Homes, Inc.", website: "https://www.lgihomes.com", termsUrl: "https://www.lgihomes.com/terms-of-use", coverage: "National — 20+ states" },
+ { slug: "toll-brothers", name: "Toll Brothers", legalName: "Toll Brothers, Inc.", website: "https://www.tollbrothers.com", termsUrl: "https://www.tollbrothers.com/terms-of-use", coverage: "National luxury — 24+ states" },
+];
+
+async function main() {
+ for (const spec of BUILDERS) {
+ const builder = await prisma.builder.upsert({
+ where: { slug: spec.slug },
+ update: { name: spec.name, legalName: spec.legalName, websiteUrl: spec.website },
+ create: {
+ slug: spec.slug,
+ name: spec.name,
+ legalName: spec.legalName,
+ websiteUrl: spec.website,
+ coverageArea: spec.coverage,
+ logoRightsStatus: "none",
+ partnershipStatus: null, // inclusion never implies sponsorship
+ isDemo: false,
+ },
+ });
+ await prisma.sourceRegistry.upsert({
+ where: { key: `${spec.slug}-site` },
+ update: { termsUrl: spec.termsUrl },
+ create: {
+ key: `${spec.slug}-site`,
+ name: `${spec.name} (builder website)`,
+ builderId: builder.id,
+ baseUrl: spec.website,
+ collectionMethod: "CRAWL", // limited factual collection; refined per recon
+ termsUrl: spec.termsUrl,
+ mediaRights: "NONE", // facts only — no photos/renderings/descriptions
+ rateLimitRpm: 10,
+ freshIntervalHours: 24,
+ agingIntervalHours: 72,
+ health: "PAUSED",
+ active: false, // nothing collects until recon findings are recorded + Steve activates
+ notes: "Awaiting recon: robots stance, terms review, feed feasibility.",
+ },
+ });
+ console.log(`registered ${spec.name}`);
+ }
+}
+
+main()
+ .catch((error) => {
+ console.error(error);
+ process.exitCode = 1;
+ })
+ .finally(() => prisma.$disconnect());
← 4dce3d2 pm2 ecosystem config, 9 ADRs, data-rights policy, source run
·
back to Homesonspec
·
top-30 builder registry + recon findings recorded; toll-brot f0e81ad →