← back to Homesonspec
collectors/common/src/live-fetch.ts
244 lines
import { setTimeout as sleep } from "node:timers/promises";
import { sha256, type RawPage, type SourceRegistryEntry } from "./adapter";
/**
* Polite live fetcher for permissioned/limited factual collection.
* - Honest User-Agent with a contact address (never disguised).
* - Rate-limited per source (registry rateLimitRpm, default 12/min).
* - Never retries a 403/401/429 challenge — bot protection means STOP;
* the caller records the source as blocked/degraded.
* - GET only. No cookies, no login, no media downloads.
*/
const USER_AGENT = "HomesOnSpecBot/0.1 (+https://homesonspec.com/bot; contact: data@homesonspec.com)";
// The product token robots.txt groups are matched against (case-insensitive).
const UA_TOKEN = "homesonspecbot";
// ---------------------------------------------------------------------------
// robots.txt enforcement (POLICY.md: "Collection respects robots.txt").
// Enforced HERE in code — not left to per-adapter discipline. For every origin
// we fetch, robots.txt is loaded once and cached; each URL's path is checked
// against the applicable group (our UA token if present, else `*`) using the
// standard longest-match Allow/Disallow precedence with `*`/`$` wildcards.
// A disallowed URL throws DisallowedError so the adapter skips+logs it (adapters
// already catch per-URL errors); a missing/unparseable robots.txt = allow all.
// ---------------------------------------------------------------------------
interface RobotsRule {
allow: boolean;
length: number; // raw pattern length — longest match wins
re: RegExp;
}
const robotsCache = new Map<string, RobotsRule[]>();
const robotsInflight = new Map<string, Promise<RobotsRule[]>>();
function patternToRegex(pattern: string): RegExp {
let p = pattern;
const anchored = p.endsWith("$");
if (anchored) p = p.slice(0, -1);
const esc = p.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
return new RegExp("^" + esc + (anchored ? "$" : ""));
}
export function parseRobots(txt: string, uaToken = UA_TOKEN): RobotsRule[] {
// Group directive lines by their preceding User-agent block(s).
const groups: { agents: string[]; rules: { allow: boolean; path: string }[] }[] = [];
let cur: (typeof groups)[number] | null = null;
let sawRuleSinceAgent = false;
for (const rawLine of txt.split(/\r?\n/)) {
const line = rawLine.replace(/#.*$/, "").trim();
if (!line) continue;
const idx = line.indexOf(":");
if (idx < 0) continue;
const field = line.slice(0, idx).trim().toLowerCase();
const value = line.slice(idx + 1).trim();
if (field === "user-agent") {
if (!cur || sawRuleSinceAgent) {
cur = { agents: [], rules: [] };
groups.push(cur);
sawRuleSinceAgent = false;
}
cur.agents.push(value.toLowerCase());
} else if ((field === "disallow" || field === "allow") && cur) {
sawRuleSinceAgent = true;
cur.rules.push({ allow: field === "allow", path: value });
}
}
// Prefer the most specific matching group (our UA token) over the `*` group.
const specific = groups.filter((g) => g.agents.some((a) => a !== "*" && uaToken.includes(a)));
const wildcard = groups.filter((g) => g.agents.includes("*"));
const chosen = specific.length ? specific : wildcard;
const rules: RobotsRule[] = [];
for (const g of chosen) {
for (const r of g.rules) {
if (r.path === "" && !r.allow) continue; // empty Disallow = allow all (no-op)
rules.push({ allow: r.allow, length: r.path.length, re: patternToRegex(r.path) });
}
}
return rules;
}
/** Returns true if `path` is allowed for our UA under the given rules. */
export function robotsAllows(rules: RobotsRule[], path: string): boolean {
let best: RobotsRule | null = null;
for (const rule of rules) {
if (!rule.re.test(path)) continue;
if (!best || rule.length > best.length || (rule.length === best.length && rule.allow && !best.allow)) {
best = rule;
}
}
return best ? best.allow : true; // no matching rule = allowed
}
async function loadRobots(origin: string): Promise<RobotsRule[]> {
const cached = robotsCache.get(origin);
if (cached) return cached;
const inflight = robotsInflight.get(origin);
if (inflight) return inflight;
const promise = (async () => {
let rules: RobotsRule[] = [];
try {
const res = await fetch(`${origin}/robots.txt`, {
method: "GET",
headers: { "User-Agent": USER_AGENT, Accept: "text/plain,*/*;q=0.8" },
redirect: "follow",
signal: AbortSignal.timeout(15_000),
});
if (res.ok) rules = parseRobots(await res.text());
// Non-2xx (404 no robots, 5xx) → allow all (empty rules), per convention.
} catch {
rules = []; // network/timeout → fail open (allow), same as "no robots.txt".
}
robotsCache.set(origin, rules);
robotsInflight.delete(origin);
return rules;
})();
robotsInflight.set(origin, promise);
return promise;
}
export class LiveFetcher {
private lastRequestAt = 0;
private readonly minIntervalMs: number;
constructor(private readonly registry: SourceRegistryEntry) {
const rpm = registry.rateLimitRpm ?? 12;
this.minIntervalMs = Math.ceil(60_000 / Math.max(1, rpm));
}
async fetch(url: string): Promise<RawPage> {
// robots.txt enforcement — check before spending the rate-limit budget.
const parsed = new URL(url);
if (parsed.pathname !== "/robots.txt") {
const rules = await loadRobots(parsed.origin);
if (!robotsAllows(rules, parsed.pathname + parsed.search)) {
throw new DisallowedError(url);
}
}
const wait = this.lastRequestAt + this.minIntervalMs - Date.now();
if (wait > 0) await sleep(wait);
this.lastRequestAt = Date.now();
const response = await fetch(url, {
method: "GET",
// Broad Accept with a */* fallback. A narrow "text/html,application/json"
// 406s on picky WAFs (D.R. Horton) that do strict content negotiation and
// reject the absence of a wildcard fallback. The honest bot UA is kept —
// this is not disguise, just a standards-compliant Accept header. */*
// still covers the JSON-API builders (Pulte) and XML sitemaps.
headers: {
"User-Agent": USER_AGENT,
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.9,*/*;q=0.8",
},
redirect: "follow",
signal: AbortSignal.timeout(25_000),
});
if (response.status === 401 || response.status === 403 || response.status === 429) {
// Access control / rate limiting — never circumvented.
throw new BlockedError(url, response.status);
}
if (!response.ok) {
throw new Error(`fetch ${url} → HTTP ${response.status}`);
}
const body = Buffer.from(await response.arrayBuffer());
return {
url,
retrievedAt: new Date().toISOString(),
contentType: response.headers.get("content-type") ?? "text/html",
body,
contentHash: sha256(body),
};
}
/**
* POST a JSON body and return the JSON response as a RawPage. Same rails as
* fetch(): honest UA (never disguised), robots.txt enforcement, per-source
* rate-limit, and STOP-on-403/401/429 (bot protection is never circumvented).
* No cookies, no login, no media. Added for JSON-search-API builders whose
* inventory feed is a POST (e.g. Sitecore Discover) rather than a GET —
* the only extra affordance is a JSON body; no bypass of any protection.
*/
async postJson(url: string, payload: unknown): Promise<RawPage> {
const parsed = new URL(url);
if (parsed.pathname !== "/robots.txt") {
const rules = await loadRobots(parsed.origin);
if (!robotsAllows(rules, parsed.pathname + parsed.search)) {
throw new DisallowedError(url);
}
}
const wait = this.lastRequestAt + this.minIntervalMs - Date.now();
if (wait > 0) await sleep(wait);
this.lastRequestAt = Date.now();
const response = await fetch(url, {
method: "POST",
headers: {
"User-Agent": USER_AGENT,
"Content-Type": "application/json",
Accept: "application/json,*/*;q=0.8",
},
body: JSON.stringify(payload),
redirect: "follow",
signal: AbortSignal.timeout(25_000),
});
if (response.status === 401 || response.status === 403 || response.status === 429) {
throw new BlockedError(url, response.status);
}
if (!response.ok) {
throw new Error(`POST ${url} → HTTP ${response.status}`);
}
const body = Buffer.from(await response.arrayBuffer());
return {
url,
retrievedAt: new Date().toISOString(),
contentType: response.headers.get("content-type") ?? "application/json",
body,
contentHash: sha256(body),
};
}
}
export class BlockedError extends Error {
constructor(
public readonly url: string,
public readonly status: number,
) {
super(`blocked by access control (HTTP ${status}) at ${url} — collection stops, source marked degraded`);
this.name = "BlockedError";
}
}
export class DisallowedError extends Error {
constructor(public readonly url: string) {
super(`disallowed by robots.txt: ${url} — URL skipped (POLICY.md robots.txt compliance)`);
this.name = "DisallowedError";
}
}