← back to Homesonspec
auto-save: 2026-07-27T19:54:12 (1 files) — collectors/common/src/live-fetch.ts
7782f9c46de86dac0f1f45e79f73bc1ea3627e80 · 2026-07-27 19:54:19 -0700 · Steve Abrams
Files touched
M collectors/common/src/live-fetch.ts
Diff
commit 7782f9c46de86dac0f1f45e79f73bc1ea3627e80
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 27 19:54:19 2026 -0700
auto-save: 2026-07-27T19:54:12 (1 files) — collectors/common/src/live-fetch.ts
---
collectors/common/src/live-fetch.ts | 122 ++++++++++++++++++++++++++++++++++++
1 file changed, 122 insertions(+)
diff --git a/collectors/common/src/live-fetch.ts b/collectors/common/src/live-fetch.ts
index 8c6d853..e153bb8 100644
--- a/collectors/common/src/live-fetch.ts
+++ b/collectors/common/src/live-fetch.ts
@@ -11,6 +11,112 @@ import { sha256, type RawPage, type SourceRegistryEntry } from "./adapter";
*/
const USER_AGENT = "HomesOnSpecBot/0.1 (+https://homesonspec.com/bot; contact: data@homesonspec.com)";
+// The product token robots.txt groups are matched against (case-insensitive).
+const UA_TOKEN = "homesonspecbot";
+
+// ---------------------------------------------------------------------------
+// robots.txt enforcement (POLICY.md: "Collection respects robots.txt").
+// Enforced HERE in code — not left to per-adapter discipline. For every origin
+// we fetch, robots.txt is loaded once and cached; each URL's path is checked
+// against the applicable group (our UA token if present, else `*`) using the
+// standard longest-match Allow/Disallow precedence with `*`/`$` wildcards.
+// A disallowed URL throws DisallowedError so the adapter skips+logs it (adapters
+// already catch per-URL errors); a missing/unparseable robots.txt = allow all.
+// ---------------------------------------------------------------------------
+
+interface RobotsRule {
+ allow: boolean;
+ length: number; // raw pattern length — longest match wins
+ re: RegExp;
+}
+
+const robotsCache = new Map<string, RobotsRule[]>();
+const robotsInflight = new Map<string, Promise<RobotsRule[]>>();
+
+function patternToRegex(pattern: string): RegExp {
+ let p = pattern;
+ const anchored = p.endsWith("$");
+ if (anchored) p = p.slice(0, -1);
+ const esc = p.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
+ return new RegExp("^" + esc + (anchored ? "$" : ""));
+}
+
+export function parseRobots(txt: string, uaToken = UA_TOKEN): RobotsRule[] {
+ // Group directive lines by their preceding User-agent block(s).
+ const groups: { agents: string[]; rules: { allow: boolean; path: string }[] }[] = [];
+ let cur: (typeof groups)[number] | null = null;
+ let sawRuleSinceAgent = false;
+ for (const rawLine of txt.split(/\r?\n/)) {
+ const line = rawLine.replace(/#.*$/, "").trim();
+ if (!line) continue;
+ const idx = line.indexOf(":");
+ if (idx < 0) continue;
+ const field = line.slice(0, idx).trim().toLowerCase();
+ const value = line.slice(idx + 1).trim();
+ if (field === "user-agent") {
+ if (!cur || sawRuleSinceAgent) {
+ cur = { agents: [], rules: [] };
+ groups.push(cur);
+ sawRuleSinceAgent = false;
+ }
+ cur.agents.push(value.toLowerCase());
+ } else if ((field === "disallow" || field === "allow") && cur) {
+ sawRuleSinceAgent = true;
+ cur.rules.push({ allow: field === "allow", path: value });
+ }
+ }
+ // Prefer the most specific matching group (our UA token) over the `*` group.
+ const specific = groups.filter((g) => g.agents.some((a) => a !== "*" && uaToken.includes(a)));
+ const wildcard = groups.filter((g) => g.agents.includes("*"));
+ const chosen = specific.length ? specific : wildcard;
+ const rules: RobotsRule[] = [];
+ for (const g of chosen) {
+ for (const r of g.rules) {
+ if (r.path === "" && !r.allow) continue; // empty Disallow = allow all (no-op)
+ rules.push({ allow: r.allow, length: r.path.length, re: patternToRegex(r.path) });
+ }
+ }
+ return rules;
+}
+
+/** Returns true if `path` is allowed for our UA under the given rules. */
+export function robotsAllows(rules: RobotsRule[], path: string): boolean {
+ let best: RobotsRule | null = null;
+ for (const rule of rules) {
+ if (!rule.re.test(path)) continue;
+ if (!best || rule.length > best.length || (rule.length === best.length && rule.allow && !best.allow)) {
+ best = rule;
+ }
+ }
+ return best ? best.allow : true; // no matching rule = allowed
+}
+
+async function loadRobots(origin: string): Promise<RobotsRule[]> {
+ const cached = robotsCache.get(origin);
+ if (cached) return cached;
+ const inflight = robotsInflight.get(origin);
+ if (inflight) return inflight;
+ const promise = (async () => {
+ let rules: RobotsRule[] = [];
+ try {
+ const res = await fetch(`${origin}/robots.txt`, {
+ method: "GET",
+ headers: { "User-Agent": USER_AGENT, Accept: "text/plain,*/*;q=0.8" },
+ redirect: "follow",
+ signal: AbortSignal.timeout(15_000),
+ });
+ if (res.ok) rules = parseRobots(await res.text());
+ // Non-2xx (404 no robots, 5xx) → allow all (empty rules), per convention.
+ } catch {
+ rules = []; // network/timeout → fail open (allow), same as "no robots.txt".
+ }
+ robotsCache.set(origin, rules);
+ robotsInflight.delete(origin);
+ return rules;
+ })();
+ robotsInflight.set(origin, promise);
+ return promise;
+}
export class LiveFetcher {
private lastRequestAt = 0;
@@ -22,6 +128,15 @@ export class LiveFetcher {
}
async fetch(url: string): Promise<RawPage> {
+ // robots.txt enforcement — check before spending the rate-limit budget.
+ const parsed = new URL(url);
+ if (parsed.pathname !== "/robots.txt") {
+ const rules = await loadRobots(parsed.origin);
+ if (!robotsAllows(rules, parsed.pathname + parsed.search)) {
+ throw new DisallowedError(url);
+ }
+ }
+
const wait = this.lastRequestAt + this.minIntervalMs - Date.now();
if (wait > 0) await sleep(wait);
this.lastRequestAt = Date.now();
@@ -69,3 +184,10 @@ export class BlockedError extends Error {
this.name = "BlockedError";
}
}
+
+export class DisallowedError extends Error {
+ constructor(public readonly url: string) {
+ super(`disallowed by robots.txt: ${url} — URL skipped (POLICY.md robots.txt compliance)`);
+ this.name = "DisallowedError";
+ }
+}
← ce2edb5 chore: v0.3.1 (session close — spechomes→homesonspec rename)
·
back to Homesonspec
·
collectors: enforce robots.txt in LiveFetcher (POLICY compli 7677fe3 →