`collectors/common/src/robots.selftest.ts`

58 lines
import { parseRobots, robotsAllows } from "./live-fetch";

const LENNAR = `User-Agent: *
Disallow: /*included
Disallow: /*nearby-places
Disallow: /*faq
Disallow: /*walkthrough
Disallow: /episerver
Sitemap: https://www.lennar.com/api/images/sitemapfe.xml`;

const KB = `User-agent: *
Sitemap: https://www.kbhome.com/sitemap.xml`;

const TRI = `User-Agent: *
Disallow: /api/
Disallow: /wp-json/
Disallow: /components
Sitemap: https://www.tripointehomes.com/sitemap.xml`;

const PULTE = `User-agent: *
Disallow: /search
Disallow: /sitecore/content/
User-agent: PetalBot
Disallow: /`;

const DRHORTON = `User-agent: *
Disallow: /community-finder
Disallow: /browse
Disallow: /scanner`;

const cases: [string, string, string, boolean][] = [
  // [label, robots, path, expectedAllowed]
  ["lennar sitemap (/api/images) allowed", LENNAR, "/api/images/sitemapfe.xml", true],
  ["lennar /new-homes community allowed", LENNAR, "/new-homes/alabama/birmingham/childersburg/college-park", true],
  ["lennar community faq sub-page BLOCKED", LENNAR, "/new-homes/alabama/x/college-park/faq", false],
  ["lennar nearby-places BLOCKED", LENNAR, "/new-homes/alabama/x/college-park/nearby-places", false],
  ["lennar /episerver BLOCKED", LENNAR, "/episerver/dashboard", false],
  ["kb everything allowed", KB, "/move-in-ready", true],
  ["tri /api/ BLOCKED", TRI, "/api/inventory", false],
  ["tri sitemap allowed", TRI, "/sitemap-communities.xml", true],
  ["tri community page allowed", TRI, "/new-homes/ca/community-x", true],
  ["pulte /api/community allowed (not in disallow)", PULTE, "/api/community/getcommunities?state=CA", true],
  ["pulte /search BLOCKED", PULTE, "/search", false],
  ["drhorton /browse BLOCKED", DRHORTON, "/browse", false],
  ["drhorton /communities allowed", DRHORTON, "/communities/al/birmingham", true],
];

let pass = 0, fail = 0;
for (const [label, robots, path, expected] of cases) {
  const rules = parseRobots(robots);
  const got = robotsAllows(rules, path);
  const ok = got === expected;
  console.log(`${ok ? "PASS" : "FAIL"}  ${label}  → allowed=${got} (want ${expected})`);
  ok ? pass++ : fail++;
}
console.log(`\n${pass} passed, ${fail} failed`);
process.exit(fail ? 1 : 0);