`scripts/probe-govdeals.ts`

62 lines
// One-off: discover live GovDeals item URLs by rendering the Angular SPA and
// harvesting item-detail links. Prints candidates so we can point scrapeUrl at
// a real, currently-open listing.
import { chromium } from "playwright";

const ENTRY = process.argv[2] || "https://www.govdeals.com/";
const HEADED = process.env.HEADED === "1";

async function main() {
  const browser = await chromium.launch({
    headless: !HEADED,
    args: ["--disable-blink-features=AutomationControlled"],
  });
  const context = await browser.newContext({
    userAgent:
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    viewport: { width: 1400, height: 900 },
    locale: "en-US",
    timezoneId: "America/Los_Angeles",
  });
  // Mask the headless/automation tells before any page script runs.
  await context.addInitScript(() => {
    Object.defineProperty(navigator, "webdriver", { get: () => undefined });
    // @ts-expect-error test fingerprint props
    window.chrome = { runtime: {} };
    Object.defineProperty(navigator, "plugins", { get: () => [1, 2, 3] });
    Object.defineProperty(navigator, "languages", { get: () => ["en-US", "en"] });
  });
  const page = await context.newPage();
  try {
    console.log("goto", ENTRY);
    await page.goto(ENTRY, { waitUntil: "networkidle", timeout: 45_000 }).catch((e) => console.log("nav warn:", e.message));
    await page.waitForTimeout(4000);
    console.log("title:", await page.title());
    console.log("url:", page.url());

    const links = await page.evaluate(() => {
      const hrefs = Array.from(document.querySelectorAll("a"))
        .map((a) => (a as HTMLAnchorElement).href)
        .filter((h) => /asset|\/item|itemid|\/listing/i.test(h));
      return Array.from(new Set(hrefs)).slice(0, 15);
    });
    console.log(`\nfound ${links.length} item-like links:`);
    links.forEach((l) => console.log("  ", l));

    // Fallback: sample any anchor hrefs so we can see the URL shape.
    if (links.length === 0) {
      const sample = await page.evaluate(() =>
        Array.from(new Set(Array.from(document.querySelectorAll("a")).map((a) => (a as HTMLAnchorElement).href)))
          .filter((h) => h.includes("govdeals.com"))
          .slice(0, 20),
      );
      console.log("\nno item links; sample of on-site anchors:");
      sample.forEach((s) => console.log("  ", s));
    }
  } finally {
    await browser.close();
  }
}

main();