`src/importers/scrape.ts`

81 lines
import type { RawListing } from "./ingest";

// Best-effort single-URL importer. Uses Playwright (a devDependency) to render
// the page, then extracts fields with host-specific selectors and generic
// fallbacks (og: tags, $-regex). Live auction-site DOMs change often, so this is
// intentionally defensive: whatever it can't find is left undefined and the
// research pipeline's AI identifier fills gaps from the title/description.

function sourceFromHost(host: string): string {
  if (host.includes("govdeals")) return "GOVDEALS";
  if (host.includes("publicsurplus")) return "PUBLIC_SURPLUS";
  if (host.includes("gsaauctions")) return "GSA_AUCTIONS";
  if (host.includes("municibid")) return "MUNICIBID";
  if (host.includes("bid4assets")) return "BID4ASSETS";
  return "OTHER";
}

function auctionIdFromUrl(u: URL): string {
  const q =
    u.searchParams.get("itemid") ||
    u.searchParams.get("auc") ||
    u.searchParams.get("id") ||
    u.searchParams.get("ac");
  if (q) return q;
  const segs = u.pathname.split("/").filter(Boolean);
  return segs[segs.length - 1] || u.pathname;
}

export async function scrapeUrl(url: string): Promise<RawListing> {
  const u = new URL(url);
  const source = sourceFromHost(u.hostname);

  // Dynamic import so the main app bundle never hard-depends on Playwright.
  let chromium: typeof import("playwright").chromium;
  try {
    ({ chromium } = await import("playwright"));
  } catch {
    throw new Error(
      "Playwright is not installed. Run `npx playwright install chromium` to enable URL import.",
    );
  }

  const browser = await chromium.launch({ headless: true });
  try {
    const page = await browser.newPage({ userAgent: "Mozilla/5.0 GovArbitrageBot/0.1" });
    await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });

    const extracted = await page.evaluate(() => {
      const meta = (p: string) =>
        (document.querySelector(`meta[property="${p}"]`) as HTMLMetaElement | null)?.content ||
        (document.querySelector(`meta[name="${p}"]`) as HTMLMetaElement | null)?.content ||
        undefined;
      const text = document.body?.innerText || "";
      const priceMatch = text.match(/\$\s?([\d,]+(?:\.\d{2})?)/);
      const images = Array.from(document.querySelectorAll("img"))
        .map((i) => (i as HTMLImageElement).src)
        .filter((s) => s && s.startsWith("http"))
        .slice(0, 6);
      const ogImage = meta("og:image");
      return {
        title: meta("og:title") || document.querySelector("h1")?.textContent?.trim() || document.title,
        description: meta("og:description") || undefined,
        currentBid: priceMatch ? Number(priceMatch[1].replace(/,/g, "")) : undefined,
        imageUrls: ogImage ? [ogImage, ...images] : images,
      };
    });

    return {
      source,
      sourceAuctionId: auctionIdFromUrl(u),
      sourceUrl: url,
      title: extracted.title || `${source} lot ${auctionIdFromUrl(u)}`,
      description: extracted.description,
      currentBid: extracted.currentBid,
      imageUrls: extracted.imageUrls,
    };
  } finally {
    await browser.close();
  }
}