`scripts/crawlAffiliateSites.ts`

241 lines
// Automated web crawler for top 20 luxury handbag affiliate sites
// Crawls for new listings, extracts images, prices, and product data

import puppeteer from "puppeteer";
import fs from "fs";
import path from "path";
import crypto from "crypto";

interface CrawledListing {
  id: string;
  brand: string;
  title: string;
  price?: number;
  currency: string;
  imageUrl: string;
  productUrl: string;
  merchant: string;
  condition: "new" | "used" | "auction";
  crawledAt: string;
  description?: string;
}

// Top affiliate merchants with good image policies
const AFFILIATE_MERCHANTS = [
  {
    id: "fashionphile",
    name: "Fashionphile",
    baseUrl: "https://www.fashionphile.com",
    searchUrl: (brand: string) => `https://www.fashionphile.com/shop/bags?designers=${encodeURIComponent(brand)}`,
    allowsImages: true,
    condition: "used" as const,
    selectors: {
      items: ".product-tile",
      title: ".product-name",
      price: ".product-price",
      image: "img[data-src]",
      link: "a.product-link",
    }
  },
  {
    id: "rebag",
    name: "Rebag",
    baseUrl: "https://www.rebag.com",
    searchUrl: (brand: string) => `https://www.rebag.com/shop?brand=${encodeURIComponent(brand)}`,
    allowsImages: true,
    condition: "used" as const,
    selectors: {
      items: "[data-testid='product-card']",
      title: "[data-testid='product-title']",
      price: "[data-testid='product-price']",
      image: "img[src]",
      link: "a[href*='/products/']",
    }
  },
  {
    id: "therealreal",
    name: "The RealReal",
    baseUrl: "https://www.therealreal.com",
    searchUrl: (brand: string) => `https://www.therealreal.com/designers/${brand.toLowerCase().replace(/ /g, "-")}/handbags`,
    allowsImages: true,
    condition: "used" as const,
    selectors: {
      items: ".product-tile",
      title: ".product-title",
      price: ".product-price",
      image: "img.product-image",
      link: "a.product-link",
    }
  },
  {
    id: "vestiaire",
    name: "Vestiaire Collective",
    baseUrl: "https://us.vestiairecollective.com",
    searchUrl: (brand: string) => `https://us.vestiairecollective.com/search/?q=${encodeURIComponent(brand + " bag")}`,
    allowsImages: true,
    condition: "used" as const,
    selectors: {
      items: "[data-testid='productCard']",
      title: "[data-testid='productTitle']",
      price: "[data-testid='price']",
      image: "img[alt]",
      link: "a[href*='/items/']",
    }
  },
  {
    id: "farfetch",
    name: "Farfetch",
    baseUrl: "https://www.farfetch.com",
    searchUrl: (brand: string) => `https://www.farfetch.com/shopping/women/bags-1/items.aspx?designers=${encodeURIComponent(brand)}`,
    allowsImages: true,
    condition: "new" as const,
    selectors: {
      items: "[data-testid='productCard']",
      title: "[data-testid='productTitle']",
      price: "[data-testid='price']",
      image: "img",
      link: "a",
    }
  }
];

function loadTopBrands(): string[] {
  const file = path.resolve("data/top_luxury_brands.json");
  if (!fs.existsSync(file)) {
    return ["HERMÈS", "CHANEL", "LOUIS VUITTON", "DIOR", "GUCCI"];
  }
  const brands = JSON.parse(fs.readFileSync(file, "utf8"));
  return brands.slice(0, 20).map((b: any) => b.brand);
}

function saveCrawledListings(listings: CrawledListing[]) {
  const file = path.resolve("data/crawled_affiliate_listings.json");
  const existing = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : [];

  const combined = [...existing, ...listings];

  // Deduplicate by ID
  const unique = Array.from(new Map(combined.map(l => [l.id, l])).values());

  fs.writeFileSync(file, JSON.stringify(unique, null, 2));
  return unique.length - existing.length;
}

async function crawlMerchant(merchant: typeof AFFILIATE_MERCHANTS[0], brand: string): Promise<CrawledListing[]> {
  console.log(`\nCrawling ${merchant.name} for ${brand}...`);

  const browser = await puppeteer.launch({
    headless: true,
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });

  try {
    const page = await browser.newPage();
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

    const url = merchant.searchUrl(brand);
    console.log(`  URL: ${url}`);

    await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });

    // Wait for products to load
    await page.waitForSelector(merchant.selectors.items, { timeout: 10000 }).catch(() => {
      console.log(`  No items found for ${brand} on ${merchant.name}`);
    });

    const listings: CrawledListing[] = await page.evaluate((selectors, merchantData) => {
      const items = document.querySelectorAll(selectors.items);
      const results: CrawledListing[] = [];

      items.forEach((item, idx) => {
        if (idx >= 50) return; // Limit to 50 per brand per merchant

        const titleEl = item.querySelector(selectors.title);
        const priceEl = item.querySelector(selectors.price);
        const imageEl = item.querySelector(selectors.image);
        const linkEl = item.querySelector(selectors.link);

        if (!titleEl || !imageEl || !linkEl) return;

        const title = titleEl.textContent?.trim() || "";
        const priceText = priceEl?.textContent?.trim() || "";
        const price = parseFloat(priceText.replace(/[^0-9.]/g, ""));
        const imageUrl = (imageEl as HTMLImageElement).src || (imageEl as HTMLImageElement).getAttribute('data-src') || "";
        const productUrl = (linkEl as HTMLAnchorElement).href || "";

        if (title && imageUrl && productUrl) {
          results.push({
            id: crypto.createHash('md5').update(productUrl).digest('hex').slice(0, 12),
            brand: merchantData.brand,
            title,
            price: price || undefined,
            currency: "USD",
            imageUrl,
            productUrl,
            merchant: merchantData.merchantName,
            condition: merchantData.condition,
            crawledAt: new Date().toISOString(),
          } as any);
        }
      });

      return results;
    }, merchant.selectors, {
      brand,
      merchantName: merchant.name,
      condition: merchant.condition
    });

    console.log(`  Found ${listings.length} items`);
    await browser.close();
    return listings;

  } catch (error) {
    console.error(`  Error crawling ${merchant.name}:`, error);
    await browser.close();
    return [];
  }
}

async function crawlAll() {
  console.log("=== Starting Affiliate Site Crawler ===\n");
  console.log("Top 20 Luxury Handbag Brands Only\n");

  const brands = loadTopBrands();
  console.log(`Brands: ${brands.join(", ")}\n`);

  let totalListings: CrawledListing[] = [];

  for (const brand of brands) {
    console.log(`\n--- ${brand} ---`);

    for (const merchant of AFFILIATE_MERCHANTS) {
      if (!merchant.allowsImages) continue;

      const listings = await crawlMerchant(merchant, brand);
      totalListings = [...totalListings, ...listings];

      // Rate limiting
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
  }

  const newCount = saveCrawledListings(totalListings);

  console.log(`\n=== Crawl Complete ===`);
  console.log(`Total items crawled: ${totalListings.length}`);
  console.log(`New items added: ${newCount}`);
  console.log(`Saved to: data/crawled_affiliate_listings.json`);
}

const isMainModule = import.meta.url === `file://${process.argv[1]}`;

if (isMainModule) {
  crawlAll().catch((err) => {
    console.error("Crawl failed:", err);
    process.exit(1);
  });
}

export { crawlAll };