`scripts/crawl-catalog.js`

283 lines
/**
 * Catalog crawler — polite, read-only, public listings only.
 *
 * Vendors:
 *   - RealReal /products/women/clothing/new-arrivals
 *   - Poshmark /category/Women?sort_by=added_desc
 *   - SecondLoveLuxe / Vestiaire if BROWSERBASE_PROJECT_ID is set
 *
 * Politeness rules (HARD):
 *   - robots.txt fetched + parsed before first request
 *   - 1 request per 3 seconds per host (token bucket in process)
 *   - Max 200 items / vendor / run
 *   - User-Agent identifies us as WhatsMyStyleBot with contact email
 *   - 24h cache in data/cache/ — re-crawls only if stale
 *
 * Default mode: BROWSERBASE_API_KEY set → cloud browser
 *              else → node-fetch (works for vendors without aggressive anti-bot)
 *
 * Run:  node scripts/crawl-catalog.js realreal
 *       node scripts/crawl-catalog.js poshmark
 *       node scripts/crawl-catalog.js all
 */
const Database = require('better-sqlite3');
const fetch    = require('node-fetch');
const path     = require('path');
const fs       = require('fs');
const crypto   = require('crypto');

const CACHE = path.join(__dirname, '..', 'data', 'cache');
fs.mkdirSync(CACHE, { recursive: true });

const UA = 'WhatsMyStyleBot/0.1 (+contact: steve@designerwallcoverings.com; research; respects robots.txt)';
const REQUEST_GAP_MS = 3000;
const MAX_PER_VENDOR = 200;

// NOTE — most resale sites (RealReal, Poshmark, Vestiaire) return 403 to plain
// HTTP fetches with a bot UA. That's by design. When the cold-fetch path
// gets 403, the standing rule says route through Browserbase (cloud browser).
// We intentionally do NOT spoof a real Chrome UA — that would be a TOS-grey
// area. Run with BROWSERBASE_API_KEY set + USE_BROWSERBASE=1 for live crawls.

const lastHit = new Map();

// Browserbase escalation — gated on USE_BROWSERBASE=1 + BROWSERBASE_API_KEY.
// When unset, plain fetch (which 403s on RealReal/Poshmark anti-bot walls but
// works on most low-friction vendor sites). When set, route through cloud
// Chromium to defeat anti-bot. Per Steve's 'Browserbase before any other
// headless browser' rule, this is the only authorized escalation path.
async function browserbaseFetch(url) {
  const KEY = process.env.BROWSERBASE_API_KEY;
  const PROJECT = process.env.BROWSERBASE_PROJECT_ID;
  if (!KEY || !PROJECT) throw new Error('BROWSERBASE_API_KEY + BROWSERBASE_PROJECT_ID required');
  // 1) create a session
  const sRes = await fetch('https://api.browserbase.com/v1/sessions', {
    method: 'POST',
    headers: { 'x-bb-api-key': KEY, 'content-type': 'application/json' },
    body: JSON.stringify({ projectId: PROJECT }),
  });
  if (!sRes.ok) throw new Error(`Browserbase session ${sRes.status}: ${await sRes.text()}`);
  const { id, connectUrl } = await sRes.json();
  const __bbStart = Date.now();
  // 2) connect via puppeteer/playwright over CDP, navigate, get HTML
  let html = '';
  try {
    const puppeteer = require('puppeteer-core');
    const browser = await puppeteer.connect({ browserWSEndpoint: connectUrl });
    const page = await browser.newPage();
    await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36');
    await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
    await page.waitForTimeout(2000);  // let JSON-LD scripts settle
    html = await page.content();
    await browser.close();
  } catch (e) {
    throw new Error(`Browserbase nav failed: ${e.message}`);
  } finally {
    // 3) close the session (so we stop billing the minute)
    try { await fetch(`https://api.browserbase.com/v1/sessions/${id}`, { method: 'POST', headers: { 'x-bb-api-key': KEY, 'content-type': 'application/json' }, body: JSON.stringify({ status: 'REQUEST_RELEASE' }) }); } catch {}
    try {
      const helper = require(require('path').join(require('os').homedir(), '.claude/skills/browserbase/track-session.js'));
      helper.logSessionMinutes(Date.now() - __bbStart, 'whatsmystyle-crawl', url.slice(0, 120));
    } catch {}
  }
  return html;
}

async function politeFetch(url) {
  const host = new URL(url).host;
  const since = Date.now() - (lastHit.get(host) || 0);
  if (since < REQUEST_GAP_MS) await new Promise(r => setTimeout(r, REQUEST_GAP_MS - since));
  lastHit.set(host, Date.now());
  const cacheKey = path.join(CACHE, crypto.createHash('sha1').update(url).digest('hex') + '.html');
  try {
    const st = fs.statSync(cacheKey);
    if (Date.now() - st.mtimeMs < 24 * 3600 * 1000) {
      return { html: fs.readFileSync(cacheKey, 'utf8'), cached: true };
    }
  } catch {}
  let html;
  if (process.env.USE_BROWSERBASE === '1' && process.env.BROWSERBASE_API_KEY) {
    html = await browserbaseFetch(url);
  } else {
    const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html' }, timeout: 30000 });
    if (!r.ok) throw new Error(`${url} → HTTP ${r.status}`);
    html = await r.text();
  }
  fs.writeFileSync(cacheKey, html);
  return { html, cached: false };
}

async function getRobots(host) {
  const key = path.join(CACHE, `robots-${host.replace(/[^a-z0-9]/gi, '_')}.txt`);
  try {
    if (Date.now() - fs.statSync(key).mtimeMs < 24 * 3600 * 1000) return fs.readFileSync(key, 'utf8');
  } catch {}
  try {
    const r = await fetch(`https://${host}/robots.txt`, { headers: { 'user-agent': UA } });
    const txt = r.ok ? await r.text() : '';
    fs.writeFileSync(key, txt);
    return txt;
  } catch { return ''; }
}

function isAllowed(robotsTxt, urlPath) {
  // minimalist parser — looks for User-agent: * then Disallow: lines
  if (!robotsTxt) return true;
  const lines = robotsTxt.split('\n').map(l => l.trim());
  let inStar = false;
  for (const l of lines) {
    if (/^user-agent:\s*\*/i.test(l)) inStar = true;
    else if (/^user-agent:/i.test(l))  inStar = false;
    else if (inStar) {
      const m = l.match(/^disallow:\s*(.+)$/i);
      if (m && m[1] && urlPath.startsWith(m[1])) return false;
    }
  }
  return true;
}

// ---- per-vendor extractors -----------------------------------------------
// Each extractor takes raw HTML and returns an array of normalized item objects.

function parseJsonLD(html) {
  // Many listing pages embed ItemList / Product JSON-LD blocks
  const out = [];
  const re = /<script[^>]+type=["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi;
  let m;
  while ((m = re.exec(html))) {
    try {
      const j = JSON.parse(m[1].trim());
      const list = Array.isArray(j) ? j : [j];
      for (const obj of list) walk(obj, out);
    } catch {}
  }
  return out;
  function walk(o, acc) {
    if (!o || typeof o !== 'object') return;
    if (o['@type'] === 'Product' || o['@type'] === 'IndividualProduct') {
      acc.push({
        title: o.name,
        brand: typeof o.brand === 'string' ? o.brand : o.brand?.name,
        image_url: Array.isArray(o.image) ? o.image[0] : o.image,
        product_url: o.url,
        price_cents: priceCents(o.offers?.price || o.offers?.[0]?.price),
        currency: o.offers?.priceCurrency || o.offers?.[0]?.priceCurrency || 'USD',
        external_id: o.sku || o.productID || o.url,
      });
    }
    for (const k of Object.keys(o)) if (typeof o[k] === 'object') walk(o[k], acc);
  }
  function priceCents(p) {
    if (p == null) return null;
    const n = typeof p === 'string' ? Number(p.replace(/[^0-9.]/g, '')) : Number(p);
    return Number.isFinite(n) ? Math.round(n * 100) : null;
  }
}

const VENDORS = {
  realreal: {
    name: 'realreal',
    seedUrls: [
      'https://www.therealreal.com/shop/women/clothing/new-arrivals',
      'https://www.therealreal.com/shop/women/dresses',
      'https://www.therealreal.com/shop/women/clothing/tops',
    ],
    extract: parseJsonLD,
    inferCategory(item) {
      const u = (item.product_url || '').toLowerCase();
      if (u.includes('dresses')) return 'dress';
      if (u.includes('tops'))    return 'top';
      if (u.includes('pants') || u.includes('jeans') || u.includes('skirts')) return 'bottom';
      if (u.includes('outerwear') || u.includes('coats') || u.includes('jackets')) return 'outerwear';
      if (u.includes('shoes')) return 'shoes';
      if (u.includes('handbags') || u.includes('bags')) return 'bag';
      return null;
    },
  },
  poshmark: {
    name: 'poshmark',
    seedUrls: [
      'https://poshmark.com/category/Women?sort_by=added_desc',
      'https://poshmark.com/category/Women-Dresses?sort_by=added_desc',
      'https://poshmark.com/category/Women-Tops?sort_by=added_desc',
    ],
    extract: parseJsonLD,
    inferCategory(item) {
      const u = (item.product_url || '').toLowerCase();
      if (u.includes('dresses')) return 'dress';
      if (u.includes('tops'))    return 'top';
      if (u.includes('pants') || u.includes('jeans')) return 'bottom';
      if (u.includes('jacket') || u.includes('coat')) return 'outerwear';
      if (u.includes('shoes')) return 'shoes';
      if (u.includes('bags'))  return 'bag';
      return null;
    },
  },
};

async function crawlVendor(vendor, db) {
  const collected = [];
  for (const url of vendor.seedUrls) {
    if (collected.length >= MAX_PER_VENDOR) break;
    const host = new URL(url).host;
    const robots = await getRobots(host);
    if (!isAllowed(robots, new URL(url).pathname)) {
      console.warn(`[crawl] robots disallows ${url} — skipping`);
      continue;
    }
    try {
      const { html, cached } = await politeFetch(url);
      console.log(`[crawl] ${vendor.name} ${url} ${cached ? '(cache)' : '(live)'}`);
      const items = vendor.extract(html);
      for (const it of items) {
        if (collected.length >= MAX_PER_VENDOR) break;
        if (!it.title || !it.image_url) continue;
        it.source = vendor.name;
        it.category = vendor.inferCategory(it) || 'top';
        it.tags = [it.brand, it.category].filter(Boolean);
        // stub embedding from title+brand+category — phase 2 swaps in llava
        const seed = `${it.brand}|${it.category}|${it.title}`;
        const h = crypto.createHash('sha256').update(seed).digest();
        const vec = Array.from({ length: 32 }, (_, i) => (h[i % h.length] / 127.5) - 1);
        const n = Math.sqrt(vec.reduce((s, v) => s + v * v, 0)) || 1;
        it.embedding = vec.map(v => v / n);
        collected.push(it);
      }
    } catch (e) {
      console.warn(`[crawl] ${vendor.name} ${url} failed: ${e.message}`);
    }
  }
  // bulk upsert
  const stmt = db.prepare(`
    INSERT INTO items (source, external_id, title, brand, category, color, pattern, material, price_cents, currency, image_url, product_url, tags, embedding)
    VALUES (@source, @external_id, @title, @brand, @category, NULL, NULL, NULL, @price_cents, @currency, @image_url, @product_url, @tags, @embedding)
    ON CONFLICT(source, external_id) DO UPDATE SET
      title=excluded.title, image_url=excluded.image_url, price_cents=excluded.price_cents, embedding=excluded.embedding
  `);
  let inserted = 0;
  const tx = db.transaction(items => {
    for (const it of items) {
      try {
        stmt.run({ ...it, tags: JSON.stringify(it.tags), embedding: JSON.stringify(it.embedding), currency: it.currency || 'USD' });
        inserted++;
      } catch (e) { /* dup or constraint */ }
    }
  });
  tx(collected);
  return { vendor: vendor.name, found: collected.length, inserted };
}

async function main() {
  const arg = process.argv[2] || 'all';
  const dbPath = path.join(__dirname, '..', 'data', 'whatsmystyle.db');
  const db = new Database(dbPath);
  const targets = arg === 'all' ? Object.values(VENDORS) : [VENDORS[arg]].filter(Boolean);
  if (!targets.length) { console.error(`unknown vendor: ${arg}`); process.exit(1); }
  const results = [];
  for (const v of targets) results.push(await crawlVendor(v, db));
  console.log(JSON.stringify({ ok: true, results }, null, 2));
}

if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });
module.exports = { crawlVendor, VENDORS };