`adapters/shopify.js`

52 lines
// Shopify feed-first adapter — paginate <base>/products.json?limit=250&page=N.
// $0 local HTTP, no login, no paid service. Retry-with-backoff on transient 503
// (Shopify/CF blips — learned on Astek 2026-07-13). Returns normalized products.
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36';

async function fetchPage(url, attempts = 4) {
  let last = 0;
  for (let i = 0; i < attempts; i++) {
    const res = await fetch(url, { headers: { 'User-Agent': UA } });
    if (res.ok) return res;
    last = res.status;
    if (i < attempts - 1) await new Promise(r => setTimeout(r, 1500 * (i + 1)));
  }
  throw new Error(`HTTP ${last} after ${attempts} attempts`);
}

// base = e.g. https://www.ellipopp.com  (trailing slash tolerated)
async function scrape(base, { maxPages = 60, politeMs = 150 } = {}) {
  const root = base.replace(/\/+$/, '');
  const feed = `${root}/products.json`;
  const out = [];
  for (let page = 1; page <= maxPages; page++) {
    const res = await fetchPage(`${feed}?limit=250&page=${page}`);
    const json = await res.json();
    const products = json.products || [];
    if (products.length === 0) break;
    for (const p of products) {
      const v0 = (p.variants || [])[0] || {};
      const img0 = (p.images || [])[0] || {};
      out.push({
        mfr_sku: v0.sku || String(p.id),
        handle: p.handle,
        title: p.title,
        product_type: p.product_type || '',
        vendor: p.vendor || '',
        price: v0.price != null ? Number(v0.price) : null,
        body_html: p.body_html || '',
        image_url: img0.src || null,
        all_images: (p.images || []).map(i => i.src),
        product_url: `${root}/products/${p.handle}`,
        tags: typeof p.tags === 'string' ? p.tags.split(',').map(s => s.trim()).filter(Boolean) : (p.tags || []),
        raw: p,
      });
    }
    if (products.length < 250) break;
    await new Promise(r => setTimeout(r, politeMs));
  }
  return out;
}

module.exports = { scrape, platform: 'shopify' };