`scripts/clothing-apis/shopify.js`

149 lines
/**
 * Shopify /products.json adapter — zero key, paginated, public.
 *
 * Every Shopify-powered DTC storefront exposes /products.json by default.
 * The endpoint is documented (Shopify's intentional API for the embedded
 * Buy Button etc.), returns JSON, and accepts ?limit + ?page query args.
 *
 * Pull pattern: 250 per page (Shopify's cap), back off if the brand returns
 * 429 or 5xx, identify ourselves in User-Agent, stop at MAX_PAGES per brand.
 *
 * Maps a Shopify product → WhatsMyStyle `items` row:
 *   id          → external_id (composite: shopify:<domain>:<product-id>)
 *   source      → 'shopify:<domain>'
 *   title       → title
 *   vendor      → brand
 *   product_type→ category (lowercased, normalized to top/bottom/dress/etc.)
 *   tags        → tags (JSON array)
 *   variants[0].price → price_cents (× 100)
 *   images[0].src → image_url
 *   handle      → product_url (https://<domain>/products/<handle>)
 *
 * One brand per call. Idempotent — uses INSERT OR IGNORE so re-running adds
 * only new SKUs and updates existing ones aren't trampled.
 */
const fetch = global.fetch || require('node-fetch');

const UA = 'WhatsMyStyleBot/0.1 (catalog importer; steve@designerwallcoverings.com)';
const PAGE_SIZE = 250;
const MAX_PAGES_PER_BRAND = 4;        // 4 × 250 = 1000 cap per brand per run
const PAGE_DELAY_MS = 600;            // be nice; ~1.6 RPS

// Heuristic mapping from Shopify product_type → our category enum.
const CATEGORY_MAP = [
  [/^t.?shirt|tee|tank|cami|blouse|shirt|button[- ]?down|top$/i, 'top'],
  [/sweater|knit|cardigan|pullover|jumper|hoodie/i, 'top'],
  [/pant|trouser|jean|chino|legging|short/i, 'bottom'],
  [/skirt|midi$|mini$/i, 'bottom'],
  [/dress|gown|jumpsuit|romper/i, 'dress'],
  [/coat|jacket|parka|trench|blazer|puffer|outerwear/i, 'outerwear'],
  [/shoe|sneaker|boot|sandal|loafer|heel|flat$/i, 'shoes'],
  [/bag|tote|purse|backpack|crossbody|clutch/i, 'bag'],
  [/scarf|hat|belt|sunglasses|jewelry|earring|necklace|ring$|bracelet|accessor/i, 'accessory'],
];
function normalizeCategory(productType, tags) {
  const text = `${productType || ''} ${(tags || []).join(' ')}`;
  for (const [re, cat] of CATEGORY_MAP) if (re.test(text)) return cat;
  return null;
}

async function fetchOnePage(domain, page) {
  const url = `https://${domain}/products.json?limit=${PAGE_SIZE}&page=${page}`;
  const r = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } });
  if (r.status === 404) return { products: [], end: true, status: 404 };
  if (r.status === 429) return { products: [], end: true, status: 429 };
  if (!r.ok) throw new Error(`shopify ${domain} page ${page} → ${r.status}`);
  // Tick 19: detect non-Shopify sites that return 200 + HTML (Next.js SPAs,
  // custom backends). Without this, we'd silently treat the homepage HTML as
  // an empty product list and report "ok: true, fetched: 0".
  const ct = (r.headers.get('content-type') || '').toLowerCase();
  if (!ct.includes('json')) {
    return { products: [], end: true, status: 'not-shopify' };
  }
  let j;
  try { j = await r.json(); }
  catch { return { products: [], end: true, status: 'invalid-json' }; }
  if (!Array.isArray(j.products)) return { products: [], end: true, status: 'no-products-array' };
  return { products: j.products, end: j.products.length < PAGE_SIZE, status: r.status };
}

async function fetchAllPages(domain) {
  const all = [];
  for (let page = 1; page <= MAX_PAGES_PER_BRAND; page++) {
    const { products, end, status } = await fetchOnePage(domain, page);
    if (status !== 200) {
      return { ok: false, status, products: all };
    }
    all.push(...products);
    if (end) break;
    await new Promise(r => setTimeout(r, PAGE_DELAY_MS));
  }
  return { ok: true, products: all };
}

function rowFromProduct(domain, brand, sustainTier, proGrade, p) {
  const variant = (p.variants || [])[0] || {};
  const price = Number(variant.price);
  const price_cents = Number.isFinite(price) ? Math.round(price * 100) : null;
  const image = (p.images || [])[0]?.src || p.image?.src || null;
  const category = normalizeCategory(p.product_type, p.tags);
  // Skip products with no image OR no price OR no recognized category — they
  // can't render in the duel grid anyway.
  if (!image || !price_cents || !category) return null;

  return {
    source: `shopify:${domain}`,
    external_id: `shopify:${domain}:${p.id}`,
    title: (p.title || '').trim().slice(0, 200),
    brand: brand || p.vendor || domain,
    category,
    color: null,
    pattern: null,
    material: null,
    price_cents,
    currency: 'USD',
    image_url: image,
    product_url: `https://${domain}/products/${p.handle}`,
    tags: JSON.stringify(p.tags || []),
    pro_grade: proGrade,
    _sustain: sustainTier,  // not a column on items; importer maps to sustainability via brand match
  };
}

async function importBrand(db, brandEntry) {
  const { name, domain, sustain_tier, pro_grade } = brandEntry;
  const t0 = Date.now();
  const { ok, status, products } = await fetchAllPages(domain);

  // Prepare a single-statement upsert. items has UNIQUE(source, external_id),
  // so INSERT OR IGNORE is the idempotent path.
  const insert = db.prepare(`
    INSERT OR IGNORE INTO items
      (source, external_id, title, brand, category, price_cents, currency, image_url, product_url, tags, pro_grade)
    VALUES
      (@source, @external_id, @title, @brand, @category, @price_cents, @currency, @image_url, @product_url, @tags, @pro_grade)
  `);

  let imported = 0, skipped = 0;
  const tx = db.transaction((rows) => {
    for (const r of rows) {
      if (!r) { skipped++; continue; }
      const res = insert.run(r);
      if (res.changes > 0) imported++;
      else skipped++;
    }
  });
  const rows = (products || []).map(p => rowFromProduct(domain, name, sustain_tier, pro_grade, p));
  tx(rows);

  return {
    brand: name, domain, ok, status,
    fetched: (products || []).length,
    imported,
    skipped,
    ms: Date.now() - t0,
  };
}

module.exports = { importBrand, fetchAllPages, normalizeCategory };