`scripts/scrape-fromental.js`

170 lines
#!/usr/bin/env node
/**
 * scrape-fromental.js — feed-first ($0) Fromental catalog scraper.
 *
 * Fromental (https://www.fromental.co.uk) is a CONFIRMED Shopify store, so we hit
 * products.json?limit=250&page=N until a page returns 0 products. No auth, no
 * proxy, no captcha, no paid API. Cost: $0 (local).
 *
 * Captures EVERYTHING per product:
 *   title -> pattern_name, handle, product_type, body_html -> description,
 *   image srcs -> image_url + gallery_images, variants (price/sku/options),
 *   tags, vendor, options.
 *
 * Pricing: QUOTE-ONLY line. Most bespoke items are $0/blank (made-to-order);
 * some Artwork / Shop / Accessory items carry real prices. We capture whatever
 * price exists (max variant price), else leave NULL and mark quote_only=true.
 *
 * mfr_sku key resolution (idempotent upsert key):
 *   1. first non-empty variant sku
 *   2. else "handle:<handle>"
 *   3. else "shopify:<id>"
 *
 * Usage: node scripts/scrape-fromental.js
 */
const { Client } = require('pg');
const { buildResolver } = require('./collection-material');

const BASE = 'https://www.fromental.co.uk';
const DB = process.env.DATABASE_URL || 'postgresql://localhost/dw_unified';

async function fetchPage(page) {
  const url = `${BASE}/products.json?limit=250&page=${page}`;
  const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh) DW-catalog-sync' } });
  if (!res.ok) throw new Error(`HTTP ${res.status} on ${url}`);
  const j = await res.json();
  return j.products || [];
}

function stripHtml(html) {
  if (!html) return null;
  return String(html)
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&#39;|&rsquo;|&lsquo;/g, "'")
    .replace(/&quot;|&ldquo;|&rdquo;/g, '"')
    .replace(/&eacute;/g, 'e')
    .replace(/\s{2,}/g, ' ')
    .trim() || null;
}

// "Myeoh - Minhwa" -> pattern "Myeoh", color/variant "Minhwa"
function splitTitle(title) {
  if (!title) return { pattern: null, color: null };
  const parts = title.split(/\s+[-–—]\s+/);
  if (parts.length >= 2) return { pattern: parts[0].trim(), color: parts.slice(1).join(' - ').trim() };
  return { pattern: title.trim(), color: null };
}

// Fromental SKU pattern e.g. AW-YH01-02-OS-A ; collection ~ the alpha prefix family
function inferCollection(product, sku) {
  // prefer a "collection" style tag if present
  const tags = product.tags || [];
  const t = (Array.isArray(tags) ? tags : String(tags).split(',')).map((s) => s.trim()).filter(Boolean);
  const col = t.find((x) => /collection|series/i.test(x));
  if (col) return col.replace(/collection[:\s-]*/i, '').trim();
  return null;
}

function pickPrice(variants) {
  const prices = (variants || [])
    .map((v) => parseFloat(v.price))
    .filter((n) => Number.isFinite(n) && n > 0);
  if (!prices.length) return { price: null, quote: true };
  return { price: Math.max(...prices), quote: false };
}

function resolveMfrSku(product) {
  const v = (product.variants || []).find((x) => x.sku && String(x.sku).trim());
  if (v) return String(v.sku).trim();
  if (product.handle) return `handle:${product.handle}`;
  return `shopify:${product.id}`;
}

async function main() {
  const client = new Client({ connectionString: DB });
  await client.connect();

  let page = 1, all = [];
  while (true) {
    const products = await fetchPage(page);
    if (!products.length) break;
    all = all.concat(products);
    process.stdout.write(`  page ${page}: ${products.length} (total ${all.length})\n`);
    page++;
    if (page > 100) break; // safety
    await new Promise((r) => setTimeout(r, 300)); // polite
  }
  console.log(`Fetched ${all.length} products across ${page - 1} pages.`);

  // Build the collection + material resolver from the storefront's own
  // collection membership (feed-first, $0). Populates collection + material so
  // those facets never render empty and never regress on a refresh.
  console.log('Building collection + material resolver from /collections feed...');
  const resolver = await buildResolver();
  console.log('  resolver:', JSON.stringify(resolver.stats));

  let up = 0;
  for (const p of all) {
    const mfr_sku = resolveMfrSku(p);
    const { pattern, color } = splitTitle(p.title);
    const images = (p.images || []).map((im) => im.src).filter(Boolean);
    const { price, quote } = pickPrice(p.variants);
    const cmp = (p.variants || [])
      .map((v) => parseFloat(v.compare_at_price))
      .filter((n) => Number.isFinite(n) && n > 0);
    const compareAt = cmp.length ? Math.max(...cmp) : null;
    const tags = Array.isArray(p.tags) ? p.tags : String(p.tags || '').split(',').map((s) => s.trim()).filter(Boolean);
    // COLLECTION + MATERIAL from the storefront's own collection membership
    // (feed-first, $0). Falls back to product_type / body_html so neither is
    // ever left empty. Prefer a tag-derived collection only if present, else
    // the resolver's named design collection.
    const collection = inferCollection(p, mfr_sku) || resolver.collectionFor(p.handle, p);
    const material = resolver.materialFor(p.handle, p);
    const productUrl = `${BASE}/products/${p.handle}`;

    await client.query(
      `INSERT INTO fromental_catalog
        (mfr_sku, shopify_id, handle, pattern_name, color_name, collection, material,
         image_url, product_url, product_type, description, tags, vendor,
         gallery_images, variants, options, price, compare_at_price, quote_only,
         published_at, vendor_created_at, last_scraped, updated_at)
       VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,NOW(),NOW())
       ON CONFLICT (mfr_sku) DO UPDATE SET
         shopify_id=EXCLUDED.shopify_id, handle=EXCLUDED.handle,
         pattern_name=EXCLUDED.pattern_name, color_name=EXCLUDED.color_name,
         collection=EXCLUDED.collection, material=EXCLUDED.material,
         image_url=EXCLUDED.image_url,
         product_url=EXCLUDED.product_url, product_type=EXCLUDED.product_type,
         description=EXCLUDED.description, tags=EXCLUDED.tags, vendor=EXCLUDED.vendor,
         gallery_images=EXCLUDED.gallery_images, variants=EXCLUDED.variants,
         options=EXCLUDED.options, price=EXCLUDED.price,
         compare_at_price=EXCLUDED.compare_at_price, quote_only=EXCLUDED.quote_only,
         published_at=EXCLUDED.published_at, vendor_created_at=EXCLUDED.vendor_created_at,
         last_scraped=NOW(), updated_at=NOW()`,
      [
        mfr_sku, p.id, p.handle, pattern, color, collection, material,
        images[0] || null, productUrl, p.product_type || null,
        stripHtml(p.body_html), tags, p.vendor || 'Fromental',
        images, JSON.stringify(p.variants || []), JSON.stringify(p.options || []),
        price, compareAt, quote,
        p.published_at ? new Date(p.published_at) : null,
        p.created_at ? new Date(p.created_at) : null,
      ]
    );
    up++;
  }
  console.log(`Upserted ${up} rows into fromental_catalog.`);
  const { rows } = await client.query(
    `SELECT count(*) total,
            count(*) FILTER (WHERE price IS NOT NULL) priced,
            count(*) FILTER (WHERE quote_only) quote
       FROM fromental_catalog`
  );
  console.log('Table state:', rows[0]);
  await client.end();
}

main().catch((e) => { console.error(e); process.exit(1); });