`scripts/pull-from-shopify.js`

112 lines
// pull-from-shopify.js — Natural Textile Wallpaper catalog fetcher
// Paginates DW Shopify /products.json, filters to natural-fiber wallcoverings:
// linen, jute, raffia, sisal, hemp, grasscloth, woven, seagrass, abaca, coir
const https = require('https');
const fs = require('fs');
const path = require('path');

const NATURAL_TAGS = /^(linen|jute|raffia|sisal|hemp|grasscloth|seagrass|abaca|coir|woven|natural fiber|natural-fiber|textured|woven natural|natural weave|burlap|wicker|rattan|bamboo|woven texture|eco|biophilic|sustainable|organic|earthy)$/i;
const NATURAL_TITLE = /\b(linen|jute|raffia|sisal|hemp|grasscloth|seagrass|abaca|coir|woven|natural fiber|natural weave|burlap|rattan|bamboo|wicker|biophilic|organic|textured weave)\b/i;
const REJECT_TITLE = /(lamp|rug|pillow|throw|tripod|frame|mirror|vase|candle|sculpture|figurine|visual.{0,3}merchandiser|bh.?90210|image[ _-]?4)/i;

function fetchPage(page, retries = 3) {
  return new Promise((resolve, reject) => {
    const url = `https://designerwallcoverings.com/products.json?limit=250&page=${page}`;
    https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 naturaltextilewallpaper-builder' } }, (res) => {
      let data = '';
      res.on('data', c => data += c);
      res.on('end', () => {
        try {
          const parsed = JSON.parse(data);
          resolve(parsed.products || []);
        } catch(e) {
          if (retries > 0) {
            console.error(`  page ${page} parse error, retrying in 2s...`);
            setTimeout(() => fetchPage(page, retries - 1).then(resolve).catch(reject), 2000);
          } else {
            reject(e);
          }
        }
      });
    }).on('error', err => {
      if (retries > 0) {
        setTimeout(() => fetchPage(page, retries - 1).then(resolve).catch(reject), 2000);
      } else {
        reject(err);
      }
    });
  });
}

function materialOf(tags, title) {
  const blob = (Array.isArray(tags) ? tags.join(' ') : '') + ' ' + (title || '');
  const b = blob.toLowerCase();
  if (/linen/.test(b)) return 'linen';
  if (/jute/.test(b)) return 'jute';
  if (/raffia/.test(b)) return 'raffia';
  if (/sisal/.test(b)) return 'sisal';
  if (/hemp/.test(b)) return 'hemp';
  if (/seagrass/.test(b)) return 'seagrass';
  if (/abaca/.test(b)) return 'abaca';
  if (/grasscloth/.test(b)) return 'grasscloth';
  if (/bamboo/.test(b)) return 'bamboo';
  if (/coir|burlap|wicker|rattan/.test(b)) return 'woven';
  if (/woven|natural fiber|natural weave/.test(b)) return 'woven';
  return 'natural';
}

(async () => {
  const all = [];
  for (let page = 1; page <= 35; page++) {
    const products = await fetchPage(page);
    if (!products.length) break;
    all.push(...products);
    process.stdout.write(`page ${page}: ${products.length} (total ${all.length})\n`);
    if (products.length < 250) break;
    await new Promise(r => setTimeout(r, 500));
  }
  console.log(`\ntotal fetched: ${all.length}`);

  const natural = all
    .filter(p => p.product_type && /wallcovering|wallpaper/i.test(p.product_type))
    .filter(p => !REJECT_TITLE.test(p.title || ''))
    .filter(p => p.images && p.images.length > 0 && p.images[0].src)
    .filter(p => {
      const hasNaturalTag = (p.tags || []).some(t => NATURAL_TAGS.test(t));
      const hasNaturalTitle = NATURAL_TITLE.test(p.title || '');
      return hasNaturalTag || hasNaturalTitle;
    })
    .map(p => {
      const maxPrice = p.variants
        ? Math.max(...p.variants.map(v => parseFloat(v.price) || 0))
        : 0;
      const minPrice = p.variants
        ? Math.min(...p.variants.filter(v => parseFloat(v.price) > 0).map(v => parseFloat(v.price)))
        : 0;
      return {
        handle: p.handle,
        sku: p.handle,
        title: p.title,
        vendor: (p.vendor || '').trim(),
        product_type: p.product_type,
        tags: p.tags || [],
        material: materialOf(p.tags, p.title),
        image_url: p.images[0].src,
        images: (p.images || []).map(i => i.src).slice(0, 4),
        max_price: maxPrice,
        min_price: minPrice > 0 ? minPrice : maxPrice,
        fetched_at: new Date().toISOString()
      };
    });

  const outPath = path.join(__dirname, '..', 'data', 'products.json');
  fs.writeFileSync(outPath, JSON.stringify(natural, null, 2));
  console.log(`\nSaved ${natural.length} natural textile wallpaper products to ${outPath}`);

  // material breakdown
  const byMaterial = {};
  for (const p of natural) byMaterial[p.material] = (byMaterial[p.material] || 0) + 1;
  console.log('\nMaterial breakdown:');
  Object.entries(byMaterial).sort((a,b) => b[1]-a[1]).forEach(([m,n]) => console.log(`  ${m}: ${n}`));
})().catch(e => { console.error(e); process.exit(1); });