`scripts/scrape.js`

95 lines
#!/usr/bin/env node
// Fentucci Naturals — Tokiwa USA (INTERNAL vendor name only, never customer-facing)
// $0 plain-fetch scrape of Wix Thunderbolt SSR pages on www.tokiwausa.com.
// Product/swatch data is server-rendered: <img src="https://static.wixstatic.com/media/<uri>/..." alt="<mfr-sku>.jpg">
// (The "warmupData" JSON on this site is empty of gallery items — SSR img tags are the real feed.)
// Emits data/raw/<category>.json

const fs = require('fs');
const path = require('path');

const BASE = 'https://www.tokiwausa.com';

// 21 product category pages (confirmed complete against the /product index page 2026-07-22)
const CATEGORIES = [
  'raffia', 'sisal', 'abaca', 'bamboo', 'hyacinth', 'fine-arrowroot',
  'heavy-jute', 'washi', 'washi2', 'japanese-silk', 'japanese-silk2',
  'metal-leaf', 'metalleaf2', 'metal-leaf-3', 'mica', 'glassbeads',
  'paperweave', 'denim', 'textile', 'wood-veneer', 'gildingwashi',
];

// Collection catalog pages — Pro Gallery lookbook spreads, scraped for completeness
// (items generally lack SKU-named alts; anything SKU-named is captured)
const COLLECTIONS = [
  'reflectionvol1', 'reflectionsvol2', 'greenbookcollection',
  'glassbeadscollection', 'textilecollection',
];

// site-chrome / non-product alts to drop
const JUNK_ALTS = new Set(['', 'Logo.JPG', 'Logo', 'Facebook', 'Instagram']);

function extractItems(html) {
  const items = [];
  const numericItems = [];
  const re = /<img[^>]*src="https:\/\/static\.wixstatic\.com\/media\/([^"\/]+)\/[^"]*"[^>]*alt="([^"]*)"[^>]*>/g;
  let m;
  while ((m = re.exec(html)) !== null) {
    const uri = m[1];
    const alt = m[2];
    if (JUNK_ALTS.has(alt)) continue;
    const base = alt.replace(/\.(jpe?g|png|webp)$/i, '');
    // heroes/banners: "_edited" crops and "Untitled"
    if (/_edited/i.test(base)) continue;
    if (/^untitled/i.test(base)) continue;
    if (/^\d{1,3}$/.test(base)) {
      // numeric filename — real SKU may live in an adjacent "XXX NO.n" rich-text label
      numericItems.push({ uri, alt, num: parseInt(base, 10) });
      continue;
    }
    items.push({ uri, alt, image_url: `https://static.wixstatic.com/media/${uri}` });
  }
  // Label-pairing pass (japanese-silk pattern): page carries rich-text labels like
  // "J-SILK NO.5" and images named "5.jpg" — alt number maps 1:1 to label number.
  if (numericItems.length) {
    const labels = [...new Set([...html.matchAll(/>([A-Z][A-Z -]{1,20}NO\.\s?(\d+))</g)].map(x => x[1]))];
    const byNum = new Map(labels.map(l => [parseInt(l.match(/NO\.\s?(\d+)/)[1], 10), l]));
    for (const it of numericItems) {
      const label = byNum.get(it.num);
      if (label) items.push({ uri: it.uri, alt: label, image_url: `https://static.wixstatic.com/media/${it.uri}`, paired_from: it.alt });
      // numeric image with no matching label = banner/hero — skip (never "Unknown")
    }
  }
  // dedupe by uri (same swatch can appear twice, e.g. og:image + grid)
  const seen = new Set();
  return items.filter(it => (seen.has(it.uri) ? false : (seen.add(it.uri), true)));
}

async function fetchPage(slug) {
  const url = `${BASE}/${slug}`;
  const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)' } });
  if (!res.ok) throw new Error(`${slug}: HTTP ${res.status}`);
  return { url, html: await res.text() };
}

(async () => {
  const outDir = path.join(__dirname, '..', 'data', 'raw');
  fs.mkdirSync(outDir, { recursive: true });
  const summary = [];
  for (const slug of [...CATEGORIES, ...COLLECTIONS]) {
    try {
      const { url, html } = await fetchPage(slug);
      const items = extractItems(html);
      const kind = COLLECTIONS.includes(slug) ? 'collection' : 'category';
      const payload = { source_page: url, category: slug, kind, scraped_at: new Date().toISOString(), items };
      fs.writeFileSync(path.join(outDir, `${slug}.json`), JSON.stringify(payload, null, 2));
      summary.push({ slug, kind, items: items.length });
      console.log(`${slug} (${kind}): ${items.length} items`);
    } catch (e) {
      summary.push({ slug, error: e.message });
      console.error(`${slug}: FAILED — ${e.message}`);
    }
    await new Promise(r => setTimeout(r, 800)); // be polite
  }
  fs.writeFileSync(path.join(outDir, '_summary.json'), JSON.stringify(summary, null, 2));
})();