`scripts/scrape.js`

363 lines
#!/usr/bin/env node
/**
 * Rebel Walls scraper — public storefront, no auth.
 * Modes:
 *   node scripts/scrape.js enumerate            -> data/product-urls.json
 *   node scripts/scrape.js pilot <N> [offset]   -> output/pilot-<date>.json
 *   node scripts/scrape.js crawl                -> output/crawl-<date>.json (full set, resumable, checkpointed)
 *   node scripts/scrape.js fullproduct <json>   -> data-quality scorecard
 *
 * Zero-dependency: uses global fetch (Node 18+). Polite: 1.5s delay, single-threaded.
 * Pricing model = DTD Option A (per-m² base rate; see SKILL.md).
 */
'use strict';
const fs = require('fs');
const path = require('path');

const ROOT = path.resolve(__dirname, '..');
const DATA = path.join(ROOT, 'data');
const OUT = path.join(ROOT, 'output');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
const BASE = 'https://rebelwalls.com';
const DELAY_MS = 1500;

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const today = () => new Date().toISOString().slice(0, 10);

async function get(url) {
  const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept-Language': 'en-US,en' } });
  if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
  return res.text();
}

/* ---- enumerate: sitemap -> product slugs (drop categories/blog) ---- */
// Category/collection/blog pages we must NOT treat as products.
const NON_PRODUCT_SUFFIX = /-wallpaper$|-collection$|-murals$|-guide$|-info$/;
const NON_PRODUCT_KEYWORDS = /(\bblog\b|how-to|ways-of|making-your|ideas$|inspiration|about|contact|returns|faq|trade|sample|wallpaper-for|-tips|guide-to)/i;

async function enumerate() {
  console.error('Fetching sitemap…');
  const xml = await get(`${BASE}/sitemap.xml`);
  const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim());
  // Only root-level single-segment slugs are candidate products.
  const candidates = locs.filter((u) => {
    if (!u.startsWith(BASE + '/')) return false;
    const slug = u.slice(BASE.length + 1);
    if (!slug || slug.includes('/')) return false; // skip locale/nested
    if (NON_PRODUCT_SUFFIX.test(slug)) return false;
    if (NON_PRODUCT_KEYWORDS.test(slug)) return false;
    return true;
  });
  const uniq = [...new Set(candidates)];
  fs.mkdirSync(DATA, { recursive: true });
  fs.writeFileSync(path.join(DATA, 'product-urls.json'), JSON.stringify(uniq, null, 2));
  console.error(`Enumerated ${uniq.length} candidate product URLs (from ${locs.length} sitemap locs) -> data/product-urls.json`);
  return uniq;
}

/* ---- parse a single product page ---- */
function decode(s) {
  if (s == null) return s;
  // unescape \/ and \uXXXX then strip HTML tags + html entities
  let out = s.replace(/\\\//g, '/');
  out = out.replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
  out = out.replace(/<[^>]+>/g, '');
  out = out.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
           .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ');
  return out.trim();
}

function sanitizeJsonLd(raw) {
  // Some Rebel Walls product titles carry literal control chars (TAB/newline)
  // inside JSON string values (e.g. "Backwood Birds,\tBrown"), which JSON.parse
  // rejects. Escape raw control chars that sit INSIDE a string literal.
  let out = '';
  let inStr = false, esc = false;
  for (let i = 0; i < raw.length; i++) {
    const ch = raw[i];
    const code = raw.charCodeAt(i);
    if (esc) { out += ch; esc = false; continue; }
    if (ch === '\\') { out += ch; esc = true; continue; }
    if (ch === '"') { inStr = !inStr; out += ch; continue; }
    if (inStr && code < 0x20) {
      if (ch === '\t') out += '\\t';
      else if (ch === '\n') out += '\\n';
      else if (ch === '\r') out += '\\r';
      else out += '\\u' + code.toString(16).padStart(4, '0');
      continue;
    }
    out += ch;
  }
  return out;
}

function jsonLdProduct(html) {
  for (const m of html.matchAll(/<script type="application\/ld\+json">(.*?)<\/script>/gs)) {
    let d = null;
    try { d = JSON.parse(m[1]); }
    catch (_) { try { d = JSON.parse(sanitizeJsonLd(m[1])); } catch (_) {} }
    if (d && d['@type'] === 'Product') return d;
  }
  return null;
}

function specValue(html, label) {
  // {"label":"Collection","value":"Shutterstock"}
  const re = new RegExp('\\{"label":"' + label.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') + '","value":"((?:[^"\\\\]|\\\\.)*)"\\}');
  const m = html.match(re);
  return m ? decode(m[1]) : '';
}

function specItem(html, header) {
  // {"header":"Roll Width","materials":{"std":"<p>19.7 in / 0.5 m</p>",...
  const re = new RegExp('"header":"' + header.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') + '","materials":\\{"std":"((?:[^"\\\\]|\\\\.)*)"');
  const m = html.match(re);
  return m ? decode(m[1]) : '';
}

function priceTier(html, key) {
  // "description":"non-woven-text","price":"$7.10 / sq ft","defaultPrice":"$7.10"
  const re = new RegExp('"description":"' + key + '-text","price":"([^"]+)"');
  const m = html.match(re);
  if (!m) return null;
  const num = (m[1].match(/([0-9]+(?:\.[0-9]+)?)/) || [])[1];
  return num ? parseFloat(num) : null;
}

function hiResImage(url) {
  if (!url) return url;
  // strip the watermark layer + bump width; keep auto format/quality.
  let u = url.replace(/l_common:rw_watermark[^/]*\//, '');
  u = u.replace(/[?,]?w_\d+/g, '').replace(/,h_\d+/g, '');
  // ensure a clean wide transform
  u = u.replace('/image/upload/', '/image/upload/c_limit,w_1600,f_auto,q_auto/');
  // collapse accidental double-transform
  u = u.replace(/c_limit,w_1600,f_auto,q_auto\/c_[^/]+\//, 'c_limit,w_1600,f_auto,q_auto/');
  return u;
}

function parseColorways(html) {
  const m = html.match(/"colorways":(\[.*?\}\])/s);
  if (!m) return [];
  try {
    const arr = JSON.parse(m[1].replace(/\\\//g, '/'));
    return arr.map((c) => ({ no: c.no, name: decode(c.name), link: c.link }));
  } catch (_) { return []; }
}

function splitName(name) {
  // "Abstract Blossoms, Brown" -> pattern + color (split on LAST comma)
  const i = name.lastIndexOf(',');
  if (i === -1) return { pattern: name.trim(), color: '' };
  return { pattern: name.slice(0, i).trim(), color: name.slice(i + 1).trim() };
}

async function parseProduct(url) {
  const html = await get(url);
  const ld = jsonLdProduct(html);
  if (!ld) throw new Error(`no JSON-LD Product at ${url}`);
  const { pattern, color } = splitName(ld.name || '');
  const basePerM2 = ld.offers && ld.offers.price ? parseFloat(ld.offers.price) : null;
  const sqftNonwoven = priceTier(html, 'non-woven');
  const sqftPeel = priceTier(html, 'peel-and-stick');
  const sqftComm = priceTier(html, 'commercial-grade');

  // gather interior/room images from inline state
  const slug = url.slice(BASE.length);
  const mfr = ld.sku || '';
  const imgs = [...new Set([...html.matchAll(/https:\\?\/\\?\/res\.rebelwalls\.com\/[^"\\]+/g)]
    .map((m) => m[0].replace(/\\\//g, '/'))
    .filter((u) => mfr && u.includes(mfr)))];
  const interior = imgs.filter((u) => /_interior\d*\.jpg/i.test(u)).map(hiResImage);
  const primary = hiResImage(ld.image || interior[0] || imgs[0] || '');

  return {
    mfr_sku: mfr,
    pattern_name: pattern,
    color_name: color,
    full_name: ld.name || '',
    collection: specValue(html, 'Collection'),
    base_unit: specValue(html, 'Base Unit'),
    product_type: 'Mural',
    material: 'Non-woven',
    roll_width: specItem(html, 'Roll Width'),
    grammage: specItem(html, 'Grammage'),
    fire_rating: specItem(html, 'Fire Rating'),
    light_fastness: specItem(html, 'Light Fastness'),
    cleanability: specItem(html, 'Cleanability'),
    sustainability: specItem(html, 'Sustainability'),
    paper_quality: specItem(html, 'Paper Quality'),
    manufacturer: specItem(html, 'Manufacturer'),
    repeat_h: specValue(html, 'Horizontal Repeat'),
    repeat_v: specValue(html, 'Vertical Repeat'),
    match_type: specValue(html, 'Pattern Repeat'),
    // ---- Pricing: DTD Option A ----
    price_retail: basePerM2,                 // per-m² base (non-woven)
    price_unit: 'Priced Per Square Meter',
    display_dimension: '1 m² (≈10.76 sq ft)',
    price_per_sqft_nonwoven: sqftNonwoven,
    price_per_sqft_peelstick: sqftPeel,
    price_per_sqft_commercial: sqftComm,
    currency: (ld.offers && ld.offers.priceCurrency) || 'USD',
    in_stock: !ld.offers || /InStock/i.test(ld.offers.availability || ''),
    image_url: primary,
    all_images: imgs.map(hiResImage),
    room_setting_images: interior,
    colorways: parseColorways(html),
    description: decode(ld.description || ''),
    product_url: url,
    last_scraped: new Date().toISOString(),
  };
}

/* ---- pilot ---- */
async function pilot(n, offset) {
  const urlsPath = path.join(DATA, 'product-urls.json');
  let urls;
  if (fs.existsSync(urlsPath)) urls = JSON.parse(fs.readFileSync(urlsPath, 'utf8'));
  else urls = await enumerate();
  const slice = urls.slice(offset || 0, (offset || 0) + n);
  const results = [];
  for (let i = 0; i < slice.length; i++) {
    const u = slice[i];
    process.stderr.write(`[${i + 1}/${slice.length}] ${u} … `);
    try {
      const p = await parseProduct(u);
      results.push(p);
      process.stderr.write(`OK ${p.mfr_sku} $${p.price_retail}/m²\n`);
    } catch (e) {
      results.push({ product_url: u, error: String(e.message || e) });
      process.stderr.write(`ERR ${e.message}\n`);
    }
    if (i < slice.length - 1) await sleep(DELAY_MS);
  }
  fs.mkdirSync(OUT, { recursive: true });
  const outFile = path.join(OUT, `pilot-${today()}.json`);
  fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
  console.error(`\nWrote ${results.length} products -> ${outFile}`);
  return outFile;
}

/* ---- crawl: full set, single-threaded, checkpointed + resumable ---- */
async function crawl() {
  const urlsPath = path.join(DATA, 'product-urls.json');
  let urls;
  if (fs.existsSync(urlsPath)) urls = JSON.parse(fs.readFileSync(urlsPath, 'utf8'));
  else urls = await enumerate();

  fs.mkdirSync(OUT, { recursive: true });
  const outFile = path.join(OUT, `crawl-${today()}.json`);

  // Resume support: if outFile exists, skip URLs already captured OK.
  let results = [];
  const done = new Set();
  if (fs.existsSync(outFile)) {
    try {
      results = JSON.parse(fs.readFileSync(outFile, 'utf8'));
      for (const r of results) if (r && r.product_url && !r.error) done.add(r.product_url);
      console.error(`Resume: ${done.size} already captured in ${path.basename(outFile)}; ${results.length} total rows.`);
    } catch (_) { results = []; }
  }
  // Drop prior error rows for URLs we'll retry, then re-add fresh.
  results = results.filter((r) => r && r.product_url && !r.error);

  const todo = urls.filter((u) => !done.has(u));
  console.error(`Crawling ${todo.length} of ${urls.length} URLs (${done.size} already done). ~${Math.round(todo.length * DELAY_MS / 1000 / 60)} min @ ${DELAY_MS}ms.`);

  let ok = 0, err = 0;
  for (let i = 0; i < todo.length; i++) {
    const u = todo[i];
    process.stderr.write(`[${i + 1}/${todo.length}] ${u} … `);
    let attempt = 0, captured = null, lastErr = null;
    while (attempt < 3 && !captured) {
      attempt++;
      try {
        captured = await parseProduct(u);
      } catch (e) {
        lastErr = e;
        if (attempt < 3) { process.stderr.write(`retry${attempt} `); await sleep(DELAY_MS * 2); }
      }
    }
    if (captured) {
      results.push(captured);
      ok++;
      process.stderr.write(`OK ${captured.mfr_sku} $${captured.price_retail}/m²\n`);
    } else {
      results.push({ product_url: u, error: String((lastErr && lastErr.message) || lastErr) });
      err++;
      process.stderr.write(`ERR ${(lastErr && lastErr.message) || lastErr}\n`);
    }
    // Checkpoint every 25 products (and on the last one) so a crash loses ≤25.
    if ((i + 1) % 25 === 0 || i === todo.length - 1) {
      fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
    }
    if (i < todo.length - 1) await sleep(DELAY_MS);
  }
  fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
  console.error(`\nCrawl complete: ${ok} OK, ${err} ERR this run; ${results.length} total rows -> ${outFile}`);
  return outFile;
}

/* ---- fullproduct gate ---- */
function fullproduct(jsonPath) {
  const rows = JSON.parse(fs.readFileSync(jsonPath, 'utf8')).filter((r) => !r.error);
  const total = rows.length;
  const nonEmpty = (v) => v != null && String(v).trim() !== '' && !(Array.isArray(v) && v.length === 0);
  const pct = (n) => `${n}/${total} (${total ? Math.round((n / total) * 100) : 0}%)`;
  const tier1 = {
    image_url: rows.filter((r) => nonEmpty(r.image_url)).length,
    'roll_width(width)': rows.filter((r) => nonEmpty(r.roll_width)).length,
    mfr_sku: rows.filter((r) => nonEmpty(r.mfr_sku)).length,
    pattern_name: rows.filter((r) => nonEmpty(r.pattern_name)).length,
    price_retail: rows.filter((r) => r.price_retail > 0).length,
  };
  const tier2 = {
    all_images: rows.filter((r) => nonEmpty(r.all_images)).length,
    room_setting_images: rows.filter((r) => nonEmpty(r.room_setting_images)).length,
    material: rows.filter((r) => nonEmpty(r.material)).length,
    color_name: rows.filter((r) => nonEmpty(r.color_name)).length,
    collection: rows.filter((r) => nonEmpty(r.collection)).length,
    product_url: rows.filter((r) => nonEmpty(r.product_url)).length,
    fire_rating: rows.filter((r) => nonEmpty(r.fire_rating)).length,
    grammage: rows.filter((r) => nonEmpty(r.grammage)).length,
  };
  const tier1Pass = Object.values(tier1).every((v) => v === total);
  const lines = [];
  lines.push(`FULLPRODUCT Validation: Rebel Walls (pilot)`);
  lines.push(`═══════════════════════════════`);
  lines.push(`Total products: ${total}`);
  lines.push('');
  lines.push('TIER 1 (Required — import blocked if any fail):');
  for (const [k, v] of Object.entries(tier1)) lines.push(`  ${v === total ? '✅' : '❌'} ${k}: ${pct(v)}`);
  lines.push('');
  lines.push('TIER 2 (Recommended — warn if <80%):');
  for (const [k, v] of Object.entries(tier2)) lines.push(`  ${v >= total * 0.8 ? '✅' : '⚠️'} ${k}: ${pct(v)}`);
  lines.push('');
  lines.push(`VERDICT: ${tier1Pass ? '✅ READY FOR IMPORT (Tier 1 complete)' : '❌ NOT READY (Tier 1 gap)'}`);
  const report = lines.join('\n');
  console.log(report);
  return { report, tier1, tier2, total, tier1Pass };
}

/* ---- main (only when run directly, not on require) ---- */
if (require.main === module) {
  (async () => {
    const [, , mode, a, b] = process.argv;
    try {
      if (mode === 'enumerate') await enumerate();
      else if (mode === 'pilot') await pilot(parseInt(a || '25', 10), parseInt(b || '0', 10));
      else if (mode === 'crawl') await crawl();
      else if (mode === 'fullproduct') fullproduct(a);
      else {
        console.error('Usage: scrape.js enumerate | pilot <N> [offset] | crawl | fullproduct <json>');
        process.exit(1);
      }
    } catch (e) {
      console.error('FATAL:', e.message || e);
      process.exit(1);
    }
  })();
}

module.exports = { parseProduct, hiResImage, splitName, jsonLdProduct, sanitizeJsonLd };