← back to Rebel Walls Scraper

initial scaffold: rebel-walls scraper + skill

2142acaa88d924cabf1ba823e161c8d5df2ea8ab · 2026-06-03 09:01:39 -0700 · Steve

Files touched

• A .gitignore
• A README.md
• A scripts/scrape.js

Diff

commit 2142acaa88d924cabf1ba823e161c8d5df2ea8ab
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jun 3 09:01:39 2026 -0700

    initial scaffold: rebel-walls scraper + skill
---
 .gitignore        |   8 ++
 README.md         |  12 +++
 scripts/scrape.js | 274 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 294 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..dd83fed
--- /dev/null
+++ b/README.md
@@ -0,0 +1,12 @@
+# Rebel Walls Scraper
+
+Public-storefront scraper for rebelwalls.com (designer wall murals). No auth/trade portal.
+Custom-dimension murals priced per m² (DTD Option A — see skill SKILL.md).
+
+## Usage
+    node scripts/scrape.js enumerate          # sitemap -> data/product-urls.json
+    node scripts/scrape.js pilot 25           # scrape 25 -> output/pilot-<date>.json
+    node scripts/scrape.js fullproduct output/pilot-<date>.json   # quality gate
+
+Pilot only. NO import to dw_unified/Shopify (gated — see ~/.claude/yolo-queue/pending-approval/).
+Skill: ~/.claude/skills/rebel-walls-scraper-manager/
diff --git a/scripts/scrape.js b/scripts/scrape.js
new file mode 100644
index 0000000..2c80e59
--- /dev/null
+++ b/scripts/scrape.js
@@ -0,0 +1,274 @@
+#!/usr/bin/env node
+/**
+ * Rebel Walls scraper — public storefront, no auth.
+ * Modes:
+ *   node scripts/scrape.js enumerate            -> data/product-urls.json
+ *   node scripts/scrape.js pilot <N> [offset]   -> output/pilot-<date>.json
+ *   node scripts/scrape.js fullproduct <json>   -> data-quality scorecard
+ *
+ * Zero-dependency: uses global fetch (Node 18+). Polite: 1.5s delay, single-threaded.
+ * Pricing model = DTD Option A (per-m² base rate; see SKILL.md).
+ */
+'use strict';
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.resolve(__dirname, '..');
+const DATA = path.join(ROOT, 'data');
+const OUT = path.join(ROOT, 'output');
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
+const BASE = 'https://rebelwalls.com';
+const DELAY_MS = 1500;
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const today = () => new Date().toISOString().slice(0, 10);
+
+async function get(url) {
+  const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept-Language': 'en-US,en' } });
+  if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
+  return res.text();
+}
+
+/* ---- enumerate: sitemap -> product slugs (drop categories/blog) ---- */
+// Category/collection/blog pages we must NOT treat as products.
+const NON_PRODUCT_SUFFIX = /-wallpaper$|-collection$|-murals$|-guide$|-info$/;
+const NON_PRODUCT_KEYWORDS = /(\bblog\b|how-to|ways-of|making-your|ideas$|inspiration|about|contact|returns|faq|trade|sample|wallpaper-for|-tips|guide-to)/i;
+
+async function enumerate() {
+  console.error('Fetching sitemap…');
+  const xml = await get(`${BASE}/sitemap.xml`);
+  const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1].trim());
+  // Only root-level single-segment slugs are candidate products.
+  const candidates = locs.filter((u) => {
+    if (!u.startsWith(BASE + '/')) return false;
+    const slug = u.slice(BASE.length + 1);
+    if (!slug || slug.includes('/')) return false; // skip locale/nested
+    if (NON_PRODUCT_SUFFIX.test(slug)) return false;
+    if (NON_PRODUCT_KEYWORDS.test(slug)) return false;
+    return true;
+  });
+  const uniq = [...new Set(candidates)];
+  fs.mkdirSync(DATA, { recursive: true });
+  fs.writeFileSync(path.join(DATA, 'product-urls.json'), JSON.stringify(uniq, null, 2));
+  console.error(`Enumerated ${uniq.length} candidate product URLs (from ${locs.length} sitemap locs) -> data/product-urls.json`);
+  return uniq;
+}
+
+/* ---- parse a single product page ---- */
+function decode(s) {
+  if (s == null) return s;
+  // unescape \/ and \uXXXX then strip HTML tags + html entities
+  let out = s.replace(/\\\//g, '/');
+  out = out.replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16)));
+  out = out.replace(/<[^>]+>/g, '');
+  out = out.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+           .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, ' ');
+  return out.trim();
+}
+
+function jsonLdProduct(html) {
+  for (const m of html.matchAll(/<script type="application\/ld\+json">(.*?)<\/script>/gs)) {
+    try {
+      const d = JSON.parse(m[1]);
+      if (d['@type'] === 'Product') return d;
+    } catch (_) {}
+  }
+  return null;
+}
+
+function specValue(html, label) {
+  // {"label":"Collection","value":"Shutterstock"}
+  const re = new RegExp('\\{"label":"' + label.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') + '","value":"((?:[^"\\\\]|\\\\.)*)"\\}');
+  const m = html.match(re);
+  return m ? decode(m[1]) : '';
+}
+
+function specItem(html, header) {
+  // {"header":"Roll Width","materials":{"std":"<p>19.7 in / 0.5 m</p>",...
+  const re = new RegExp('"header":"' + header.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&') + '","materials":\\{"std":"((?:[^"\\\\]|\\\\.)*)"');
+  const m = html.match(re);
+  return m ? decode(m[1]) : '';
+}
+
+function priceTier(html, key) {
+  // "description":"non-woven-text","price":"$7.10 / sq ft","defaultPrice":"$7.10"
+  const re = new RegExp('"description":"' + key + '-text","price":"([^"]+)"');
+  const m = html.match(re);
+  if (!m) return null;
+  const num = (m[1].match(/([0-9]+(?:\.[0-9]+)?)/) || [])[1];
+  return num ? parseFloat(num) : null;
+}
+
+function hiResImage(url) {
+  if (!url) return url;
+  // strip the watermark layer + bump width; keep auto format/quality.
+  let u = url.replace(/l_common:rw_watermark[^/]*\//, '');
+  u = u.replace(/[?,]?w_\d+/g, '').replace(/,h_\d+/g, '');
+  // ensure a clean wide transform
+  u = u.replace('/image/upload/', '/image/upload/c_limit,w_1600,f_auto,q_auto/');
+  // collapse accidental double-transform
+  u = u.replace(/c_limit,w_1600,f_auto,q_auto\/c_[^/]+\//, 'c_limit,w_1600,f_auto,q_auto/');
+  return u;
+}
+
+function parseColorways(html) {
+  const m = html.match(/"colorways":(\[.*?\}\])/s);
+  if (!m) return [];
+  try {
+    const arr = JSON.parse(m[1].replace(/\\\//g, '/'));
+    return arr.map((c) => ({ no: c.no, name: decode(c.name), link: c.link }));
+  } catch (_) { return []; }
+}
+
+function splitName(name) {
+  // "Abstract Blossoms, Brown" -> pattern + color (split on LAST comma)
+  const i = name.lastIndexOf(',');
+  if (i === -1) return { pattern: name.trim(), color: '' };
+  return { pattern: name.slice(0, i).trim(), color: name.slice(i + 1).trim() };
+}
+
+async function parseProduct(url) {
+  const html = await get(url);
+  const ld = jsonLdProduct(html);
+  if (!ld) throw new Error(`no JSON-LD Product at ${url}`);
+  const { pattern, color } = splitName(ld.name || '');
+  const basePerM2 = ld.offers && ld.offers.price ? parseFloat(ld.offers.price) : null;
+  const sqftNonwoven = priceTier(html, 'non-woven');
+  const sqftPeel = priceTier(html, 'peel-and-stick');
+  const sqftComm = priceTier(html, 'commercial-grade');
+
+  // gather interior/room images from inline state
+  const slug = url.slice(BASE.length);
+  const mfr = ld.sku || '';
+  const imgs = [...new Set([...html.matchAll(/https:\\?\/\\?\/res\.rebelwalls\.com\/[^"\\]+/g)]
+    .map((m) => m[0].replace(/\\\//g, '/'))
+    .filter((u) => mfr && u.includes(mfr)))];
+  const interior = imgs.filter((u) => /_interior\d*\.jpg/i.test(u)).map(hiResImage);
+  const primary = hiResImage(ld.image || interior[0] || imgs[0] || '');
+
+  return {
+    mfr_sku: mfr,
+    pattern_name: pattern,
+    color_name: color,
+    full_name: ld.name || '',
+    collection: specValue(html, 'Collection'),
+    base_unit: specValue(html, 'Base Unit'),
+    product_type: 'Mural',
+    material: 'Non-woven',
+    roll_width: specItem(html, 'Roll Width'),
+    grammage: specItem(html, 'Grammage'),
+    fire_rating: specItem(html, 'Fire Rating'),
+    light_fastness: specItem(html, 'Light Fastness'),
+    cleanability: specItem(html, 'Cleanability'),
+    sustainability: specItem(html, 'Sustainability'),
+    paper_quality: specItem(html, 'Paper Quality'),
+    manufacturer: specItem(html, 'Manufacturer'),
+    repeat_h: specValue(html, 'Horizontal Repeat'),
+    repeat_v: specValue(html, 'Vertical Repeat'),
+    match_type: specValue(html, 'Pattern Repeat'),
+    // ---- Pricing: DTD Option A ----
+    price_retail: basePerM2,                 // per-m² base (non-woven)
+    price_unit: 'Priced Per Square Meter',
+    display_dimension: '1 m² (≈10.76 sq ft)',
+    price_per_sqft_nonwoven: sqftNonwoven,
+    price_per_sqft_peelstick: sqftPeel,
+    price_per_sqft_commercial: sqftComm,
+    currency: (ld.offers && ld.offers.priceCurrency) || 'USD',
+    in_stock: !ld.offers || /InStock/i.test(ld.offers.availability || ''),
+    image_url: primary,
+    all_images: imgs.map(hiResImage),
+    room_setting_images: interior,
+    colorways: parseColorways(html),
+    description: decode(ld.description || ''),
+    product_url: url,
+    last_scraped: new Date().toISOString(),
+  };
+}
+
+/* ---- pilot ---- */
+async function pilot(n, offset) {
+  const urlsPath = path.join(DATA, 'product-urls.json');
+  let urls;
+  if (fs.existsSync(urlsPath)) urls = JSON.parse(fs.readFileSync(urlsPath, 'utf8'));
+  else urls = await enumerate();
+  const slice = urls.slice(offset || 0, (offset || 0) + n);
+  const results = [];
+  for (let i = 0; i < slice.length; i++) {
+    const u = slice[i];
+    process.stderr.write(`[${i + 1}/${slice.length}] ${u} … `);
+    try {
+      const p = await parseProduct(u);
+      results.push(p);
+      process.stderr.write(`OK ${p.mfr_sku} $${p.price_retail}/m²\n`);
+    } catch (e) {
+      results.push({ product_url: u, error: String(e.message || e) });
+      process.stderr.write(`ERR ${e.message}\n`);
+    }
+    if (i < slice.length - 1) await sleep(DELAY_MS);
+  }
+  fs.mkdirSync(OUT, { recursive: true });
+  const outFile = path.join(OUT, `pilot-${today()}.json`);
+  fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
+  console.error(`\nWrote ${results.length} products -> ${outFile}`);
+  return outFile;
+}
+
+/* ---- fullproduct gate ---- */
+function fullproduct(jsonPath) {
+  const rows = JSON.parse(fs.readFileSync(jsonPath, 'utf8')).filter((r) => !r.error);
+  const total = rows.length;
+  const nonEmpty = (v) => v != null && String(v).trim() !== '' && !(Array.isArray(v) && v.length === 0);
+  const pct = (n) => `${n}/${total} (${total ? Math.round((n / total) * 100) : 0}%)`;
+  const tier1 = {
+    image_url: rows.filter((r) => nonEmpty(r.image_url)).length,
+    'roll_width(width)': rows.filter((r) => nonEmpty(r.roll_width)).length,
+    mfr_sku: rows.filter((r) => nonEmpty(r.mfr_sku)).length,
+    pattern_name: rows.filter((r) => nonEmpty(r.pattern_name)).length,
+    price_retail: rows.filter((r) => r.price_retail > 0).length,
+  };
+  const tier2 = {
+    all_images: rows.filter((r) => nonEmpty(r.all_images)).length,
+    room_setting_images: rows.filter((r) => nonEmpty(r.room_setting_images)).length,
+    material: rows.filter((r) => nonEmpty(r.material)).length,
+    color_name: rows.filter((r) => nonEmpty(r.color_name)).length,
+    collection: rows.filter((r) => nonEmpty(r.collection)).length,
+    product_url: rows.filter((r) => nonEmpty(r.product_url)).length,
+    fire_rating: rows.filter((r) => nonEmpty(r.fire_rating)).length,
+    grammage: rows.filter((r) => nonEmpty(r.grammage)).length,
+  };
+  const tier1Pass = Object.values(tier1).every((v) => v === total);
+  const lines = [];
+  lines.push(`FULLPRODUCT Validation: Rebel Walls (pilot)`);
+  lines.push(`═══════════════════════════════`);
+  lines.push(`Total products: ${total}`);
+  lines.push('');
+  lines.push('TIER 1 (Required — import blocked if any fail):');
+  for (const [k, v] of Object.entries(tier1)) lines.push(`  ${v === total ? '✅' : '❌'} ${k}: ${pct(v)}`);
+  lines.push('');
+  lines.push('TIER 2 (Recommended — warn if <80%):');
+  for (const [k, v] of Object.entries(tier2)) lines.push(`  ${v >= total * 0.8 ? '✅' : '⚠️'} ${k}: ${pct(v)}`);
+  lines.push('');
+  lines.push(`VERDICT: ${tier1Pass ? '✅ READY FOR IMPORT (Tier 1 complete)' : '❌ NOT READY (Tier 1 gap)'}`);
+  const report = lines.join('\n');
+  console.log(report);
+  return { report, tier1, tier2, total, tier1Pass };
+}
+
+/* ---- main ---- */
+(async () => {
+  const [, , mode, a, b] = process.argv;
+  try {
+    if (mode === 'enumerate') await enumerate();
+    else if (mode === 'pilot') await pilot(parseInt(a || '25', 10), parseInt(b || '0', 10));
+    else if (mode === 'fullproduct') fullproduct(a);
+    else {
+      console.error('Usage: scrape.js enumerate | pilot <N> [offset] | fullproduct <json>');
+      process.exit(1);
+    }
+  } catch (e) {
+    console.error('FATAL:', e.message || e);
+    process.exit(1);
+  }
+})();
+
+module.exports = { parseProduct, hiResImage, splitName };

(oldest)  ·  back to Rebel Walls Scraper  ·  Rebel Walls scraper: enumerate 4309 + 25-SKU pilot (fullprod f8fa280 →