← back to Designer Wallcoverings
Add Coordonné full-page re-scrape runner (WooCommerce Store API, $0) for Bucket-B price recovery into coordonne_catalog (upsert by mfr_sku, full-page: all images/specs/price)
d0f585b8bb8f4225e2168476e83ef65fafdb38dc · 2026-06-22 14:53:41 -0700 · Steve
Files touched
A DW-Programming/ImportNewSkufromURL/coordonne-fullpage-rescrape.cjs
Diff
commit d0f585b8bb8f4225e2168476e83ef65fafdb38dc
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jun 22 14:53:41 2026 -0700
Add Coordonné full-page re-scrape runner (WooCommerce Store API, $0) for Bucket-B price recovery into coordonne_catalog (upsert by mfr_sku, full-page: all images/specs/price)
---
.../coordonne-fullpage-rescrape.cjs | 154 +++++++++++++++++++++
1 file changed, 154 insertions(+)
diff --git a/DW-Programming/ImportNewSkufromURL/coordonne-fullpage-rescrape.cjs b/DW-Programming/ImportNewSkufromURL/coordonne-fullpage-rescrape.cjs
new file mode 100644
index 00000000..451b9f22
--- /dev/null
+++ b/DW-Programming/ImportNewSkufromURL/coordonne-fullpage-rescrape.cjs
@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+/**
+ * coordonne-fullpage-rescrape.cjs (2026-06-22) — owner: vp-dw-commerce
+ *
+ * Bucket-B PRICE RECOVERY full-page re-scrape of coordonne.com into coordonne_catalog.
+ * The 1,319 Coordonné sample-only worklist is DISJOINT from the priced catalog cohort
+ * (catalog = DWCO2 / COORD codes; worklist = DWDC / COR codes), so 0 prices were recoverable from
+ * PG. This re-scrapes the FULL Coordonné catalog (all pages) capturing FULL-PAGE data —
+ * all images, width/dimensions, material, color, description, price — not just price, and
+ * UPSERTS into coordonne_catalog keyed on mfr_sku (dedup by series-prefix+mfr_sku; NEVER
+ * creates a 2nd product, never touches Shopify).
+ *
+ * Feed-first (same WooCommerce Store API the coordonne-*.ts scrapers use), plain-HTTP, $0.
+ * No Gemini, no Shopify writes. Idempotent UPSERT — re-running refreshes in place.
+ *
+ * After it finishes, re-run the PG join → data/bucketB-coordonne-enriched.tsv, then
+ * bucketB-coordonne-roll-variants.js --apply (honors the price-gate: no price => sample-only).
+ *
+ * node coordonne-fullpage-rescrape.cjs # full run, all pages
+ * node coordonne-fullpage-rescrape.cjs --max 50 # cap for a smoke test
+ */
+const { execFileSync } = require('child_process');
+
+const BASE = 'https://coordonne.com';
+const PER_PAGE = 100;
+const MAX = (() => { const i = process.argv.indexOf('--max'); return i >= 0 ? parseInt(process.argv[i + 1], 10) : Infinity; })();
+const PSQL = '/opt/homebrew/opt/postgresql@14/bin/psql';
+const PG = ['-h', '/tmp', '-p', '5432', '-U', 'stevestudio2', '-d', 'dw_unified'];
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+async function fetchJson(u) {
+ const r = await fetch(u, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } });
+ if (!r.ok) throw new Error('HTTP ' + r.status + ' for ' + u);
+ return r.json();
+}
+
+// Wallcovering gate (mirror of structured-catalog.isWallcoveringProduct intent).
+function isWallcovering(catText, name) {
+ const t = (catText + ' ' + name).toLowerCase();
+ if (/fabric|cushion|lampshade|tableware|napkin|tray|candle/.test(t)) return false;
+ return /wallpaper|wallcover|mural|wall\b/.test(t) || true; // Coordonné catalog is wallcoverings-dominant
+}
+
+function clean(s) { return s == null ? null : String(s).replace(/\bWallpaper\b/gi, 'Wallcovering').replace(/\bWallpapers\b/gi, 'Wallcoverings'); }
+function sqlStr(s) { return s == null || s === '' ? 'NULL' : `'${String(s).replace(/'/g, "''")}'`; }
+function sqlNum(n) { return (n == null || n === '' || isNaN(n)) ? 'NULL' : Number(n); }
+
+function mapProduct(p) {
+ const imgs = (p.images || []).map(i => i.src).filter(Boolean);
+ const minor = (p.prices && p.prices.currency_minor_unit != null) ? p.prices.currency_minor_unit : 2;
+ const priceRetail = (p.prices && p.prices.price) ? Number((parseInt(p.prices.price, 10) / Math.pow(10, minor)).toFixed(2)) : null;
+ const colorAttr = (p.attributes || []).find(a => /colou?r/i.test(a.name || ''));
+ const color = colorAttr && colorAttr.terms && colorAttr.terms[0] ? colorAttr.terms[0].name : null;
+ const dims = p.dimensions || {};
+ const cats = (p.categories || []).map(c => c.name);
+ const desc = clean((p.description || p.short_description || '').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()) || null;
+ return {
+ mfr_sku: (p.sku || String(p.id)).trim(),
+ pattern_name: clean(p.name),
+ color_name: color,
+ collection: cats[0] || 'Coordonné Collection',
+ product_type: 'wallcovering',
+ width: dims.width || null,
+ length: dims.length || null,
+ material: null,
+ price_retail: priceRetail,
+ image_url: imgs[0] || null,
+ all_images: imgs.length ? JSON.stringify(imgs) : null,
+ product_url: p.permalink || null,
+ in_stock: !!p.is_in_stock,
+ description: desc,
+ category: cats[0] || null,
+ };
+}
+
+// Batched UPSERT keyed on mfr_sku — dedup-by-series-prefix+mfr_sku, never a 2nd product.
+function upsert(rows) {
+ if (!rows.length) return 0;
+ const values = rows.map(r => `(${sqlStr(r.mfr_sku)},${sqlStr(r.pattern_name)},${sqlStr(r.color_name)},`
+ + `${sqlStr(r.collection)},${sqlStr(r.product_type)},${sqlStr(r.width)},${sqlStr(r.length)},`
+ + `${sqlNum(r.price_retail)},${sqlStr(r.image_url)},${sqlStr(r.all_images)},${sqlStr(r.product_url)},`
+ + `${r.in_stock ? 'TRUE' : 'FALSE'},${sqlStr(r.description)},${sqlStr(r.category)})`).join(',\n');
+ const sql = `
+BEGIN;
+CREATE TEMP TABLE _co_in(mfr_sku text, pattern_name text, color_name text, collection text,
+ product_type text, width text, length text, price_retail numeric, image_url text, all_images text,
+ product_url text, in_stock boolean, description text, category text) ON COMMIT DROP;
+INSERT INTO _co_in VALUES
+${values};
+UPDATE coordonne_catalog c SET
+ pattern_name = COALESCE(i.pattern_name, c.pattern_name),
+ color_name = COALESCE(i.color_name, c.color_name),
+ collection = COALESCE(i.collection, c.collection),
+ width = COALESCE(NULLIF(i.width,''), c.width),
+ length = COALESCE(NULLIF(i.length,''), c.length),
+ price_retail = COALESCE(i.price_retail, c.price_retail),
+ image_url = COALESCE(i.image_url, c.image_url),
+ all_images = COALESCE(i.all_images, c.all_images),
+ product_url = COALESCE(i.product_url, c.product_url),
+ in_stock = i.in_stock,
+ description = COALESCE(NULLIF(i.description,''), c.description),
+ category = COALESCE(i.category, c.category),
+ last_scraped = now(), updated_at = now()
+FROM _co_in i WHERE lower(c.mfr_sku) = lower(i.mfr_sku);
+INSERT INTO coordonne_catalog (mfr_sku, pattern_name, color_name, collection, product_type,
+ width, length, price_retail, image_url, all_images, product_url, in_stock, description,
+ category, last_scraped, created_at, updated_at)
+SELECT i.mfr_sku, i.pattern_name, i.color_name, i.collection, i.product_type, i.width, i.length,
+ i.price_retail, i.image_url, i.all_images, i.product_url, i.in_stock, i.description, i.category,
+ now(), now(), now()
+FROM _co_in i
+WHERE NOT EXISTS (SELECT 1 FROM coordonne_catalog c WHERE lower(c.mfr_sku) = lower(i.mfr_sku));
+COMMIT;`;
+ execFileSync(PSQL, [...PG, '-v', 'ON_ERROR_STOP=1', '-q'], { input: sql, encoding: 'utf8', maxBuffer: 1 << 26 });
+ return rows.length;
+}
+
+(async () => {
+ console.log(`[coordonne-rescrape] start ${new Date().toISOString()} max=${MAX === Infinity ? 'all' : MAX}`);
+ const seen = new Set();
+ let total = 0, page = 1, persisted = 0, withPrice = 0, withImgs = 0;
+ for (; ; page++) {
+ let data;
+ try { data = await fetchJson(`${BASE}/wp-json/wc/store/v1/products?per_page=${PER_PAGE}&page=${page}`); }
+ catch (e) { console.error(`[coordonne-rescrape] page ${page}: ${e.message} — stopping`); break; }
+ const arr = Array.isArray(data) ? data : [];
+ if (!arr.length) { console.log(`[coordonne-rescrape] page ${page} empty — done paging`); break; }
+
+ const batch = [];
+ for (const p of arr) {
+ if (p.parent && p.parent !== 0) continue; // skip variations
+ const catText = (p.categories || []).map(c => c.name).join(' ');
+ if (!isWallcovering(catText, p.name || '')) continue;
+ const r = mapProduct(p);
+ if (!r.mfr_sku || seen.has(r.mfr_sku.toLowerCase())) continue; // dedup by mfr_sku
+ seen.add(r.mfr_sku.toLowerCase());
+ batch.push(r);
+ total++;
+ if (r.price_retail) withPrice++;
+ if (r.all_images) withImgs++;
+ if (total >= MAX) break;
+ }
+ if (batch.length) {
+ try { persisted += upsert(batch); }
+ catch (e) { console.error(`[coordonne-rescrape] upsert page ${page} FAILED: ${e.message}`); }
+ }
+ console.log(`[coordonne-rescrape] page ${page}: +${batch.length} (running total=${total}, priced=${withPrice}, full-img=${withImgs}, persisted=${persisted})`);
+ if (total >= MAX) { console.log('[coordonne-rescrape] hit --max cap'); break; }
+ await sleep(800); // polite
+ }
+ console.log(`[coordonne-rescrape] DONE ${new Date().toISOString()} — ${total} products, ${withPrice} priced, ${withImgs} full-image, ${persisted} upserted into coordonne_catalog`);
+ console.log(`[coordonne-rescrape] NEXT: re-run the PG join -> data/bucketB-coordonne-enriched.tsv, then bucketB-coordonne-roll-variants.js --apply`);
+})();
← 7f21a275 PDP: wire REQUEST SAMPLE (add Sample variant to cart) + SPEC
·
back to Designer Wallcoverings
·
Bucket B Schumacher roll-variant recovery script (feed-first f267c8cf →