← back to Wolfgordon Crawl
ROOT-CAUSE FIX: createProduct ships full desc+metafields from crawl; scraper captures room/installation images (was swatch-only)
24bc92721117c7a9358fd604a9baa96af8583c76 · 2026-06-19 16:11:17 -0700 · Steve
Files touched
M push-shopify.jsM scrape-wolfgordon.js
Diff
commit 24bc92721117c7a9358fd604a9baa96af8583c76
Author: Steve <steve@designerwallcoverings.com>
Date: Fri Jun 19 16:11:17 2026 -0700
ROOT-CAUSE FIX: createProduct ships full desc+metafields from crawl; scraper captures room/installation images (was swatch-only)
---
push-shopify.js | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++--
scrape-wolfgordon.js | 23 ++++++++++++++++-
2 files changed, 91 insertions(+), 3 deletions(-)
diff --git a/push-shopify.js b/push-shopify.js
index dd0aa11..9f1eb44 100644
--- a/push-shopify.js
+++ b/push-shopify.js
@@ -177,7 +177,10 @@ async function buildPlan(storeBySku) {
const cat = (await pool.query(`
SELECT id, mfr_sku, dw_sku, pattern_name, color_name, collection, product_type,
width, width_inches, material, price_retail, price_trade, image_url,
- all_images, product_url, discontinued, shopify_product_id
+ all_images, product_url, discontinued, shopify_product_id,
+ -- full crawled enrichment so createProduct ships COMPLETE products
+ repeat_v, repeat_h, match_type, fire_rating, finish, design,
+ color_hex, dominant_color_hex, ai_description, ai_styles, ai_patterns
FROM wolf_gordon_catalog`)).rows;
// Live WG legacy products (the reconcile source) for UPDATE/ARCHIVE matching.
@@ -270,6 +273,65 @@ function widthInches(row) {
return m ? m[1] : null;
}
+// ----------------------------------------------------------------------------
+// FULL-ENRICHMENT builders. ROOT-CAUSE FIX (2026-06-19): the original
+// createProduct() shipped only a `custom.width` metafield and NO description,
+// producing 479 ACTIVE bare-shell products even though the full crawled record
+// (ai_description, material, fire_rating, collection, mfr_sku, colors, specs)
+// was already in wolf_gordon_catalog. These builders pull that crawled data so
+// every product is created COMPLETE. They mirror backfill-enrichment.js so the
+// repair path and the create path can never diverge again.
+// Steve's standing rules honored: never AI-invent (description comes from the
+// crawled ai_description only), word "Wallpaper" banned, never "Unknown".
+function buildDescriptionHtml(row) {
+ let d = (row.ai_description || '').trim();
+ if (!d || d.length < 20) return null; // no crawled description -> none (never AI-invent)
+ d = banWallpaper(d).replace(/</g, '<').replace(/>/g, '>');
+ return `<p>${d}</p>`;
+}
+function firstOfJson(j) { try { const a = typeof j === 'string' ? JSON.parse(j) : j; return Array.isArray(a) && a.length ? String(a[0]) : null; } catch { return null; } }
+function buildFullMetafields(row) {
+ const mf = [];
+ const add = (namespace, key, type, value) => {
+ if (value == null) return;
+ const v = String(value).trim();
+ if (!v || /^unknown$/i.test(v)) return;
+ mf.push({ namespace, key, type, value: banWallpaper(v) });
+ };
+ const wIn = widthInches(row);
+ const widthDisp = row.width ? String(row.width).trim() : (wIn ? `${wIn}"` : null);
+ // global.* (store convention for WG)
+ add('global', 'width', 'single_line_text_field', widthDisp);
+ add('global', 'material', 'single_line_text_field', row.material);
+ add('global', 'color', 'single_line_text_field', row.color_name);
+ add('global', 'Collection', 'single_line_text_field', row.collection);
+ add('global', 'finish', 'single_line_text_field', row.finish);
+ if (row.repeat_v) add('global', 'repeat', 'single_line_text_field', row.repeat_v);
+ add('global', 'title_tag', 'single_line_text_field',
+ `${titleCase(row.pattern_name || '')} - ${titleCase(row.color_name || '')} Wallcovering by Wolf Gordon Wallcoverings`.trim());
+ add('global', 'description_tag', 'single_line_text_field',
+ `Authorized Dealer of ${row.mfr_sku || row.dw_sku} by WOLF GORDON WALLCOVERINGS at Designer Wallcoverings`);
+ // custom.*
+ add('custom', 'width', 'single_line_text_field', wIn ? `${wIn} Inches` : widthDisp);
+ add('custom', 'pattern_name', 'single_line_text_field', row.pattern_name);
+ add('custom', 'manufacturer_sku', 'single_line_text_field', row.mfr_sku);
+ add('custom', 'material', 'multi_line_text_field', row.material);
+ add('custom', 'collection_name', 'single_line_text_field', row.collection);
+ add('custom', 'color', 'single_line_text_field', row.color_name);
+ add('custom', 'fire_rating', 'single_line_text_field', row.fire_rating);
+ add('custom', 'color_hex', 'single_line_text_field', row.color_hex || row.dominant_color_hex);
+ if (row.match_type) add('custom', 'pattern_repeat', 'single_line_text_field', row.match_type);
+ // dwc.* (manufacturer_sku backup + real vendor + pattern/color)
+ add('dwc', 'manufacturer_sku', 'single_line_text_field', row.mfr_sku);
+ add('dwc', 'pattern_name', 'single_line_text_field', row.pattern_name);
+ add('dwc', 'color', 'single_line_text_field', row.color_name);
+ add('dwc', 'brand', 'single_line_text_field', 'Wolf Gordon');
+ // specs.*
+ add('specs', 'style', 'single_line_text_field', firstOfJson(row.ai_styles));
+ add('specs', 'pattern', 'single_line_text_field', firstOfJson(row.ai_patterns) || row.design);
+ return mf;
+}
+
// Create a NEW product (active+published if image+width present, else draft+tag).
async function createProduct(row, { draft } = {}) {
const title = buildTitle(row);
@@ -282,12 +344,16 @@ async function createProduct(row, { draft } = {}) {
if (!width) tags.push('Needs-Width');
if (draft) tags.push('Quote-Only');
+ const descriptionHtml = buildDescriptionHtml(row);
+ const fullMetafields = buildFullMetafields(row);
+
const productSet = {
title,
vendor: VENDOR,
productType: row.product_type || 'Wallcovering',
status: isDraft ? 'DRAFT' : 'ACTIVE',
tags,
+ ...(descriptionHtml ? { descriptionHtml } : {}),
productOptions: [{ name: 'Variant', values: [{ name: 'Yard' }, { name: 'Sample' }] }],
variants: [
{
@@ -303,7 +369,8 @@ async function createProduct(row, { draft } = {}) {
inventoryPolicy: 'CONTINUE',
},
],
- metafields: width ? [{ namespace: 'custom', key: 'width', type: 'single_line_text_field', value: `${width} in` }] : [],
+ // FULL metafield set from the crawled record (was: only custom.width).
+ metafields: fullMetafields,
files: imgs.map(u => ({ originalSource: u, contentType: 'IMAGE' })),
};
diff --git a/scrape-wolfgordon.js b/scrape-wolfgordon.js
index b3bf2c6..edfef69 100644
--- a/scrape-wolfgordon.js
+++ b/scrape-wolfgordon.js
@@ -184,6 +184,24 @@ function parseProductPage(html, url) {
if (!imgMap[key]) imgMap[key] = im[0].replace(/&/g, '&');
}
+ // ROOM / INSTALLATION shot — shared per pattern (not per colorway). Lives under
+ // production/product/installation/<...>/<pattern>.jpg with a signed query string.
+ // ROOT-CAUSE FIX (2026-06-19): the old parser captured ONLY the swatch, so
+ // all_images was always a single image and products published with no room
+ // shot. Capture every distinct installation image and prepend it to all_images
+ // for each colorway, honoring the standing rule "always pull all data + images".
+ // De-dup by image PATH (ignore the resolution/signature query), keeping the
+ // first (largest in srcset order) signed URL per distinct installation file.
+ const roomByPath = {};
+ const roomRe = /https:\/\/cdn2-optimize\.wolfgordon\.com\/production\/product\/installation\/[^"'?\s,)]+\.jpg\?[^"'\s,)]+/g;
+ let rm;
+ while ((rm = roomRe.exec(html)) !== null) {
+ const u = rm[0].replace(/&/g, '&').replace(/,+$/, '');
+ const path = u.split('?')[0];
+ if (!roomByPath[path]) roomByPath[path] = u; // first signed URL wins
+ }
+ const roomImgs = Object.values(roomByPath);
+
// All colorways from swatch aria-labels: "MAO 6080 - Primavera"
const seen = new Set();
const colorways = [];
@@ -208,6 +226,9 @@ function parseProductPage(html, url) {
purl = base.slice(0, base.length - mainSlug.length) + slug(c.sku);
}
const img = imgMap[nk(c.sku)] || cdnImage(c.sku);
+ // all_images = the colorway swatch (primary) + every shared room/installation
+ // shot. De-dup, swatch first so image_url stays the primary product image.
+ const allImgs = [img, ...roomImgs.filter(u => u !== img)];
products.push({
mfr_sku: c.sku,
pattern_name: productName,
@@ -221,7 +242,7 @@ function parseProductPage(html, url) {
match_type: matchType || undefined,
fire_rating: fire || undefined,
image_url: img,
- all_images: JSON.stringify([img]),
+ all_images: JSON.stringify(allImgs),
product_url: purl,
in_stock: true,
});
← e2e3062 add find-draft-candidates audit (8 WG rows fail width gate -
·
back to Wolfgordon Crawl
·
Add targeted room-image re-scrape ($0 HTTP, resumable): re-f 55477d3 →