← back to Dw Validator Debug TK11314
auto-data-snapshot: 2026-08-31T13:51:21 (1 data files) — DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000
7fc79f2d80bd4464dc3db8e46c3d210f611d4d7e · 2026-08-31 13:53:23 -0700 · auto-commit-fleet
Files touched
A DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000
Diff
commit 7fc79f2d80bd4464dc3db8e46c3d210f611d4d7e
Author: auto-commit-fleet <steve@designerwallcoverings.com>
Date: Mon Aug 31 13:53:23 2026 -0700
auto-data-snapshot: 2026-08-31T13:51:21 (1 data files) — DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000
---
...a-evo-pvcfree-scraper.js.backup-20260831-132000 | 284 +++++++++++++++++++++
1 file changed, 284 insertions(+)
diff --git a/DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000 b/DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000
new file mode 100644
index 00000000..386aa228
--- /dev/null
+++ b/DW-Programming/versa-evo-pvcfree-scraper.js.backup-20260831-132000
@@ -0,0 +1,284 @@
+const { chromium } = require('playwright');
+const { Client } = require('pg');
+
+const DB_URL = (process.env.DATABASE_URL || 'postgresql://dw_admin@127.0.0.1:5432/dw_unified');
+const BASE_URL = 'https://www.versadesignedsurfaces.com';
+const COLLECTION_NAME = 'Versa 20 oz Evo PVC-free';
+
+// 25 products from collection listing page (pre-extracted via WebFetch)
+const KNOWN_PRODUCTS = [
+ { pattern: 'Kaolin', sku: 'AVF29-128', slug: 'kaolin-afv29-usa/avf29-128' },
+ { pattern: 'Alsace', sku: 'AVF23-898-89', slug: 'alsace-avf23-usa/avf23-898-89' },
+ { pattern: 'Lithic Texture', sku: 'AVF11-310-79', slug: 'lithic-texture-avf11-usa/avf11-310-79' },
+ { pattern: 'Lithic', sku: 'AVF12-827-80', slug: 'lithic-avf12-usa/avf12-827-80' },
+ { pattern: 'Elka Texture', sku: 'AVF05-837-74', slug: 'elka-texture-avf05-usa/avf05-837-74' },
+ { pattern: 'Elka', sku: 'AVF09-024-77', slug: 'elka-avf09-usa/avf09-024-77' },
+ { pattern: 'Barege', sku: 'AVF15-885-83', slug: 'barege-avf15-usa/avf15-885-83' },
+ { pattern: 'High Plains', sku: 'AVF06-773-75', slug: 'high-plains-avf06-usa/avf06-773-75' },
+ { pattern: 'Barege Texture', sku: 'AVF14-885-82', slug: 'barege-texture-avf14-usa/avf14-885-82' },
+ { pattern: 'Via Monte', sku: 'AVF17-530-85', slug: 'via-monte-avf17-usa/avf17-530-85' },
+ { pattern: 'Terra', sku: 'AVF01-153-70', slug: 'terra-avf01-usa/avf01-153-70' },
+ { pattern: 'Tesoro', sku: 'AVF04-153-73', slug: 'tesoro-avf04-usa/avf04-153-73' },
+ { pattern: 'Terramo', sku: 'AVF24-892-90', slug: 'terramo-avf24-usa/avf24-892-90' },
+ { pattern: 'Sisley', sku: 'AVF18-881-86', slug: 'sisley-avf18-usa/avf18-881-86' },
+ { pattern: 'Mirelli', sku: 'AVF07-531-76', slug: 'mirelli-avf07-usa/avf07-531-76' },
+ { pattern: 'Lino', sku: 'AVF10-378-78', slug: 'lino-avf10-usa/avf10-378-78' },
+ { pattern: 'Linden', sku: 'AVF16-748-84', slug: 'linden-avf16-usa/avf16-748-84' },
+ { pattern: 'Haku', sku: 'AVF21-778-88', slug: 'haku-avf21-usa/avf21-778-88' },
+ { pattern: 'Englewood', sku: 'AVF20-722-87', slug: 'englewood-avf20-usa/avf20-722-87' },
+ { pattern: 'Douro', sku: 'AVF25-205-91', slug: 'douro-avf25-usa/avf25-205-91' },
+ { pattern: 'Cape Town', sku: 'AVF02-024-71', slug: 'cape-town-avf02-usa/avf02-024-71' },
+ { pattern: 'Aspect', sku: 'AVF13-825-81', slug: 'aspect-avf13-usa/avf13-825-81' },
+ { pattern: 'Lenox', sku: 'AVF26-851-92', slug: 'lenox-avf26-usa/avf26-851-92' },
+ { pattern: 'Pallas', sku: 'AVF27-893-93', slug: 'pallas-avf27-usa/avf27-893-93' },
+ { pattern: 'Verda', sku: 'AVF03-874-72', slug: 'verda-avf03-usa/avf03-874-72' },
+];
+
+async function scrapeCollection() {
+ const browser = await chromium.launch({ headless: true });
+ const page = await browser.newPage();
+ const db = new Client({ connectionString: DB_URL });
+ await db.connect();
+
+ const allProducts = [];
+
+ console.log(`Scraping ${KNOWN_PRODUCTS.length} products from Versa Evo PVC-free collection...\n`);
+
+ for (const prod of KNOWN_PRODUCTS) {
+ const detailUrl = `${BASE_URL}/north-america/products/product-detail/${prod.slug}`;
+ console.log(`Scraping: ${prod.pattern} (${prod.sku})`);
+
+ try {
+ await page.goto(detailUrl, { waitUntil: 'domcontentloaded', timeout: 30000 });
+ await page.waitForTimeout(3000); // wait for JS rendering
+
+ const productData = await page.evaluate(() => {
+ const data = { title: '', colorways: [], images: [], specs: {} };
+
+ // Page title / h1
+ const h1 = document.querySelector('h1');
+ if (h1) data.title = h1.textContent.trim();
+
+ // Title from <title> tag as fallback
+ if (!data.title) {
+ const titleTag = document.querySelector('title');
+ if (titleTag) data.title = titleTag.textContent.trim().split('|')[0].trim();
+ }
+
+ // Find all colorway links on the page (sibling colors in this pattern)
+ const seenSkus = new Set();
+ document.querySelectorAll('a[href*="/product-detail/"]').forEach(link => {
+ const href = link.getAttribute('href') || '';
+ // Match AVF SKU pattern at end of URL
+ const m = href.match(/\/(avf[\d]+-[\d-]+)$/i);
+ if (m) {
+ const sku = m[1].toUpperCase();
+ if (!seenSkus.has(sku)) {
+ seenSkus.add(sku);
+ // Try to get color name from link text
+ let colorName = '';
+ const textContent = link.textContent.trim();
+ // Filter out generic nav text
+ if (textContent && textContent.length < 80 && !textContent.includes('\n')) {
+ colorName = textContent;
+ }
+ const img = link.querySelector('img');
+ let imgSrc = '';
+ if (img) imgSrc = img.getAttribute('src') || img.getAttribute('data-src') || '';
+
+ data.colorways.push({ sku, colorName, url: href, imgSrc });
+ }
+ }
+ });
+
+ // Collect all product images
+ document.querySelectorAll('img').forEach(img => {
+ const src = img.getAttribute('src') || img.getAttribute('data-src') || '';
+ if (src && !src.startsWith('data:') &&
+ (src.toLowerCase().includes('avf') || src.toLowerCase().includes('evo') ||
+ src.toLowerCase().includes('pvc-free') || src.includes('csm_'))) {
+ data.images.push(src);
+ }
+ });
+
+ // Try to extract specs from page text
+ const body = document.body?.innerText || '';
+ data.bodySnippet = body.substring(0, 4000);
+
+ return data;
+ });
+
+ // Determine color name from title
+ // Title is like "Kaolin Earthen" — pattern is "Kaolin", color is "Earthen"
+ let colorName = '';
+ const title = productData.title || '';
+ if (title && prod.pattern) {
+ // Remove pattern from title to get color
+ const colorPart = title.replace(new RegExp(`^${prod.pattern}\\s*`, 'i'), '').trim();
+ // Also remove trailing brand/type text
+ colorName = colorPart
+ .replace(/PVC-free/gi, '')
+ .replace(/Evo®?/gi, '')
+ .replace(/Type\s*II/gi, '')
+ .replace(/20\s*oz\.?/gi, '')
+ .replace(/Versa/gi, '')
+ .replace(/Wallcovering/gi, '')
+ .trim();
+ }
+
+ // Get main image URL
+ let imageUrl = '';
+ if (productData.images.length > 0) {
+ imageUrl = productData.images[0];
+ if (imageUrl && !imageUrl.startsWith('http')) {
+ imageUrl = `${BASE_URL}${imageUrl.startsWith('/') ? '' : '/'}${imageUrl}`;
+ }
+ }
+
+ const product = {
+ mfr_sku: prod.sku,
+ pattern_name: prod.pattern,
+ color_name: colorName,
+ collection: COLLECTION_NAME,
+ brand: 'Versa Designed Surfaces',
+ product_type: 'Type II 20 oz PVC-free',
+ width: '54"',
+ weight: '20 oz',
+ material: 'Evo PVC-free',
+ fire_rating: '',
+ finish: '',
+ cleaning_code: '',
+ backing: '',
+ match_type: '',
+ image_url: imageUrl,
+ product_url: detailUrl,
+ manufacturing_location: 'USA',
+ sustainability: 'PVC-free (Evo technology)',
+ };
+
+ allProducts.push(product);
+ console.log(` -> ${prod.pattern} ${colorName || '(no color found)'} | Image: ${imageUrl ? 'YES' : 'NO'} | Colorways on page: ${productData.colorways.length}`);
+
+ // Check for additional colorways on the page (other colors of same pattern)
+ for (const cw of productData.colorways) {
+ if (cw.sku === prod.sku) continue; // skip current
+ // Only include if it's an AVF SKU that starts with same prefix
+ if (!cw.sku.startsWith('AVF')) continue;
+
+ let cwImg = cw.imgSrc || '';
+ if (cwImg && !cwImg.startsWith('http')) {
+ cwImg = `${BASE_URL}${cwImg.startsWith('/') ? '' : '/'}${cwImg}`;
+ }
+
+ // Clean up colorway name
+ let cwColor = cw.colorName || '';
+ if (cwColor) {
+ cwColor = cwColor.replace(prod.pattern, '').replace(/PVC-free/gi, '').replace(/Evo®?/gi, '').trim();
+ }
+
+ const cwUrl = cw.url ? (cw.url.startsWith('http') ? cw.url : `${BASE_URL}${cw.url}`) : '';
+
+ allProducts.push({
+ ...product,
+ mfr_sku: cw.sku,
+ color_name: cwColor,
+ image_url: cwImg,
+ product_url: cwUrl,
+ });
+ console.log(` -> Extra colorway: ${cw.sku} - ${cwColor || '(unknown color)'}`);
+ }
+
+ } catch (err) {
+ console.error(` ERROR: ${err.message}`);
+ // Still insert with what we know from the listing
+ allProducts.push({
+ mfr_sku: prod.sku,
+ pattern_name: prod.pattern,
+ color_name: '',
+ collection: COLLECTION_NAME,
+ brand: 'Versa Designed Surfaces',
+ product_type: 'Type II 20 oz PVC-free',
+ width: '54"',
+ weight: '20 oz',
+ material: 'Evo PVC-free',
+ fire_rating: '',
+ finish: '',
+ cleaning_code: '',
+ backing: '',
+ match_type: '',
+ image_url: '',
+ product_url: `${BASE_URL}/north-america/products/product-detail/${prod.slug}`,
+ manufacturing_location: 'USA',
+ sustainability: 'PVC-free (Evo technology)',
+ });
+ }
+
+ // Rate limit
+ await page.waitForTimeout(1500);
+ }
+
+ await browser.close();
+
+ console.log(`\n--- Upserting ${allProducts.length} products into versa_catalog ---\n`);
+
+ let inserted = 0, updated = 0, errors = 0;
+
+ for (const p of allProducts) {
+ try {
+ const result = await db.query(`
+ INSERT INTO versa_catalog (
+ mfr_sku, pattern_name, color_name, collection, brand, product_type,
+ width, weight, material, fire_rating, finish, cleaning_code, backing,
+ match_type, image_url, product_url, manufacturing_location, sustainability,
+ last_scraped, updated_at
+ ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,NOW(),NOW())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name = COALESCE(NULLIF(EXCLUDED.pattern_name,''), versa_catalog.pattern_name),
+ color_name = COALESCE(NULLIF(EXCLUDED.color_name,''), versa_catalog.color_name),
+ collection = EXCLUDED.collection,
+ brand = EXCLUDED.brand,
+ product_type = EXCLUDED.product_type,
+ width = COALESCE(NULLIF(EXCLUDED.width,''), versa_catalog.width),
+ weight = COALESCE(NULLIF(EXCLUDED.weight,''), versa_catalog.weight),
+ material = COALESCE(NULLIF(EXCLUDED.material,''), versa_catalog.material),
+ image_url = COALESCE(NULLIF(EXCLUDED.image_url,''), versa_catalog.image_url),
+ product_url = COALESCE(NULLIF(EXCLUDED.product_url,''), versa_catalog.product_url),
+ manufacturing_location = COALESCE(NULLIF(EXCLUDED.manufacturing_location,''), versa_catalog.manufacturing_location),
+ sustainability = COALESCE(NULLIF(EXCLUDED.sustainability,''), versa_catalog.sustainability),
+ last_scraped = NOW(),
+ updated_at = NOW()
+ RETURNING (xmax = 0) AS is_insert
+ `, [
+ p.mfr_sku, p.pattern_name, p.color_name, p.collection, p.brand, p.product_type,
+ p.width, p.weight, p.material, p.fire_rating, p.finish, p.cleaning_code, p.backing,
+ p.match_type, p.image_url, p.product_url, p.manufacturing_location, p.sustainability
+ ]);
+
+ if (result.rows[0]?.is_insert) {
+ inserted++;
+ console.log(` INSERT: ${p.mfr_sku} - ${p.pattern_name} ${p.color_name}`);
+ } else {
+ updated++;
+ console.log(` UPDATE: ${p.mfr_sku} - ${p.pattern_name} ${p.color_name}`);
+ }
+ } catch (err) {
+ errors++;
+ console.error(` DB ERROR [${p.mfr_sku}]: ${err.message}`);
+ }
+ }
+
+ console.log(`\n========== RESULTS ==========`);
+ console.log(`Total processed: ${allProducts.length}`);
+ console.log(`Inserted (new): ${inserted}`);
+ console.log(`Updated (existing): ${updated}`);
+ console.log(`Errors: ${errors}`);
+ console.log(`Collection: ${COLLECTION_NAME}`);
+ console.log(`==============================`);
+
+ await db.end();
+}
+
+scrapeCollection().catch(err => {
+ console.error('Fatal error:', err);
+ process.exit(1);
+});
← 23131e83 TK-11047 add verified Carnegie spec bulk runner
·
back to Dw Validator Debug TK11314
·
TK-11047 verify 100 Carnegie spec updates 8406563d →