← back to Designer Wallcoverings
china_seas scraper: capture all_images=[imageUrl] (single-image source) to clear completeness gate
90279f43d98d67d381ed55ccea6a61f43ee92aaf · 2026-06-10 10:43:40 -0700 · Steve
China Seas (Quadrille) lists exactly one image per product on the flat
Wallpaper.php listing (no detail pages / extra angles / colorway swatches).
The scraper never wrote all_images -> 100% empty -> every net-new row failed
the 'always pull all data and images' completeness gate. Persist the single
src as a JSON array so all 2650 rows (incl. 842 net-new) clear the gate on
the next re-crawl. Staged locally; deploy+re-crawl parked for approval.
Files touched
A vendor-scrapers/scrape-chinaseas.js
Diff
commit 90279f43d98d67d381ed55ccea6a61f43ee92aaf
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jun 10 10:43:40 2026 -0700
china_seas scraper: capture all_images=[imageUrl] (single-image source) to clear completeness gate
China Seas (Quadrille) lists exactly one image per product on the flat
Wallpaper.php listing (no detail pages / extra angles / colorway swatches).
The scraper never wrote all_images -> 100% empty -> every net-new row failed
the 'always pull all data and images' completeness gate. Persist the single
src as a JSON array so all 2650 rows (incl. 842 net-new) clear the gate on
the next re-crawl. Staged locally; deploy+re-crawl parked for approval.
---
vendor-scrapers/scrape-chinaseas.js | 162 ++++++++++++++++++++++++++++++++++++
1 file changed, 162 insertions(+)
diff --git a/vendor-scrapers/scrape-chinaseas.js b/vendor-scrapers/scrape-chinaseas.js
new file mode 100644
index 00000000..b0ba7a59
--- /dev/null
+++ b/vendor-scrapers/scrape-chinaseas.js
@@ -0,0 +1,162 @@
+#!/usr/bin/env node
+// ============================================================================
+// China Seas (Quadrille Fabrics) Catalog Scraper — Codename: Cassie
+// ============================================================================
+// Strategy: HTTP fetch of quadrillefabrics.com/Wallpaper.html
+// Products are in div.InternalCollection with fabric-name-description spans
+// REFRESH scrape — china_seas_catalog already has 1,808 products
+// ============================================================================
+
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const https = require('https');
+const { pool, upsertProduct, wait, getCatalogCount, closePool } = require('./scraper-utils');
+
+const TABLE = 'china_seas_catalog';
+const BASE_URL = 'https://www.quadrillefabrics.com';
+const WALLPAPER_URL = `${BASE_URL}/Wallpaper.html`;
+
+let totalInserted = 0;
+let totalSkipped = 0;
+let totalErrors = 0;
+
+function fetchPage(url) {
+ return new Promise((resolve, reject) => {
+ https.get(url, {
+ timeout: 30000,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+ 'Accept': 'text/html,application/xhtml+xml',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ }
+ }, (res) => {
+ if (res.statusCode === 301 || res.statusCode === 302) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : `${BASE_URL}${res.headers.location}`;
+ return fetchPage(loc).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', chunk => data += chunk);
+ res.on('end', () => resolve(data));
+ res.on('error', reject);
+ }).on('error', reject);
+ });
+}
+
+function parseWallpaperPage(html) {
+ const products = [];
+ const blockRegex = /<div\s+class="InternalCollection">\s*<span\s+class="fabric-name-description">([\s\S]*?)<\/span>\s*<span\s+class="fabric-name-description">([\s\S]*?)<\/span>\s*<\/div>/gi;
+
+ let match;
+ while ((match = blockRegex.exec(html)) !== null) {
+ const imgBlock = match[1];
+ const textBlock = match[2];
+
+ if (textBlock.includes('Click to see full size')) continue;
+
+ const imgMatch = imgBlock.match(/src="([^"]+)"/i);
+ const imageUrl = imgMatch ? imgMatch[1] : null;
+
+ const lines = textBlock
+ .replace(/<br\s*\/?>/gi, '\n')
+ .replace(/<[^>]*>/g, '')
+ .split('\n')
+ .map(l => l.trim())
+ .filter(l => l.length > 0);
+
+ if (lines.length < 2) continue;
+
+ const sku = lines[lines.length - 1].trim();
+ const patternName = lines[0].trim();
+ const colorName = lines.length > 2 ? lines.slice(1, -1).join(' ').trim() : null;
+
+ if (!sku || sku.length < 3 || sku.length > 50) continue;
+ if (sku.includes('Click') || sku.includes('click')) continue;
+
+ // Images — China Seas (Quadrille) lists exactly ONE image per product on
+ // the Wallpaper listing page; there are no detail pages, extra angles, or
+ // per-colorway swatch sets. Persist that single src as a JSON array string
+ // in all_images so the row clears the "always pull all data and images"
+ // completeness gate (was previously never written -> 100% empty all_images).
+ products.push({
+ mfr_sku: sku,
+ pattern_name: patternName || null,
+ color_name: colorName || null,
+ collection: 'China Seas',
+ product_type: 'wallcovering',
+ image_url: imageUrl || null,
+ all_images: imageUrl ? JSON.stringify([imageUrl]) : null,
+ source_url: WALLPAPER_URL,
+ });
+ }
+
+ return products;
+}
+
+async function main() {
+ console.log('='.repeat(70));
+ console.log(' CHINA SEAS (QUADRILLE) WALLPAPER SCRAPER - Cassie');
+ console.log('='.repeat(70));
+ console.log(` Source: ${WALLPAPER_URL}`);
+ console.log(` Table: ${TABLE}`);
+ console.log('='.repeat(70));
+
+ const startTime = Date.now();
+ const beforeCount = await getCatalogCount(TABLE);
+ console.log(`\n Current products in ${TABLE}: ${beforeCount}`);
+
+ console.log('\n[1] Fetching wallpaper catalog page...');
+ let html;
+ try {
+ html = await fetchPage(WALLPAPER_URL);
+ console.log(` Page fetched: ${(html.length / 1024).toFixed(0)} KB`);
+ } catch (err) {
+ console.error(` FATAL: Could not fetch page: ${err.message}`);
+ await closePool();
+ process.exit(1);
+ }
+
+ console.log('\n[2] Parsing products...');
+ const products = parseWallpaperPage(html);
+ console.log(` Found ${products.length} wallpaper products`);
+
+ console.log('\n Sample products:');
+ for (const p of products.slice(0, 5)) {
+ console.log(` ${p.mfr_sku.padEnd(20)} | ${(p.pattern_name || '').padEnd(30)} | ${(p.color_name || '').padEnd(25)} | img: ${p.image_url ? 'YES' : 'no'}`);
+ }
+
+ console.log(`\n[3] Upserting ${products.length} products to ${TABLE}...`);
+ for (let i = 0; i < products.length; i++) {
+ const p = products[i];
+ try {
+ const id = await upsertProduct(TABLE, p);
+ if (id) totalInserted++;
+ else totalSkipped++;
+ } catch (err) {
+ totalErrors++;
+ if (totalErrors <= 5) console.error(` Error for ${p.mfr_sku}: ${err.message}`);
+ }
+ if ((i + 1) % 200 === 0) console.log(` Progress: ${i + 1}/${products.length} (inserted: ${totalInserted})`);
+ }
+
+ const afterCount = await getCatalogCount(TABLE);
+ const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+
+ console.log('\n' + '='.repeat(70));
+ console.log(' CHINA SEAS SCRAPE COMPLETE');
+ console.log('='.repeat(70));
+ console.log(` Products found: ${products.length}`);
+ console.log(` Inserted/updated: ${totalInserted}`);
+ console.log(` Skipped: ${totalSkipped}`);
+ console.log(` Errors: ${totalErrors}`);
+ console.log(` Before: ${beforeCount}`);
+ console.log(` After: ${afterCount}`);
+ console.log(` Net new: ${afterCount - beforeCount}`);
+ console.log(` Duration: ${elapsed}s`);
+ console.log('='.repeat(70));
+
+ await closePool();
+}
+
+main().catch(err => {
+ console.error('FATAL:', err);
+ closePool().then(() => process.exit(1));
+});
← 6b33ebc1 PJ vendor_catalog spec scraper: tear-sheet specs for all phi
·
back to Designer Wallcoverings
·
PJ scraper pass 2: fix detail--row regex (ng-if attrs hid Fi 426995c5 →