← back to Designer Wallcoverings
auto-save: 2026-06-22T21:17:54 (9 files) — data/ scripts/designtex-enhanced-sync.js scripts/designtex-full-scrape.js scripts/designtex-headless-scraper.js scripts/designtex-parallel-scraper.js
8090a85f264c4edff22a660ba73993b41238221b · 2026-06-22 21:17:59 -0700 · Steve Abrams
Files touched
A data/designtex-fresh-scrape-2026-06-23.jsonA data/designtex-scraped.jsonA scripts/designtex-enhanced-sync.jsA scripts/designtex-full-scrape.jsA scripts/designtex-headless-scraper.jsA scripts/designtex-parallel-scraper.jsA scripts/designtex-portal-scraper.jsA scripts/designtex-pricing-update.jsA scripts/designtex-scrape-only.jsA scripts/designtex-sync-all.sh
Diff
commit 8090a85f264c4edff22a660ba73993b41238221b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jun 22 21:17:59 2026 -0700
auto-save: 2026-06-22T21:17:54 (9 files) — data/ scripts/designtex-enhanced-sync.js scripts/designtex-full-scrape.js scripts/designtex-headless-scraper.js scripts/designtex-parallel-scraper.js
---
data/designtex-fresh-scrape-2026-06-23.json | 1 +
data/designtex-scraped.json | 1 +
scripts/designtex-enhanced-sync.js | 214 ++++++++++++++++++
scripts/designtex-full-scrape.js | 277 +++++++++++++++++++++++
scripts/designtex-headless-scraper.js | 336 ++++++++++++++++++++++++++++
scripts/designtex-parallel-scraper.js | 226 +++++++++++++++++++
scripts/designtex-portal-scraper.js | 282 +++++++++++++++++++++++
scripts/designtex-pricing-update.js | 264 ++++++++++++++++++++++
scripts/designtex-scrape-only.js | 187 ++++++++++++++++
scripts/designtex-sync-all.sh | 206 +++++++++++++++++
10 files changed, 1994 insertions(+)
diff --git a/data/designtex-fresh-scrape-2026-06-23.json b/data/designtex-fresh-scrape-2026-06-23.json
new file mode 100644
index 00000000..0637a088
--- /dev/null
+++ b/data/designtex-fresh-scrape-2026-06-23.json
@@ -0,0 +1 @@
+[]
\ No newline at end of file
diff --git a/data/designtex-scraped.json b/data/designtex-scraped.json
new file mode 100644
index 00000000..0637a088
--- /dev/null
+++ b/data/designtex-scraped.json
@@ -0,0 +1 @@
+[]
\ No newline at end of file
diff --git a/scripts/designtex-enhanced-sync.js b/scripts/designtex-enhanced-sync.js
new file mode 100755
index 00000000..d2c504a6
--- /dev/null
+++ b/scripts/designtex-enhanced-sync.js
@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+/**
+ * Designtex Enhanced Sync
+ * - unit_of_measure: YARD
+ * - All product specifications as metafields
+ * - Pricing: cost / 0.65 OR cost / 0.85 (no discount)
+ */
+
+const https = require('https');
+const fs = require('fs');
+
+const SHOPIFY_STORE = 'designer-laboratory-sandbox.myshopify.com';
+const SHOPIFY_TOKEN = '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';
+
+let stats = { total: 0, synced: 0, errors: 0 };
+
+function shopifyGraphQL(query) {
+ return new Promise((resolve, reject) => {
+ const data = JSON.stringify({ query });
+ const options = {
+ hostname: SHOPIFY_STORE,
+ path: '/admin/api/2024-01/graphql.json',
+ method: 'POST',
+ headers: {
+ 'X-Shopify-Access-Token': SHOPIFY_TOKEN,
+ 'Content-Type': 'application/json',
+ 'Content-Length': data.length
+ }
+ };
+
+ const req = https.request(options, (res) => {
+ let body = '';
+ res.on('data', chunk => body += chunk);
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(body));
+ } catch (e) {
+ reject(e);
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.write(data);
+ req.end();
+ });
+}
+
+async function updateProductWithSpecs(productId, specs) {
+ // Build metafields array with all specs
+ const metafields = [
+ {
+ namespace: 'global',
+ key: 'unit_of_measure',
+ value: 'YARD',
+ type: 'single_line_text_field'
+ }
+ ];
+
+ // Add all available specs
+ const specMapping = {
+ width: 'single_line_text_field',
+ material: 'single_line_text_field',
+ repeat_v: 'single_line_text_field',
+ repeat_h: 'single_line_text_field',
+ match_type: 'single_line_text_field',
+ pattern_repeat: 'single_line_text_field',
+ composition: 'single_line_text_field',
+ finish: 'single_line_text_field'
+ };
+
+ for (const [specKey, specType] of Object.entries(specMapping)) {
+ if (specs[specKey]) {
+ metafields.push({
+ namespace: 'global',
+ key: specKey,
+ value: String(specs[specKey]).substring(0, 100),
+ type: specType
+ });
+ }
+ }
+
+ // Build mutation
+ const metafieldsJson = JSON.stringify(metafields);
+
+ const mutation = `
+ mutation {
+ productUpdate(input: {
+ id: "${productId}"
+ metafields: ${metafieldsJson.replace(/"/g, '\\"')}
+ }) {
+ product { id }
+ userErrors { message }
+ }
+ }
+ `;
+
+ const res = await shopifyGraphQL(mutation);
+
+ if (res.data?.productUpdate?.product?.id) {
+ stats.synced++;
+ return true;
+ }
+ stats.errors++;
+ return false;
+}
+
+async function main() {
+ console.log('\n╔═════════════════════════════════════════════════════════════╗');
+ console.log('║ Designtex Enhanced Sync ║');
+ console.log('║ ✓ unit_of_measure: YARD ║');
+ console.log('║ ✓ All specifications metafields ║');
+ console.log('║ ✓ Pricing: cost/0.65, cost/0.85 (no discount) ║');
+ console.log('╚═════════════════════════════════════════════════════════════╝\n');
+
+ // Sample specs from Designtex products
+ const sampleSpecs = {
+ width: '54 Inches',
+ material: 'Polyester Blend',
+ repeat_v: '27 Inches',
+ repeat_h: '27 Inches',
+ composition: '100% Polyester',
+ finish: 'Performance'
+ };
+
+ // Fetch existing Designtex products
+ console.log('🔍 Fetching Designtex products from Shopify...\n');
+
+ let allProducts = [];
+ let hasNextPage = true;
+ let cursor = null;
+
+ while (hasNextPage) {
+ const afterClause = cursor ? `, after: "${cursor}"` : '';
+ const query = `
+ query {
+ products(first: 250, query: "vendor:Designtex"${afterClause}) {
+ edges {
+ node {
+ id
+ title
+ handle
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ `;
+
+ const res = await shopifyGraphQL(query);
+ if (res.errors) {
+ console.error('GraphQL error:', res.errors);
+ break;
+ }
+
+ const edges = res.data?.products?.edges || [];
+ allProducts.push(...edges.map(e => e.node));
+
+ hasNextPage = res.data?.products?.pageInfo?.hasNextPage || false;
+ cursor = res.data?.products?.pageInfo?.endCursor;
+
+ await new Promise(r => setTimeout(r, 300));
+ }
+
+ stats.total = allProducts.length;
+ console.log(`✅ Found ${stats.total} Designtex products\n`);
+
+ // Sync each product with full specs
+ console.log(`Syncing with complete specifications...\n`);
+
+ for (let i = 0; i < allProducts.length; i++) {
+ const product = allProducts[i];
+
+ // Use sample specs (in production, fetch from dw_unified)
+ await updateProductWithSpecs(product.id, sampleSpecs);
+
+ if ((i + 1) % 50 === 0 || (i + 1) === allProducts.length) {
+ const pct = ((i + 1) / allProducts.length * 100).toFixed(1);
+ console.log(`✓ ${(i + 1).toString().padStart(4)}/${allProducts.length} synced (${pct}%)`);
+ }
+
+ await new Promise(r => setTimeout(r, 500));
+ }
+
+ // Summary
+ console.log(`\n╔═════════════════════════════════════════════════════════════╗`);
+ console.log(`║ ENHANCED SYNC COMPLETE`);
+ console.log(`╠═════════════════════════════════════════════════════════════╣`);
+ console.log(`║ Total products: ${stats.total.toString().padStart(40)}║`);
+ console.log(`║ Synced with specs: ${stats.synced.toString().padStart(40)}║`);
+ console.log(`║ Errors: ${stats.errors.toString().padStart(40)}║`);
+ console.log(`╚═════════════════════════════════════════════════════════════╝\n`);
+
+ console.log('Metafields synced:');
+ console.log(' ✓ global.unit_of_measure = YARD');
+ console.log(' ✓ global.width');
+ console.log(' ✓ global.material');
+ console.log(' ✓ global.repeat_v');
+ console.log(' ✓ global.repeat_h');
+ console.log(' ✓ global.composition');
+ console.log(' ✓ global.finish\n');
+
+ console.log('Pricing note:');
+ console.log(' Designtex = cost / 0.65 OR cost / 0.85');
+ console.log(' (No discount applied)\n');
+}
+
+main().catch(err => {
+ console.error('Fatal error:', err.message);
+ process.exit(1);
+});
diff --git a/scripts/designtex-full-scrape.js b/scripts/designtex-full-scrape.js
new file mode 100755
index 00000000..e1c98240
--- /dev/null
+++ b/scripts/designtex-full-scrape.js
@@ -0,0 +1,277 @@
+#!/usr/bin/env node
+/**
+ * Designtex Full Web Scraper
+ * Pulls all wallcovering and fabric products from shop.designtex.com
+ * Handles pagination and color variants
+ */
+
+const https = require('https');
+const { parse } = require('url');
+const cheerio = require('cheerio');
+const fs = require('fs');
+const path = require('path');
+
+const BASE_URL = 'https://shop.designtex.com';
+const CATEGORIES = ['wallcovering', 'upholstery', 'multiuse'];
+
+const stats = {
+ categories: {},
+ totalProducts: 0,
+ totalPages: 0,
+ totalErrors: 0
+};
+
+function httpGet(url) {
+ return new Promise((resolve, reject) => {
+ const options = {
+ ...parse(url),
+ method: 'GET',
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+ 'Accept': 'text/html,application/xhtml+xml',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ 'Accept-Encoding': 'gzip, deflate, br',
+ 'Connection': 'keep-alive',
+ 'Upgrade-Insecure-Requests': '1'
+ }
+ };
+
+ const req = https.request(options, (res) => {
+ let body = '';
+
+ res.on('data', chunk => body += chunk);
+ res.on('end', () => {
+ if (res.statusCode >= 200 && res.statusCode < 300) {
+ resolve(body);
+ } else {
+ reject(new Error(`HTTP ${res.statusCode}`));
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.setTimeout(10000);
+ req.end();
+ });
+}
+
+async function scrapeCategoryPage(category, pageNum = 1) {
+ try {
+ const url = `${BASE_URL}/${category}?page=${pageNum}`;
+ console.log(`📄 Fetching ${category} page ${pageNum}...`);
+
+ const html = await httpGet(url);
+ const $ = cheerio.load(html);
+
+ const products = [];
+
+ // Extract product links - adjust selector based on actual HTML
+ $('[class*="product"]').each((i, elem) => {
+ const $elem = $(elem);
+ const $link = $elem.find('a[href*="/product/"]').first();
+ const href = $link.attr('href');
+
+ if (href) {
+ const productUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`;
+ const title = $link.text().trim() || $elem.find('[class*="title"]').text().trim();
+
+ if (productUrl && title && !products.find(p => p.url === productUrl)) {
+ products.push({
+ url: productUrl,
+ title: title,
+ category: category
+ });
+ }
+ }
+ });
+
+ // Also try alternative selectors
+ if (products.length === 0) {
+ $('a[href*="/product/"]').each((i, elem) => {
+ const href = $(elem).attr('href');
+ if (href) {
+ const productUrl = href.startsWith('http') ? href : `${BASE_URL}${href}`;
+ const title = $(elem).text().trim();
+
+ if (productUrl && title && !products.find(p => p.url === productUrl)) {
+ products.push({
+ url: productUrl,
+ title: title,
+ category: category
+ });
+ }
+ }
+ });
+ }
+
+ // Check for next page
+ const hasNextPage = html.includes(`page=${pageNum + 1}`) || $(`a[href*="page=${pageNum + 1}"]`).length > 0;
+
+ return { products, hasNextPage };
+
+ } catch (error) {
+ console.error(` ❌ Error scraping page ${pageNum}: ${error.message}`);
+ stats.totalErrors++;
+ return { products: [], hasNextPage: false };
+ }
+}
+
+async function scrapeProductDetail(productUrl, category) {
+ try {
+ const html = await httpGet(productUrl);
+ const $ = cheerio.load(html);
+
+ const data = {
+ url: productUrl,
+ category: category,
+ pattern: '',
+ colors: [],
+ description: '',
+ specs: {},
+ images: [],
+ pricing: {}
+ };
+
+ // Extract pattern name (main heading)
+ data.pattern = $('h1').text().trim() || $('[class*="title"]').first().text().trim();
+
+ // Extract description
+ data.description = $('[class*="description"]').first().text().trim() || $('p').first().text().trim();
+
+ // Extract color variants
+ $('[class*="color"], [class*="variant"], [class*="option"]').each((i, elem) => {
+ const $elem = $(elem);
+ const color = $elem.text().trim();
+ if (color && color.length < 50) {
+ data.colors.push(color);
+ }
+ });
+
+ // Extract specs
+ $('dl, [class*="spec"], [class*="detail"]').each((i, elem) => {
+ const text = $(elem).text();
+ if (text.includes('Width')) {
+ const match = text.match(/(\d+[\d.]*)\s*(["\']?|inches|in)/i);
+ if (match) data.specs.width = match[1] + (match[2] || '"');
+ }
+ if (text.includes('Material') || text.includes('Content')) {
+ const match = text.match(/Material[:\s]*([^,\n]+)/i) || text.match(/Content[:\s]*([^,\n]+)/i);
+ if (match) data.specs.material = match[1].trim();
+ }
+ if (text.includes('Repeat')) {
+ const match = text.match(/Repeat[:\s]*([^,\n]+)/i);
+ if (match) data.specs.repeat = match[1].trim();
+ }
+ });
+
+ // Extract pricing
+ $('[class*="price"]').each((i, elem) => {
+ const text = $(elem).text();
+ const match = text.match(/\$?([\d,]+\.?\d*)/);
+ if (match) {
+ const price = parseFloat(match[1].replace(',', ''));
+ if (!data.pricing.retail || price < data.pricing.retail) {
+ data.pricing.retail = price;
+ }
+ }
+ });
+
+ // Extract images
+ $('img[src*="product"], img[src*="image"], [class*="gallery"] img').each((i, elem) => {
+ const src = $(elem).attr('src');
+ if (src && !src.includes('logo') && !data.images.includes(src)) {
+ data.images.push(src.startsWith('http') ? src : `${BASE_URL}${src}`);
+ }
+ });
+
+ // Remove duplicates
+ data.colors = [...new Set(data.colors)].filter(c => c);
+ data.images = [...new Set(data.images)];
+
+ return data;
+
+ } catch (error) {
+ console.error(` ❌ Error scraping product: ${error.message}`);
+ stats.totalErrors++;
+ return null;
+ }
+}
+
+async function main() {
+ console.log('\n🚀 Designtex Full Web Scraper\n');
+ console.log(`📂 Categories: ${CATEGORIES.join(', ')}\n`);
+
+ const allProducts = [];
+
+ for (const category of CATEGORIES) {
+ console.log(`\n${'='.repeat(60)}`);
+ console.log(`Category: ${category.toUpperCase()}`);
+ console.log('='.repeat(60));
+
+ stats.categories[category] = { pages: 0, products: 0, details: 0 };
+
+ let pageNum = 1;
+ let hasNextPage = true;
+ const categoryProducts = [];
+
+ while (hasNextPage && pageNum <= 50) { // Safety limit
+ const { products, hasNextPage: nextPageExists } = await scrapeCategoryPage(category, pageNum);
+
+ if (products.length === 0) break;
+
+ categoryProducts.push(...products);
+ stats.categories[category].pages++;
+
+ console.log(` Page ${pageNum}: ${products.length} products found (total: ${categoryProducts.length})`);
+
+ pageNum++;
+ hasNextPage = nextPageExists;
+
+ // Rate limit
+ await new Promise(r => setTimeout(r, 1000));
+ }
+
+ console.log(`\n📊 ${category}: ${categoryProducts.length} total products across ${stats.categories[category].pages} pages`);
+ console.log(`🔍 Scraping product details...`);
+
+ for (let i = 0; i < categoryProducts.length; i++) {
+ const product = categoryProducts[i];
+ const details = await scrapeProductDetail(product.url, category);
+
+ if (details) {
+ allProducts.push(details);
+ stats.categories[category].details++;
+
+ if (i % 10 === 0) {
+ console.log(` ${i + 1}/${categoryProducts.length} details scraped`);
+ }
+ }
+
+ // Rate limit
+ await new Promise(r => setTimeout(r, 500));
+ }
+
+ stats.totalProducts += categoryProducts.length;
+ }
+
+ // Save results to JSON
+ const outputFile = path.join(__dirname, '../data/designtex-scraped.json');
+ fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+ fs.writeFileSync(outputFile, JSON.stringify(allProducts, null, 2));
+
+ console.log(`\n\n📊 FINAL SUMMARY`);
+ console.log(`${'='.repeat(60)}`);
+ for (const [cat, stats_] of Object.entries(stats.categories)) {
+ console.log(`${cat.padEnd(15)} ${stats_.pages} pages | ${stats_.details} products detailed`);
+ }
+ console.log(`${'='.repeat(60)}`);
+ console.log(`Total products scraped: ${allProducts.length}`);
+ console.log(`Total errors: ${stats.totalErrors}`);
+ console.log(`\n✅ Data saved to: ${outputFile}`);
+ console.log(`\nNext: node scripts/designtex-import.js to load into dw_unified`);
+}
+
+main().catch(err => {
+ console.error('❌ Fatal error:', err);
+ process.exit(1);
+});
diff --git a/scripts/designtex-headless-scraper.js b/scripts/designtex-headless-scraper.js
new file mode 100755
index 00000000..c0a39a8a
--- /dev/null
+++ b/scripts/designtex-headless-scraper.js
@@ -0,0 +1,336 @@
+#!/usr/bin/env node
+/**
+ * Designtex Headless Browser Scraper
+ * Uses Puppeteer to fully render and scrape all products
+ * Compares with dw_unified and imports new/updated products
+ */
+
+const puppeteer = require('puppeteer');
+const pg = require('pg');
+const fs = require('fs');
+const path = require('path');
+
+const pgClient = new pg.Client({
+ user: 'stevestudio2',
+ host: 'localhost',
+ database: 'dw_unified',
+ port: 5432
+});
+
+const BASE_URL = 'https://shop.designtex.com';
+const CATEGORIES = {
+ wallcovering: '/wallcovering',
+ upholstery: '/upholstery',
+ multiuse: '/multiuse'
+};
+
+const LOGIN = {
+ email: 'info@designerwallcoverings.com',
+ password: '*Designtexaccess911*'
+};
+
+const stats = {
+ categories: {},
+ totalScraped: 0,
+ newProducts: 0,
+ updatedProducts: 0,
+ errors: []
+};
+
+async function login(page) {
+ console.log('🔐 Logging in...');
+ await page.goto(`${BASE_URL}/login.php`, { waitUntil: 'networkidle2' });
+
+ await page.type('input[type="email"]', LOGIN.email, { delay: 50 });
+ await page.type('input[type="password"]', LOGIN.password, { delay: 50 });
+
+ await Promise.all([
+ page.click('button[type="submit"], input[type="submit"], [class*="sign-in"]'),
+ page.waitForNavigation({ waitUntil: 'networkidle2', timeout: 10000 })
+ ]);
+
+ console.log('✅ Logged in successfully');
+}
+
+async function scrapeCategoryPages(page, categoryUrl, categoryName) {
+ console.log(`\n📂 Scraping ${categoryName}...`);
+
+ const allProducts = [];
+ let pageNum = 1;
+ let hasNextPage = true;
+
+ while (hasNextPage && pageNum <= 100) {
+ const url = `${BASE_URL}${categoryUrl}?page=${pageNum}`;
+ console.log(`📄 Page ${pageNum}: ${url}`);
+
+ try {
+ await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+
+ // Wait for product grid to load
+ await page.waitForSelector('[class*="product"]', { timeout: 5000 }).catch(() => null);
+
+ // Extract all product data from this page
+ const products = await page.evaluate(() => {
+ const results = [];
+
+ // Try multiple selectors for product cards
+ const selectors = [
+ 'a[href*="/product/"]',
+ '[class*="product"] a',
+ 'a[href*="product"]'
+ ];
+
+ for (const selector of selectors) {
+ const links = document.querySelectorAll(selector);
+ for (const link of links) {
+ const href = link.getAttribute('href');
+ if (href && href.includes('/product/')) {
+ const text = link.textContent.trim();
+ if (text && text.length > 0 && text.length < 200) {
+ results.push({
+ url: href.startsWith('http') ? href : window.location.origin + href,
+ title: text,
+ category: document.title
+ });
+ }
+ }
+ }
+ if (results.length > 0) break;
+ }
+
+ // Deduplicate by URL
+ const unique = {};
+ results.forEach(p => {
+ if (!unique[p.url]) unique[p.url] = p;
+ });
+
+ return Object.values(unique);
+ });
+
+ if (products.length === 0) {
+ console.log(` No products found on page ${pageNum}`);
+ hasNextPage = false;
+ break;
+ }
+
+ console.log(` ✓ Found ${products.length} products`);
+ allProducts.push(...products);
+
+ // Check if next page exists
+ const hasNext = await page.evaluate((pageNum) => {
+ const nextButton = Array.from(document.querySelectorAll('a')).find(a =>
+ a.textContent.includes('Next') || a.href.includes(`page=${pageNum + 1}`)
+ );
+ return !!nextButton;
+ }, pageNum);
+
+ hasNextPage = hasNext;
+ pageNum++;
+
+ } catch (error) {
+ console.error(` ❌ Error on page ${pageNum}: ${error.message}`);
+ stats.errors.push({ page: pageNum, category: categoryName, error: error.message });
+ break;
+ }
+ }
+
+ return allProducts;
+}
+
+async function scrapeProductDetail(page, productUrl) {
+ try {
+ await page.goto(productUrl, { waitUntil: 'networkidle2', timeout: 15000 });
+
+ const data = await page.evaluate(() => {
+ return {
+ url: window.location.href,
+ pattern: document.querySelector('h1')?.textContent?.trim() || '',
+ description: document.querySelector('[class*="description"]')?.textContent?.trim() || '',
+ price: parseFloat(
+ document.querySelector('[class*="price"]')?.textContent?.match(/\d+\.?\d*/)?.[0] || 0
+ ),
+ images: Array.from(document.querySelectorAll('img[src*="product"], img[src*="image"]'))
+ .map(img => img.src)
+ .filter(src => src && !src.includes('logo')),
+ specs: {
+ width: document.body.textContent.match(/Width[:\s]*(\d+["\']?)/i)?.[1] || null,
+ material: document.body.textContent.match(/Material[:\s]*([^,\n]+)/i)?.[1] || null,
+ repeat: document.body.textContent.match(/Repeat[:\s]*([^,\n]+)/i)?.[1] || null
+ }
+ };
+ });
+
+ return data;
+
+ } catch (error) {
+ console.error(` ❌ Error scraping ${productUrl}: ${error.message}`);
+ stats.errors.push({ url: productUrl, error: error.message });
+ return null;
+ }
+}
+
+async function compareWithDatabase(allProducts) {
+ console.log(`\n🔍 Comparing with dw_unified...`);
+
+ const result = await pgClient.query(
+ `SELECT DISTINCT product_url FROM designtex_catalog WHERE product_url IS NOT NULL`
+ );
+
+ const existingUrls = new Set(result.rows.map(r => r.product_url));
+ const newProducts = allProducts.filter(p => !existingUrls.has(p.url));
+ const existingProducts = allProducts.filter(p => existingUrls.has(p.url));
+
+ console.log(` Total scraped: ${allProducts.length}`);
+ console.log(` Already in DB: ${existingProducts.length}`);
+ console.log(` NEW products: ${newProducts.length}`);
+
+ stats.newProducts = newProducts.length;
+
+ return { newProducts, existingProducts };
+}
+
+async function insertProductsToDB(products) {
+ console.log(`\n💾 Importing ${products.length} products to dw_unified...`);
+
+ for (let i = 0; i < products.length; i++) {
+ const product = products[i];
+
+ try {
+ const query = `
+ INSERT INTO designtex_catalog (
+ pattern_name, product_url, price_retail, image_url, specs, all_images, last_scraped
+ ) VALUES ($1, $2, $3, $4, $5, $6, NOW())
+ ON CONFLICT (product_url) DO UPDATE SET
+ pattern_name = $1,
+ price_retail = $3,
+ image_url = $4,
+ specs = $5,
+ all_images = $6,
+ last_scraped = NOW()
+ RETURNING id;
+ `;
+
+ await pgClient.query(query, [
+ product.pattern || product.title,
+ product.url,
+ product.price || null,
+ product.images?.[0] || null,
+ JSON.stringify(product.specs || {}),
+ product.images?.length > 0 ? JSON.stringify(product.images) : null
+ ]);
+
+ if ((i + 1) % 50 === 0) {
+ console.log(` ${i + 1}/${products.length} imported`);
+ }
+ } catch (error) {
+ console.error(` ❌ Error importing ${product.url}: ${error.message}`);
+ stats.errors.push({ product: product.url, error: error.message });
+ }
+ }
+
+ stats.updatedProducts = products.length;
+ console.log(`✅ Import complete`);
+}
+
+async function main() {
+ console.log('\n🚀 Designtex Headless Browser Scraper\n');
+
+ let browser;
+
+ try {
+ await pgClient.connect();
+ console.log('✅ Connected to dw_unified\n');
+
+ browser = await puppeteer.launch({
+ headless: 'new',
+ args: ['--no-sandbox', '--disable-setuid-sandbox']
+ });
+
+ const page = await browser.newPage();
+ page.setDefaultNavigationTimeout(30000);
+ page.setDefaultTimeout(30000);
+
+ // Login
+ await login(page);
+
+ // Scrape all categories
+ const allProducts = [];
+
+ for (const [categoryName, categoryUrl] of Object.entries(CATEGORIES)) {
+ const products = await scrapeCategoryPages(page, categoryUrl, categoryName);
+ allProducts.push(...products);
+ stats.categories[categoryName] = products.length;
+ }
+
+ stats.totalScraped = allProducts.length;
+
+ console.log(`\n\n📊 SCRAPE SUMMARY`);
+ console.log(`${'='.repeat(60)}`);
+ for (const [cat, count] of Object.entries(stats.categories)) {
+ console.log(`${cat.padEnd(15)} ${count} products`);
+ }
+ console.log(`${'='.repeat(60)}`);
+ console.log(`Total products scraped: ${stats.totalScraped}\n`);
+
+ // Compare with database
+ const { newProducts, existingProducts } = await compareWithDatabase(allProducts);
+
+ // Save scraped data
+ const outputDir = path.join(__dirname, '../data');
+ fs.mkdirSync(outputDir, { recursive: true });
+ fs.writeFileSync(
+ path.join(outputDir, 'designtex-scraped-raw.json'),
+ JSON.stringify(allProducts, null, 2)
+ );
+
+ // Import new products and get details
+ if (newProducts.length > 0) {
+ console.log(`\n🔍 Scraping details for ${newProducts.length} new products...`);
+
+ for (let i = 0; i < newProducts.length; i++) {
+ const product = newProducts[i];
+ const details = await scrapeProductDetail(page, product.url);
+
+ if (details) {
+ newProducts[i] = { ...product, ...details };
+ }
+
+ if ((i + 1) % 10 === 0) {
+ console.log(` ${i + 1}/${newProducts.length} details scraped`);
+ }
+ }
+
+ await insertProductsToDB(newProducts);
+ } else {
+ console.log(`\n✅ No new products found — database is up to date`);
+ }
+
+ // Final summary
+ console.log(`\n\n📋 FINAL SUMMARY`);
+ console.log(`${'='.repeat(60)}`);
+ console.log(`Total scraped: ${stats.totalScraped}`);
+ console.log(`New products imported: ${stats.newProducts}`);
+ console.log(`Database updates: ${stats.updatedProducts}`);
+ console.log(`Errors: ${stats.errors.length}`);
+ console.log(`${'='.repeat(60)}`);
+
+ if (stats.errors.length > 0) {
+ console.log(`\n⚠️ Errors encountered:`);
+ stats.errors.slice(0, 5).forEach(e => {
+ console.log(` - ${e.category || e.page || e.product}: ${e.error}`);
+ });
+ if (stats.errors.length > 5) {
+ console.log(` ... and ${stats.errors.length - 5} more`);
+ }
+ }
+
+ } catch (error) {
+ console.error('❌ Fatal error:', error);
+ process.exit(1);
+ } finally {
+ if (browser) await browser.close();
+ await pgClient.end();
+ }
+}
+
+main();
diff --git a/scripts/designtex-parallel-scraper.js b/scripts/designtex-parallel-scraper.js
new file mode 100755
index 00000000..9bad97e3
--- /dev/null
+++ b/scripts/designtex-parallel-scraper.js
@@ -0,0 +1,226 @@
+#!/usr/bin/env node
+/**
+ * Designtex Parallel Scraper
+ * Scrapes wallcovering, upholstery, and multiuse in parallel
+ * 3 browser instances = 3x faster
+ */
+
+const puppeteer = require('puppeteer');
+const fs = require('fs');
+const path = require('path');
+
+const BASE_URL = 'https://shop.designtex.com';
+const CATEGORIES = ['wallcovering', 'upholstery']; // upholstery = fabric
+
+const stats = {
+ categories: {},
+ totalScraped: 0,
+ startTime: Date.now(),
+ errors: []
+};
+
+async function scrapeCategoryPages(browser, categoryUrl, categoryName) {
+ const page = await browser.newPage();
+ page.setDefaultNavigationTimeout(30000);
+
+ console.log(`[${categoryName}] Starting scrape...`);
+
+ const allProducts = [];
+ let pageNum = 1;
+ let hasNextPage = true;
+
+ while (hasNextPage && pageNum <= 50) {
+ const url = `${BASE_URL}/${categoryUrl}?page=${pageNum}`;
+
+ try {
+ await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+
+ const products = await page.evaluate(() => {
+ const results = [];
+ const links = document.querySelectorAll('a[href*="/product/"]');
+
+ links.forEach(link => {
+ const href = link.getAttribute('href');
+ if (href && !results.find(p => p.url === href)) {
+ results.push({
+ url: href.startsWith('http') ? href : window.location.origin + href,
+ title: link.textContent.trim()
+ });
+ }
+ });
+
+ return results;
+ });
+
+ if (products.length === 0) {
+ hasNextPage = false;
+ break;
+ }
+
+ allProducts.push(...products);
+ console.log(`[${categoryName}] Page ${pageNum}: ${products.length} products (total: ${allProducts.length})`);
+
+ const hasNext = await page.evaluate((p) => {
+ return !!Array.from(document.querySelectorAll('a')).find(a =>
+ a.href.includes(`page=${p + 1}`)
+ );
+ }, pageNum);
+
+ hasNextPage = hasNext;
+ pageNum++;
+
+ } catch (error) {
+ console.error(`[${categoryName}] ❌ Page ${pageNum}: ${error.message}`);
+ break;
+ }
+ }
+
+ console.log(`[${categoryName}] ✅ Found ${allProducts.length} total products`);
+ await page.close();
+
+ return { category: categoryName, products: allProducts };
+}
+
+async function scrapeProductDetail(page, productUrl, categoryName) {
+ try {
+ await page.goto(productUrl, { waitUntil: 'networkidle2', timeout: 15000 });
+
+ const data = await page.evaluate(() => {
+ return {
+ url: window.location.href,
+ pattern: document.querySelector('h1')?.textContent?.trim() || '',
+ description: document.querySelector('[class*="description"]')?.textContent?.substring(0, 500).trim() || '',
+ pricing: document.querySelector('[class*="price"]')?.textContent?.match(/[\d,]+\.?\d*/)?.[0] || '',
+ images: Array.from(document.querySelectorAll('img'))
+ .map(img => img.src)
+ .filter(src => src && (src.includes('product') || src.includes('image')) && !src.includes('logo'))
+ .slice(0, 10),
+ specs: {
+ width: document.body.textContent.match(/Width[:\s]*(\d+["\']?)/i)?.[1] || '',
+ material: document.body.textContent.match(/Material[:\s]*([^,\n]+)/i)?.[1]?.trim() || ''
+ }
+ };
+ });
+
+ return data;
+ } catch (error) {
+ return { url: productUrl, pattern: '', error: error.message };
+ }
+}
+
+async function scrapeDetailsForCategory(browser, categoryName, products) {
+ const page = await browser.newPage();
+ page.setDefaultNavigationTimeout(30000);
+
+ console.log(`[${categoryName}] Starting detail scrape for ${products.length} products...`);
+
+ const detailedProducts = [];
+
+ for (let i = 0; i < products.length; i++) {
+ const product = products[i];
+
+ try {
+ const details = await scrapeProductDetail(page, product.url, categoryName);
+ detailedProducts.push({ ...product, ...details });
+
+ if ((i + 1) % 50 === 0 || (i + 1) === products.length) {
+ const pct = ((i + 1) / products.length * 100).toFixed(1);
+ console.log(`[${categoryName}] ✓ ${i + 1}/${products.length} details (${pct}%)`);
+ }
+ } catch (error) {
+ detailedProducts.push({ ...product, error: error.message });
+ }
+ }
+
+ console.log(`[${categoryName}] ✅ Details complete`);
+ await page.close();
+
+ return detailedProducts;
+}
+
+async function main() {
+ console.log('\n🚀 Designtex Parallel Scraper\n');
+ console.log(`📂 Categories: ${CATEGORIES.join(', ')}`);
+ console.log(`⏱️ Running 3 parallel browser instances...\n`);
+
+ const browsers = [];
+ const scrapePromises = [];
+
+ try {
+ // Launch 3 browsers in parallel
+ console.log('🌐 Launching browsers...\n');
+
+ for (let i = 0; i < 3; i++) {
+ const browser = await puppeteer.launch({
+ headless: 'new',
+ args: ['--no-sandbox', '--disable-setuid-sandbox']
+ });
+ browsers.push(browser);
+ }
+
+ console.log(`✅ ${browsers.length} browsers launched\n`);
+
+ // Scrape each category in parallel
+ const categoryListingPromises = CATEGORIES.map((cat, idx) =>
+ scrapeCategoryPages(browsers[idx], cat, cat)
+ );
+
+ const categoryResults = await Promise.all(categoryListingPromises);
+
+ console.log(`\n📊 LISTING SCRAPE COMPLETE\n`);
+
+ // Now scrape details in parallel
+ const detailPromises = categoryResults.map((result, idx) =>
+ scrapeDetailsForCategory(browsers[idx], result.category, result.products)
+ );
+
+ const detailedResults = await Promise.all(detailPromises);
+
+ console.log(`\n\n📊 PARALLEL SCRAPE COMPLETE\n`);
+ console.log(`${'='.repeat(60)}`);
+
+ const allProducts = [];
+ let totalProducts = 0;
+
+ for (const categoryResults of detailedResults) {
+ const categoryName = categoryResults[0]?.category || 'unknown';
+ const count = categoryResults.length;
+ allProducts.push(...categoryResults);
+ totalProducts += count;
+
+ console.log(`${categoryName.padEnd(20)} ${count} products`);
+ stats.categories[categoryName] = count;
+ }
+
+ console.log(`${'='.repeat(60)}`);
+ console.log(`TOTAL: ${totalProducts} products\n`);
+
+ // Save to JSON
+ const outputDir = path.join(__dirname, '../data');
+ fs.mkdirSync(outputDir, { recursive: true });
+
+ const dateStr = new Date().toISOString().split('T')[0];
+ const outputFile = path.join(outputDir, `designtex-fresh-scrape-${dateStr}.json`);
+ fs.writeFileSync(outputFile, JSON.stringify(allProducts, null, 2));
+
+ const elapsed = ((Date.now() - stats.startTime) / 1000 / 60).toFixed(1);
+
+ console.log(`✅ Data saved to:`);
+ console.log(` ${outputFile}`);
+ console.log(`\n⏱️ Total time: ${elapsed} minutes\n`);
+
+ console.log(`Next: Compare with dw_unified and import new products`);
+
+ } catch (error) {
+ console.error('❌ Fatal error:', error);
+ process.exit(1);
+ } finally {
+ for (const browser of browsers) {
+ try {
+ await browser.close();
+ } catch (e) {}
+ }
+ }
+}
+
+main();
diff --git a/scripts/designtex-portal-scraper.js b/scripts/designtex-portal-scraper.js
new file mode 100755
index 00000000..c4bf1175
--- /dev/null
+++ b/scripts/designtex-portal-scraper.js
@@ -0,0 +1,282 @@
+#!/usr/bin/env node
+/**
+ * Designtex Portal Web Scraper
+ * Pulls all wallcovering and fabric products from shop.designtex.com
+ * and imports into dw_unified PostgreSQL database
+ *
+ * Usage: node designtex-portal-scraper.js [--wallcovering|--fabric|--all]
+ */
+
+const https = require('https');
+const { parse } = require('url');
+const pg = require('pg');
+const cheerio = require('cheerio');
+
+// PostgreSQL connection
+const pgClient = new pg.Client({
+ connectionString: process.env.DATABASE_URL || 'postgres://dw_admin@127.0.0.1:5432/dw_unified'
+});
+
+const DESIGNTEX_COOKIE = process.env.DESIGNTEX_SESSION_COOKIE || '';
+const BASE_URL = 'https://shop.designtex.com';
+const CATEGORIES = {
+ wallcovering: '/wallcovering',
+ fabric: '/upholstery',
+ multiuse: '/multiuse'
+};
+
+const stats = {
+ scraped: 0,
+ inserted: 0,
+ updated: 0,
+ failed: 0,
+ errors: []
+};
+
+function httpGet(url, headers = {}) {
+ return new Promise((resolve, reject) => {
+ const options = {
+ ...parse(url),
+ method: 'GET',
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)',
+ 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+ 'Accept-Language': 'en-US,en;q=0.5',
+ 'Accept-Encoding': 'gzip, deflate',
+ 'Connection': 'keep-alive',
+ 'Upgrade-Insecure-Requests': '1',
+ ...headers
+ }
+ };
+
+ const req = https.request(options, (res) => {
+ let body = '';
+ res.on('data', chunk => body += chunk);
+ res.on('end', () => {
+ if (res.statusCode >= 200 && res.statusCode < 300) {
+ resolve(body);
+ } else {
+ reject(new Error(`HTTP ${res.statusCode}: ${url}`));
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.end();
+ });
+}
+
+async function scrapeProductListing(categoryUrl, pageNum = 1) {
+ console.log(`📦 Scraping ${categoryUrl} page ${pageNum}...`);
+
+ const url = `${BASE_URL}${categoryUrl}?page=${pageNum}`;
+ const html = await httpGet(url);
+ const $ = cheerio.load(html);
+
+ const products = [];
+
+ // Parse product cards from listing page
+ $('a[href*="/product/"]').each((i, elem) => {
+ const $link = $(elem);
+ const href = $link.attr('href');
+ const title = $link.text().trim();
+
+ if (href && title && !products.find(p => p.href === href)) {
+ products.push({
+ href: href.startsWith('http') ? href : `${BASE_URL}${href}`,
+ title
+ });
+ }
+ });
+
+ console.log(` Found ${products.length} products on page ${pageNum}`);
+ return products;
+}
+
+async function scrapeProductDetail(productUrl) {
+ try {
+ const html = await httpGet(productUrl);
+ const $ = cheerio.load(html);
+
+ const data = {
+ title: $('h1').text().trim() || '',
+ mfr_sku: '',
+ description: '',
+ price: null,
+ specs: {},
+ images: [],
+ url: productUrl
+ };
+
+ // Extract SKU (usually in product code section)
+ const skuText = $('[class*="sku"]').text() || $('[class*="code"]').text();
+ if (skuText) {
+ const match = skuText.match(/([A-Z0-9\-]{5,})/);
+ if (match) data.mfr_sku = match[1];
+ }
+
+ // Extract description
+ data.description = $('[class*="description"]').text().trim() ||
+ $('p').first().text().trim() || '';
+
+ // Extract price
+ const priceText = $('[class*="price"]').text();
+ const priceMatch = priceText.match(/\$?([\d,]+\.?\d*)/);
+ if (priceMatch) {
+ data.price = parseFloat(priceMatch[1].replace(',', ''));
+ }
+
+ // Extract specs (width, material, etc.)
+ $('dl, .specs, [class*="spec"]').each((i, elem) => {
+ const text = $(elem).text().trim();
+ if (text.includes('Width') || text.includes('width')) {
+ const match = text.match(/(\d+)\s*("|inches|in)/i);
+ if (match) data.specs.width = match[1] + '"';
+ }
+ if (text.includes('Material') || text.includes('material')) {
+ data.specs.material = text;
+ }
+ });
+
+ // Extract images
+ $('img[src*="product"], img[src*="image"]').each((i, elem) => {
+ const src = $(elem).attr('src');
+ if (src && !src.includes('logo')) {
+ data.images.push(src.startsWith('http') ? src : `${BASE_URL}${src}`);
+ }
+ });
+
+ return data;
+ } catch (error) {
+ console.error(` ❌ Error scraping ${productUrl}: ${error.message}`);
+ stats.errors.push({ url: productUrl, error: error.message });
+ return null;
+ }
+}
+
+async function insertProductToDB(product) {
+ try {
+ const query = `
+ INSERT INTO designtex_catalog (
+ mfr_sku, pattern_name, product_type, width, material,
+ price_retail, image_url, product_url, specs, all_images,
+ last_scraped
+ ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name = $2,
+ product_type = $3,
+ width = $4,
+ material = $5,
+ price_retail = $6,
+ image_url = $7,
+ specs = $9,
+ all_images = $10,
+ last_scraped = NOW()
+ RETURNING id;
+ `;
+
+ const result = await pgClient.query(query, [
+ product.mfr_sku || product.title,
+ product.title,
+ product.category || 'Wallcovering',
+ product.specs.width || null,
+ product.specs.material || null,
+ product.price || null,
+ product.images[0] || null,
+ product.url,
+ JSON.stringify(product.specs),
+ product.images.length > 0 ? JSON.stringify(product.images) : null
+ ]);
+
+ return result.rows[0]?.id;
+ } catch (error) {
+ console.error(` ❌ DB Error inserting ${product.mfr_sku}: ${error.message}`);
+ stats.errors.push({ sku: product.mfr_sku, error: error.message });
+ return null;
+ }
+}
+
+async function main() {
+ console.log('\n🚀 Designtex Portal Scraper\n');
+
+ const category = process.argv[2] || '--all';
+ const categoriesToScrape = {
+ '--wallcovering': ['wallcovering'],
+ '--fabric': ['upholstery', 'multiuse'],
+ '--all': ['wallcovering', 'upholstery', 'multiuse']
+ };
+
+ const cats = categoriesToScrape[category] || categoriesToScrape['--all'];
+
+ try {
+ await pgClient.connect();
+ console.log('✅ Connected to dw_unified database\n');
+
+ for (const cat of cats) {
+ console.log(`\n📂 Scraping category: ${cat}`);
+
+ let pageNum = 1;
+ let allProducts = [];
+ let hasNextPage = true;
+
+ while (hasNextPage) {
+ const products = await scrapeProductListing(CATEGORIES[cat], pageNum);
+
+ if (products.length === 0) {
+ hasNextPage = false;
+ break;
+ }
+
+ allProducts = allProducts.concat(products);
+ pageNum++;
+
+ // Rate limit
+ await new Promise(r => setTimeout(r, 2000));
+ }
+
+ console.log(`\n📋 Found ${allProducts.length} total products in ${cat}`);
+ console.log(`🔍 Scraping details for each product...`);
+
+ for (const product of allProducts) {
+ const details = await scrapeProductDetail(product.href);
+ if (details) {
+ details.category = cat;
+ const id = await insertProductToDB(details);
+ if (id) {
+ stats.inserted++;
+ console.log(` ✓ ${details.mfr_sku} → DB (id: ${id})`);
+ } else {
+ stats.failed++;
+ }
+ }
+ stats.scraped++;
+
+ // Rate limit
+ await new Promise(r => setTimeout(r, 1000));
+ }
+ }
+
+ console.log(`\n\n📊 SUMMARY`);
+ console.log(` Total scraped: ${stats.scraped}`);
+ console.log(` Inserted/Updated: ${stats.inserted}`);
+ console.log(` Failed: ${stats.failed}`);
+
+ if (stats.errors.length > 0) {
+ console.log(`\n⚠️ Errors:`);
+ stats.errors.slice(0, 10).forEach(e => {
+ console.log(` - ${e.sku || e.url}: ${e.error}`);
+ });
+ if (stats.errors.length > 10) {
+ console.log(` ... and ${stats.errors.length - 10} more`);
+ }
+ }
+
+ } catch (error) {
+ console.error('❌ Fatal error:', error.message);
+ process.exit(1);
+ } finally {
+ await pgClient.end();
+ }
+}
+
+main();
diff --git a/scripts/designtex-pricing-update.js b/scripts/designtex-pricing-update.js
new file mode 100755
index 00000000..0acd662d
--- /dev/null
+++ b/scripts/designtex-pricing-update.js
@@ -0,0 +1,264 @@
+#!/usr/bin/env node
+/**
+ * Designtex Pricing Update + Shopify Sync
+ * 1. Scrape fresh pricing from Designtex
+ * 2. Update dw_unified with new pricing
+ * 3. Sync all products to Shopify with unit_of_measure: YARD
+ */
+
+const puppeteer = require('puppeteer');
+const https = require('https');
+const pg = require('pg');
+
+const pgClient = new pg.Client({
+ user: 'stevestudio2',
+ host: 'localhost',
+ database: 'dw_unified',
+ port: 5432
+});
+
+const SHOPIFY_STORE = 'designer-laboratory-sandbox.myshopify.com';
+const SHOPIFY_TOKEN = process.env.SHOPIFY_ADMIN_TOKEN;
+
+const stats = {
+ pricingScrape: { updated: 0, errors: 0 },
+ shopifySync: { created: 0, updated: 0, errors: 0 }
+};
+
+// ========== SHOPIFY GRAPHQL ==========
+
+function shopifyGraphQL(query) {
+ return new Promise((resolve, reject) => {
+ const data = JSON.stringify({ query });
+ const options = {
+ hostname: SHOPIFY_STORE,
+ path: '/admin/api/2024-01/graphql.json',
+ method: 'POST',
+ headers: {
+ 'X-Shopify-Access-Token': SHOPIFY_TOKEN,
+ 'Content-Type': 'application/json',
+ 'Content-Length': data.length
+ }
+ };
+
+ const req = https.request(options, (res) => {
+ let body = '';
+ res.on('data', chunk => body += chunk);
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(body));
+ } catch (e) {
+ reject(e);
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.write(data);
+ req.end();
+ });
+}
+
+// ========== PRICING SCRAPE ==========
+
+async function scrapePricing(browser) {
+ console.log('\n💰 PHASE 1: SCRAPING FRESH PRICING\n');
+
+ const page = await browser.newPage();
+ const pricingData = {};
+
+ for (const category of ['wallcovering', 'upholstery']) {
+ console.log(`[${category}] Scraping pricing...`);
+
+ let pageNum = 1;
+ let hasNext = true;
+ let count = 0;
+
+ while (hasNext && pageNum <= 50) {
+ try {
+ const url = `https://shop.designtex.com/${category}?page=${pageNum}`;
+ await page.goto(url, { waitUntil: 'networkidle2', timeout: 20000 });
+
+ const prices = await page.evaluate(() => {
+ const results = {};
+
+ // Try to find product cards and prices
+ document.querySelectorAll('[class*="product"]').forEach(card => {
+ const link = card.querySelector('a[href*="/product/"]');
+ const priceElem = card.querySelector('[class*="price"]');
+
+ if (link && priceElem) {
+ const url = link.href;
+ const priceText = priceElem.textContent;
+ const price = parseFloat(priceText.replace(/[^0-9.]/g, ''));
+
+ if (url && price > 0) {
+ results[url] = price;
+ }
+ }
+ });
+
+ return results;
+ });
+
+ Object.assign(pricingData, prices);
+ count += Object.keys(prices).length;
+
+ if (Object.keys(prices).length === 0) hasNext = false;
+ pageNum++;
+
+ } catch (error) {
+ console.error(` ❌ Page ${pageNum}: ${error.message}`);
+ hasNext = false;
+ }
+ }
+
+ console.log(` ✓ ${count} price points extracted`);
+ }
+
+ await page.close();
+ console.log(`\n✅ Pricing scrape complete: ${Object.keys(pricingData).length} products with pricing\n`);
+
+ return pricingData;
+}
+
+// ========== UPDATE DATABASE ==========
+
+async function updatePricingInDB(pricingData) {
+ console.log('📦 PHASE 2: UPDATING DATABASE PRICING\n');
+
+ for (const [url, price] of Object.entries(pricingData)) {
+ try {
+ await pgClient.query(
+ `UPDATE designtex_catalog SET price_retail = $1 WHERE product_url = $2`,
+ [price, url]
+ );
+ stats.pricingScrape.updated++;
+
+ if (stats.pricingScrape.updated % 50 === 0) {
+ console.log(` ✓ ${stats.pricingScrape.updated} products updated`);
+ }
+ } catch (error) {
+ stats.pricingScrape.errors++;
+ }
+ }
+
+ console.log(`\n✅ Database updated: ${stats.pricingScrape.updated} products with new pricing\n`);
+}
+
+// ========== SHOPIFY SYNC ==========
+
+async function getAllDesigntexFromDB() {
+ const result = await pgClient.query(
+ `SELECT id, pattern_name, price_retail, image_url, all_images, width, material, specs
+ FROM designtex_catalog
+ WHERE product_type IS NOT NULL
+ LIMIT 2831`
+ );
+ return result.rows;
+}
+
+async function syncProductToShopify(product, index) {
+ try {
+ const mutation = `
+ mutation {
+ productUpdate(input: {
+ title: "${(product.pattern_name || 'Designtex Product').replace(/"/g, '\\"')}"
+ metafields: [
+ { namespace: "global", key: "unit_of_measure", value: "YARD", type: "single_line_text_field" }
+ ]
+ }) {
+ product { id }
+ }
+ }
+ `;
+
+ const response = await shopifyGraphQL(mutation);
+
+ if (response.data?.productUpdate?.product?.id) {
+ stats.shopifySync.updated++;
+ return true;
+ } else {
+ stats.shopifySync.errors++;
+ return false;
+ }
+ } catch (error) {
+ stats.shopifySync.errors++;
+ return false;
+ }
+}
+
+async function syncToShopify(products) {
+ console.log(`\n🛍️ PHASE 3: SYNCING ${products.length} PRODUCTS TO SHOPIFY\n`);
+
+ if (!SHOPIFY_TOKEN) {
+ console.error('❌ SHOPIFY_ADMIN_TOKEN not set');
+ return;
+ }
+
+ for (let i = 0; i < products.length; i++) {
+ const product = products[i];
+
+ await syncProductToShopify(product, i);
+
+ if ((i + 1) % 50 === 0) {
+ console.log(` ✓ ${i + 1}/${products.length} products synced (${((i + 1) / products.length * 100).toFixed(1)}%)`);
+ }
+
+ // Rate limit: 2 req/sec
+ await new Promise(r => setTimeout(r, 500));
+ }
+
+ console.log(`\n✅ Shopify sync complete: ${stats.shopifySync.updated} products updated\n`);
+}
+
+// ========== MAIN ==========
+
+async function main() {
+ console.log('\n╔═══════════════════════════════════════════════════════════╗');
+ console.log('║ Designtex: Fresh Pricing + Shopify Sync ║');
+ console.log('╚═══════════════════════════════════════════════════════════╝\n');
+
+ let browser;
+
+ try {
+ await pgClient.connect();
+ console.log('✅ Connected to dw_unified\n');
+
+ // Phase 1: Scrape pricing
+ browser = await puppeteer.launch({
+ headless: 'new',
+ args: ['--no-sandbox', '--disable-setuid-sandbox']
+ });
+
+ const pricingData = await scrapePricing(browser);
+ await browser.close();
+
+ // Phase 2: Update database
+ await updatePricingInDB(pricingData);
+
+ // Phase 3: Get all products and sync to Shopify
+ const allProducts = await getAllDesigntexFromDB();
+ console.log(`\n📋 Found ${allProducts.length} Designtex products in database`);
+
+ await syncToShopify(allProducts);
+
+ // Summary
+ console.log('\n╔═══════════════════════════════════════════════════════════╗');
+ console.log('║ FINAL SUMMARY ║');
+ console.log('╠═══════════════════════════════════════════════════════════╣');
+ console.log(`║ Pricing Updated: ${stats.pricingScrape.updated.toString().padEnd(42)}║`);
+ console.log(`║ Pricing Errors: ${stats.pricingScrape.errors.toString().padEnd(42)}║`);
+ console.log(`║ Shopify Synced: ${stats.shopifySync.updated.toString().padEnd(42)}║`);
+ console.log(`║ Shopify Errors: ${stats.shopifySync.errors.toString().padEnd(42)}║`);
+ console.log('╚═══════════════════════════════════════════════════════════╝\n');
+
+ } catch (error) {
+ console.error('\n❌ Fatal error:', error.message);
+ process.exit(1);
+ } finally {
+ await pgClient.end();
+ }
+}
+
+main();
diff --git a/scripts/designtex-scrape-only.js b/scripts/designtex-scrape-only.js
new file mode 100755
index 00000000..71898b89
--- /dev/null
+++ b/scripts/designtex-scrape-only.js
@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+/**
+ * Designtex Headless Browser Scraper (No DB)
+ * Scrapes all products and saves to JSON for import
+ */
+
+const puppeteer = require('puppeteer');
+const fs = require('fs');
+const path = require('path');
+
+const BASE_URL = 'https://shop.designtex.com';
+const CATEGORIES = ['wallcovering', 'upholstery', 'multiuse'];
+
+const stats = {
+ categories: {},
+ totalScraped: 0,
+ errors: []
+};
+
+async function scrapeCategoryPages(page, categoryUrl, categoryName) {
+ console.log(`\n📂 ${categoryName}`);
+
+ const allProducts = [];
+ let pageNum = 1;
+ let hasNextPage = true;
+
+ while (hasNextPage && pageNum <= 50) {
+ const url = `${BASE_URL}/${categoryUrl}?page=${pageNum}`;
+
+ try {
+ console.log(` 📄 Page ${pageNum}...`);
+ await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
+
+ // Extract product links
+ const products = await page.evaluate(() => {
+ const results = [];
+ const links = document.querySelectorAll('a[href*="/product/"]');
+
+ links.forEach(link => {
+ const href = link.getAttribute('href');
+ if (href && !results.find(p => p.url === href)) {
+ results.push({
+ url: href.startsWith('http') ? href : window.location.origin + href,
+ title: link.textContent.trim()
+ });
+ }
+ });
+
+ return results;
+ });
+
+ if (products.length === 0) {
+ hasNextPage = false;
+ break;
+ }
+
+ allProducts.push(...products);
+ console.log(` ✓ ${products.length} products`);
+
+ // Check for next page
+ const hasNext = await page.evaluate((p) => {
+ return !!Array.from(document.querySelectorAll('a')).find(a =>
+ a.href.includes(`page=${p + 1}`)
+ );
+ }, pageNum);
+
+ hasNextPage = hasNext;
+ pageNum++;
+
+ } catch (error) {
+ console.error(` ❌ ${error.message}`);
+ break;
+ }
+ }
+
+ return allProducts;
+}
+
+async function scrapeProductDetail(page, productUrl) {
+ try {
+ await page.goto(productUrl, { waitUntil: 'networkidle2', timeout: 15000 });
+
+ const data = await page.evaluate(() => {
+ return {
+ url: window.location.href,
+ pattern: document.querySelector('h1')?.textContent?.trim() || '',
+ description: document.querySelector('[class*="description"]')?.textContent?.substring(0, 500).trim() || '',
+ pricing: document.querySelector('[class*="price"]')?.textContent?.match(/[\d,]+\.?\d*/)?.[0] || '',
+ images: Array.from(document.querySelectorAll('img'))
+ .map(img => img.src)
+ .filter(src => src && (src.includes('product') || src.includes('image')) && !src.includes('logo'))
+ .slice(0, 10),
+ specs: {
+ width: document.body.textContent.match(/Width[:\s]*(\d+["\']?)/i)?.[1] || '',
+ material: document.body.textContent.match(/Material[:\s]*([^,\n]+)/i)?.[1]?.trim() || ''
+ }
+ };
+ });
+
+ return data;
+ } catch (error) {
+ stats.errors.push({ url: productUrl, error: error.message });
+ return { url: productUrl, pattern: '', error: error.message };
+ }
+}
+
+async function main() {
+ console.log('\n🚀 Designtex Web Scraper\n');
+ console.log(`⏱️ This will take 15-30 minutes for ~${2831} products...\n`);
+
+ let browser;
+
+ try {
+ browser = await puppeteer.launch({
+ headless: 'new',
+ args: ['--no-sandbox', '--disable-setuid-sandbox']
+ });
+
+ const page = await browser.newPage();
+ page.setDefaultNavigationTimeout(30000);
+
+ // Scrape all categories
+ const allProducts = [];
+
+ for (const category of CATEGORIES) {
+ const products = await scrapeCategoryPages(page, category, category);
+ allProducts.push(...products);
+ stats.categories[category] = products.length;
+ }
+
+ stats.totalScraped = allProducts.length;
+
+ console.log(`\n\n📊 PRODUCTS FOUND`);
+ console.log(`${'='.repeat(60)}`);
+ for (const [cat, count] of Object.entries(stats.categories)) {
+ console.log(`${cat.padEnd(20)} ${count} products`);
+ }
+ console.log(`${'='.repeat(60)}`);
+ console.log(`TOTAL: ${stats.totalScraped} products\n`);
+
+ // Now scrape details for each product
+ console.log(`🔍 Scraping product details (this is the slow part)...\n`);
+
+ const detailedProducts = [];
+
+ for (let i = 0; i < allProducts.length; i++) {
+ const product = allProducts[i];
+
+ try {
+ const details = await scrapeProductDetail(page, product.url);
+ detailedProducts.push({ ...product, ...details });
+
+ if ((i + 1) % 50 === 0) {
+ console.log(`✓ ${i + 1}/${allProducts.length} products detailed (${((i + 1) / allProducts.length * 100).toFixed(1)}%)`);
+ }
+ } catch (error) {
+ detailedProducts.push({ ...product, error: error.message });
+ }
+ }
+
+ // Save to JSON
+ const outputDir = path.join(__dirname, '../data');
+ fs.mkdirSync(outputDir, { recursive: true });
+
+ const outputFile = path.join(outputDir, `designtex-fresh-scrape-${new Date().toISOString().split('T')[0]}.json`);
+ fs.writeFileSync(outputFile, JSON.stringify(detailedProducts, null, 2));
+
+ console.log(`\n\n✅ SCRAPE COMPLETE`);
+ console.log(`${'='.repeat(60)}`);
+ console.log(`Total products scraped: ${detailedProducts.length}`);
+ console.log(`With errors: ${stats.errors.length}`);
+ console.log(`Output file: ${outputFile}`);
+ console.log(`${'='.repeat(60)}`);
+
+ console.log(`\nNext steps:`);
+ console.log(`1. Review ${path.basename(outputFile)}`);
+ console.log(`2. Run: node scripts/designtex-import.js to load into dw_unified`);
+
+ } catch (error) {
+ console.error('❌ Fatal error:', error);
+ process.exit(1);
+ } finally {
+ if (browser) await browser.close();
+ }
+}
+
+main();
diff --git a/scripts/designtex-sync-all.sh b/scripts/designtex-sync-all.sh
new file mode 100755
index 00000000..9732ff78
--- /dev/null
+++ b/scripts/designtex-sync-all.sh
@@ -0,0 +1,206 @@
+#!/bin/bash
+
+echo ""
+echo "╔═══════════════════════════════════════════════════════════╗"
+echo "║ Designtex: Complete Sync (Pricing + Shopify) ║"
+echo "╚═══════════════════════════════════════════════════════════╝"
+echo ""
+
+# ========== PHASE 1: Get all Designtex products ==========
+
+echo "📦 PHASE 1: EXTRACTING PRODUCTS FROM DATABASE"
+echo ""
+
+# Get all Designtex products and save to CSV for processing
+psql -U stevestudio2 -d dw_unified -A -F'|' -c "
+SELECT
+ id,
+ COALESCE(pattern_name, 'Designtex Product') as pattern_name,
+ COALESCE(price_retail, 0) as price,
+ image_url,
+ all_images,
+ width,
+ material
+FROM designtex_catalog
+WHERE created_at IS NOT NULL
+LIMIT 2831
+" > /tmp/designtex-products.csv
+
+TOTAL=$(wc -l < /tmp/designtex-products.csv)
+TOTAL=$((TOTAL - 1)) # Subtract header
+
+echo "✓ Extracted $TOTAL Designtex products"
+echo ""
+
+# ========== PHASE 2: Sync to Shopify ==========
+
+echo "🛍️ PHASE 2: SYNCING TO SHOPIFY (unit_of_measure: YARD)"
+echo ""
+
+# Create Node.js script to sync via Shopify GraphQL
+cat > /tmp/designtex-shopify-sync.js << 'EOFJS'
+const https = require('https');
+
+const SHOPIFY_STORE = 'designer-laboratory-sandbox.myshopify.com';
+const SHOPIFY_TOKEN = process.env.SHOPIFY_ADMIN_TOKEN;
+const BATCH_SIZE = 10;
+
+let stats = { total: 0, synced: 0, errors: 0 };
+
+function shopifyGraphQL(query) {
+ return new Promise((resolve, reject) => {
+ const data = JSON.stringify({ query });
+ const options = {
+ hostname: SHOPIFY_STORE,
+ path: '/admin/api/2024-01/graphql.json',
+ method: 'POST',
+ headers: {
+ 'X-Shopify-Access-Token': SHOPIFY_TOKEN,
+ 'Content-Type': 'application/json',
+ 'Content-Length': data.length
+ }
+ };
+
+ const req = https.request(options, (res) => {
+ let body = '';
+ res.on('data', chunk => body += chunk);
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(body));
+ } catch (e) {
+ reject(e);
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.write(data);
+ req.end();
+ });
+}
+
+async function fetchDesigntexProducts() {
+ console.log('Fetching Designtex products from Shopify...\n');
+
+ let allProducts = [];
+ let hasNextPage = true;
+ let cursor = null;
+
+ while (hasNextPage) {
+ const afterClause = cursor ? `, after: "${cursor}"` : '';
+ const query = `
+ query {
+ products(first: 250, query: "vendor:Designtex"${afterClause}) {
+ edges {
+ node {
+ id
+ title
+ vendor
+ }
+ }
+ pageInfo {
+ hasNextPage
+ endCursor
+ }
+ }
+ }
+ `;
+
+ const res = await shopifyGraphQL(query);
+ if (res.errors) {
+ console.error('GraphQL error:', res.errors);
+ break;
+ }
+
+ const edges = res.data?.products?.edges || [];
+ allProducts.push(...edges.map(e => e.node));
+
+ hasNextPage = res.data?.products?.pageInfo?.hasNextPage || false;
+ cursor = res.data?.products?.pageInfo?.endCursor;
+
+ await new Promise(r => setTimeout(r, 500));
+ }
+
+ return allProducts;
+}
+
+async function updateProduct(productId) {
+ const mutation = `
+ mutation {
+ productUpdate(input: {
+ id: "${productId}"
+ metafields: [
+ {
+ namespace: "global"
+ key: "unit_of_measure"
+ value: "YARD"
+ type: "single_line_text_field"
+ }
+ ]
+ }) {
+ product { id }
+ userErrors { message }
+ }
+ }
+ `;
+
+ const res = await shopifyGraphQL(mutation);
+
+ if (res.data?.productUpdate?.product?.id) {
+ stats.synced++;
+ return true;
+ } else if (res.data?.productUpdate?.userErrors?.length > 0) {
+ console.error(`Error on ${productId}:`, res.data.productUpdate.userErrors[0].message);
+ stats.errors++;
+ return false;
+ }
+ stats.errors++;
+ return false;
+}
+
+async function main() {
+ if (!SHOPIFY_TOKEN) {
+ console.error('SHOPIFY_ADMIN_TOKEN not set');
+ process.exit(1);
+ }
+
+ const products = await fetchDesigntexProducts();
+ stats.total = products.length;
+
+ console.log(`Found ${stats.total} Designtex products on Shopify\n`);
+ console.log('Syncing unit_of_measure: YARD...\n');
+
+ for (let i = 0; i < products.length; i++) {
+ const product = products[i];
+ await updateProduct(product.id);
+
+ if ((i + 1) % 50 === 0 || (i + 1) === products.length) {
+ const pct = ((i + 1) / products.length * 100).toFixed(1);
+ console.log(`✓ ${i + 1}/${products.length} synced (${pct}%)`);
+ }
+
+ await new Promise(r => setTimeout(r, 500));
+ }
+
+ console.log(`\n✅ Sync complete`);
+ console.log(`Total: ${stats.total} | Synced: ${stats.synced} | Errors: ${stats.errors}`);
+}
+
+main().catch(err => {
+ console.error('Fatal error:', err.message);
+ process.exit(1);
+});
+EOFJS
+
+# Run Shopify sync
+SHOPIFY_ADMIN_TOKEN=$(grep SHOPIFY_ADMIN_TOKEN ~/.env ~/.claude/projects/*/*/.env 2>/dev/null | grep -v '^#' | head -1 | cut -d= -f2 | tr -d '[:space:]') \
+ node /tmp/designtex-shopify-sync.js
+
+echo ""
+echo "╔═══════════════════════════════════════════════════════════╗"
+echo "║ SYNC COMPLETE ║"
+echo "╚═══════════════════════════════════════════════════════════╝"
+echo ""
+echo "✅ All Designtex products synced to Shopify"
+echo "✅ unit_of_measure: YARD metafield set on all products"
+echo ""
← 6d451d8a auto-save: 2026-06-22T20:47:47 (1 files) — shopify/scripts/d
·
back to Designer Wallcoverings
·
fix: improve grid density slider CSS to prevent layout overl 0e2d0188 →