← back to Crezana Scraper
initial scaffold: crezana-scraper (Crezana onboard)
078064ed47ff4a03e00994dc171f161433fdc3bd · 2026-07-09 09:12:12 -0700 · Steve
Files touched
A .gitignoreA enrich.mjsA package-lock.jsonA package.jsonA scrape.mjs
Diff
commit 078064ed47ff4a03e00994dc171f161433fdc3bd
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jul 9 09:12:12 2026 -0700
initial scaffold: crezana-scraper (Crezana onboard)
---
.gitignore | 8 +++
enrich.mjs | 85 ++++++++++++++++++++++++++++
package-lock.json | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
package.json | 16 ++++++
scrape.mjs | 142 +++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 413 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/enrich.mjs b/enrich.mjs
new file mode 100644
index 0000000..405904d
--- /dev/null
+++ b/enrich.mjs
@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+// Enrich crezana_catalog rows with qwen2.5vl:7b vision on Mac1 Ollama. $0 local.
+// Fills: ai_colors (hex+name), color_hex, dominant_color_hex, ai_styles, ai_tags, description.
+import pg from 'pg';
+
+const OLLAMA = process.env.OLLAMA_URL || 'http://192.168.1.133:11434';
+const MODEL = 'qwen2.5vl:7b';
+const PGHOST = process.env.PGHOST || '/tmp';
+const CONCURRENCY = 3;
+
+const PROMPT = `You are a textile/fabric merchandiser. Look at this fabric or textile swatch image and return STRICT JSON only, no prose:
+{
+ "description": "one vivid sentence (max 30 words) describing this fabric — texture, motif, mood",
+ "colors": [{"name":"color name","hex":"#RRGGBB"}, ...up to 4, ordered by dominance],
+ "dominant_hex": "#RRGGBB",
+ "styles": ["style tag", ...up to 3 e.g. Modern, Traditional, Damask, Floral, Geometric, Textured, Embroidered],
+ "tags": ["descriptive tag", ...up to 6]
+}`;
+
+async function fetchImageB64(url) {
+ const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0' } });
+ if (!r.ok) throw new Error(`img ${r.status}`);
+ const buf = Buffer.from(await r.arrayBuffer());
+ return buf.toString('base64');
+}
+
+async function visionCall(b64) {
+ const r = await fetch(`${OLLAMA}/api/generate`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ model: MODEL, prompt: PROMPT, images: [b64], stream: false, format: 'json', options: { temperature: 0.1 } }),
+ });
+ if (!r.ok) throw new Error(`ollama ${r.status}`);
+ const j = await r.json();
+ return JSON.parse(j.response);
+}
+
+const HEX = /^#[0-9a-fA-F]{6}$/;
+function cleanHex(h) { return (typeof h === 'string' && HEX.test(h.trim())) ? h.trim().toLowerCase() : null; }
+
+async function main() {
+ const onlyNew = process.argv.includes('--new');
+ const client = new pg.Client({ host: PGHOST, database: 'dw_unified' });
+ await client.connect();
+ const where = onlyNew ? "AND ai_accepted_at IS NULL" : "";
+ const { rows } = await client.query(
+ `SELECT id, mfr_sku, image_url FROM crezana_catalog WHERE image_url IS NOT NULL ${where} ORDER BY id`);
+ console.log(`Enriching ${rows.length} rows with ${MODEL} (concurrency ${CONCURRENCY}) — $0 (local)`);
+
+ let done = 0, ok = 0, fail = 0;
+ const queue = [...rows];
+ async function worker() {
+ while (queue.length) {
+ const row = queue.shift();
+ try {
+ // request a reasonably sized render from the isteam CDN
+ const url = row.image_url + (row.image_url.includes('/:/') ? '' : '/:/rs=w:600');
+ const b64 = await fetchImageB64(url);
+ const v = await visionCall(b64);
+ const colors = Array.isArray(v.colors) ? v.colors.filter(c => c && c.name).map(c => ({ name: String(c.name).slice(0,40), hex: cleanHex(c.hex) })) : [];
+ const domHex = cleanHex(v.dominant_hex) || (colors[0] && colors[0].hex) || null;
+ const styles = Array.isArray(v.styles) ? v.styles.map(s => String(s).slice(0,40)).slice(0,3) : [];
+ const tags = Array.isArray(v.tags) ? v.tags.map(t => String(t).slice(0,40)).slice(0,6) : [];
+ const desc = typeof v.description === 'string' ? v.description.slice(0, 300) : null;
+ await client.query(
+ `UPDATE crezana_catalog SET
+ ai_colors=$1, color_hex=$2, dominant_color_hex=$2,
+ ai_styles=$3, ai_tags=$4, description=COALESCE($5, description), ai_description=$5,
+ ai_accepted_at=now(), updated_at=now()
+ WHERE id=$6`,
+ [JSON.stringify(colors), domHex, JSON.stringify(styles), JSON.stringify(tags), desc, row.id]);
+ ok++;
+ } catch (e) {
+ fail++;
+ if (fail <= 5) console.error(` ! ${row.mfr_sku}: ${e.message}`);
+ }
+ done++;
+ if (done % 25 === 0) console.log(` ${done}/${rows.length} (ok=${ok} fail=${fail})`);
+ }
+ }
+ await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+ await client.end();
+ console.log(`Done. ok=${ok} fail=${fail}. Cost: $0 (local)`);
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 0000000..71e1484
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,162 @@
+{
+ "name": "crezana-scraper",
+ "version": "1.0.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "crezana-scraper",
+ "version": "1.0.0",
+ "license": "ISC",
+ "dependencies": {
+ "pg": "^8.22.0"
+ }
+ },
+ "node_modules/pg": {
+ "version": "8.22.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
+ "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-connection-string": "^2.14.0",
+ "pg-pool": "^3.14.0",
+ "pg-protocol": "^1.15.0",
+ "pg-types": "2.2.0",
+ "pgpass": "1.0.5"
+ },
+ "engines": {
+ "node": ">= 16.0.0"
+ },
+ "optionalDependencies": {
+ "pg-cloudflare": "^1.4.0"
+ },
+ "peerDependencies": {
+ "pg-native": ">=3.0.1"
+ },
+ "peerDependenciesMeta": {
+ "pg-native": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/pg-cloudflare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
+ "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
+ "license": "MIT",
+ "optional": true
+ },
+ "node_modules/pg-connection-string": {
+ "version": "2.14.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
+ "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
+ "license": "MIT"
+ },
+ "node_modules/pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+ "license": "ISC",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/pg-pool": {
+ "version": "3.14.0",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
+ "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
+ "license": "MIT",
+ "peerDependencies": {
+ "pg": ">=8.0"
+ }
+ },
+ "node_modules/pg-protocol": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
+ "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
+ "license": "MIT"
+ },
+ "node_modules/pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "license": "MIT",
+ "dependencies": {
+ "split2": "^4.1.0"
+ }
+ },
+ "node_modules/postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/postgres-bytea": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
+ "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "license": "MIT",
+ "dependencies": {
+ "xtend": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/split2": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+ "license": "ISC",
+ "engines": {
+ "node": ">= 10.x"
+ }
+ },
+ "node_modules/xtend": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4"
+ }
+ }
+ }
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..6c5ae7a
--- /dev/null
+++ b/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "crezana-scraper",
+ "version": "1.0.0",
+ "description": "",
+ "main": "index.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "type": "commonjs",
+ "dependencies": {
+ "pg": "^8.22.0"
+ }
+}
diff --git a/scrape.mjs b/scrape.mjs
new file mode 100644
index 0000000..c67b338
--- /dev/null
+++ b/scrape.mjs
@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+// Crezana Design feed-first scraper — GoDaddy Website Builder (DPS) editorial galleries.
+// $0 plain-fetch. No SKUs/prices on the site → derive deterministic mfr_sku from image hash.
+// Vendor-scoped to Crezana ONLY. product_type = 'Fabric'. QUOTE-ONLY.
+import pg from 'pg';
+
+const SITE = 'https://crezanadesign.com';
+const SITE_UUID = '1f48ce84-09d7-40fc-8a82-23198f7239f7';
+const PGHOST = process.env.PGHOST || '/tmp';
+
+// Category pages discovered from sitemap.website.xml. Path -> friendly collection label.
+const CATEGORIES = [
+ { path: '/', collection: 'Home' },
+ { path: '/printed', collection: 'Printed' },
+ { path: '/textures', collection: 'Textures' },
+ { path: '/embroideries', collection: 'Embroideries' },
+ { path: '/embroideries-ii',collection: 'Embroideries II' },
+ { path: '/madagascars', collection: 'Madagascars' },
+ { path: '/grommets%2Fstuds', collection: 'Grommets & Studs' },
+ { path: '/showrooms', collection: 'Showrooms' },
+];
+
+async function fetchText(url) {
+ const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' } });
+ if (!r.ok) throw new Error(`${r.status} ${url}`);
+ return await r.text();
+}
+
+// GoDaddy stores captions as escaped DraftJS JSON. Decode to plain text.
+function decodeCaption(escaped) {
+ if (!escaped) return '';
+ try {
+ // The JS bundle double-escapes; unescape backslashes then parse.
+ let s = escaped.replace(/\\"/g, '"').replace(/\\\\/g, '\\');
+ const j = JSON.parse(s);
+ if (j && j.blocks) return j.blocks.map(b => b.text).join(' ').trim();
+ } catch { /* fall through */ }
+ return '';
+}
+
+// Boilerplate disclaimer text we want to ignore as a "name".
+const BOILERPLATE = /inconsistencies may occur|dye lot|request sample|patterns and colors are approximate/i;
+
+// Parse a gpub script bundle for gallery images with captions.
+// Structure (escaped): "galleryImages":[{"image":{... "image":"//img1.wsimg.com/isteam/ip/UUID/HASH.jpg" ...},"caption":"{draftjs}"}]
+function parseGpub(js) {
+ const out = [];
+ // Find each image object + its following caption.
+ // Match: "image":"//img1.wsimg.com/isteam/ip/UUID/HASH.ext" ... "caption":"..."
+ const imgRe = new RegExp('\\\\?"image\\\\?":\\\\?"(\\/\\/img1\\.wsimg\\.com\\/isteam\\/ip\\/' + SITE_UUID + '\\/[a-f0-9]+\\.(?:jpg|jpeg|png|webp))\\\\?"', 'gi');
+ let m;
+ const seen = new Set();
+ while ((m = imgRe.exec(js)) !== null) {
+ let url = m[1];
+ if (seen.has(url)) continue;
+ seen.add(url);
+ // Try to grab a caption within the next ~600 chars after this image.
+ const tail = js.slice(m.index, m.index + 1200);
+ const capM = tail.match(/\\?"caption\\?":\\?"(\{.*?entityMap.*?\})\\?"/);
+ let caption = '';
+ if (capM) caption = decodeCaption(capM[1]);
+ if (BOILERPLATE.test(caption)) caption = '';
+ out.push({ url: 'https:' + url, caption });
+ }
+ return out;
+}
+
+function hashFromUrl(url) {
+ const m = url.match(/\/([a-f0-9]{16,})\.(?:jpg|jpeg|png|webp)/i);
+ return m ? m[1] : null;
+}
+
+async function main() {
+ const items = []; // {mfr_sku, collection, image_url, product_url, caption}
+ const globalSeenHash = new Set();
+
+ for (const cat of CATEGORIES) {
+ const pageUrl = SITE + cat.path;
+ let html;
+ try { html = await fetchText(pageUrl); }
+ catch (e) { console.error(` ! page ${cat.path}: ${e.message}`); continue; }
+
+ const hashes = [...html.matchAll(/gpub\/([a-f0-9]+)\/script\.js/g)].map(x => x[1]);
+ const uniqHashes = [...new Set(hashes)];
+ let catImages = [];
+ for (const h of uniqHashes) {
+ try {
+ const js = await fetchText(`https://img1.wsimg.com/blobby/go/${SITE_UUID}/gpub/${h}/script.js`);
+ catImages = catImages.concat(parseGpub(js));
+ } catch (e) { console.error(` ! gpub ${h}: ${e.message}`); }
+ }
+
+ let n = 0;
+ for (const img of catImages) {
+ const hash = hashFromUrl(img.url);
+ if (!hash) continue;
+ if (globalSeenHash.has(hash)) continue; // an image appears in one canonical collection only
+ globalSeenHash.add(hash);
+ n++;
+ const mfr_sku = `CREZ-${hash.slice(0, 12).toUpperCase()}`;
+ const collSlug = cat.collection.replace(/[^A-Za-z0-9]+/g, '-').replace(/-+$/,'');
+ const pattern_name = img.caption && img.caption.length > 2 && img.caption.length < 120
+ ? img.caption
+ : `Crezana ${cat.collection} ${String(n).padStart(3, '0')}`;
+ items.push({
+ mfr_sku,
+ pattern_name,
+ collection: cat.collection,
+ image_url: img.url,
+ product_url: pageUrl,
+ });
+ }
+ console.log(` ${cat.collection}: ${n} images`);
+ }
+
+ console.log(`\nTotal unique gallery images: ${items.length}`);
+
+ // Upsert
+ const client = new pg.Client({ host: PGHOST, database: 'dw_unified' });
+ await client.connect();
+ let ins = 0, upd = 0;
+ for (const it of items) {
+ const r = await client.query(
+ `INSERT INTO crezana_catalog (mfr_sku, pattern_name, collection, image_url, product_url, product_type, crawled_at, last_scraped, updated_at)
+ VALUES ($1,$2,$3,$4,$5,'Fabric',now(),now(),now())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name=EXCLUDED.pattern_name,
+ collection=EXCLUDED.collection,
+ image_url=EXCLUDED.image_url,
+ product_url=EXCLUDED.product_url,
+ last_scraped=now(),
+ updated_at=now()
+ RETURNING (xmax=0) AS inserted`,
+ [it.mfr_sku, it.pattern_name, it.collection, it.image_url, it.product_url]
+ );
+ if (r.rows[0].inserted) ins++; else upd++;
+ }
+ await client.end();
+ console.log(`Upserted: ${ins} inserted, ${upd} updated. Cost: $0 (local)`);
+}
+
+main().catch(e => { console.error(e); process.exit(1); });
(oldest)
·
back to Crezana Scraper
·
Auto-assign DWCZ- series SKU to new rows on scrape (idempote f721c89 →