← back to Gracie Scraper
initial scaffold: Gracie scraper (feed-first scraper + local enrich)
f8c8306e164c8db5d6442021d881e33b271b1f20 · 2026-07-09 09:15:06 -0700 · Steve
Files touched
A .gitignoreA node_modulesA package.jsonA scripts/enrich.mjsA scripts/scrape.mjs
Diff
commit f8c8306e164c8db5d6442021d881e33b271b1f20
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jul 9 09:15:06 2026 -0700
initial scaffold: Gracie scraper (feed-first scraper + local enrich)
---
.gitignore | 9 +++
node_modules | 1 +
package.json | 8 +++
scripts/enrich.mjs | 108 ++++++++++++++++++++++++++++
scripts/scrape.mjs | 202 +++++++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 328 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ce356f3
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,9 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
+data/*.log
diff --git a/node_modules b/node_modules
new file mode 120000
index 0000000..a3d2394
--- /dev/null
+++ b/node_modules
@@ -0,0 +1 @@
+/Users/macstudio3/Projects/all-designerwallcoverings/node_modules
\ No newline at end of file
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..e53bf8d
--- /dev/null
+++ b/package.json
@@ -0,0 +1,8 @@
+{
+ "name": "gracie-scraper",
+ "version": "1.0.0",
+ "private": true,
+ "type": "module",
+ "description": "Gracie Studio feed-first scraper (dw_unified.gracie_catalog)",
+ "scripts": { "scrape": "node scripts/scrape.mjs" }
+}
diff --git a/scripts/enrich.mjs b/scripts/enrich.mjs
new file mode 100644
index 0000000..ac55e0d
--- /dev/null
+++ b/scripts/enrich.mjs
@@ -0,0 +1,108 @@
+#!/usr/bin/env node
+/**
+ * Gracie image enrichment — LOCAL qwen2.5vl:7b vision on Mac1 Ollama.
+ * Reads gracie_catalog rows with an image_url, downloads the packshot,
+ * asks qwen2.5vl for ai_colors (hex+name), color_hex (dominant), ai_styles,
+ * ai_tags, and a clean 1-2 sentence description. Writes back to PG.
+ * Cost: $0 (local Ollama). No paid API.
+ *
+ * Resumable: only enriches rows where ai_accepted_at IS NULL (or --all to redo).
+ */
+import { Client } from 'pg';
+
+const OLLAMA = process.env.OLLAMA || 'http://192.168.1.133:11434';
+const MODEL = 'qwen2.5vl:7b';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120 Safari/537.36';
+const REDO = process.argv.includes('--all');
+const LIMIT = (() => { const i = process.argv.indexOf('--limit'); return i > -1 ? parseInt(process.argv[i + 1], 10) : null; })();
+
+// Sanity CDN lets us request a smaller render for faster vision — append ?w=768
+function smallImg(url) {
+ if (!url) return url;
+ return url.includes('?') ? url : `${url}?w=768&q=75`;
+}
+
+async function fetchB64(url) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA } });
+ if (!res.ok) throw new Error(`img HTTP ${res.status}`);
+ const buf = Buffer.from(await res.arrayBuffer());
+ return buf.toString('base64');
+}
+
+const PROMPT = `You are a luxury interior-design cataloguer looking at a hand-painted wallcovering panel image.
+Return STRICT JSON only, no prose, with this exact shape:
+{
+ "colors": [{"name":"<color name e.g. Celadon>","hex":"#rrggbb"}, ... up to 5, ordered by prominence],
+ "dominant_hex": "#rrggbb",
+ "background_hex": "#rrggbb",
+ "styles": ["<e.g. Chinoiserie, Scenic, Traditional>"],
+ "tags": ["<motif/subject words e.g. flowering-tree, songbird, pagoda>"],
+ "description": "<one or two elegant sentences describing the scene and palette>"
+}
+Use real color names an interior designer would use. Hex must be valid 6-digit. Do not include markdown fences.`;
+
+function extractJson(txt) {
+ if (!txt) return null;
+ let s = txt.trim().replace(/^```json?/i, '').replace(/```$/,'').trim();
+ const a = s.indexOf('{'), b = s.lastIndexOf('}');
+ if (a === -1 || b === -1) return null;
+ try { return JSON.parse(s.slice(a, b + 1)); } catch { return null; }
+}
+
+async function enrichOne(imgUrl) {
+ const b64 = await fetchB64(smallImg(imgUrl));
+ const res = await fetch(`${OLLAMA}/api/generate`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ model: MODEL, prompt: PROMPT, images: [b64], stream: false,
+ options: { temperature: 0.2 }, format: 'json',
+ }),
+ });
+ if (!res.ok) throw new Error(`ollama HTTP ${res.status}`);
+ const j = await res.json();
+ return extractJson(j.response);
+}
+
+async function main() {
+ const pg = new Client({ connectionString: process.env.PG || 'postgresql://localhost/dw_unified' });
+ await pg.connect();
+ const where = REDO ? 'image_url IS NOT NULL' : 'image_url IS NOT NULL AND ai_accepted_at IS NULL';
+ const { rows } = await pg.query(
+ `SELECT id, mfr_sku, image_url FROM gracie_catalog WHERE ${where} ORDER BY id ${LIMIT ? `LIMIT ${LIMIT}` : ''}`
+ );
+ console.log(`[enrich] ${rows.length} rows to enrich via ${MODEL} @ ${OLLAMA}. Cost: $0 (local)`);
+ let ok = 0, fail = 0;
+ for (const r of rows) {
+ try {
+ const e = await enrichOne(r.image_url);
+ if (!e || !Array.isArray(e.colors)) { fail++; console.warn(` ✗ ${r.mfr_sku}: no JSON`); continue; }
+ const hex = (e.dominant_hex && /^#[0-9a-fA-F]{6}$/.test(e.dominant_hex)) ? e.dominant_hex.toLowerCase() : null;
+ await pg.query(
+ `UPDATE gracie_catalog SET
+ ai_colors=$2, color_hex=COALESCE($3,color_hex), dominant_color_hex=COALESCE($3,dominant_color_hex),
+ ai_background_color=$4, ai_styles=$5, ai_tags=COALESCE($6, ai_tags),
+ ai_description=$7, ai_accepted_at=NOW(), agent_updated_at=NOW()
+ WHERE id=$1`,
+ [
+ r.id,
+ JSON.stringify(e.colors || []),
+ hex,
+ (e.background_hex && /^#[0-9a-fA-F]{6}$/.test(e.background_hex)) ? e.background_hex.toLowerCase() : null,
+ JSON.stringify(e.styles || []),
+ Array.isArray(e.tags) && e.tags.length ? JSON.stringify(e.tags) : null,
+ e.description || null,
+ ]
+ );
+ ok++;
+ if (ok % 10 === 0) console.log(` … ${ok}/${rows.length}`);
+ } catch (err) {
+ fail++;
+ console.warn(` ✗ ${r.mfr_sku}: ${err.message}`);
+ }
+ }
+ console.log(`[enrich] DONE. ok=${ok} fail=${fail}. Cost: $0 (local)`);
+ await pg.end();
+}
+
+main().catch((e) => { console.error('[enrich] FATAL', e); process.exit(1); });
diff --git a/scripts/scrape.mjs b/scripts/scrape.mjs
new file mode 100644
index 0000000..578a2a7
--- /dev/null
+++ b/scripts/scrape.mjs
@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+/**
+ * Gracie Studio scraper — FEED-FIRST, $0 plain-fetch.
+ *
+ * Gracie is a Next.js site over a Sanity CMS backend. There is NO usable
+ * sitemap.xml (it returns the SPA HTML). The catalog is exposed as Next.js
+ * page-data JSON:
+ * https://graciestudio.com/_next/data/<buildId>/collection/<slug>.json?slug=<slug>
+ *
+ * pageProps.collectionContent.wallpapers[] is the full product array.
+ * Each wallpaper: title, subTitle (Gracie product code = mfr_sku), slug,
+ * tags[], rightBody (rich-text description), image / hoverImage / imageList[]
+ * (Sanity asset refs).
+ *
+ * Sanity image URL: https://cdn.sanity.io/images/<projectId>/<dataset>/<assetId>-<WxH>.<ext>
+ * from ref image-<assetId>-<WxH>-<ext>.
+ *
+ * The buildId is discovered live from the homepage __NEXT_DATA__, so a Gracie
+ * redeploy (new buildId) does not break the scraper.
+ *
+ * Upserts into dw_unified.gracie_catalog keyed on mfr_sku (ON CONFLICT).
+ * Cost: $0 (local) — pure HTTP fetch, no Browserbase / proxy / captcha / paid API.
+ */
+import { Client } from 'pg';
+
+const UA =
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
+const BASE = 'https://graciestudio.com';
+const SANITY = { projectId: 'fiu2vnvt', dataset: 'production' };
+const DELAY_MS = 1200; // polite delay between collection fetches
+
+// Sections that are wallcovering scenic panels vs fabric vs prints.
+// Gracie is quote-only bespoke; we treat all as 'wallcovering' EXCEPT the
+// explicit fabric collection.
+const FABRIC_COLLECTIONS = new Set(['gracie-fabric']);
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+async function fetchText(url) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json,text/html' } });
+ if (!res.ok) throw new Error(`HTTP ${res.status} ${url}`);
+ return res.text();
+}
+
+async function getBuildIdAndCollections() {
+ const html = await fetchText(`${BASE}/`);
+ const bid = (html.match(/"buildId":"([^"]+)"/) || [])[1];
+ if (!bid) throw new Error('could not find Next.js buildId on homepage');
+ const cols = [...new Set([...html.matchAll(/href="\/collection\/([^"]+)"/g)].map((m) => m[1]))];
+ return { buildId: bid, collections: cols };
+}
+
+// image-<assetId>-<WxH>-<ext> -> cdn URL
+function refToUrl(ref) {
+ if (!ref || typeof ref !== 'string') return null;
+ const m = ref.match(/^image-([a-f0-9]+)-(\d+x\d+)-(\w+)$/);
+ if (!m) return null;
+ const [, assetId, dims, ext] = m;
+ return `https://cdn.sanity.io/images/${SANITY.projectId}/${SANITY.dataset}/${assetId}-${dims}.${ext}`;
+}
+
+function imgUrl(imgObj) {
+ return refToUrl(imgObj?.asset?._ref);
+}
+
+// flatten Sanity portable-text rich body to a plain string
+function flattenBody(body) {
+ if (!Array.isArray(body)) return '';
+ const parts = [];
+ for (const blk of body) {
+ for (const ch of blk.children || []) if (ch.text) parts.push(ch.text);
+ }
+ return parts.join(' ').replace(/\s+/g, ' ').trim();
+}
+
+// strip the boilerplate CTA sentences Gracie appends to every description
+function cleanDescription(txt) {
+ if (!txt) return '';
+ return txt
+ .replace(/Complete the form below[^.]*\./gi, '')
+ .replace(/Click the link below[^.]*\./gi, '')
+ .replace(/Typically stocked pre-painted[^.]*\./gi, '')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+// Banned-word rule: OUR titles/descriptions say "Wallcovering", never "Wallpaper".
+function sanitize(txt) {
+ if (!txt) return txt;
+ return txt
+ .replace(/\bwallpapers\b/gi, 'wallcoverings')
+ .replace(/\bwallpaper\b/gi, 'wallcovering');
+}
+
+function parseWallpaper(w, collectionSlug, collectionTitle) {
+ const mfr = (w.subTitle || w.slug?.current || w._id || '').trim();
+ if (!mfr) return null;
+ const pattern = (w.title || mfr).trim();
+ const gallery = (w.imageList || []).map((im) => imgUrl(im)).filter(Boolean);
+ const primary = imgUrl(w.image) || imgUrl(w.hoverImage) || gallery[0] || null;
+ const rawDesc = flattenBody(w.rightBody);
+ const desc = sanitize(cleanDescription(rawDesc));
+ const tags = (w.tags || []).map((t) => String(t)).filter(Boolean);
+ const productUrl = w.slug?.current
+ ? `${BASE}/${w.slug.current}`
+ : `${BASE}/collection/${collectionSlug}`;
+ const isFabric = FABRIC_COLLECTIONS.has(collectionSlug);
+ return {
+ mfr_sku: mfr,
+ pattern_name: sanitize(pattern),
+ collection: sanitize(collectionTitle || collectionSlug),
+ image_url: primary,
+ product_url: productUrl,
+ product_type: isFabric ? 'fabric' : 'wallcovering',
+ material: 'Handpainted',
+ description: desc,
+ all_images: gallery.join('|'),
+ gallery_images: gallery,
+ specs: {
+ gracie_id: w._id,
+ subtitle: w.subTitle || null,
+ tags,
+ hover_image: imgUrl(w.hoverImage),
+ created_at: w._createdAt,
+ updated_at: w._updatedAt,
+ },
+ ai_tags: tags.length ? tags : null,
+ };
+}
+
+async function main() {
+ const { buildId, collections } = await getBuildIdAndCollections();
+ console.log(`[gracie] buildId=${buildId} collections=${collections.length}: ${collections.join(', ')}`);
+
+ const pg = new Client({ connectionString: process.env.PG || 'postgresql://localhost/dw_unified' });
+ await pg.connect();
+
+ const seen = new Set();
+ let upserts = 0;
+ const perCol = {};
+
+ for (const slug of collections) {
+ const url = `${BASE}/_next/data/${buildId}/collection/${encodeURIComponent(slug)}.json?slug=${encodeURIComponent(slug)}`;
+ let json;
+ try {
+ json = JSON.parse(await fetchText(url));
+ } catch (e) {
+ console.warn(`[gracie] SKIP ${slug}: ${e.message}`);
+ continue;
+ }
+ const cc = json?.pageProps?.collectionContent;
+ if (!cc) {
+ console.warn(`[gracie] ${slug}: no collectionContent`);
+ continue;
+ }
+ const wallpapers = cc.wallpapers || [];
+ perCol[slug] = 0;
+ for (const w of wallpapers) {
+ const rec = parseWallpaper(w, slug, cc.title);
+ if (!rec) continue;
+ if (seen.has(rec.mfr_sku)) continue; // dedup within run (patterns repeat across collections)
+ seen.add(rec.mfr_sku);
+ await pg.query(
+ `INSERT INTO gracie_catalog
+ (mfr_sku, pattern_name, collection, image_url, product_url, product_type,
+ material, description, all_images, gallery_images, specs, ai_tags, last_scraped, crawled_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,NOW(),NOW())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name=EXCLUDED.pattern_name,
+ collection=EXCLUDED.collection,
+ image_url=COALESCE(EXCLUDED.image_url, gracie_catalog.image_url),
+ product_url=EXCLUDED.product_url,
+ product_type=EXCLUDED.product_type,
+ material=EXCLUDED.material,
+ description=EXCLUDED.description,
+ all_images=EXCLUDED.all_images,
+ gallery_images=EXCLUDED.gallery_images,
+ specs=EXCLUDED.specs,
+ ai_tags=COALESCE(EXCLUDED.ai_tags, gracie_catalog.ai_tags),
+ last_scraped=NOW(), updated_at=NOW()`,
+ [
+ rec.mfr_sku, rec.pattern_name, rec.collection, rec.image_url, rec.product_url,
+ rec.product_type, rec.material, rec.description, rec.all_images, rec.gallery_images,
+ JSON.stringify(rec.specs), rec.ai_tags ? JSON.stringify(rec.ai_tags) : null,
+ ]
+ );
+ upserts++;
+ perCol[slug]++;
+ }
+ console.log(`[gracie] ${slug}: ${perCol[slug]} patterns (of ${wallpapers.length} raw)`);
+ await sleep(DELAY_MS);
+ }
+
+ const { rows } = await pg.query('SELECT COUNT(*) c, COUNT(image_url) img FROM gracie_catalog');
+ console.log(`[gracie] DONE. upserts this run=${upserts}. table total=${rows[0].c}, with image=${rows[0].img}. Cost: $0 (local)`);
+ await pg.end();
+}
+
+main().catch((e) => {
+ console.error('[gracie] FATAL', e);
+ process.exit(1);
+});
(oldest)
·
back to Gracie Scraper
·
(newest)