← back to Designer Wallcoverings
Color collection cleaner v2: Gemini blueness gate + reversible ledger + banner re-crawl queue + resumable
b4a3d85e247457ed8a9fd1bda7ca79fd39b5ad2b · 2026-06-12 12:13:06 -0700 · Steve Abrams
Files touched
A DW-Programming/color-collection-cleaner-v2.js
Diff
commit b4a3d85e247457ed8a9fd1bda7ca79fd39b5ad2b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Jun 12 12:13:06 2026 -0700
Color collection cleaner v2: Gemini blueness gate + reversible ledger + banner re-crawl queue + resumable
---
DW-Programming/color-collection-cleaner-v2.js | 221 ++++++++++++++++++++++++++
1 file changed, 221 insertions(+)
diff --git a/DW-Programming/color-collection-cleaner-v2.js b/DW-Programming/color-collection-cleaner-v2.js
new file mode 100644
index 00000000..641374df
--- /dev/null
+++ b/DW-Programming/color-collection-cleaner-v2.js
@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+/**
+ * Color Collection Cleaner v2 (Gemini vision, reversible, banner-aware)
+ *
+ * For each product in a color smart-collection, ask Gemini:
+ * (1) is it PRIMARILY this color? -> if NO, strip the color-family tags
+ * so it drops out of the collection.
+ * (2) does the IMAGE have a burned-in vendor banner/watermark/url/swatch-strip?
+ * -> if YES, append to the re-crawl queue.
+ *
+ * Safety:
+ * - Never removes tags on a Gemini error (skip).
+ * - Every removal is logged to a REVERSIBLE ledger (data/color-clean-ledger.jsonl)
+ * containing the full before/after tag arrays -> one-command rollback.
+ * - Resumable: processed product ids per color are tracked; re-runs skip them.
+ *
+ * Env: SHOPIFY_ADMIN_TOKEN, GEMINI_API_KEY
+ * Usage:
+ * node color-collection-cleaner-v2.js --color blue [--dry-run] [--limit N] [--conc 8]
+ * node color-collection-cleaner-v2.js --all [--dry-run]
+ * node color-collection-cleaner-v2.js --rollback data/color-clean-ledger.jsonl
+ */
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+const SHOPIFY_DOMAIN = 'designer-laboratory-sandbox.myshopify.com';
+const SHOPIFY_TOKEN = process.env.SHOPIFY_ADMIN_TOKEN;
+const GEMINI_KEY = process.env.GEMINI_API_KEY;
+const API_VERSION = '2024-01';
+
+const args = process.argv.slice(2);
+const DRY_RUN = args.includes('--dry-run');
+const TARGET_COLOR = args.find((_, i, a) => a[i - 1] === '--color') || null;
+const ALL_COLORS = args.includes('--all');
+const LIMIT = parseInt(args.find((_, i, a) => a[i - 1] === '--limit') || '0', 10) || 0;
+const CONC = parseInt(args.find((_, i, a) => a[i - 1] === '--conc') || '8', 10);
+const ROLLBACK = args.find((_, i, a) => a[i - 1] === '--rollback') || null;
+
+const DATA_DIR = path.join(__dirname, 'data');
+const STATE_DIR = path.join(DATA_DIR, 'color-clean-state');
+fs.mkdirSync(STATE_DIR, { recursive: true });
+const LEDGER = path.join(DATA_DIR, 'color-clean-ledger.jsonl');
+const BANNER_QUEUE = path.join(DATA_DIR, 'banner-recrawl-queue.jsonl');
+
+const COLOR_COLLECTIONS = {
+ blue: { collectionId: 95278104641, tags:['blue','navy','sky','indigo','cobalt','azure','cerulean','sapphire'], word:'BLUE', consider:'navy, indigo, cobalt, sky blue, teal-blue, sapphire, royal blue, periwinkle', notColors:'brown, gold, silver, black, white, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange' },
+ green: { collectionId: 164060037171, tags:['green','emerald','olive','sage','forest','mint','lime','jade','hunter'], word:'GREEN', consider:'emerald, olive, sage, forest, mint, lime, jade, hunter, moss, fern', notColors:'brown, gold, silver, black, white, blue, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange' },
+ yellow: { collectionId: 283180367923, tags:['yellow','mustard','lemon','saffron','amber','marigold','sunflower','buttercup'], word:'YELLOW', consider:'mustard, lemon, saffron, amber, marigold, sunflower, buttercup, canary', notColors:'brown, silver, black, white, blue, red, bronze, copper, beige, tan, grey, cork, natural, cream, green, pink, purple, orange' },
+ cream: { collectionId: 283307828659, tags:['cream','ivory','off-white','vanilla','eggshell','champagne'], word:'CREAM or IVORY', consider:'off-white, vanilla, eggshell, champagne, warm white', notColors:'brown, silver, black, blue, red, bronze, copper, tan, grey, cork, green, yellow, pink, purple, orange, navy, bright white' },
+ pink: { collectionId: null, tags:['pink','rose','blush','fuchsia','magenta','salmon','coral pink','mauve'], word:'PINK', consider:'rose, blush, fuchsia, magenta, salmon, hot pink, mauve, dusty rose', notColors:'brown, gold, silver, black, white, blue, green, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, purple, orange' },
+ red: { collectionId: 264122269747, tags:['red','crimson','burgundy','scarlet','ruby','cherry','maroon','wine'], word:'RED', consider:'crimson, burgundy, scarlet, ruby, cherry, maroon, wine, vermilion', notColors:'brown, gold, silver, black, white, blue, green, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange' },
+ purple: { collectionId: 283307573299, tags:['purple','violet','lavender','plum','amethyst','lilac','aubergine','mauve'], word:'PURPLE', consider:'violet, lavender, plum, amethyst, lilac, aubergine, mauve, eggplant', notColors:'brown, gold, silver, black, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, orange' },
+ gold: { collectionId: 164059906099, tags:['gold','golden','gilded','brass'], word:'GOLD or GOLDEN', consider:'brass, gilded, antique gold, warm metallic gold', notColors:'silver, black, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange, brown' },
+ grey: { collectionId: 164059938867, tags:['grey','gray','charcoal','slate','silver','pewter','ash','graphite'], word:'GREY/GRAY', consider:'charcoal, slate, silver, pewter, ash, graphite, gunmetal', notColors:'brown, gold, black, white, blue, green, red, bronze, copper, beige, tan, cork, natural, cream, yellow, pink, purple, orange' },
+ black: { collectionId: 283285651507, tags:['black','ebony','onyx','jet','noir'], word:'BLACK', consider:'ebony, onyx, jet, noir, charcoal-black', notColors:'brown, gold, silver, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange' },
+ brown: { collectionId: 283307868211, tags:['brown','chocolate','cocoa','mocha','coffee','walnut','chestnut','espresso','umber'], word:'BROWN', consider:'chocolate, cocoa, mocha, coffee, walnut, chestnut, espresso, umber, sienna', notColors:'gold, silver, black, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple, orange' },
+ orange: { collectionId: 283307999283, tags:['orange','tangerine','burnt orange','rust','terracotta','sienna','pumpkin'], word:'ORANGE', consider:'tangerine, burnt orange, rust, terracotta, sienna, pumpkin, paprika', notColors:'brown, gold, silver, black, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple' },
+ coral: { collectionId: 282281377843, tags:['coral','peach','apricot'], word:'CORAL or PEACH', consider:'warm salmon-orange, apricot, soft orange-pink', notColors:'brown, gold, silver, black, white, blue, green, red, bronze, copper, beige, tan, grey, cork, natural, cream, yellow, pink, purple' },
+};
+
+function shopifyREST(method, p) {
+ return new Promise((resolve, reject) => {
+ const req = https.request({ hostname: SHOPIFY_DOMAIN, path: `/admin/api/${API_VERSION}${p}`, method,
+ headers: { 'X-Shopify-Access-Token': SHOPIFY_TOKEN }, timeout: 30000 }, (res) => {
+ if (res.statusCode === 429) {
+ const wait = parseFloat(res.headers['retry-after'] || '2') * 1000;
+ res.on('data', () => {}); res.on('end', () => setTimeout(() => shopifyREST(method, p).then(resolve).catch(reject), wait));
+ return;
+ }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch { resolve({}); } });
+ });
+ req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('rest timeout')); }); req.end();
+ });
+}
+function shopifyGraphQL(query, variables) {
+ return new Promise((resolve, reject) => {
+ const body = JSON.stringify({ query, variables });
+ const req = https.request({ hostname: SHOPIFY_DOMAIN, path: `/admin/api/${API_VERSION}/graphql.json`, method: 'POST',
+ headers: { 'Content-Type': 'application/json', 'X-Shopify-Access-Token': SHOPIFY_TOKEN, 'Content-Length': Buffer.byteLength(body) }, timeout: 30000 }, (res) => {
+ let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } });
+ });
+ req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('gql timeout')); }); req.write(body); req.end();
+ });
+}
+function downloadImage(url) {
+ return new Promise((resolve) => {
+ https.get(url, { timeout: 15000 }, (r) => {
+ if ([301, 302].includes(r.statusCode) && r.headers.location) { return downloadImage(r.headers.location).then(resolve); }
+ const c = []; r.on('data', x => c.push(x)); r.on('end', () => resolve(Buffer.concat(c).toString('base64')));
+ }).on('error', () => resolve(null)).on('timeout', function () { this.destroy(); resolve(null); });
+ });
+}
+function callGemini(body) {
+ return new Promise((resolve, reject) => {
+ const req = https.request({ hostname: 'generativelanguage.googleapis.com',
+ path: `/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_KEY}`, method: 'POST',
+ headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) }, timeout: 30000 }, (res) => {
+ let d = ''; res.on('data', c => d += c); res.on('end', () => {
+ if (res.statusCode === 429) return reject({ code: 429, retryAfter: parseFloat(res.headers['retry-after'] || '15') });
+ if (res.statusCode >= 500) return reject({ code: res.statusCode });
+ try { const t = JSON.parse(d).candidates?.[0]?.content?.parts?.[0]?.text; t ? resolve(t.trim()) : reject({ code: 'NO_TEXT' }); }
+ catch (e) { reject({ code: 'PARSE' }); }
+ });
+ });
+ req.on('error', () => reject({ code: 'NET' })); req.on('timeout', () => { req.destroy(); reject({ code: 'TIMEOUT' }); });
+ req.write(body); req.end();
+ });
+}
+async function analyze(imageUrl, cfg) {
+ const b64 = await downloadImage(imageUrl);
+ if (!b64 || b64.length < 100) return { err: 'no-image' };
+ const body = JSON.stringify({ contents: [{ parts: [
+ { text: `You are auditing a wallcovering product photo for an e-commerce catalog.
+Q1 COLOR: Is the PRODUCT primarily ${cfg.word}? Consider ${cfg.consider} as this color. Do NOT count ${cfg.notColors} as this color.
+Q2 BANNER: Does the IMAGE have any burned-in vendor banner, watermark, website URL, logo text, label box, or a strip/row of color swatches overlaid on the photo? A clean edge-to-edge wallpaper pattern with NO overlay = NO.
+Answer on ONE line EXACTLY: "COLOR=YES|NO (dominant) BANNER=YES|NO"` },
+ { inline_data: { mime_type: 'image/jpeg', data: b64 } } ] }],
+ generationConfig: { temperature: 0.1, maxOutputTokens: 40 } });
+ for (let a = 1; a <= 5; a++) {
+ try { const v = await callGemini(body);
+ return { color: /COLOR=YES/i.test(v), banner: /BANNER=YES/i.test(v), raw: v };
+ } catch (e) {
+ if (e.code === 429) { await new Promise(r => setTimeout(r, Math.min(e.retryAfter || 15, 60) * 1000)); continue; }
+ if (e.code >= 500 && a < 3) { await new Promise(r => setTimeout(r, 4000)); continue; }
+ return { err: e.code || 'unknown' };
+ }
+ }
+ return { err: 'max-retries' };
+}
+async function fetchCollectionProducts(collectionId) {
+ const all = []; let sinceId = 0;
+ while (true) {
+ const url = `/collections/${collectionId}/products.json?limit=250&fields=id,title,vendor,image,tags${sinceId ? '&since_id=' + sinceId : ''}`;
+ const r = await shopifyREST('GET', url); const ps = r.products || [];
+ if (!ps.length) break; all.push(...ps); sinceId = ps[ps.length - 1].id;
+ if (ps.length < 250) break; await new Promise(r => setTimeout(r, 300));
+ }
+ return all;
+}
+const append = (f, o) => fs.appendFileSync(f, JSON.stringify(o) + '\n');
+const loadProcessed = (color) => { const f = path.join(STATE_DIR, `${color}.done`); try { return new Set(fs.readFileSync(f, 'utf8').split('\n').filter(Boolean).map(Number)); } catch { return new Set(); } };
+const markProcessed = (color, id) => fs.appendFileSync(path.join(STATE_DIR, `${color}.done`), id + '\n');
+
+async function processColor(color) {
+ const cfg = COLOR_COLLECTIONS[color];
+ if (!cfg) { console.log(`unknown color ${color}`); return; }
+ if (!cfg.collectionId) { console.log(`[${color}] no collection id — skip`); return; }
+ console.log(`\n===== ${color.toUpperCase()} (collection ${cfg.collectionId}) =====`);
+ let products = await fetchCollectionProducts(cfg.collectionId);
+ const done = loadProcessed(color);
+ products = products.filter(p => !done.has(p.id));
+ if (LIMIT) products = products.slice(0, LIMIT);
+ console.log(` to process: ${products.length} (already done ${done.size})`);
+ const tagSet = new Set(cfg.tags.map(t => t.toLowerCase()));
+ let kept = 0, removed = 0, errors = 0, skipped = 0, banners = 0, idx = 0;
+
+ const queue = [...products];
+ async function worker() {
+ while (queue.length) {
+ const p = queue.shift(); idx++; const n = idx;
+ const src = p.image?.src;
+ if (!src) { skipped++; markProcessed(color, p.id); continue; }
+ const res = await analyze(src, cfg);
+ if (res.err) { errors++; if (n % 25 === 0) console.log(` [${n}] SKIP(err:${res.err}) ${p.title.slice(0,40)}`); continue; } // do NOT mark done -> retried next run
+ if (res.banner) {
+ banners++; append(BANNER_QUEUE, { ts: new Date().toISOString(), color, product_id: p.id, title: p.title, vendor: p.vendor, image_src: src, raw: res.raw });
+ }
+ if (res.color) {
+ kept++; markProcessed(color, p.id);
+ if (n <= 15 || n % 100 === 0) console.log(` [${n}] KEEP ${p.title.slice(0,42)} — ${res.raw}`);
+ } else {
+ const tags = (p.tags || '').split(',').map(t => t.trim()).filter(Boolean);
+ const cleaned = tags.filter(t => !tagSet.has(t.toLowerCase()));
+ const removedTags = tags.filter(t => tagSet.has(t.toLowerCase()));
+ if (removedTags.length) {
+ if (!DRY_RUN) {
+ const r = await shopifyGraphQL(`mutation u($input: ProductInput!){ productUpdate(input:$input){ product{id} userErrors{message} } }`,
+ { input: { id: `gid://shopify/Product/${p.id}`, tags: cleaned } });
+ const ue = r?.data?.productUpdate?.userErrors;
+ if (ue && ue.length) { errors++; console.log(` [${n}] WRITE-ERR ${p.title.slice(0,30)}: ${ue[0].message}`); continue; }
+ }
+ removed++;
+ append(LEDGER, { ts: new Date().toISOString(), color, product_id: p.id, title: p.title, removed_tags: removedTags, tags_before: tags, tags_after: cleaned, verdict: res.raw, dry_run: DRY_RUN });
+ markProcessed(color, p.id);
+ console.log(` [${n}] ${DRY_RUN ? 'DRY-REMOVE' : 'REMOVE'} ${p.title.slice(0,40)} — ${res.raw} — [${removedTags.join(', ')}]`);
+ } else { kept++; markProcessed(color, p.id); }
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: CONC }, worker));
+ const sum = ` ${color.toUpperCase()}: kept ${kept}, removed ${removed}, banners ${banners}, skipped ${skipped}, errors ${errors}`;
+ console.log(sum); return sum;
+}
+
+async function rollback(file) {
+ const lines = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(JSON.parse).filter(e => !e.dry_run);
+ console.log(`Rolling back ${lines.length} removals (restoring tags_before)...`);
+ let ok = 0;
+ for (const e of lines) {
+ const r = await shopifyGraphQL(`mutation u($input: ProductInput!){ productUpdate(input:$input){ product{id} userErrors{message} } }`,
+ { input: { id: `gid://shopify/Product/${e.product_id}`, tags: e.tags_before } });
+ if (!r?.data?.productUpdate?.userErrors?.length) ok++;
+ if (ok % 50 === 0) console.log(` restored ${ok}/${lines.length}`);
+ }
+ console.log(`Rollback done: ${ok}/${lines.length} restored.`);
+}
+
+async function main() {
+ if (ROLLBACK) return rollback(ROLLBACK);
+ if (!SHOPIFY_TOKEN) { console.error('Missing SHOPIFY_ADMIN_TOKEN'); process.exit(1); }
+ if (!GEMINI_KEY) { console.error('Missing GEMINI_API_KEY'); process.exit(1); }
+ console.log(`Color Collection Cleaner v2 — ${DRY_RUN ? 'DRY RUN' : 'LIVE'} — conc=${CONC}${LIMIT ? ' limit=' + LIMIT : ''}`);
+ const colors = TARGET_COLOR ? [TARGET_COLOR] : (ALL_COLORS ? Object.keys(COLOR_COLLECTIONS) : null);
+ if (!colors) { console.log('Usage: --color blue | --all [--dry-run] [--limit N] [--conc 8]'); return; }
+ const sums = [];
+ for (const c of colors) { try { const s = await processColor(c); if (s) sums.push(s); } catch (e) { console.error(`err ${c}: ${e.message}`); } }
+ console.log('\n===== SUMMARY ====='); sums.forEach(s => console.log(s));
+ console.log(`\nLedger: ${LEDGER}\nBanner re-crawl queue: ${BANNER_QUEUE}`);
+}
+main().catch(console.error);
← 67801868 copygen: env-configurable Ollama host/model/prompt-prefix (O
·
back to Designer Wallcoverings
·
color cleaner v2: use gemini-2.5-flash (2.0-flash retired 20 631cb8bf →