← back to Letsbegin
cohort-tag-cleanup.js
400 lines
#!/usr/bin/env node
/**
* cohort-tag-cleanup.js — Class-wide, idempotent tag/entity cleanup for the
* last-20-day Shopify cohort (1,003 products) on the sandbox store.
*
* Four deterministic fixes applied to TITLE, product TAGS, and image ALT text.
* It does NOT touch cost/price, and does NOT touch the color metafields
* (custom.dominant_color / color_name / color_family). Read-only on those.
*
* FIX 1 HTML-entity decode: "&" and the mis-cased "&Amp;"/"&" -> "&"
* in title, every tag, and every image alt.
* FIX 2 Strip Title-Case (poetic) color-name tags from the tag array.
* A tag is stripped if it equals (case-insensitive) the product's
* color_name / dominant_color metafield, OR every token is drawn from
* a curated color vocabulary AND it is NOT a simple single-word basic
* color (those are kept) and NOT a structural/collection/style tag.
* FIX 3 Lowercase + case-insensitive dedupe of the whole tag array.
* FIX 4 Style-tag lemma map: singularize clear plurals + collapse synonyms
* (stripes->stripe, trees->tree, animals->animal, kids/childrens->
* children, leaves/foliage->leaf, flowers->flower, etc.).
*
* Order of operations per product:
* decode -> strip-color-tags -> lemma-map -> lowercase -> dedupe.
* Title-Case color match for FIX 2 runs on the DECODED tag (so "Black & White"
* is compared post-decode), but the simple-color KEEP and structural KEEP are
* applied before lowercasing so we can tell a poetic Title-Case color tag from
* a genuine lowercase style tag.
*
* IDEMPOTENT + RESUMABLE: a cleaned product produces identical output, so the
* mutation is skipped (no write) — safe to re-run any number of times.
*
* Usage:
* node cohort-tag-cleanup.js --limit 5 # validate: print before/after, NO writes unless --write
* node cohort-tag-cleanup.js --limit 5 --write # validate WITH writes on first 5
* node cohort-tag-cleanup.js --write # full cohort run (writes)
* node cohort-tag-cleanup.js # full cohort DRY-RUN (no writes)
*
* Env: SHOPIFY_ADMIN_TOKEN, SHOPIFY_STORE. PG* optional.
*/
const https = require('https');
const { Client } = require('pg');
const STORE = process.env.SHOPIFY_STORE || 'designer-laboratory-sandbox.myshopify.com';
const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN || process.env.SHOPIFY_PRODUCT_TOKEN;
const API = '/admin/api/2024-10/graphql.json';
if (!TOKEN) { console.error('FATAL: SHOPIFY_ADMIN_TOKEN env var required'); process.exit(1); }
// HARD GUARD: sandbox store only (matches rw-cost-write.js / rw-material-options.js).
if (STORE !== 'designer-laboratory-sandbox.myshopify.com') {
console.error(`FATAL: refusing to write to non-sandbox store "${STORE}"`); process.exit(1);
}
const argv = process.argv.slice(2);
const WRITE = argv.includes('--write');
const limIdx = argv.indexOf('--limit');
const LIMIT = limIdx >= 0 ? parseInt(argv[limIdx + 1], 10) : null;
const VERBOSE = argv.includes('--verbose') || (LIMIT != null && LIMIT <= 10);
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
// ---------- Shopify GraphQL (helpers mirrored from enrich-write.js) ----------
function gqlRaw(body) {
return new Promise((resolve, reject) => {
const data = JSON.stringify(body);
const req = https.request({
hostname: STORE, path: API, method: 'POST',
headers: { 'X-Shopify-Access-Token': TOKEN, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
}, res => { let c = ''; res.on('data', d => c += d); res.on('end', () => { try { resolve(JSON.parse(c)); } catch { resolve({ error: c.slice(0, 300) }); } }); });
req.on('error', reject);
req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); });
req.write(data); req.end();
});
}
async function gql(body, retries = 5) {
for (let attempt = 1; attempt <= retries; attempt++) {
try {
const result = await gqlRaw(body);
const throttled = result?.errors?.some(e => /Throttled/i.test(e.message || ''));
const lowBudget = (result?.extensions?.cost?.throttleStatus?.currentlyAvailable || 9999) < 200;
if (throttled) { await sleep(3000); continue; }
if (lowBudget) await sleep(1500);
return result;
} catch (e) {
if (attempt < retries) { await sleep(attempt * 2000); continue; }
throw e;
}
}
}
// ============================================================================
// FIX 1 — HTML entity decode
// ============================================================================
// Decode & in any case (& &Amp; &) plus the numeric forms to "&".
// Also handle double-encoding (&amp;) by looping until stable.
function decodeEntities(s) {
if (s == null) return s;
let out = String(s);
let prev;
do {
prev = out;
out = out
.replace(/&/gi, '&') // & / &Amp; / &
.replace(/�*38;/g, '&') // &
.replace(/�*26;/gi, '&'); // &
} while (out !== prev);
return out;
}
// ============================================================================
// Vocabulary for FIX 2 — color-name detection
// ============================================================================
// Color NOUNS (the head of a color name) — single lowercase ones are KEPT as
// genuine facet tags; but they're also valid tokens inside a multi-word poetic
// color name like "sage green" / "warm taupe".
const COLOR_NOUNS = new Set([
'red','orange','yellow','green','blue','purple','violet','pink','brown','black',
'white','gray','grey','beige','tan','cream','navy','teal','olive','sage','taupe',
'mauve','lavender','lilac','burgundy','crimson','scarlet','maroon','rust',
'terracotta','peach','coral','salmon','rose','blush','ivory','charcoal','graphite',
'slate','steel','cobalt','sapphire','indigo','periwinkle','turquoise','aqua','cyan',
'mint','emerald','jade','forest','lime','mustard','amber','gold','golden','bronze',
'copper','silver','platinum','pewter','stone','sand','khaki','camel','chocolate',
'mahogany','walnut','espresso','caramel','honey','butter','lemon','marigold',
'plum','aubergine','eggplant','magenta','fuchsia','raspberry','cherry','wine',
'oxblood','brick','clay','ochre','umber','sepia','greige','oatmeal','linen',
'porcelain','alabaster','snow','pearl','smoke','ash','dove','fog','mist','cloud',
'midnight','onyx','jet','ebony','petrol','marine','ocean','sky','powder','denim',
'cornflower','azure','cerulean','seafoam','pistachio','moss','fern','hunter',
'army','avocado','chartreuse','citrus','tangerine','apricot','melon','blush',
'multicolor','multi','neutral','natural','birch','tangerine','sunshine','sunflower',
]);
// Color MODIFIERS (adjectives that prefix a color noun in poetic names).
const COLOR_MODS = new Set([
'light','dark','deep','pale','soft','bright','muted','dusty','warm','cool','hot',
'rich','vivid','vibrant','faded','washed','smoky','smokey','dusky','misty','hazy',
'pastel','neon','electric','royal','classic','antique','vintage','burnt','toasted',
'sun','sunny','frosted','icy','crisp','clean','pure','medium','mid','off','baby',
'powder','dirty','ash','stone','steel','autumn','spring','summer','winter','dusk',
'dawn','shadow','moody','jewel','earthy','earth','desert','tropical','arctic',
]);
// Single-word seasonal / poetic color_name leftovers that are NOT basic colors and
// should be stripped only when they equal the metafield (handled by metafield match);
// listed here so the vocab path also catches them if they slip through as tags.
const POETIC_SINGLE = new Set(['spring','autumn','breeze','sunshine','haze','dusk','dawn','color','soft','dark','light','warm','cool','mist','fog','smoke','shadow','glow','sunset','sunrise']);
// Simple BASIC single-word colors we KEEP as lowercase facet tags (standing rule).
const KEEP_SIMPLE_COLORS = new Set([
'red','orange','yellow','green','blue','purple','pink','brown','black','white',
'gray','grey','beige','navy','teal','tan','cream','multi','multicolor','neutral',
]);
// Structural / system / collection tags that must NEVER be touched by the
// color-strip logic (they survive into the lowercase+dedupe step).
function isStructural(tag) {
return /^collection:/i.test(tag) ||
/^(mural|wall mural|non-woven|non woven|priced per square meter|rebel walls|cole & son|glitter walls|glitterwalls|panel|panels|wallcovering|commercial wallcovering|class a fire rated|display_variant|bundle|wallpaper-bundle|wallco-ai-bundle|wallco-original|digital-download|seamless-pattern|seamless-wallpaper|printable-wallpaper|wallpaper-download|home-decor|commercial-use|craft-pattern|curated-pattern|designer-wallpaper|pattern-collection|pattern-pack|spoonflower)$/i.test(tag) ||
/fire rated/i.test(tag) ||
/\.cs\.\d/i.test(tag) || // sku-like junk e.g. 95/1001.cs.0
/^colour_/i.test(tag) ||
/^ai-analyzed/i.test(tag);
}
// Tokenize a tag for vocab matching: lowercase, split on space/hyphen, drop "&" and "and".
function colorTokens(tag) {
return tag.toLowerCase()
.replace(/&/g, ' ')
.split(/[\s\-]+/)
.map(t => t.trim())
.filter(t => t && t !== 'and');
}
// Is this tag a poetic color name (vocab path)? Multi-word where EVERY token is a
// color noun or modifier (e.g. "Warm Taupe", "Sage Green", "Periwinkle Blue",
// "Black & White"), OR a single poetic leftover. Single basic colors are NOT
// flagged here (kept), and structural tags are never flagged.
function isColorVocabTag(tag) {
if (isStructural(tag)) return false;
const toks = colorTokens(tag);
if (toks.length === 0) return false;
if (toks.length === 1) {
// single token: only strip if it's a poetic-single leftover, NOT a basic color
if (KEEP_SIMPLE_COLORS.has(toks[0])) return false;
return POETIC_SINGLE.has(toks[0]);
}
// multi-word: strip only if EVERY token is in the color vocabulary
return toks.every(t => COLOR_NOUNS.has(t) || COLOR_MODS.has(t) || POETIC_SINGLE.has(t));
}
// ============================================================================
// FIX 4 — explicit lemma map (plurals -> singular, synonyms -> canonical).
// Kept deliberately small + clear. Keys are lowercase; applied AFTER lowercasing.
// ============================================================================
const LEMMA = {
stripes: 'stripe',
trees: 'tree',
tiles: 'tile',
animals: 'animal',
flowers: 'flower',
florals: 'floral',
birds: 'bird',
leaves: 'leaf',
foliage: 'leaf',
shapes: 'shape',
motifs: 'motif',
arches: 'arch',
dogs: 'dog',
cats: 'cat',
// synonyms -> canonical
kids: 'children',
childrens: 'children',
'kids-room': 'children',
illustrative: 'illustrated',
illustration: 'illustrated',
minimalist: 'minimal',
'art-deco': 'art deco',
'art deco': 'art deco',
};
function lemma(tag) {
return Object.prototype.hasOwnProperty.call(LEMMA, tag) ? LEMMA[tag] : tag;
}
// ============================================================================
// Core transform: given raw tags + color name(s), produce the cleaned tag list.
// Returns { tags, stripped, lemmatized, decodedTag } for change accounting.
// ============================================================================
function cleanTags(rawTags, colorNames) {
const colorSet = new Set(colorNames.filter(Boolean).map(c => decodeEntities(c).toLowerCase().trim()));
let strippedAny = false, lemmatizedAny = false, decodedTag = false;
// 1) decode entities on each tag
const decoded = rawTags.map(t => {
const d = decodeEntities(t);
if (d !== t) decodedTag = true;
return d;
});
// 2) strip color-name tags (metafield match OR vocab match) — but keep structural + simple colors
const afterStrip = decoded.filter(t => {
if (isStructural(t)) return true;
const low = t.toLowerCase().trim();
if (colorSet.has(low)) { strippedAny = true; return false; } // matches product's own color name
if (KEEP_SIMPLE_COLORS.has(low)) return true; // keep simple lowercase basic color
if (isColorVocabTag(t)) { strippedAny = true; return false; } // poetic color vocab tag
return true;
});
// 3) lowercase + lemma map
const mapped = afterStrip.map(t => {
const low = t.toLowerCase().trim();
const lem = lemma(low);
if (lem !== low) lemmatizedAny = true;
return lem;
});
// 4) case-insensitive dedupe (already lowercased), preserve first-seen order
const seen = new Set();
const final = [];
for (const t of mapped) {
if (!t) continue;
if (seen.has(t)) continue;
seen.add(t);
final.push(t);
}
return { tags: final, stripped: strippedAny, lemmatized: lemmatizedAny, decodedTag };
}
// ---------- product fetch / mutate ----------
const PRODUCT_QUERY = (id) => ({ query: `{
product(id:"${id}") {
id title tags
media(first:30){ edges{ node{ ... on MediaImage { id alt } } } }
metafields(first:50){ edges{ node{ namespace key value } } }
}
}` });
const PRODUCT_UPDATE = 'mutation ($i: ProductInput!) { productUpdate(input: $i) { product { id } userErrors { message field } } }';
async function processOne(id, counters) {
const fr = await gql(PRODUCT_QUERY(id));
const p = fr?.data?.product;
if (!p) { counters.notfound++; console.log(` ${id} NOT FOUND`); return; }
const mf = {};
for (const e of (p.metafields?.edges || [])) { if (e.node.namespace === 'custom') mf[e.node.key] = e.node.value; }
const colorNames = [mf.color_name, mf.dominant_color];
const origTitle = p.title || '';
const origTags = Array.isArray(p.tags) ? p.tags : [];
// --- title ---
const newTitle = decodeEntities(origTitle);
const titleChanged = newTitle !== origTitle;
// --- tags ---
const { tags: newTags, stripped, lemmatized, decodedTag } = cleanTags(origTags, colorNames);
const tagsChanged = newTags.length !== origTags.length || newTags.some((t, i) => t !== origTags[i]);
// --- image alts ---
const mediaEdges = p.media?.edges || [];
const altUpdates = [];
for (const e of mediaEdges) {
const n = e.node; if (!n?.id) continue;
const newAlt = decodeEntities(n.alt || '');
if (newAlt !== (n.alt || '')) altUpdates.push({ id: n.id, alt: newAlt });
}
const altChanged = altUpdates.length > 0;
// account per-fix (per product, count once each)
if (decodedTag || titleChanged || altChanged) counters.entityDecoded++;
if (stripped) counters.colorStripped++;
if (lemmatized) counters.lemmatized++;
// case-dedupe: did lowercasing/dedup change anything beyond strip/lemma/decode?
// detect by comparing a pure-lowercase-dedupe of origTags (post-decode) vs origTags
{
const decodedOrig = origTags.map(decodeEntities);
const lowSeen = new Set(); let caseChanged = false;
for (const t of decodedOrig) { const l = t.toLowerCase(); if (t !== l || lowSeen.has(l)) caseChanged = true; lowSeen.add(l); }
if (caseChanged) counters.caseDeduped++;
}
if (!titleChanged && !tagsChanged && !altChanged) { counters.noop++; if (VERBOSE) console.log(` ${id} no-op`); return; }
if (VERBOSE) {
console.log(` ${id}`);
if (titleChanged) console.log(` TITLE "${origTitle}" -> "${newTitle}"`);
if (tagsChanged) {
console.log(` TAGS before: [${origTags.join(', ')}]`);
console.log(` after: [${newTags.join(', ')}]`);
}
if (altChanged) altUpdates.forEach(a => console.log(` ALT -> "${a.alt}"`));
}
counters.changed++;
if (!WRITE) return;
// write title + tags
if (titleChanged || tagsChanged) {
const input = { id };
if (titleChanged) input.title = newTitle;
if (tagsChanged) input.tags = newTags;
const r = await gql({ query: PRODUCT_UPDATE, variables: { i: input } });
const errs = r?.data?.productUpdate?.userErrors || [];
if (errs.length) { counters.errors++; console.log(` ${id} TITLE/TAGS ERR: ${errs.map(e => e.message).join('; ')}`); }
await sleep(350);
}
// write alts
if (altChanged) {
const mediaJson = altUpdates.map(m => `{id: "${m.id}", alt: "${m.alt.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"}`).join(', ');
const r = await gql({ query: `mutation { productUpdateMedia(productId: "${id}", media: [${mediaJson}]) { media { ... on MediaImage { id } } mediaUserErrors { message } } }` });
const errs = r?.data?.productUpdateMedia?.mediaUserErrors || [];
if (errs.length) { counters.errors++; console.log(` ${id} ALT ERR: ${errs.map(e => e.message).join('; ')}`); }
await sleep(350);
}
}
async function main() {
// cohort from PG
const pg = new Client({ user: process.env.PGUSER || 'stevestudio2', database: process.env.PGDATABASE || 'dw_unified', host: process.env.PGHOST || '/tmp' });
await pg.connect();
const res = await pg.query(`SELECT shopify_id FROM shopify_products WHERE created_at_shopify >= now()-interval '20 days' ORDER BY created_at_shopify DESC`);
await pg.end();
let ids = res.rows.map(r => r.shopify_id).filter(Boolean);
// --ids "id1,id2" restricts to specific cohort members (validation aid; still must be in cohort)
const idsIdx = argv.indexOf('--ids');
if (idsIdx >= 0 && argv[idsIdx + 1]) {
const want = new Set(argv[idsIdx + 1].split(',').map(s => {
const t = s.trim();
return /^gid:\/\//.test(t) ? t : `gid://shopify/Product/${t.replace(/\D/g, '')}`;
}));
ids = ids.filter(id => want.has(id));
}
if (LIMIT != null) ids = ids.slice(0, LIMIT);
console.log(`cohort-tag-cleanup: ${ids.length} products | WRITE=${WRITE ? 'YES' : 'DRY-RUN'}${LIMIT != null ? ` | limit=${LIMIT}` : ''}`);
const counters = { changed: 0, noop: 0, notfound: 0, errors: 0, entityDecoded: 0, colorStripped: 0, caseDeduped: 0, lemmatized: 0 };
let n = 0;
for (const id of ids) {
n++;
try { await processOne(id, counters); }
catch (e) { counters.errors++; console.log(` ${id} EXC: ${e.message}`); }
if (n % 50 === 0) console.log(` ...${n}/${ids.length} (changed=${counters.changed} noop=${counters.noop} err=${counters.errors})`);
await sleep(120); // ~2 req/s pacing baseline; gql() adds backoff
}
console.log('\n=== SUMMARY ===');
console.log(`total: ${ids.length}`);
console.log(`changed: ${counters.changed}`);
console.log(`no-op (clean): ${counters.noop}`);
console.log(`not found: ${counters.notfound}`);
console.log(`errors: ${counters.errors}`);
console.log('--- per-fix product counts (pre-clean state) ---');
console.log(`entity-decoded: ${counters.entityDecoded}`);
console.log(`color-tags strip: ${counters.colorStripped}`);
console.log(`case-deduped: ${counters.caseDeduped}`);
console.log(`lemmatized: ${counters.lemmatized}`);
}
main().catch(e => { console.error(`FATAL: ${e.message}\n${e.stack}`); process.exit(1); });