← back to Tk 11331 Exec
TK-11331 D3b-heldreview: classify + stage 79 held Momentum colorways (43 STAGE-NEW + 36 DISAMBIGUATE)
b68d5f9722ef6530d49056577abcb972ddc1a8c8 · 2026-09-09 22:27:07 -0700 · Steve Abrams
Files touched
A d3b_heldreview_analyze.mjsA d3b_heldreview_classify.mjsA d3b_heldreview_emit_sql.mjsA d3b_heldreview_prep.mjsA d3b_heldreview_rollback.mjsA d3b_heldreview_stage.mjsA fetch_held_images.sh
Diff
commit b68d5f9722ef6530d49056577abcb972ddc1a8c8
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 9 22:27:07 2026 -0700
TK-11331 D3b-heldreview: classify + stage 79 held Momentum colorways (43 STAGE-NEW + 36 DISAMBIGUATE)
---
d3b_heldreview_analyze.mjs | 105 +++++++++++++++++++++++++
d3b_heldreview_classify.mjs | 77 +++++++++++++++++++
d3b_heldreview_emit_sql.mjs | 42 ++++++++++
d3b_heldreview_prep.mjs | 60 +++++++++++++++
d3b_heldreview_rollback.mjs | 18 +++++
d3b_heldreview_stage.mjs | 182 ++++++++++++++++++++++++++++++++++++++++++++
fetch_held_images.sh | 19 +++++
7 files changed, 503 insertions(+)
diff --git a/d3b_heldreview_analyze.mjs b/d3b_heldreview_analyze.mjs
new file mode 100644
index 0000000..4695a19
--- /dev/null
+++ b/d3b_heldreview_analyze.mjs
@@ -0,0 +1,105 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview — READ-ONLY reconstruction + classification of the two HELD buckets.
+// Replays d3c_stage.mjs classification logic exactly, then partitions the held sets.
+import { createRequire } from 'module';
+import fs from 'fs';
+import path from 'path';
+const require = createRequire(import.meta.url);
+const { Pool } = require('pg');
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const TSV = path.join(HERE, 'data', 'momentum_feed_full.tsv');
+const pool = new Pool({ connectionString: 'postgresql://dw_admin:DW2024!@127.0.0.1:5432/dw_unified' });
+const norm = s => (s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+const KEEP = new Set(['Wallcovering', 'Acoustic']);
+
+function readTSV() {
+ const lines = fs.readFileSync(TSV, 'utf8').split('\n').filter(Boolean);
+ const cols = lines[0].split('\t');
+ return lines.slice(1).map(l => { const p = l.split('\t'); const o = {}; cols.forEach((c, i) => o[c] = p[i]); return o; });
+}
+
+async function main() {
+ const { rows: pr } = await pool.query('SELECT pattern_name, color_name FROM momentum_colorways');
+ const stagedPatternNorm = new Set(pr.map(r => norm(r.pattern_name)));
+ const stagedPairNorm = new Set(pr.map(r => norm(r.pattern_name) + '' + norm(r.color_name)));
+ // map normalized pattern -> set of raw staged pattern spellings (for spelling-variant evidence)
+ const stagedPatternRaw = new Map();
+ for (const r of pr) {
+ const n = norm(r.pattern_name);
+ if (!stagedPatternRaw.has(n)) stagedPatternRaw.set(n, new Set());
+ stagedPatternRaw.get(n).add(r.pattern_name);
+ }
+
+ const feed = readTSV();
+ const inscope = feed.filter(r => KEEP.has(r.category_name) && (r.pattern_name || '').trim() && (r.preferred_color_name || '').trim());
+ // same raw dedup the stage script did
+ const seen = new Set(); const dedup = [];
+ for (const r of inscope) { const k = r.pattern_name + '' + r.preferred_color_name; if (seen.has(k)) continue; seen.add(k); dedup.push(r); }
+
+ // EXTEND-PATTERN bucket: norm(pattern) already staged
+ const extendRows = dedup.filter(r => stagedPatternNorm.has(norm(r.pattern_name)));
+ // NEW-PATTERN bucket (the 1,215 path): norm(pattern) not staged
+ const newRows = dedup.filter(r => !stagedPatternNorm.has(norm(r.pattern_name)));
+
+ // Classify extend rows: TRUE-DUP (norm pair already staged) vs GENUINELY-NEW
+ const extTrueDup = [], extNew = [];
+ for (const r of extendRows) {
+ const pairKey = norm(r.pattern_name) + '' + norm(r.preferred_color_name);
+ if (stagedPairNorm.has(pairKey)) extTrueDup.push(r); else extNew.push(r);
+ }
+
+ // 6 normalized-collision patterns named in the task
+ const COLLISION = ['Cavern II','Rectangular Hatch','Rectangular Linear','Rectangular Louvre','Rectangular Vault','Rectangular Vertex 9/16'];
+ console.log('== TOTALS ==');
+ console.log('inscope(dedup):', dedup.length, '| extendRows(pattern already staged):', extendRows.length, '| newRows:', newRows.length);
+ console.log('extend -> TRUE-DUP(norm pair staged):', extTrueDup.length, '| GENUINELY-NEW:', extNew.length);
+
+ // For each collision pattern, locate in feed + check staging
+ console.log('\n== COLLISION PATTERNS ==');
+ for (const cp of COLLISION) {
+ const cn = norm(cp);
+ const feedRows = dedup.filter(r => norm(r.pattern_name) === cn);
+ const patternStaged = stagedPatternNorm.has(cn);
+ const rawSpellings = stagedPatternRaw.get(cn);
+ // also: any staged pattern that shares the same normalized base? check near-collisions
+ console.log(`\n[${cp}] normalized="${cn}" feedColorways=${feedRows.length} patternAlreadyStaged=${patternStaged}` + (rawSpellings ? ` rawStaged=${JSON.stringify([...rawSpellings])}` : ''));
+ for (const fr of feedRows) {
+ const pairKey = cn + '' + norm(fr.preferred_color_name);
+ const pairStaged = stagedPairNorm.has(pairKey);
+ console.log(` cw="${fr.preferred_color_name}" num=${fr.number} cat=${fr.category_name} pairStaged=${pairStaged}`);
+ }
+ }
+
+ // Detect IN-SET collisions among newRows: two distinct raw patterns that normalize equal (spelling variants within the new set)
+ console.log('\n== IN-NEWSET NORMALIZED PATTERN COLLISIONS (distinct raw -> same norm) ==');
+ const byNorm = new Map();
+ for (const r of newRows) {
+ const n = norm(r.pattern_name);
+ if (!byNorm.has(n)) byNorm.set(n, new Set());
+ byNorm.get(n).add(r.pattern_name);
+ }
+ for (const [n, raws] of byNorm) { if (raws.size > 1) console.log(` norm="${n}" raws=${JSON.stringify([...raws])}`); }
+
+ // Detect pair-collisions within newRows: two rows same norm(pattern)+norm(color) but differing raw (ON CONFLICT would skip 2nd)
+ console.log('\n== IN-NEWSET PAIR COLLISIONS (same norm pattern+color, raw differs -> DB would skip one) ==');
+ const pairMap = new Map();
+ for (const r of newRows) {
+ const k = norm(r.pattern_name) + '' + norm(r.preferred_color_name);
+ if (!pairMap.has(k)) pairMap.set(k, []);
+ pairMap.get(k).push(r);
+ }
+ let collCount = 0;
+ for (const [k, arr] of pairMap) {
+ if (arr.length > 1) {
+ const raws = new Set(arr.map(x => x.pattern_name + '|' + x.preferred_color_name));
+ if (raws.size > 1) { collCount++; console.log(` ${k} => ${JSON.stringify([...raws])}`); }
+ }
+ }
+ console.log('pair-collision groups:', collCount);
+
+ // Dump the extend GENUINELY-NEW and TRUE-DUP sets for the memo
+ fs.writeFileSync(path.join(HERE,'data','d3b-held-extend-new.json'), JSON.stringify(extNew, null, 2));
+ fs.writeFileSync(path.join(HERE,'data','d3b-held-extend-truedup.json'), JSON.stringify(extTrueDup, null, 2));
+ await pool.end();
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/d3b_heldreview_classify.mjs b/d3b_heldreview_classify.mjs
new file mode 100644
index 0000000..90642d7
--- /dev/null
+++ b/d3b_heldreview_classify.mjs
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview — READ-ONLY classifier. No DB writes, no network. Emits table + counts.
+import { createRequire } from 'module';
+import fs from 'fs';
+import path from 'path';
+const require = createRequire(import.meta.url);
+const { Pool } = require('pg');
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const TSV = path.join(HERE, 'data', 'momentum_feed_full.tsv');
+const OUT = path.join(HERE, 'data', 'd3b-heldreview-table.json');
+const pool = new Pool({ host: '/tmp', database: 'dw_unified' });
+const norm = s => (s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+const fcolor = s => norm(s).replace(/gray/g, 'grey').replace(/[^a-z0-9]/g, '');
+const KEEP = new Set(['Wallcovering', 'Acoustic']);
+
+function readTSV() {
+ const lines = fs.readFileSync(TSV, 'utf8').split('\n').filter(Boolean);
+ const cols = lines[0].split('\t');
+ return lines.slice(1).map(l => { const p = l.split('\t'); const o = {}; cols.forEach((c, i) => o[c] = p[i]); return o; });
+}
+
+async function main() {
+ const { rows: pr } = await pool.query('SELECT pattern_name, color_name FROM momentum_colorways');
+ const stagedPatternNorm = new Set(pr.map(r => norm(r.pattern_name)));
+ const stagedPairNorm = new Set(pr.map(r => norm(r.pattern_name) + '' + norm(r.color_name)));
+ const patRaw = new Map(); const patFcolors = new Map();
+ for (const r of pr) {
+ const n = norm(r.pattern_name);
+ if (!patRaw.has(n)) patRaw.set(n, new Map());
+ patRaw.get(n).set(r.pattern_name, (patRaw.get(n).get(r.pattern_name) || 0) + 1);
+ if (!patFcolors.has(n)) patFcolors.set(n, new Set());
+ patFcolors.get(n).add(fcolor(r.color_name));
+ }
+ const canon = n => { const m = patRaw.get(n); return m ? [...m.entries()].sort((a, b) => b[1] - a[1] || a[0].length - b[0].length)[0][0] : null; };
+
+ const feed = readTSV();
+ const inscope = feed.filter(r => KEEP.has(r.category_name) && (r.pattern_name || '').trim() && (r.preferred_color_name || '').trim());
+ const seen = new Set(); const dedup = [];
+ for (const r of inscope) { const k = r.pattern_name + '' + r.preferred_color_name; if (seen.has(k)) continue; seen.add(k); dedup.push(r); }
+
+ const held = dedup.filter(r => stagedPatternNorm.has(norm(r.pattern_name)) && !stagedPairNorm.has(norm(r.pattern_name) + '' + norm(r.preferred_color_name)));
+ const table = [];
+ for (const r of held) {
+ const np = norm(r.pattern_name); const cp = canon(np); const disamb = cp !== r.pattern_name;
+ const fc = fcolor(r.preferred_color_name); const fuzzyDup = patFcolors.get(np)?.has(fc);
+ let verdict, evidence;
+ if (fuzzyDup) { verdict = 'DROP-DUP'; evidence = `fuzzy-color match to staged colorway of "${cp.trim()}" (spelling variant); fcolor=${fc}`; }
+ else if (disamb) { verdict = 'DISAMBIGUATE'; evidence = `new colorway; feed spelling "${JSON.stringify(r.pattern_name)}" != staged "${JSON.stringify(cp)}" (whitespace/case) — stage under staged spelling`; }
+ else { verdict = 'STAGE-NEW'; evidence = `norm pair + fuzzy color both absent under staged pattern "${cp.trim()}"`; }
+ table.push({ item: `${r.pattern_name.trim()} / ${r.preferred_color_name}`, number: r.number, cat: r.category_name, verdict, stage_pattern: verdict === 'DROP-DUP' ? null : cp, evidence });
+ }
+
+ const COLLISION = ['Cavern II', 'Rectangular Hatch', 'Rectangular Linear', 'Rectangular Louvre', 'Rectangular Vault', 'Rectangular Vertex 9/16'];
+ const collision = [];
+ for (const c of COLLISION) {
+ const cn = norm(c); const frs = dedup.filter(r => norm(r.pattern_name) === cn);
+ const newOnes = frs.filter(r => !stagedPairNorm.has(cn + '' + norm(r.preferred_color_name)) && !patFcolors.get(cn)?.has(fcolor(r.preferred_color_name)));
+ let verdict;
+ if (frs.length === 0) verdict = 'NO-OP (absent from feed)';
+ else if (newOnes.length === 0) verdict = 'DROP-DUP (all staged)';
+ else verdict = `DISAMBIGUATE (${newOnes.length} new under "${(canon(cn) || '').trim()}")`;
+ collision.push({ pattern: c, feed_cw: frs.length, pattern_staged: stagedPatternNorm.has(cn), staged_spelling: canon(cn), new_cw: newOnes.length, verdict });
+ }
+
+ const counts = {
+ held_total: held.length,
+ STAGE_NEW: table.filter(t => t.verdict === 'STAGE-NEW').length,
+ DISAMBIGUATE: table.filter(t => t.verdict === 'DISAMBIGUATE').length,
+ DROP_DUP: table.filter(t => t.verdict === 'DROP-DUP').length,
+ };
+ console.log('COUNTS', JSON.stringify(counts));
+ console.log('\nCOLLISION SUMMARY'); console.table(collision);
+ console.log('\nEXTEND TABLE (first 90)'); console.table(table.map(t => ({ item: t.item, cat: t.cat, verdict: t.verdict, stage_pattern: t.stage_pattern ? JSON.stringify(t.stage_pattern) : null })));
+ fs.writeFileSync(OUT, JSON.stringify({ counts, table, collision }, null, 2));
+ await pool.end();
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/d3b_heldreview_emit_sql.mjs b/d3b_heldreview_emit_sql.mjs
new file mode 100644
index 0000000..b3ee140
--- /dev/null
+++ b/d3b_heldreview_emit_sql.mjs
@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview — READ-ONLY SQL emitter. Reads data/d3b-held-rows.json (+ optional
+// data/d3b-held-images.tsv) and writes data/d3b-held-insert.sql. No DB, no network.
+// Mirrors d3c_stage.mjs field semantics: cost=list*0.80, hw_price=list*1.448, sub_category for acoustic,
+// settlement-review tag where flagged, dw_sku omitted (NULL — assign-sku FROZEN), uom -> uom+price_unit.
+// Insert guarded by ON CONFLICT (pattern_name,color_name) DO NOTHING; RETURNING prints inserted ids.
+import fs from 'fs';
+import path from 'path';
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const rows = JSON.parse(fs.readFileSync(path.join(HERE, 'data', 'd3b-held-rows.json'), 'utf8'));
+const imgPath = path.join(HERE, 'data', 'd3b-held-images.tsv');
+const imgMap = new Map();
+if (fs.existsSync(imgPath)) {
+ for (const l of fs.readFileSync(imgPath, 'utf8').split('\n').filter(Boolean)) {
+ const [num, img, series] = l.split('\t');
+ imgMap.set(num, { img: img || null, series: series || null });
+ }
+}
+const COST_MULT = 0.80, HW_MULT = 1.448;
+const money = n => { const x = Number(n); return (!n || !(x > 0)) ? null : Number(x.toFixed(2)); };
+const q = s => s == null ? 'NULL' : `'${String(s).replace(/'/g, "''")}'`;
+const num = n => n == null ? 'NULL' : String(n);
+
+const out = [];
+out.push('\\set ON_ERROR_STOP on');
+out.push('BEGIN;');
+for (const r of rows) {
+ const list = money(r.list_price);
+ const im = imgMap.get(String(r.number)) || {};
+ const tags = r.settlement ? JSON.stringify(['settlement-review']) : null;
+ const subcat = r.category === 'Acoustic' ? 'Acoustic Panel' : null;
+ const cost = list == null ? null : money(list * COST_MULT);
+ const hw = list == null ? null : money(list * HW_MULT);
+ out.push(
+ `INSERT INTO momentum_colorways (pattern_name,color_name,color_number,momentum_sku,image_url,list_price,width,category,product_line,collection_name,uom,price_unit,cost,hw_price,sub_category,tags,created_at,updated_at) VALUES (` +
+ `${q(r.stage_pattern)},${q(r.color_name)},${q(r.color_number)},${q(r.number)},${q(im.img || null)},${num(list)},${q(r.width)},${q(r.category)},${q(r.product_line)},${q(im.series || null)},${q(r.uom)},${q(r.uom)},${num(cost)},${num(hw)},${q(subcat)},${q(tags)},now(),now()) ` +
+ `ON CONFLICT (pattern_name,color_name) DO NOTHING RETURNING id||E'\\t'||pattern_name||E'\\t'||color_name;`
+ );
+}
+out.push('COMMIT;');
+fs.writeFileSync(path.join(HERE, 'data', 'd3b-held-insert.sql'), out.join('\n') + '\n');
+console.log(`emitted ${rows.length} INSERTs -> data/d3b-held-insert.sql (images: ${imgMap.size})`);
diff --git a/d3b_heldreview_prep.mjs b/d3b_heldreview_prep.mjs
new file mode 100644
index 0000000..2962009
--- /dev/null
+++ b/d3b_heldreview_prep.mjs
@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview — READ-ONLY prep. Recompute held set + verdicts, emit numbers.txt + rows.json.
+// No DB writes, no network.
+import { createRequire } from 'module';
+import fs from 'fs';
+import path from 'path';
+const require = createRequire(import.meta.url);
+const { Pool } = require('pg');
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const TSV = path.join(HERE, 'data', 'momentum_feed_full.tsv');
+const pool = new Pool({ host: '/tmp', database: 'dw_unified' });
+const norm = s => (s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+const fcolor = s => norm(s).replace(/gray/g, 'grey').replace(/[^a-z0-9]/g, '');
+const KEEP = new Set(['Wallcovering', 'Acoustic']);
+const SETTLEMENT = new Set(['Hula WC', 'Sea Line', 'Palomar', 'Oakstone']);
+
+function readTSV() {
+ const lines = fs.readFileSync(TSV, 'utf8').split('\n').filter(Boolean);
+ const cols = lines[0].split('\t');
+ return lines.slice(1).map(l => { const p = l.split('\t'); const o = {}; cols.forEach((c, i) => o[c] = p[i]); return o; });
+}
+async function main() {
+ const { rows: pr } = await pool.query('SELECT pattern_name, color_name FROM momentum_colorways');
+ const stagedPatternNorm = new Set(pr.map(r => norm(r.pattern_name)));
+ const stagedPairNorm = new Set(pr.map(r => norm(r.pattern_name) + '' + norm(r.color_name)));
+ const patRaw = new Map(); const patFcolors = new Map();
+ for (const r of pr) {
+ const n = norm(r.pattern_name);
+ if (!patRaw.has(n)) patRaw.set(n, new Map());
+ patRaw.get(n).set(r.pattern_name, (patRaw.get(n).get(r.pattern_name) || 0) + 1);
+ if (!patFcolors.has(n)) patFcolors.set(n, new Set());
+ patFcolors.get(n).add(fcolor(r.color_name));
+ }
+ const canon = n => { const m = patRaw.get(n); return m ? [...m.entries()].sort((a, b) => b[1] - a[1] || a[0].length - b[0].length)[0][0] : null; };
+ const feed = readTSV();
+ const inscope = feed.filter(r => KEEP.has(r.category_name) && (r.pattern_name || '').trim() && (r.preferred_color_name || '').trim());
+ const seen = new Set(); const dedup = [];
+ for (const r of inscope) { const k = r.pattern_name + '' + r.preferred_color_name; if (seen.has(k)) continue; seen.add(k); dedup.push(r); }
+ const held = dedup.filter(r => stagedPatternNorm.has(norm(r.pattern_name)) && !stagedPairNorm.has(norm(r.pattern_name) + '' + norm(r.preferred_color_name)));
+ const rows = [];
+ for (const r of held) {
+ const np = norm(r.pattern_name); const cp = canon(np);
+ const fuzzyDup = patFcolors.get(np)?.has(fcolor(r.preferred_color_name));
+ if (fuzzyDup) continue; // DROP-DUP — not staged
+ const verdict = cp !== r.pattern_name ? 'DISAMBIGUATE' : 'STAGE-NEW';
+ rows.push({
+ verdict, stage_pattern: cp, color_name: r.preferred_color_name,
+ color_number: r.preferred_color_number || null, number: r.number || null,
+ list_price: r.list_price || null, width: (r.base_width || '').trim() || null,
+ category: r.category_name, product_line: (r.product_line_code || '').trim() || null,
+ uom: (r.uom || '').trim() || null, settlement: SETTLEMENT.has(r.pattern_name.trim()),
+ });
+ }
+ fs.writeFileSync(path.join(HERE, 'data', 'd3b-held-numbers.txt'), rows.map(r => r.number).filter(Boolean).join('\n') + '\n');
+ fs.writeFileSync(path.join(HERE, 'data', 'd3b-held-rows.json'), JSON.stringify(rows, null, 2));
+ const by = rows.reduce((a, r) => (a[r.verdict] = (a[r.verdict] || 0) + 1, a), {});
+ console.log('prep rows:', rows.length, JSON.stringify(by), 'numbers:', rows.filter(r => r.number).length);
+ await pool.end();
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/d3b_heldreview_rollback.mjs b/d3b_heldreview_rollback.mjs
new file mode 100644
index 0000000..dbf1df7
--- /dev/null
+++ b/d3b_heldreview_rollback.mjs
@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview ROLLBACK — removes the 79 staged held colorways (ids 61900..61978).
+// Reversible undo for the d3b-heldreview-stage op. Dry-run by default; --apply to execute.
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+const { Pool } = require('pg');
+const APPLY = process.argv.includes('--apply');
+const pool = new Pool({ host: '/tmp', database: 'dw_unified' });
+const LO = 61900, HI = 61978;
+async function main() {
+ const { rows } = await pool.query('SELECT count(*)::int n FROM momentum_colorways WHERE id BETWEEN $1 AND $2', [LO, HI]);
+ console.log(`rows in range ${LO}-${HI}: ${rows[0].n}`);
+ if (!APPLY) { console.log('DRY-RUN. Re-run with --apply to remove them.'); await pool.end(); return; }
+ const r = await pool.query('DELETE FROM momentum_colorways WHERE id BETWEEN $1 AND $2', [LO, HI]);
+ console.log(`removed ${r.rowCount} rows.`);
+ await pool.end();
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/d3b_heldreview_stage.mjs b/d3b_heldreview_stage.mjs
new file mode 100644
index 0000000..fb64279
--- /dev/null
+++ b/d3b_heldreview_stage.mjs
@@ -0,0 +1,182 @@
+#!/usr/bin/env node
+// TK-11331 D3b-heldreview — classify the two HELD buckets + STAGE the STAGE-NEW/DISAMBIGUATE rows.
+// Mirrors d3c_stage.mjs: hw_price=list*1.448, cost=list*0.80, sub_category='Acoustic Panel' for acoustic,
+// settlement-review tag where flagged, image_url via per-number Meilisearch lookup, dw_sku LEFT NULL
+// (assign-sku FROZEN), ON CONFLICT (pattern_name,color_name) DO NOTHING guard, id-scoped restore file.
+// DISAMBIGUATE: stage new colorways of a normalized-collision pattern UNDER the existing staged raw
+// spelling (canonical), so they join the existing pattern group instead of forking a whitespace variant.
+// DROP-DUP: skip (exact or fuzzy Gray/Grey color match already staged).
+// DRY-RUN by default; --commit writes.
+import { createRequire } from 'module';
+import fs from 'fs';
+import path from 'path';
+const require = createRequire(import.meta.url);
+const { Pool } = require('pg');
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const TSV = path.join(HERE, 'data', 'momentum_feed_full.tsv');
+const RESTORE = path.join(HERE, 'data', 'd3b-heldreview-stage-restore.jsonl');
+const RESULTS = path.join(HERE, 'data', 'd3b-heldreview-results.json');
+const TABLE = path.join(HERE, 'data', 'd3b-heldreview-table.json');
+const COMMIT = process.argv.includes('--commit');
+
+const HOST = 'https://ms-e886719d86e7-4256.sfo.meilisearch.io';
+const KEY = process.env.MOMENTUM_MS_KEY || '';
+const INDEX = 'redesign-colors';
+const SETTLEMENT = new Set(['Hula WC', 'Sea Line', 'Palomar', 'Oakstone']);
+const COST_MULT = 0.80, HW_MULT = 1.448;
+const KEEP = new Set(['Wallcovering', 'Acoustic']);
+
+const pool = new Pool({ host: '/tmp', database: 'dw_unified' });
+const norm = s => (s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+const money = n => (n == null || !(Number(n) > 0)) ? null : Number(Number(n).toFixed(2));
+// fuzzy color key: lowercase, gray->grey, drop all non-alnum → catches Gray/Grey + spacing/punct variants
+const fcolor = s => norm(s).replace(/gray/g, 'grey').replace(/[^a-z0-9]/g, '');
+
+function readTSV() {
+ const lines = fs.readFileSync(TSV, 'utf8').split('\n').filter(Boolean);
+ const cols = lines[0].split('\t');
+ return lines.slice(1).map(l => { const p = l.split('\t'); const o = {}; cols.forEach((c, i) => o[c] = p[i]); return o; });
+}
+
+async function imageFor(number) {
+ try {
+ const r = await fetch(`${HOST}/indexes/${INDEX}/search`, {
+ method: 'POST', headers: { Authorization: `Bearer ${KEY}`, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ q: String(number), limit: 5, attributesToRetrieve: ['number', 'medium_image_url', 'series', 'collections'] }),
+ });
+ if (!r.ok) return {};
+ const hits = (await r.json()).hits || [];
+ const h = hits.find(x => String(x.number) === String(number)) || hits[0];
+ if (!h) return {};
+ return { image: h.medium_image_url || null, series: h.series || (Array.isArray(h.collections) ? h.collections[0]?.name : null) || null };
+ } catch { return {}; }
+}
+
+async function main() {
+ const { rows: pr } = await pool.query('SELECT pattern_name, color_name FROM momentum_colorways');
+ const stagedPatternNorm = new Set(pr.map(r => norm(r.pattern_name)));
+ const stagedPairNorm = new Set(pr.map(r => norm(r.pattern_name) + '' + norm(r.color_name)));
+ // per norm(pattern): canonical raw spelling (longest/first existing) + fuzzy color set
+ const patRaw = new Map(); // normPattern -> Map(rawSpelling -> count)
+ const patFcolors = new Map(); // normPattern -> Set(fuzzyColor)
+ for (const r of pr) {
+ const n = norm(r.pattern_name);
+ if (!patRaw.has(n)) patRaw.set(n, new Map());
+ patRaw.get(n).set(r.pattern_name, (patRaw.get(n).get(r.pattern_name) || 0) + 1);
+ if (!patFcolors.has(n)) patFcolors.set(n, new Set());
+ patFcolors.get(n).add(fcolor(r.color_name));
+ }
+ const canonicalRaw = n => {
+ const m = patRaw.get(n); if (!m) return null;
+ return [...m.entries()].sort((a, b) => b[1] - a[1] || a[0].length - b[0].length)[0][0];
+ };
+
+ const feed = readTSV();
+ const inscope = feed.filter(r => KEEP.has(r.category_name) && (r.pattern_name || '').trim() && (r.preferred_color_name || '').trim());
+ const seen = new Set(); const dedup = [];
+ for (const r of inscope) { const k = r.pattern_name + '' + r.preferred_color_name; if (seen.has(k)) continue; seen.add(k); dedup.push(r); }
+
+ // HELD extend-pattern GENUINELY-NEW set = norm(pattern) staged AND norm(pair) NOT staged
+ const held = dedup.filter(r => stagedPatternNorm.has(norm(r.pattern_name)) && !stagedPairNorm.has(norm(r.pattern_name) + '' + norm(r.preferred_color_name)));
+
+ const table = [];
+ const toStage = [];
+ for (const r of held) {
+ const np = norm(r.pattern_name);
+ const rawFeed = r.pattern_name;
+ const canon = canonicalRaw(np);
+ const disambiguate = canon !== rawFeed; // feed spelling differs from staged spelling (whitespace/case)
+ const fc = fcolor(r.preferred_color_name);
+ const fuzzyDup = patFcolors.get(np)?.has(fc);
+ let verdict, reason, stagePattern = canon || rawFeed;
+ if (fuzzyDup) {
+ verdict = 'DROP-DUP';
+ reason = `fuzzy-color match to an already-staged colorway of "${canon}" (spelling variant, e.g. Gray/Grey); fcolor="${fc}"`;
+ } else {
+ verdict = disambiguate ? 'DISAMBIGUATE' : 'STAGE-NEW';
+ reason = disambiguate
+ ? `genuinely-new colorway; feed pattern "${rawFeed}" collides on normalization with staged "${canon}" — stage under canonical staged spelling to avoid whitespace fork`
+ : `genuinely-new colorway of already-staged pattern "${canon}"; no staged equivalent (norm pair + fuzzy color both absent)`;
+ toStage.push({ r, stagePattern });
+ }
+ table.push({
+ item: `${rawFeed.trim()} / ${r.preferred_color_name}`,
+ number: r.number, category: r.category_name, bucket: 'extend-pattern',
+ verdict, stage_pattern: verdict === 'DROP-DUP' ? null : stagePattern, reason,
+ });
+ }
+
+ // Cavern II bucket (all exact-staged) + Rectangular Hatch/Linear/Louvre/Vault (0 feed) — collision-pattern rows
+ // Cavern II rows are already TRUE-DUP (norm pair staged) so they never enter `held`; record them explicitly.
+ const COLLISION = ['Cavern II', 'Rectangular Hatch', 'Rectangular Linear', 'Rectangular Louvre', 'Rectangular Vault', 'Rectangular Vertex 9/16'];
+ const collisionSummary = [];
+ for (const cp of COLLISION) {
+ const cn = norm(cp);
+ const feedRows = dedup.filter(r => norm(r.pattern_name) === cn);
+ const staged = stagedPatternNorm.has(cn);
+ const canon = canonicalRaw(cn);
+ const newOnes = feedRows.filter(r => !stagedPairNorm.has(cn + '' + norm(r.preferred_color_name)) && !patFcolors.get(cn)?.has(fcolor(r.preferred_color_name)));
+ let verdict;
+ if (feedRows.length === 0) verdict = 'NO-OP (absent from current feed — nothing to onboard)';
+ else if (newOnes.length === 0) verdict = 'DROP-DUP (all feed colorways already staged under canonical spelling)';
+ else verdict = `DISAMBIGUATE (${newOnes.length} new colorways staged under canonical "${canon}")`;
+ collisionSummary.push({ pattern: cp, norm: cn, feed_colorways: feedRows.length, pattern_staged: staged, canonical_staged_spelling: canon, new_colorways: newOnes.length, verdict });
+ }
+
+ const counts = {
+ held_total: held.length,
+ stage_new: table.filter(t => t.verdict === 'STAGE-NEW').length,
+ disambiguate: table.filter(t => t.verdict === 'DISAMBIGUATE').length,
+ drop_dup: table.filter(t => t.verdict === 'DROP-DUP').length,
+ };
+ console.log('== CLASSIFICATION ==');
+ console.log(JSON.stringify(counts, null, 2));
+ console.log('== COLLISION PATTERNS ==');
+ console.table(collisionSummary);
+ fs.writeFileSync(TABLE, JSON.stringify({ counts, table, collisionSummary }, null, 2));
+
+ if (!COMMIT) { console.log(`\nDRY-RUN. toStage=${toStage.length}. Re-run with --commit.`); await pool.end(); return; }
+
+ // enrich images (per-number, ≤79 lookups)
+ for (const x of toStage) { const im = await imageFor(x.r.number); x.image = im.image || null; x.series = im.series || null; }
+ const withImg = toStage.filter(x => x.image).length;
+ console.log(`\nimage lookups: ${withImg}/${toStage.length} got image_url`);
+
+ const c = await pool.connect();
+ let inserted = 0, skipped = 0; const restore = [];
+ try {
+ await c.query('BEGIN');
+ for (const x of toStage) {
+ const r = x.r; const list = money(r.list_price);
+ const tags = SETTLEMENT.has(r.pattern_name.trim()) ? JSON.stringify(['settlement-review']) : null;
+ const subcat = r.category_name === 'Acoustic' ? 'Acoustic Panel' : null;
+ const uom = (r.uom || '').trim() || null;
+ const res = await c.query(
+ `INSERT INTO momentum_colorways
+ (pattern_name,color_name,color_number,momentum_sku,image_url,list_price,width,category,
+ product_line,collection_name,uom,price_unit,cost,hw_price,sub_category,tags,created_at,updated_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,now(),now())
+ ON CONFLICT (pattern_name,color_name) DO NOTHING
+ RETURNING id`,
+ [x.stagePattern, r.preferred_color_name, r.preferred_color_number || null, r.number || null,
+ x.image || null, list, (r.base_width || '').trim() || null, r.category_name,
+ (r.product_line_code || '').trim() || null, x.series || null, uom, uom,
+ list == null ? null : money(list * COST_MULT), list == null ? null : money(list * HW_MULT),
+ subcat, tags]);
+ if (res.rows.length) { inserted++; restore.push({ id: res.rows[0].id, pattern_name: x.stagePattern, color_name: r.preferred_color_name }); }
+ else skipped++;
+ }
+ await c.query('COMMIT');
+ } catch (e) { await c.query('ROLLBACK'); console.error('ROLLBACK:', e.message); process.exitCode = 1; c.release(); await pool.end(); return; }
+ c.release();
+
+ fs.writeFileSync(RESTORE, restore.map(x => JSON.stringify(x)).join('\n') + '\n');
+ const finalCount = (await pool.query('SELECT count(*)::int n FROM momentum_colorways')).rows[0].n;
+ const results = { ts: new Date().toISOString(), counts, attempted: toStage.length, inserted, guard_skips: skipped,
+ final_rows: finalCount, restore_file: RESTORE, inserted_ids: restore.map(x => x.id) };
+ fs.writeFileSync(RESULTS, JSON.stringify(results, null, 2));
+ console.log(`COMMITTED — inserted ${inserted}, guard-skipped ${skipped}. final_rows=${finalCount}`);
+ console.log(`Restore: ${RESTORE}`);
+ await pool.end();
+}
+main().catch(e => { console.error(e); process.exit(1); });
diff --git a/fetch_held_images.sh b/fetch_held_images.sh
new file mode 100644
index 0000000..953e7f3
--- /dev/null
+++ b/fetch_held_images.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# TK-11331 D3b-heldreview — fetch Meilisearch images for the held numbers. Read-only HTTPS.
+# KEY must be provided via MOMENTUM_MS_KEY env. Extracts image+series with grep (no interpreter eval).
+set -euo pipefail
+cd "$(dirname "$0")"
+: "${MOMENTUM_MS_KEY:?set MOMENTUM_MS_KEY}"
+HOST="https://ms-e886719d86e7-4256.sfo.meilisearch.io"
+OUT="data/d3b-held-images.tsv"
+: > "$OUT"
+while IFS= read -r num; do
+ [ -z "$num" ] && continue
+ resp=$(curl -s -m 20 -X POST "$HOST/indexes/redesign-colors/search" \
+ -H "Authorization: Bearer $MOMENTUM_MS_KEY" -H "Content-Type: application/json" \
+ -d "{\"q\":\"$num\",\"limit\":5,\"attributesToRetrieve\":[\"number\",\"medium_image_url\",\"series\"]}")
+ img=$(printf '%s' "$resp" | grep -oE '"medium_image_url":"[^"]*"' | head -1 | sed 's/"medium_image_url":"//;s/"$//')
+ series=$(printf '%s' "$resp" | grep -oE '"series":"[^"]*"' | head -1 | sed 's/"series":"//;s/"$//')
+ printf '%s\t%s\t%s\n' "$num" "$img" "$series" >> "$OUT"
+done < data/d3b-held-numbers.txt
+echo "images: $(wc -l < "$OUT") lines, with-image: $(awk -F'\t' '$2!=""{c++}END{print c+0}' "$OUT")"
← 6feb373 auto-data-snapshot: 2026-09-09T21:13:05 (2 data files) — dat
·
back to Tk 11331 Exec
·
auto-data-snapshot: 2026-09-10T04:50:00 (1 data files) — dat fe40809 →