← back to Rebel Walls Push
dedup-guard push: skip 759 patterns already live under older Rebel Walls SKUs (DTD verdict A)
1e7025864c83cbe7444f01e502230a682ea9ed82 · 2026-06-09 11:16:11 -0700 · SteveStudio2
- dedup-flag.js: tightened pattern+exact-color match vs older live cohort (DWRW-72201..451820);
flags 759 true dups, leaves 2,874 net-new pushable via reversible dedup_skip column
- push.js fetchRows: AND dedup_skip IS NOT TRUE (null-safe) so resume pushes only net-new
- verify-live.js / reconcile-from-shopify.js: read-only Shopify count + sku->id helpers
Files touched
A scripts/dedup-flag.jsM scripts/push.jsA scripts/reconcile-from-shopify.jsA scripts/verify-live.js
Diff
commit 1e7025864c83cbe7444f01e502230a682ea9ed82
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Tue Jun 9 11:16:11 2026 -0700
dedup-guard push: skip 759 patterns already live under older Rebel Walls SKUs (DTD verdict A)
- dedup-flag.js: tightened pattern+exact-color match vs older live cohort (DWRW-72201..451820);
flags 759 true dups, leaves 2,874 net-new pushable via reversible dedup_skip column
- push.js fetchRows: AND dedup_skip IS NOT TRUE (null-safe) so resume pushes only net-new
- verify-live.js / reconcile-from-shopify.js: read-only Shopify count + sku->id helpers
---
scripts/dedup-flag.js | 60 +++++++++++++++++++++++++++++++++++++++
scripts/push.js | 4 ++-
scripts/reconcile-from-shopify.js | 60 +++++++++++++++++++++++++++++++++++++++
scripts/verify-live.js | 15 ++++++++++
4 files changed, 138 insertions(+), 1 deletion(-)
diff --git a/scripts/dedup-flag.js b/scripts/dedup-flag.js
new file mode 100644
index 0000000..b9fbbc2
--- /dev/null
+++ b/scripts/dedup-flag.js
@@ -0,0 +1,60 @@
+'use strict';
+/* Tightened dedup: flag rebelwalls_catalog rows whose (pattern[+exact color]) is ALREADY
+ * live in the OLDER Rebel Walls cohort. Adds reversible dedup_skip boolean. Local-PG write only.
+ * --apply actually writes the flag (default = dry report) */
+const { gqlRetry, psql } = require(process.env.HOME+'/Projects/rebel-walls-push/scripts/_shop');
+const APPLY = process.argv.includes('--apply');
+const norm = s => (s||'').toLowerCase().replace(/[^a-z0-9]+/g,' ').trim();
+
+const Q=`query($cursor:String){products(first:250,query:"vendor:'Rebel Walls'",after:$cursor){pageInfo{hasNextPage endCursor}edges{node{title variants(first:5){edges{node{sku}}}}}}}`;
+(async()=>{
+ // ---- older-live cohort: pattern -> Set(colors) ----
+ let cursor=null; const patColors=new Map(); const patAny=new Set(); let oldN=0;
+ do{const r=await gqlRetry(Q,{cursor},'f');const d=r.json.data.products;
+ for(const e of d.edges){
+ const mains=e.node.variants.edges.map(v=>(v.node.sku||'').replace(/-Sample$/i,'')).filter(Boolean);
+ const dwrw=mains.find(s=>/^DWRW-\d+$/.test(s));
+ const n=dwrw?parseInt(dwrw.split('-')[1],10):null;
+ const isOld = !dwrw || n<360000 || n>364132;
+ if(!isOld) continue;
+ oldN++;
+ let t=e.node.title.replace(/\s*\|\s*Rebel Walls\s*$/i,'').replace(/wallcovering/ig,'').replace(/\s*\d+\s*cm\s*x\s*\d+\s*cm\s*/ig,' ').trim();
+ const ci=t.indexOf(', ');
+ let pat, col;
+ if(ci>=0){ pat=norm(t.slice(0,ci)); col=norm(t.slice(ci+2)); }
+ else { pat=norm(t); col=''; }
+ patAny.add(pat);
+ if(!patColors.has(pat)) patColors.set(pat,new Set());
+ if(col) patColors.get(pat).add(col);
+ }
+ cursor=d.pageInfo.hasNextPage?d.pageInfo.endCursor:null;
+ }while(cursor);
+
+ // ---- classify unpushed new rows ----
+ const rows = psql("SELECT dw_sku, COALESCE(pattern_name,''), COALESCE(color_name,'') FROM rebelwalls_catalog WHERE (shopify_product_id IS NULL OR shopify_product_id='') ORDER BY dw_sku;").trim().split('\n').filter(Boolean).map(l=>l.split('\t'));
+ const dupSkus=[], netSkus=[];
+ for(const [sku,pat,col] of rows){
+ const np=norm(pat), nc=norm(col);
+ let isDup=false;
+ if(np && patAny.has(np)){
+ if(nc){ isDup = patColors.get(np).has(nc); } // has color: dup only if exact colorway live
+ else { isDup = true; } // no color: conservative -> dup if pattern live
+ }
+ (isDup?dupSkus:netSkus).push(sku);
+ }
+ console.log(`older cohort: ${oldN} products, ${patAny.size} distinct patterns`);
+ console.log(`unpushed new rows: ${rows.length}`);
+ console.log(` TRUE duplicates (tightened): ${dupSkus.length}`);
+ console.log(` NET-NEW to push: ${netSkus.length}`);
+
+ if(!APPLY){ console.log('\nDRY — no PG writes. Re-run with --apply to flag.'); return; }
+
+ psql("ALTER TABLE rebelwalls_catalog ADD COLUMN IF NOT EXISTS dedup_skip boolean;");
+ psql("UPDATE rebelwalls_catalog SET dedup_skip=false WHERE (shopify_product_id IS NULL OR shopify_product_id='');");
+ for(let i=0;i<dupSkus.length;i+=1000){
+ const chunk=dupSkus.slice(i,i+1000).map(s=>`'${s}'`).join(',');
+ psql(`UPDATE rebelwalls_catalog SET dedup_skip=true WHERE dw_sku IN (${chunk});`);
+ }
+ const after=psql("SELECT count(*) FILTER (WHERE dedup_skip=true), count(*) FILTER (WHERE dedup_skip=false AND (shopify_product_id IS NULL OR shopify_product_id='')) FROM rebelwalls_catalog;").trim();
+ console.log(`\nFLAGGED. dedup_skip=true \\t pushable-net-new: ${after}`);
+})().catch(e=>{console.error('ERR',e.message);process.exit(1);});
diff --git a/scripts/push.js b/scripts/push.js
index ee6d976..a4b57a3 100644
--- a/scripts/push.js
+++ b/scripts/push.js
@@ -72,7 +72,9 @@ function psql(sql) {
return out.toString();
}
function fetchRows() {
- let where = "(shopify_product_id IS NULL OR shopify_product_id='')";
+ // dedup_skip guard: never push rows flagged as duplicates of the older live Rebel Walls
+ // cohort (DTD verdict A, 2026-06-09). `IS NOT TRUE` is null-safe (pre-flag rows pass).
+ let where = "(shopify_product_id IS NULL OR shopify_product_id='') AND dedup_skip IS NOT TRUE";
if (IDS) where = `id IN (${IDS.split(',').map(n => parseInt(n, 10)).filter(Number.isFinite).join(',')})`;
let limitClause = '';
if (CANARY) limitClause = `LIMIT ${parseInt(CANARY, 10)}`;
diff --git a/scripts/reconcile-from-shopify.js b/scripts/reconcile-from-shopify.js
new file mode 100644
index 0000000..22c6c04
--- /dev/null
+++ b/scripts/reconcile-from-shopify.js
@@ -0,0 +1,60 @@
+'use strict';
+/* READ Shopify live Rebel Walls products -> stamp local PG shopify_product_id by dw_sku.
+ * Local-PG write ONLY. No Shopify writes. Fixes the PG<->Shopify drift before any resume. */
+const { gqlRetry, psql } = require('./_shop');
+const DRY = process.argv.includes('--dry');
+
+const Q = `query($cursor:String){
+ products(first:250, query:"vendor:'Rebel Walls'", after:$cursor){
+ pageInfo{ hasNextPage endCursor }
+ edges{ node{ id status variants(first:5){ edges{ node{ sku } } } } }
+ }
+}`;
+
+(async () => {
+ let cursor = null, page = 0, seen = 0;
+ const map = []; // {sku, id}
+ do {
+ const r = await gqlRetry(Q, { cursor }, 'recon');
+ if (r.json.errors) throw new Error(JSON.stringify(r.json.errors).slice(0,300));
+ const d = r.json.data.products;
+ for (const e of d.edges) {
+ const numId = e.node.id.split('/').pop();
+ // main variant SKU = the DWRW-xxxxxx that does NOT end in -Sample
+ let sku = null;
+ for (const v of e.node.variants.edges) {
+ const s = (v.node.sku||'').trim();
+ if (/^DWRW-\d+$/.test(s)) { sku = s; break; }
+ }
+ if (sku) { map.push({ sku, id: numId }); }
+ seen++;
+ }
+ page++;
+ process.stderr.write(`\r page ${page} | products seen ${seen} | matched DWRW ${map.length}`);
+ cursor = d.pageInfo.hasNextPage ? d.pageInfo.endCursor : null;
+ } while (cursor);
+ process.stderr.write('\n');
+
+ console.log(`Shopify Rebel Walls products: ${seen}, with DWRW main SKU: ${map.length}`);
+ if (DRY) { console.log('DRY — no PG writes. sample:', JSON.stringify(map.slice(0,3))); return; }
+
+ // Bulk update via a single VALUES join (chunked).
+ let updated = 0;
+ for (let i = 0; i < map.length; i += 500) {
+ const chunk = map.slice(i, i + 500);
+ const values = chunk.map(m => `('${m.sku.replace(/'/g,"''")}','${m.id}')`).join(',');
+ const sql = `UPDATE rebelwalls_catalog t SET shopify_product_id=v.id, updated_at=NOW()
+ FROM (VALUES ${values}) AS v(sku,id)
+ WHERE t.dw_sku = v.sku
+ AND (t.shopify_product_id IS NULL OR t.shopify_product_id='' OR t.shopify_product_id <> v.id);`;
+ const out = psql(sql + " \\echo done"); // psql prints UPDATE n? use RETURNING count instead
+ updated += chunk.length;
+ process.stderr.write(`\r stamping ${Math.min(i+500,map.length)}/${map.length}`);
+ }
+ process.stderr.write('\n');
+
+ const after = psql(`SELECT count(*) FILTER (WHERE shopify_product_id IS NOT NULL AND shopify_product_id<>'') ,
+ count(*) FILTER (WHERE shopify_product_id IS NULL OR shopify_product_id='')
+ FROM rebelwalls_catalog;`).trim();
+ console.log('PG after reconcile (stamped \\t unstamped):', after);
+})().catch(e => { console.error('\nERR', e.message); process.exit(1); });
diff --git a/scripts/verify-live.js b/scripts/verify-live.js
new file mode 100644
index 0000000..5986335
--- /dev/null
+++ b/scripts/verify-live.js
@@ -0,0 +1,15 @@
+'use strict';
+const { gqlRetry } = require('./_shop');
+(async () => {
+ const q = async (query) => {
+ const r = await gqlRetry(`query($q:String!){ productsCount(query:$q){ count } }`, { q: query }, 'count');
+ if (r.json.errors) { console.error(query, '->', JSON.stringify(r.json.errors).slice(0,200)); return null; }
+ return r.json.data.productsCount.count;
+ };
+ const vendor = "vendor:'Rebel Walls'";
+ const total = await q(vendor);
+ const active = await q(`${vendor} AND status:active`);
+ const draft = await q(`${vendor} AND status:draft`);
+ const arch = await q(`${vendor} AND status:archived`);
+ console.log(JSON.stringify({ vendor_total: total, active, draft, archived: arch }, null, 2));
+})().catch(e => { console.error('ERR', e.message); process.exit(1); });
← c54afd5 Rebel Walls: bake live-on-create (ACTIVE + all channels + in
·
back to Rebel Walls Push
·
daily-push.sh: capped daily slice wrapper for the staggered d3719d0 →