`scripts/dedup-flag.js`

61 lines
'use strict';
/* Tightened dedup: flag rebelwalls_catalog rows whose (pattern[+exact color]) is ALREADY
 * live in the OLDER Rebel Walls cohort. Adds reversible dedup_skip boolean. Local-PG write only.
 *   --apply  actually writes the flag (default = dry report) */
const { gqlRetry, psql } = require(process.env.HOME+'/Projects/rebel-walls-push/scripts/_shop');
const APPLY = process.argv.includes('--apply');
const norm = s => (s||'').toLowerCase().replace(/[^a-z0-9]+/g,' ').trim();

const Q=`query($cursor:String){products(first:250,query:"vendor:'Rebel Walls'",after:$cursor){pageInfo{hasNextPage endCursor}edges{node{title variants(first:5){edges{node{sku}}}}}}}`;
(async()=>{
  // ---- older-live cohort: pattern -> Set(colors) ----
  let cursor=null; const patColors=new Map(); const patAny=new Set(); let oldN=0;
  do{const r=await gqlRetry(Q,{cursor},'f');const d=r.json.data.products;
    for(const e of d.edges){
      const mains=e.node.variants.edges.map(v=>(v.node.sku||'').replace(/-Sample$/i,'')).filter(Boolean);
      const dwrw=mains.find(s=>/^DWRW-\d+$/.test(s));
      const n=dwrw?parseInt(dwrw.split('-')[1],10):null;
      const isOld = !dwrw || n<360000 || n>364132;
      if(!isOld) continue;
      oldN++;
      let t=e.node.title.replace(/\s*\|\s*Rebel Walls\s*$/i,'').replace(/wallcovering/ig,'').replace(/\s*\d+\s*cm\s*x\s*\d+\s*cm\s*/ig,' ').trim();
      const ci=t.indexOf(', ');
      let pat, col;
      if(ci>=0){ pat=norm(t.slice(0,ci)); col=norm(t.slice(ci+2)); }
      else { pat=norm(t); col=''; }
      patAny.add(pat);
      if(!patColors.has(pat)) patColors.set(pat,new Set());
      if(col) patColors.get(pat).add(col);
    }
    cursor=d.pageInfo.hasNextPage?d.pageInfo.endCursor:null;
  }while(cursor);

  // ---- classify unpushed new rows ----
  const rows = psql("SELECT dw_sku, COALESCE(pattern_name,''), COALESCE(color_name,'') FROM rebelwalls_catalog WHERE (shopify_product_id IS NULL OR shopify_product_id='') ORDER BY dw_sku;").trim().split('\n').filter(Boolean).map(l=>l.split('\t'));
  const dupSkus=[], netSkus=[];
  for(const [sku,pat,col] of rows){
    const np=norm(pat), nc=norm(col);
    let isDup=false;
    if(np && patAny.has(np)){
      if(nc){ isDup = patColors.get(np).has(nc); }   // has color: dup only if exact colorway live
      else  { isDup = true; }                         // no color: conservative -> dup if pattern live
    }
    (isDup?dupSkus:netSkus).push(sku);
  }
  console.log(`older cohort: ${oldN} products, ${patAny.size} distinct patterns`);
  console.log(`unpushed new rows: ${rows.length}`);
  console.log(`  TRUE duplicates (tightened): ${dupSkus.length}`);
  console.log(`  NET-NEW to push:             ${netSkus.length}`);

  if(!APPLY){ console.log('\nDRY — no PG writes. Re-run with --apply to flag.'); return; }

  psql("ALTER TABLE rebelwalls_catalog ADD COLUMN IF NOT EXISTS dedup_skip boolean;");
  psql("UPDATE rebelwalls_catalog SET dedup_skip=false WHERE (shopify_product_id IS NULL OR shopify_product_id='');");
  for(let i=0;i<dupSkus.length;i+=1000){
    const chunk=dupSkus.slice(i,i+1000).map(s=>`'${s}'`).join(',');
    psql(`UPDATE rebelwalls_catalog SET dedup_skip=true WHERE dw_sku IN (${chunk});`);
  }
  const after=psql("SELECT count(*) FILTER (WHERE dedup_skip=true), count(*) FILTER (WHERE dedup_skip=false AND (shopify_product_id IS NULL OR shopify_product_id='')) FROM rebelwalls_catalog;").trim();
  console.log(`\nFLAGGED. dedup_skip=true \\t pushable-net-new:  ${after}`);
})().catch(e=>{console.error('ERR',e.message);process.exit(1);});