cadence: honor newwall_catalog dedup_skip so Tres Tintas imports net-new-only (Option A, Steve-approved)

0b67db27261d1ce3ed667aae822f7cea7a0a4955 · 2026-06-23 10:46:32 -0700 · Steve
Files touched

M shopify/scripts/cadence/cadence-import.js
Diff

commit 0b67db27261d1ce3ed667aae822f7cea7a0a4955
Author: Steve <steve@designerwallcoverings.com>
Date:   Tue Jun 23 10:46:32 2026 -0700

    cadence: honor newwall_catalog dedup_skip so Tres Tintas imports net-new-only (Option A, Steve-approved)
---
 shopify/scripts/cadence/cadence-import.js | 45 ++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 4 deletions(-)

diff --git a/shopify/scripts/cadence/cadence-import.js b/shopify/scripts/cadence/cadence-import.js
index 0e1dd2ed..048b59e0 100644
--- a/shopify/scripts/cadence/cadence-import.js
+++ b/shopify/scripts/cadence/cadence-import.js
@@ -49,8 +49,8 @@ const TODAY = new Date().toISOString().slice(0, 10);
 // import these (belt-and-suspenders for sub-brand/registry drift). Today vendors.js already
 // excludes them by omission; this hard-blocks a future accidental add. (compliance officer, 2026-06-17)
 const DENY_VENDORS = new Set(['Nicolette Mayer','Phillip Jeffries','Et Cie','Spoonflower','Colefax & Fowler','Carlisle','Paul Montgomery','Cowtan & Tout','Rebel Walls','Koroseal','Newmor','Wolf Gordon','Scalamandre','Desima',
-  'Romo',   // Romo: vendors.js note "confirm tariff treatment w/ Steve before bulk import" → hold from the auto-uploader until confirmed
-  'York Wallcoverings']);   // York (york_catalog): wired in vendors.js 2026-06-21 (DTD verdict A) but HELD — re-import needs the ~395 dangling shopify_product_ids cleared + paced run; gated for Steve (pending-approval/york-reimport-cadence-wiring.md)
+  // Romo LIFTED 2026-06-23 (Steve: "WE WANT ROMO") — tariff-treatment hold cleared; cost = romo_catalog landed (trade+tariff) per vendors.js.
+  'York Wallcoverings']);   // York (york_catalog): HELD — Steve 2026-06-23 "york just needs to create skus in our private label program first"; yes in principle, but blocked until the private-label SKUs exist (+ ~395 dangling shopify_product_ids cleared). Re-enable after PL SKU creation.
   // Innovations LIFTED 2026-06-18 (Steve): cost on innovations_catalog.price_trade (10% off net, per-yd),
   // wired in vendors.js (soldBy:Yard). Imported the costed-not-on-Shopify rows as DRAFTs (no --activate).
 const INV_LOCATION = 'gid://shopify/Location/5795643504';  // 15442 Ventura Blvd. (inventory-set-2026.js)
@@ -122,6 +122,22 @@ function confirmedDiscountVendors() {
   const out = psql(`SELECT vendor_name FROM vendor_registry WHERE discount_confirmed IS TRUE;`);
   return new Set(out ? out.split('\n') : []);
 }
+// ---- dedup_skip support (cross-prefix dedup, rebel-walls model; Steve-approved Option A 2026-06-23) ----
+// A catalog row flagged dedup_skip=true is a duplicate of a pattern already live on Shopify under a
+// DIFFERENT dw_sku prefix (e.g. Tres Tintas DWXW newwall rows that duplicate live DWTB/DWTS patterns).
+// The cadence's create-time guard keys on dw_sku, so it can't see across prefixes — this column closes
+// that gap. Filter is applied ONLY to tables that actually have the column (most don't), so it's a no-op
+// for every vendor except those explicitly deduped. Detected once per table (cheap catalog query, cached).
+const _dedupColCache = {};
+function dedupSkipClause(table) {
+  const t = String(table).replace(/[^a-z0-9_]/gi, '');
+  if (!(t in _dedupColCache)) {
+    let has = false;
+    try { has = (psql(`SELECT 1 FROM information_schema.columns WHERE table_name='${t}' AND column_name='dedup_skip' LIMIT 1;`) || '').trim() === '1'; } catch { has = false; }
+    _dedupColCache[t] = has;
+  }
+  return _dedupColCache[t] ? ' AND dedup_skip IS NOT TRUE' : '';
+}
 
 // ---- shopify ----
 function gql(query, variables) {
@@ -167,7 +183,7 @@ function gate() {
     if (!confirmed.has(vendor)) { rows.push({ vendor, cfg, ready: false, why: 'discount-not-confirmed', netnew: 0 }); continue; }
     let netnew = 0, why = 'READY';
     try {
-      const c = psql(`SELECT count(*) FROM ${cfg.table} WHERE coalesce(shopify_product_id::text,'')='' AND (${cfg.costExpr})::numeric > 0;`);
+      const c = psql(`SELECT count(*) FROM ${cfg.table} WHERE coalesce(shopify_product_id::text,'')='' AND (${cfg.costExpr})::numeric > 0${dedupSkipClause(cfg.table)};`);
       netnew = parseInt(c || '0', 10);
       if (!netnew) why = 'no-net-new-costed-skus';
     } catch (e) { why = 'gate-query-error: ' + String(e.message).slice(0,60); }
@@ -176,6 +192,18 @@ function gate() {
   return rows;
 }
 
+// ---- skip vendors heavily represented in the recent last-500 uploads (Steve 2026-06-23:
+// "skip vendors we recently uploaded among the last 500; start showing more vendors").
+// Keeps New diverse — a vendor that just flooded the recent feed rests until it ages out of
+// the last-500 window. Threshold = >=80 of the last 500 (~16%+). Fail-open: if every READY
+// vendor is heavy (or the query errors), keep the full list so the slot never starves.
+function recentlyHeavyVendors() {
+  try {
+    const out = psql(`SELECT vendor FROM (SELECT vendor FROM shopify_products WHERE created_at_shopify IS NOT NULL ORDER BY created_at_shopify DESC LIMIT 500) t GROUP BY vendor HAVING count(*) >= 80;`);
+    return out ? new Set(out.split('\n').map(l => l.trim()).filter(Boolean)) : new Set();
+  } catch { return new Set(); }
+}
+
 // ---- rotation cursor so successive slots/days pick different vendors ----
 function loadCursor() { try { return JSON.parse(fs.readFileSync(CURSOR,'utf8')); } catch { return { idx: 0 }; } }
 function saveCursor(c) { fs.mkdirSync(DATADIR,{recursive:true}); fs.writeFileSync(CURSOR, JSON.stringify(c,null,1)); }
@@ -205,7 +233,7 @@ function selectSkus(cfg, limit) {
   const allImgSel = hasAllImg ? `, (${ALLIMG})::text AS all_images` : '';
   const sql = `SELECT ${cols}, (${cfg.costExpr})::numeric AS cost${sellSel}${allImgSel}
     FROM ${cfg.table}
-    WHERE coalesce(shopify_product_id::text,'')='' AND (${cfg.costExpr})::numeric > 0
+    WHERE coalesce(shopify_product_id::text,'')='' AND (${cfg.costExpr})::numeric > 0${dedupSkipClause(cfg.table)}
     ORDER BY dw_sku LIMIT ${limit};`;
   const out = psql(sql);
   if (!out) return [];
@@ -455,6 +483,15 @@ async function createProduct(table, row, payload) {
   if (ONLY) ready = ready.filter(r => r.vendor.toLowerCase() === ONLY.toLowerCase());
   if (!ready.length) { console.log(`\nNo READY vendors${ONLY?` matching --only ${ONLY}`:''} with net-new costed SKUs. Nothing to import.`); return; }
 
+  // skip-recent: drop vendors that already flooded the last-500 uploads (unless --only).
+  if (!ONLY) {
+    const heavy = recentlyHeavyVendors();
+    const dropped = ready.filter(r => heavy.has(r.vendor)).map(r => r.vendor);
+    const kept = ready.filter(r => !heavy.has(r.vendor));
+    if (dropped.length && kept.length) { console.log(`skip-recent: resting ${dropped.join(', ')} (heavy in last 500) → ${kept.length} vendor(s) left`); ready = kept; }
+    else if (dropped.length) console.log(`skip-recent: all READY vendors heavy in last 500 — keeping full list (no starvation)`);
+  }
+
   // rotate — ROUND-ROBIN one-new-vendor-per-slot (Steve 2026-06-23: "every 18/hr should be a
   // new vendor"). The cursor (data/cadence-cursor.json idx) names the lead vendor for THIS slot;
   // every slot starts on the NEXT vendor in the READY list and the cursor advances by exactly the
← a727c094 Fix spec-sheet Download PDF: render via hidden iframe (fetch · back to Designer Wallcoverings · auto-save: 2026-06-23T10:51:10 (5 files) — shopify/scripts/c b7e3a00a →