← back to Dw Fleet Refresh
dw-fleet-weekly-refresh engine: snapshot-driven per-site refresh+prune+topup with safety gates
e2a9503dce30999de1b6c3c0252c9c76d9d80fa6 · 2026-08-12 10:39:02 -0700 · Steve Abrams
Files touched
A .gitignoreA fleet-weekly-refresh.js
Diff
commit e2a9503dce30999de1b6c3c0252c9c76d9d80fa6
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 12 10:39:02 2026 -0700
dw-fleet-weekly-refresh engine: snapshot-driven per-site refresh+prune+topup with safety gates
---
.gitignore | 2 +
fleet-weekly-refresh.js | 136 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 138 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..790c219
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+node_modules/
+.env*
diff --git a/fleet-weekly-refresh.js b/fleet-weekly-refresh.js
new file mode 100644
index 0000000..9dfb300
--- /dev/null
+++ b/fleet-weekly-refresh.js
@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * dw-fleet-weekly-refresh — keep every DW sister site's catalog live + current.
+ * Per weekly run: build ONE authoritative Shopify Admin snapshot (all active
+ * products -> handle->record), then for each /var/www/<domain>/data/products.json:
+ * REFRESH existing (live+image -> refresh URL; archived/gone -> drop)
+ * TOP-UP with new niche-matching active products (site's own NICHE_POS/NEG)
+ * GATE (sample images 200; anti-collapse floor) then BACKUP+swap+pm2 reload.
+ * Flags: --dry (no writes), --all, --site <slug>, --limit-sites N, --no-topup.
+ * Env: SHOPIFY_ADMIN_TOKEN (required for snapshot).
+ */
+const fs = require('fs'), path = require('path'), cp = require('child_process');
+const DRY = process.argv.includes('--dry');
+const NOTOPUP = process.argv.includes('--no-topup');
+const ONE = (i=>i>=0?process.argv[i+1]:null)(process.argv.indexOf('--site'));
+const LIMIT = (i=>i>=0?parseInt(process.argv[i+1]):null)(process.argv.indexOf('--limit-sites'));
+const ROOT = '/var/www';
+const STORE = 'designer-laboratory-sandbox.myshopify.com';
+const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN;
+const API = `https://${STORE}/admin/api/2024-10`;
+const STYLE_TAGS = ['traditional','transitional','modern','contemporary','minimalist','art deco','victorian','mid-century','mid century','rustic','industrial','farmhouse','bohemian','boho','geometric','floral','botanical','damask','toile','grasscloth','silk','linen','vinyl','natural'];
+const sleep = ms => new Promise(r=>setTimeout(r,ms));
+const styleOf = t => { t=t.map(x=>String(x).toLowerCase()); for(const s of STYLE_TAGS) if(t.some(x=>x.includes(s))) return s; return ''; };
+
+async function buildSnapshot(){
+ if(!TOKEN) throw new Error('SHOPIFY_ADMIN_TOKEN missing');
+ const map = new Map(); let url = `${API}/products.json?limit=250&status=active&fields=handle,status,published_at,images,tags,variants,title,vendor,product_type`;
+ let pages=0;
+ while(url){
+ const r = await fetch(url,{headers:{'X-Shopify-Access-Token':TOKEN}});
+ if(r.status===429){ await sleep((parseFloat(r.headers.get('retry-after')||'2'))*1000+300); continue; }
+ if(!r.ok) throw new Error('admin snapshot http '+r.status);
+ const d = await r.json();
+ for(const p of (d.products||[])){
+ if(!p.published_at) continue; // must be live on Online Store
+ const img = p.images && p.images[0] && p.images[0].src; if(!img) continue;
+ const prices=(p.variants||[]).map(v=>parseFloat(v.price)).filter(n=>!isNaN(n));
+ map.set(p.handle, { img, tags:String(p.tags||'').split(',').map(s=>s.trim()).filter(Boolean),
+ maxp: prices.length?Math.max(...prices):0, title:p.title||'', vendor:p.vendor||'', ptype:p.product_type||'Wallcovering' });
+ }
+ pages++;
+ const link=r.headers.get('link')||''; const m=link.match(/<([^>]+)>;\s*rel="next"/); url=m?m[1]:null;
+ await sleep(120);
+ if(pages%50===0) console.error(` snapshot… ${pages} pages, ${map.size} live products`);
+ }
+ console.error(`snapshot complete: ${map.size} live products, ${pages} pages`);
+ return map;
+}
+function parseNiche(dir){
+ try{ const s=fs.readFileSync(path.join(dir,'server.js'),'utf8');
+ const grab=n=>{ const m=s.match(new RegExp(n+'\\s*=\\s*\\[([^\\]]*)\\]')); return m?m[1].split(',').map(x=>x.replace(/['"\s]/g,'').toLowerCase()).filter(Boolean):null; };
+ const pos=grab('NICHE_POS'), neg=grab('NICHE_NEG'); if(pos) return {pos, neg:neg||[]};
+ }catch(e){}
+ return null;
+}
+function nicheFit(niche,title,tags){ const blob=(title+' '+tags.join(' ')).toLowerCase();
+ if(niche.neg.some(n=>blob.includes(n))) return false; return niche.pos.length===0||niche.pos.some(k=>blob.includes(k)); }
+
+function pmMap(){ // proc name -> cwd
+ try{ const j=JSON.parse(cp.execSync('pm2 jlist 2>/dev/null',{maxBuffer:1e8}).toString());
+ const m={}; for(const p of j){ const cwd=(p.pm2_env&&(p.pm2_env.pm_cwd))||''; m[cwd]=p.name; } return m;
+ }catch(e){ return {}; }
+}
+
+function refreshSite(dir, snap, PM){
+ const dataFile = path.join(dir,'data','products.json');
+ if(!fs.existsSync(dataFile)) return { skip:'no products.json' };
+ let raw; try{ raw=JSON.parse(fs.readFileSync(dataFile,'utf8')); }catch(e){ return {skip:'bad json'}; }
+ const arr = Array.isArray(raw)?raw:(raw.products||raw.items||[]);
+ const prior = arr.length;
+ const have = new Set();
+ const kept = [];
+ let refreshed=0, dropped=0;
+ for(const p of arr){
+ const h=p.handle||p.sku; if(!h) { dropped++; continue; }
+ const rec = snap.get(h);
+ if(!rec){ dropped++; continue; } // archived/gone
+ have.add(h);
+ const img = rec.img;
+ if(img!==p.image_url) refreshed++;
+ kept.push({ ...p, image_url: img, max_price: (p.max_price||rec.maxp||0) });
+ }
+ // top-up
+ let added=0; const niche = NOTOPUP?null:parseNiche(dir);
+ if(niche){
+ const target = Math.max(prior, kept.length); // restore toward prior size
+ for(const [h,rec] of snap){
+ if(kept.length>=target) break;
+ if(have.has(h)) continue;
+ if(!nicheFit(niche, rec.title, rec.tags)) continue;
+ kept.push({ sku:h, handle:h, title:rec.title, vendor:rec.vendor, product_type:rec.ptype,
+ image_url:rec.img, tags:rec.tags, max_price:rec.maxp, aesthetic:styleOf(rec.tags),
+ product_url:`https://designerwallcoverings.com/products/${h}` });
+ have.add(h); added++;
+ }
+ }
+ // GATES
+ const finalCount = kept.length;
+ const gates = [];
+ if(finalCount===0) gates.push('EMPTY');
+ if(prior>=50 && finalCount < prior*0.4) gates.push(`COLLAPSE(${prior}->${finalCount})`);
+ const pass = gates.length===0;
+ return { prior, refreshed, dropped, added, finalCount, nicheKnown:!!niche, pass, gates, kept, dataFile, dir };
+}
+
+(async () => {
+ const snap = await buildSnapshot();
+ let dirs = fs.readdirSync(ROOT).filter(d=>{ try{ return fs.existsSync(path.join(ROOT,d,'data','products.json')); }catch(e){return false;} });
+ dirs = dirs.filter(d=>!/\.bak/i.test(d));
+ if(ONE) dirs = dirs.filter(d=>d.includes(ONE));
+ if(LIMIT) dirs = dirs.slice(0,LIMIT);
+ const PM = pmMap();
+ const ledger=[];
+ for(const d of dirs){
+ const dir=path.join(ROOT,d);
+ const res = refreshSite(dir, snap, PM);
+ if(res.skip){ ledger.push({site:d, skip:res.skip}); continue; }
+ // sample-verify a few final images are 200
+ let deadSample=0; const s=[]; const step=Math.max(1,Math.floor(res.kept.length/12));
+ for(let i=0;i<res.kept.length&&s.length<12;i+=step) s.push(res.kept[i].image_url);
+ for(const u of s){ try{ const r=await fetch(u); if(r.status===404) deadSample++; }catch(e){} }
+ const verifyOk = deadSample<=1;
+ let deployed=false, procName=PM[dir]||d.replace(/\.com$/,'');
+ if(!DRY && res.pass && verifyOk){
+ const ts=Date.now();
+ fs.copyFileSync(res.dataFile, res.dataFile+'.bak.weekly.'+ts);
+ fs.writeFileSync(res.dataFile, JSON.stringify(res.kept));
+ try{ cp.execSync(`pm2 reload ${procName} --update-env`,{stdio:'ignore'}); deployed=true; }
+ catch(e){ try{ cp.execSync(`pm2 restart ${procName}`,{stdio:'ignore'}); deployed=true; }catch(e2){} }
+ }
+ ledger.push({ site:d, prior:res.prior, refreshed:res.refreshed, dropped:res.dropped, added:res.added,
+ final:res.finalCount, nicheKnown:res.nicheKnown, deadSample, gate:res.pass&&verifyOk?'PASS':(res.gates.concat(verifyOk?[]:['IMG_VERIFY']).join('+')), deployed, proc:procName });
+ console.error(`[${d}] prior=${res.prior} -> final=${res.finalCount} (refresh ${res.refreshed}, drop ${res.dropped}, add ${res.added}) gate=${ledger[ledger.length-1].gate} deployed=${deployed}`);
+ }
+ console.log(JSON.stringify({ ts:new Date().toISOString(), dry:DRY, sites:ledger.length, snapshot:snap.size, ledger }, null, 2));
+})();
(oldest)
·
back to Dw Fleet Refresh
·
chore: add weekly driver + VERSION v1.0.0 (session close) df268b0 →