← back to Carnegie Reprice
Carnegie Phase 3 full-line rollout: resumable builder + factual desc fallback + storage guard
af22a7d8cd8f87c2f59ec024af87d74165a5dd71 · 2026-08-18 13:19:03 -0700 · steve
Files touched
A build-one-proof.mjsA proof-one.jsonA rebuild-ledger.jsonlA rebuild-line.mjs
Diff
commit af22a7d8cd8f87c2f59ec024af87d74165a5dd71
Author: steve <steve@designerwallcoverings.com>
Date: Tue Aug 18 13:19:03 2026 -0700
Carnegie Phase 3 full-line rollout: resumable builder + factual desc fallback + storage guard
---
build-one-proof.mjs | 61 +++++++++++++++
proof-one.json | 8 ++
rebuild-ledger.jsonl | 3 +
rebuild-line.mjs | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 280 insertions(+)
diff --git a/build-one-proof.mjs b/build-one-proof.mjs
new file mode 100644
index 0000000..c2033ea
--- /dev/null
+++ b/build-one-proof.mjs
@@ -0,0 +1,61 @@
+// build-one-proof — create ONE perfect Carnegie product off to the side (unlinked, deletable)
+// from the now-complete carnegie_catalog, to prove what the rebuild looks like.
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+const ENV = `${process.env.HOME}/Projects/secrets-manager/.env`;
+const env = k => { const m = fs.readFileSync(ENV,'utf8').split('\n').find(l=>l.startsWith(k+'=')); return m? m.slice(k.length+1).trim().replace(/^["']|["']$/g,''):''; };
+const TOKEN = env('SHOPIFY_ADMIN_TOKEN'); let SHOP = env('SHOPIFY_STORE_DOMAIN')||env('SHOPIFY_STORE'); if(SHOP&&!SHOP.includes('.'))SHOP+='.myshopify.com';
+const API = `https://${SHOP}/admin/api/2024-10`, H = { 'X-Shopify-Access-Token':TOKEN, 'Content-Type':'application/json' };
+const PSQL = ['/opt/homebrew/opt/postgresql@14/bin/psql','/usr/local/opt/postgresql@14/bin/psql','psql'].find(p=>{try{execFileSync(p,['--version'],{stdio:'ignore'});return 1}catch{return 0}})||'psql';
+const q1 = sql => execFileSync(PSQL,['postgresql:///dw_unified?host=/tmp','-At','-c',sql],{encoding:'utf8'}).trim();
+const sleep = ms => new Promise(r=>setTimeout(r,ms));
+async function shop(path,opts={}){ const r=await fetch(`${API}${path}`,{headers:H,...opts}); if(!r.ok) throw new Error(`HTTP ${r.status} ${path} :: ${(await r.text()).slice(0,300)}`); return r.json(); }
+
+const SKU='DWAG-379296';
+const row = JSON.parse(q1(`select row_to_json(t) from (select * from carnegie_catalog where dw_sku='${SKU}') t`));
+const specs = row.specs||{};
+const cost=+row.price, retail=Math.round(cost/0.65/0.85);
+const colorName = (row.color_tags&&row.color_tags[0])||row.color_bucket||('Color '+row.color_number);
+const big = u => u? u.replace(/height=\d*&width=\d*/,'height=1400&width=1400').replace(/height=&width=/,'height=1400&width=1400') : null;
+const imgs = (row.all_images||[]).filter(u=>/_puf|_pud|siltech/i.test(u)).slice(0,3).map(big);
+
+// --- create product (unlinked proof) ---
+const title = `Carnegie Siltech Grain — ${colorName}`;
+const payload = { product: {
+ title, body_html: row.description_text, vendor:'Carnegie', product_type:'Upholstery',
+ handle: 'carnegie-siltech-grain-proof-ivory', status:'active', published_scope:'web',
+ tags: [colorName, row.color_bucket, 'Carnegie','Fabric','Upholstery','Carnegie Textiles','PROOF-DELETE-ME'].filter(Boolean).join(', '),
+ options:[{name:'Format'}],
+ images: imgs.map((src,i)=>({src, position:i+1})),
+ variants:[
+ { option1:'Per Yard', price: retail.toFixed(2), sku: SKU, requires_shipping:true, taxable:true, inventory_management:null },
+ { option1:'Memo Sample', price: '4.25', sku: `${SKU}-Sample`, requires_shipping:true, taxable:true, inventory_management:null },
+ ],
+}};
+const { product } = await shop('/products.json',{method:'POST',body:JSON.stringify(payload)});
+console.log('created product id', product.id, 'handle', product.handle);
+
+// --- metafields: identity + FULL specs across the namespaces the theme may read ---
+const spec = {
+ Width: row.width, Content: row.content, Contents: row.content, Durability: row.durability_wyzenbeek,
+ 'Cleaning Code': row.cleaning_code, Cleaning: row.cleaning_code, Finish: row.finish, Backing: row.backing,
+ Flammability: row.flammability, Origin: row.origin, 'Country of Origin': row.origin,
+ Type: specs.Type, Warranty: specs.Warranty, Weight: specs['Weight per Linear Yard'],
+ Use:'Upholstery', Repeat: (row.repeat_v||row.repeat_h||'')||undefined,
+};
+const ident = { dw_sku:SKU, manufacturer_sku:row.mfr_sku, brand:'Carnegie', vendor:'Carnegie',
+ pattern_name:row.pattern_name, color:colorName, color_number:String(row.color_number),
+ product_class:'Fabric', collection_name:'Carnegie Textiles' };
+const mfs=[];
+for(const ns of ['global','custom','specifications','specs','dwc'])
+ for(const [k,v] of Object.entries(spec)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v)});
+for(const ns of ['global','custom','dwc'])
+ for(const [k,v] of Object.entries(ident)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v)});
+mfs.push({namespace:'custom',key:'color_hex',type:'single_line_text_field',value:row.primary_hex||''});
+
+let ok=0,err=0;
+for(const m of mfs){ try{ await shop(`/products/${product.id}/metafields.json`,{method:'POST',body:JSON.stringify({metafield:m})}); ok++; }catch(e){ err++; } await sleep(120); }
+console.log(`metafields: ${ok} set, ${err} err`);
+fs.writeFileSync(`${new URL('.',import.meta.url).pathname}proof-one.json`, JSON.stringify({product_id:product.id, handle:product.handle, sku:SKU, colorName, retail, url:`https://www.designerwallcoverings.com/products/${product.handle}`},null,2));
+console.log('URL: https://www.designerwallcoverings.com/products/'+product.handle);
+console.log('title:', title, '| color:', colorName, '| retail $'+retail, '| images:', imgs.length, '| specs on page:', Object.values(spec).filter(Boolean).length);
diff --git a/proof-one.json b/proof-one.json
new file mode 100644
index 0000000..c8a36d8
--- /dev/null
+++ b/proof-one.json
@@ -0,0 +1,8 @@
+{
+ "product_id": 7923100680243,
+ "handle": "carnegie-siltech-grain-proof-ivory",
+ "sku": "DWAG-379296",
+ "colorName": "Ivory",
+ "retail": 136,
+ "url": "https://www.designerwallcoverings.com/products/carnegie-siltech-grain-proof-ivory"
+}
\ No newline at end of file
diff --git a/rebuild-ledger.jsonl b/rebuild-ledger.jsonl
new file mode 100644
index 0000000..92fb854
--- /dev/null
+++ b/rebuild-ledger.jsonl
@@ -0,0 +1,3 @@
+{"dw_sku":"DWAG-381677","product_id":7923105497139,"handle":"carnegie-abbey-oatmeal-61","color":"Oatmeal","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":116,"mf_err":7,"at":"2026-08-18T20:16:45.230Z"}
+{"dw_sku":"DWAG-381678","product_id":7923105595443,"handle":"carnegie-abbey-blush-62","color":"Blush","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":111,"mf_err":12,"at":"2026-08-18T20:17:22.455Z"}
+{"dw_sku":"DWAG-381679","product_id":7923105660979,"handle":"carnegie-abbey-greige-63","color":"Greige","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":106,"mf_err":17,"at":"2026-08-18T20:17:59.708Z"}
diff --git a/rebuild-line.mjs b/rebuild-line.mjs
new file mode 100644
index 0000000..396a1fb
--- /dev/null
+++ b/rebuild-line.mjs
@@ -0,0 +1,208 @@
+// rebuild-line.mjs — Phase 3 FULL-LINE Carnegie rollout (TK-10671)
+// Generalizes build-one-proof.mjs (id 7923100680243) to the whole 5,928-SKU line.
+// RESUMABLE (ledger), STORAGE-GUARDED (halt on FILE_STORAGE_LIMIT_EXCEEDED), REVERSIBLE.
+//
+// Usage:
+// node rebuild-line.mjs # run the create batch (skips ledgered dw_skus)
+// node rebuild-line.mjs --limit 20 # cap creates (smoke test)
+// node rebuild-line.mjs --status # print progress from the ledger and exit
+//
+// Archiving old products + folding the proof is a SEPARATE step: archive-old.mjs (end, after verify).
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const DIR = new URL('.', import.meta.url).pathname;
+const ENV = `${process.env.HOME}/Projects/secrets-manager/.env`;
+const env = k => { const m = fs.readFileSync(ENV,'utf8').split('\n').find(l=>l.startsWith(k+'=')); return m? m.slice(k.length+1).trim().replace(/^["']|["']$/g,''):''; };
+const TOKEN = env('SHOPIFY_ADMIN_TOKEN');
+let SHOP = env('SHOPIFY_STORE_DOMAIN')||env('SHOPIFY_STORE'); if(SHOP&&!SHOP.includes('.'))SHOP+='.myshopify.com';
+const API = `https://${SHOP}/admin/api/2024-10`;
+const H = { 'X-Shopify-Access-Token':TOKEN, 'Content-Type':'application/json' };
+const PSQL = ['/opt/homebrew/opt/postgresql@14/bin/psql','/usr/local/opt/postgresql@14/bin/psql','psql'].find(p=>{try{execFileSync(p,['--version'],{stdio:'ignore'});return 1}catch{return 0}})||'psql';
+const DB = 'postgresql:///dw_unified?host=/tmp';
+const q1 = sql => execFileSync(PSQL,[DB,'-At','-c',sql],{encoding:'utf8',maxBuffer:1<<30}).trim();
+const sleep = ms => new Promise(r=>setTimeout(r,ms));
+
+const LEDGER = path.join(DIR,'rebuild-ledger.jsonl');
+const HALT = path.join(DIR,'rebuild-HALTED.flag');
+const args = process.argv.slice(2);
+const argVal = f => { const i=args.indexOf(f); return i>=0? args[i+1]:null; };
+const LIMIT = argVal('--limit') ? +argVal('--limit') : Infinity;
+
+// ---- ledger helpers ----
+function ledgerMap(){ const m=new Map(); if(fs.existsSync(LEDGER)){ for(const line of fs.readFileSync(LEDGER,'utf8').split('\n')){ if(!line.trim())continue; try{const r=JSON.parse(line); if(r.dw_sku) m.set(r.dw_sku, r);}catch{} } } return m; }
+function ledgerAppend(rec){ fs.appendFileSync(LEDGER, JSON.stringify(rec)+'\n'); }
+
+if(args.includes('--status')){
+ const m=ledgerMap();
+ const ok=[...m.values()].filter(r=>r.product_id).length;
+ const errs=[...m.values()].filter(r=>r.error).length;
+ const total=+q1(`select count(*) from carnegie_catalog`);
+ console.log(`ledger: ${ok} created, ${errs} errored, ${total-ok} remaining of ${total}`);
+ console.log(`halted: ${fs.existsSync(HALT)?'YES — '+fs.readFileSync(HALT,'utf8'):'no'}`);
+ process.exit(0);
+}
+if(fs.existsSync(HALT)){ console.error('REFUSING TO RUN — halt flag present:', fs.readFileSync(HALT,'utf8')); process.exit(2); }
+
+async function shop(p,opts={}){ const r=await fetch(`${API}${p}`,{headers:H,...opts}); const t=await r.text(); if(!r.ok) throw new Error(`HTTP ${r.status} ${p} :: ${t.slice(0,300)}`); return t?JSON.parse(t):{}; }
+
+// ---- image selection: robust, priority-ordered, NEVER zero ----
+const bump = u => u ? u.replace(/height=\d*&width=\d*/,'height=1400&width=1400').replace(/height=&width=/,'height=1400&width=1400') : null;
+// priority: best fabric shots first, swatch last (still better than nothing)
+const IMG_PRIORITY = [/_puf\./i,/_pud\./i,/_pdp\./i,/_pd[a-z]\./i,/_detail\./i,/_repeat\./i,/_iii\./i,/_web\./i,/_out\./i,/lifestyle/i,/_uhr\./i,/_neu\./i,/_swatch/i];
+function selectImages(all){
+ const clean = (all||[]).filter(Boolean);
+ if(!clean.length) return [];
+ const scored = clean.map(u=>{ let s=IMG_PRIORITY.findIndex(re=>re.test(u)); if(s<0)s=IMG_PRIORITY.length; return {u,s}; });
+ scored.sort((a,b)=>a.s-b.s);
+ // take up to 4 best, de-duped, bumped
+ const seen=new Set(); const out=[];
+ for(const {u} of scored){ const b=bump(u); if(!seen.has(b)){ seen.add(b); out.push(b); if(out.length>=4)break; } }
+ return out;
+}
+
+// ---- clean spec keys only (allowlist — avoids the scraped "Product Specifications..." junk keys) ----
+const SPEC_ALLOW = ['Type','Width','Content','Contents','Backing','Repeat','Warranty','Weight per Linear Yard',
+ 'Cleaning Code','Finish/es (as stocked)','Flammability','Manufactured In','Durability','Acoustics',
+ 'Lightfastness','Hydrolysis','Performance','Uses','ACT Symbols','Available Backing(s)','Available Finish(es)',
+ 'Standards and Certifications','Free of','Additional Details','IMO Certification Type'];
+
+function buildPayload(row){
+ const specs = row.specs || {};
+ const cost = +row.price;
+ if(!(cost>0)) throw new Error(`no price for ${row.dw_sku}`);
+ const retail = Math.round(cost/0.65/0.85);
+ const colorName = (row.color_tags && row.color_tags[0]) || row.color_name || row.color_bucket || (row.mfr_sku) || ('Color '+row.color_number);
+ const ptype = row.product_type || 'Fabric';
+ const imgs = selectImages(row.all_images);
+ const title = `Carnegie ${row.pattern_name} — ${colorName}`;
+ // description: use scraped text; else compose a FACTUAL description from real catalog data (no fabricated claims)
+ let body = (row.description_text||'').trim() || (row.body_html||'').trim();
+ if(!body){
+ const bits = [];
+ bits.push(`<p>${row.pattern_name} in ${colorName} from Carnegie${row.mfr_sku?` (${row.mfr_sku})`:''}.`);
+ const facts=[];
+ if(row.content) facts.push(`${row.content} content`);
+ if(row.width) facts.push(`${row.width} wide`);
+ if(row.durability_wyzenbeek) facts.push(`${row.durability_wyzenbeek} durability`);
+ if(specs.Type) facts.push(String(specs.Type).toLowerCase());
+ if(facts.length) bits.push(` A ${ptype.toLowerCase()} textile featuring ${facts.join(', ')}.`);
+ else bits.push(` A ${ptype.toLowerCase()} textile.`);
+ bits.push(`</p>`);
+ body = bits.join('');
+ }
+ // handle: unique via color_number so no collisions across a pattern's colorways
+ const slug = s => String(s||'').toLowerCase().replace(/[^a-z0-9]+/g,'-').replace(/^-|-$/g,'').slice(0,60);
+ const handle = `carnegie-${slug(row.pattern_name)}-${slug(colorName)}-${slug(row.color_number||row.dw_sku)}`.replace(/-+/g,'-').slice(0,120);
+
+ const tags = [
+ ...(row.color_tags||[]), row.color_bucket, ...(row.style_tags||[]),
+ 'Carnegie','carnegie-textile','Fabric', ptype, 'Carnegie Textiles',
+ 'display_variant',
+ ].filter(Boolean);
+ const tagStr = [...new Set(tags.map(t=>String(t).trim()).filter(Boolean))].join(', ');
+
+ const payload = { product: {
+ title, body_html: body, vendor:'Carnegie', product_type: ptype,
+ handle, status: imgs.length ? 'active' : 'draft', published_scope:'web', tags: tagStr,
+ options:[{name:'Format'}],
+ images: imgs.map((src,i)=>({src, position:i+1})),
+ variants:[
+ { option1:'Per Yard', price: retail.toFixed(2), sku: row.dw_sku, requires_shipping:true, taxable:true, inventory_management:null },
+ { option1:'Memo Sample', price: '4.25', sku: `${row.dw_sku}-Sample`, requires_shipping:true, taxable:true, inventory_management:null },
+ ],
+ }};
+
+ // spec metafields — dedicated columns first, then clean allowlisted jsonb keys
+ const spec = {
+ Width: row.width, Content: row.content, Contents: row.content, Durability: row.durability_wyzenbeek,
+ 'Cleaning Code': row.cleaning_code, Finish: row.finish, Backing: row.backing,
+ Flammability: row.flammability, Origin: row.origin, 'Country of Origin': row.origin,
+ Repeat: (row.repeat_v||row.repeat_h||'')||undefined,
+ Type: specs.Type, Warranty: specs.Warranty, Weight: specs['Weight per Linear Yard'], Use: ptype,
+ };
+ for(const k of SPEC_ALLOW){ const v=specs[k]; if(v && !spec[k] && String(v).length<600) spec[k]=v; }
+ const ident = { dw_sku:row.dw_sku, manufacturer_sku:row.mfr_sku, brand:'Carnegie', vendor:'Carnegie',
+ pattern_name:row.pattern_name, color:colorName, color_number:String(row.color_number||''),
+ product_class:'Fabric', collection_name:'Carnegie Textiles' };
+
+ return { payload, spec, ident, colorName, retail, handle, imgCount: imgs.length, primary_hex: row.primary_hex };
+}
+
+async function setMetafields(pid, {spec, ident, primary_hex}){
+ const mfs=[];
+ for(const ns of ['global','custom','specifications','specs','dwc'])
+ for(const [k,v] of Object.entries(spec)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v).slice(0,900)});
+ for(const ns of ['global','custom','dwc'])
+ for(const [k,v] of Object.entries(ident)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v)});
+ if(primary_hex) mfs.push({namespace:'custom',key:'color_hex',type:'single_line_text_field',value:primary_hex});
+ let ok=0,err=0;
+ for(const m of mfs){ try{ await shop(`/products/${pid}/metafields.json`,{method:'POST',body:JSON.stringify({metafield:m})}); ok++; }catch{ err++; } await sleep(60); }
+ return {ok,err};
+}
+
+// ---- storage guard: confirm a freshly-created product actually has images attached ----
+async function verifyImages(pid){
+ try{ const {product}=await shop(`/products/${pid}.json?fields=id,images`); return (product.images||[]).length; }catch{ return -1; }
+}
+
+async function main(){
+ const done = ledgerMap();
+ const skus = q1(`select dw_sku from carnegie_catalog order by pattern_name, color_number`).split('\n').filter(Boolean);
+ const todo = skus.filter(s=>!(done.get(s)&&done.get(s).product_id));
+ console.log(`catalog ${skus.length} | already created ${[...done.values()].filter(r=>r.product_id).length} | to create ${todo.length} (limit ${LIMIT})`);
+
+ let made=0, sinceCheck=0, imgFailStreak=0;
+ for(const sku of todo){
+ if(made>=LIMIT) break;
+ let row;
+ try{ row = JSON.parse(q1(`select row_to_json(t) from (select * from carnegie_catalog where dw_sku='${sku.replace(/'/g,"''")}') t`)); }
+ catch(e){ ledgerAppend({dw_sku:sku, error:'row-fetch: '+e.message, at:new Date().toISOString()}); continue; }
+ let built;
+ try{ built = buildPayload(row); }
+ catch(e){ ledgerAppend({dw_sku:sku, error:'build: '+e.message, at:new Date().toISOString()}); continue; }
+
+ let product;
+ try{ ({product} = await shop('/products.json',{method:'POST',body:JSON.stringify(built.payload)})); }
+ catch(e){
+ // storage-cap detection on create
+ if(/FILE_STORAGE_LIMIT_EXCEEDED|storage limit/i.test(e.message)){ haltStorage(sku, e.message); return; }
+ ledgerAppend({dw_sku:sku, error:'create: '+e.message.slice(0,200), at:new Date().toISOString()}); continue;
+ }
+
+ const mf = await setMetafields(product.id, built);
+ // storage guard: if we EXPECTED images but the product has none, that's the cap symptom
+ let attached = product.images ? product.images.length : 0;
+ if(built.imgCount>0 && attached===0){ attached = await verifyImages(product.id); }
+ const imageStarved = built.imgCount>0 && attached<=0;
+ imgFailStreak = imageStarved ? imgFailStreak+1 : 0;
+
+ ledgerAppend({ dw_sku:sku, product_id:product.id, handle:built.handle, color:built.colorName,
+ retail:built.retail, imgExpected:built.imgCount, imgAttached:attached, status:product.status,
+ mf_ok:mf.ok, mf_err:mf.err, at:new Date().toISOString() });
+ made++; sinceCheck++;
+
+ if(imgFailStreak>=3){ haltStorage(sku, `3 consecutive image-starved creates (expected imgs, got 0) — Shopify storage cap suspected`); return; }
+ if(made%100===0) console.log(` ... ${made} created (last: ${built.handle} $${built.retail} imgs ${attached}/${built.imgCount})`);
+
+ // periodic explicit storage check every ~50
+ if(sinceCheck>=50){
+ sinceCheck=0;
+ const cnt = await verifyImages(product.id);
+ console.log(` [guard @${made}] latest product images=${cnt} status=${product.status}`);
+ }
+ await sleep(250); // gentle pace, well under any burst limit
+ }
+ console.log(`DONE this run: ${made} created. Ledger: ${LEDGER}`);
+}
+
+function haltStorage(sku, msg){
+ const text = `HALTED at ${new Date().toISOString()} on ${sku}: ${msg}`;
+ fs.writeFileSync(HALT, text+'\n');
+ console.error('!!! STORAGE HALT !!! '+text);
+ try{ execFileSync('bash',['-lc',`export TK_AGENT=vp-dw-commerce; tk log TK-10671 ${JSON.stringify('STORAGE HALT: '+text)} 2>/dev/null || true`]); }catch{}
+ try{ execFileSync('bash',['-lc',`curl -s -X POST http://127.0.0.1:3333/api/wins -H 'Content-Type: application/json' -d ${JSON.stringify(JSON.stringify({project:'carnegie-reprice',title:'Carnegie rollout HALTED: Shopify storage cap',summary:text}))} >/dev/null 2>&1 || true`]); }catch{}
+}
+
+main().catch(e=>{ console.error('FATAL', e); process.exit(1); });
← 4a8631c Carnegie Phase 2 PoC: Acapella de-fragment tool (14 per-colo
·
back to Carnegie Reprice
·
handle: append dw_sku tail for guaranteed uniqueness (2424 c c05ae91 →