← back to Carnegie Reprice
rebuild-line.mjs
268 lines
// rebuild-line.mjs — Phase 3 FULL-LINE Carnegie rollout (TK-10671)
// Generalizes build-one-proof.mjs (id 7923100680243) to the whole 5,928-SKU line.
// RESUMABLE (ledger), STORAGE-GUARDED (halt on FILE_STORAGE_LIMIT_EXCEEDED), REVERSIBLE.
//
// Usage:
// node rebuild-line.mjs # run the create batch (skips ledgered dw_skus)
// node rebuild-line.mjs --limit 20 # cap creates (smoke test)
// node rebuild-line.mjs --status # print progress from the ledger and exit
//
// Archiving old products + folding the proof is a SEPARATE step: archive-old.mjs (end, after verify).
//
// TK-10792 stop-the-bleed: mfr_sku gate — products without a real vendor mfr_sku
// (null, empty, or DWAG-* internal placeholder) are created as DRAFT with the
// Needs-Mfr-SKU tag. They are NEVER set to ACTIVE until Steve resolves the real code.
import { execFileSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
// TK-10792: canonical mfr_sku gate — one source of truth, no inline copy
import { mfrSkuValid, bodyHtmlValid, addSkipTag, SKIP_TAG as NEEDS_MFR_TAG, NEEDS_DESC_TAG } from './carnegie-mfr-gate.mjs';
const DIR = new URL('.', import.meta.url).pathname;
const ENV = `${process.env.HOME}/Projects/secrets-manager/.env`;
const env = k => { const m = fs.readFileSync(ENV,'utf8').split('\n').find(l=>l.startsWith(k+'=')); return m? m.slice(k.length+1).trim().replace(/^["']|["']$/g,''):''; };
const TOKEN = env('SHOPIFY_ADMIN_TOKEN');
let SHOP = env('SHOPIFY_STORE_DOMAIN')||env('SHOPIFY_STORE'); if(SHOP&&!SHOP.includes('.'))SHOP+='.myshopify.com';
const API = `https://${SHOP}/admin/api/2024-10`;
const H = { 'X-Shopify-Access-Token':TOKEN, 'Content-Type':'application/json' };
const PSQL = ['/opt/homebrew/opt/postgresql@14/bin/psql','/usr/local/opt/postgresql@14/bin/psql','psql'].find(p=>{try{execFileSync(p,['--version'],{stdio:'ignore'});return 1}catch{return 0}})||'psql';
const DB = 'postgresql:///dw_unified?host=/tmp';
const q1 = sql => execFileSync(PSQL,[DB,'-At','-c',sql],{encoding:'utf8',maxBuffer:1<<30}).trim();
const sleep = ms => new Promise(r=>setTimeout(r,ms));
const LEDGER = path.join(DIR,'rebuild-ledger.jsonl');
const HALT = path.join(DIR,'rebuild-HALTED.flag');
const args = process.argv.slice(2);
const argVal = f => { const i=args.indexOf(f); return i>=0? args[i+1]:null; };
const LIMIT = argVal('--limit') ? +argVal('--limit') : Infinity;
// --shard N/M : this worker handles SKUs where (index % M === N-1). Disjoint slices, ledger-safe.
let SHARD_N=0, SHARD_M=1;
if(argVal('--shard')){ const [n,m]=argVal('--shard').split('/').map(Number); SHARD_N=n-1; SHARD_M=m; }
// ---- ledger helpers ----
function ledgerMap(){ const m=new Map(); if(fs.existsSync(LEDGER)){ for(const line of fs.readFileSync(LEDGER,'utf8').split('\n')){ if(!line.trim())continue; try{const r=JSON.parse(line); if(r.dw_sku) m.set(r.dw_sku, r);}catch{} } } return m; }
function ledgerAppend(rec){ fs.appendFileSync(LEDGER, JSON.stringify(rec)+'\n'); }
if(args.includes('--status')){
const m=ledgerMap();
const ok=[...m.values()].filter(r=>r.product_id).length;
const errs=[...m.values()].filter(r=>r.error).length;
const total=+q1(`select count(*) from carnegie_catalog`);
console.log(`ledger: ${ok} created, ${errs} errored, ${total-ok} remaining of ${total}`);
console.log(`halted: ${fs.existsSync(HALT)?'YES — '+fs.readFileSync(HALT,'utf8'):'no'}`);
process.exit(0);
}
if(fs.existsSync(HALT)){ console.error('REFUSING TO RUN — halt flag present:', fs.readFileSync(HALT,'utf8')); process.exit(2); }
async function shop(p,opts={}){
for(let attempt=0;attempt<6;attempt++){
const r=await fetch(`${API}${p}`,{headers:H,...opts});
if(r.status===429){ const ra=+(r.headers.get('retry-after')||2); await sleep(Math.max(ra*1000, 1000*(attempt+1))); continue; }
const t=await r.text();
if(!r.ok) throw new Error(`HTTP ${r.status} ${p} :: ${t.slice(0,300)}`);
return t?JSON.parse(t):{};
}
throw new Error(`HTTP 429 exhausted ${p}`);
}
// ---- image selection: robust, priority-ordered, NEVER zero ----
const bump = u => u ? u.replace(/height=\d*&width=\d*/,'height=1400&width=1400').replace(/height=&width=/,'height=1400&width=1400') : null;
// priority: best fabric shots first, swatch last (still better than nothing)
const IMG_PRIORITY = [/_puf\./i,/_pud\./i,/_pdp\./i,/_pd[a-z]\./i,/_detail\./i,/_repeat\./i,/_iii\./i,/_web\./i,/_out\./i,/lifestyle/i,/_uhr\./i,/_neu\./i,/_swatch/i];
function selectImages(all){
const clean = (all||[]).filter(Boolean);
if(!clean.length) return [];
const scored = clean.map(u=>{ let s=IMG_PRIORITY.findIndex(re=>re.test(u)); if(s<0)s=IMG_PRIORITY.length; return {u,s}; });
scored.sort((a,b)=>a.s-b.s);
// take up to 4 best, de-duped, bumped
const seen=new Set(); const out=[];
for(const {u} of scored){ const b=bump(u); if(!seen.has(b)){ seen.add(b); out.push(b); if(out.length>=4)break; } }
return out;
}
// ---- clean spec keys only (allowlist — avoids the scraped "Product Specifications..." junk keys) ----
const SPEC_ALLOW = ['Type','Width','Content','Contents','Backing','Repeat','Warranty','Weight per Linear Yard',
'Cleaning Code','Finish/es (as stocked)','Flammability','Manufactured In','Durability','Acoustics',
'Lightfastness','Hydrolysis','Performance','Uses','ACT Symbols','Available Backing(s)','Available Finish(es)',
'Standards and Certifications','Free of','Additional Details','IMO Certification Type'];
function buildPayload(row){
const specs = row.specs || {};
const cost = +row.price;
if(!(cost>0)) throw new Error(`no price for ${row.dw_sku}`);
// 2 decimals (TK-11434, 2026-09-11): was Math.round(), i.e. whole dollars. The
// reprice fixer writes round(cost/0.65/0.85, 2), so whole-dollar rounding here meant the
// same cost produced a different price depending on whether a product was CREATED or
// REPRICED ($71 vs $70.59 on a $39 cost). All four Carnegie create paths now agree.
// Keep retail a NUMBER — line 151 and the log at :239 both call .toFixed(2) on it.
const retail = Math.round(cost/0.65/0.85*100)/100;
const colorName = (row.color_tags && row.color_tags[0]) || row.color_name || row.color_bucket || (row.mfr_sku) || ('Color '+row.color_number);
const ptype = row.product_type || 'Fabric';
const imgs = selectImages(row.all_images);
const title = `Carnegie ${row.pattern_name} — ${colorName}`;
// description: use scraped text; else compose a FACTUAL description from real catalog data (no fabricated claims)
let body = (row.description_text||'').trim() || (row.body_html||'').trim();
if(!body){
const bits = [];
bits.push(`<p>${row.pattern_name} in ${colorName} from Carnegie${row.mfr_sku?` (${row.mfr_sku})`:''}.`);
const facts=[];
if(row.content) facts.push(`${row.content} content`);
if(row.width) facts.push(`${row.width} wide`);
if(row.durability_wyzenbeek) facts.push(`${row.durability_wyzenbeek} durability`);
if(specs.Type) facts.push(String(specs.Type).toLowerCase());
if(facts.length) bits.push(` A ${ptype.toLowerCase()} textile featuring ${facts.join(', ')}.`);
else bits.push(` A ${ptype.toLowerCase()} textile.`);
bits.push(`</p>`);
body = bits.join('');
}
// handle: dw_sku tail is the GUARANTEED-unique discriminator (2,424 pattern+color+color_number
// collisions exist = distinct sellable SKUs, e.g. different backings of one colorway)
const slug = s => String(s||'').toLowerCase().replace(/[^a-z0-9]+/g,'-').replace(/^-|-$/g,'').slice(0,50);
const handle = `carnegie-${slug(row.pattern_name)}-${slug(colorName)}-${slug(row.dw_sku)}`.replace(/-+/g,'-').slice(0,120);
const tags = [
...(row.color_tags||[]), row.color_bucket, ...(row.style_tags||[]),
'Carnegie','carnegie-textile','Fabric', ptype, 'Carnegie Textiles',
'display_variant',
].filter(Boolean);
let tagStr = [...new Set(tags.map(t=>String(t).trim()).filter(Boolean))].join(', ');
// TK-10792/TK-10820: only allow ACTIVE if images exist AND mfr_sku is a real vendor code
// AND the final body_html is a real, non-blank description (a whitespace/entity shell like
// <p> </p> is truthy at L100 so the fallback composer never fires — this catches it).
const hasMfr = mfrSkuValid(row.mfr_sku);
const bodyValid = bodyHtmlValid(body);
const finalStatus = (imgs.length && hasMfr && bodyValid) ? 'active' : 'draft';
if (!hasMfr) {
tagStr = addSkipTag(tagStr);
console.warn(` [MFR-GATE] SKIP-ACTIVE dw_sku=${row.dw_sku} mfr_sku=${JSON.stringify(row.mfr_sku)} — keeping DRAFT+${NEEDS_MFR_TAG}`);
}
if (!bodyValid) {
if (!tagStr.split(',').map(t=>t.trim()).includes(NEEDS_DESC_TAG)) tagStr = [tagStr, NEEDS_DESC_TAG].filter(Boolean).join(', ');
console.warn(` [DESC-GATE] SKIP-ACTIVE dw_sku=${row.dw_sku} — blank body_html — keeping DRAFT+${NEEDS_DESC_TAG}`);
}
const payload = { product: {
title, body_html: body, vendor:'Carnegie', product_type: ptype,
handle, status: finalStatus, published_scope:'web', tags: tagStr,
options:[{name:'Format'}],
images: imgs.map((src,i)=>({src, position:i+1})),
variants:[
{ option1:'Per Yard', price: retail.toFixed(2), sku: row.dw_sku, requires_shipping:true, taxable:true, inventory_management:null },
{ option1:'Memo Sample', price: '4.25', sku: `${row.dw_sku}-Sample`, requires_shipping:true, taxable:true, inventory_management:null },
],
}};
// spec metafields — dedicated columns first, then clean allowlisted jsonb keys
const spec = {
Width: row.width, Content: row.content, Contents: row.content, Durability: row.durability_wyzenbeek,
'Cleaning Code': row.cleaning_code, Finish: row.finish, Backing: row.backing,
Flammability: row.flammability, Origin: row.origin, 'Country of Origin': row.origin,
Repeat: (row.repeat_v||row.repeat_h||'')||undefined,
Type: specs.Type, Warranty: specs.Warranty, Weight: specs['Weight per Linear Yard'], Use: ptype,
};
for(const k of SPEC_ALLOW){ const v=specs[k]; if(v && !spec[k] && String(v).length<600) spec[k]=v; }
const ident = { dw_sku:row.dw_sku, manufacturer_sku:row.mfr_sku, brand:'Carnegie', vendor:'Carnegie',
pattern_name:row.pattern_name, color:colorName, color_number:String(row.color_number||''),
product_class:'Fabric', collection_name:'Carnegie Textiles' };
return { payload, spec, ident, colorName, retail, handle, imgCount: imgs.length, bodyValid, primary_hex: row.primary_hex };
}
// REST metafields — preserves the EXACT space-containing keys the proof rendered with
// (global/custom/specifications/specs for specs; global/custom for identity — matching proof-one).
async function setMetafields(pid, {spec, ident, primary_hex}){
const mfs=[];
for(const ns of ['global','custom','specifications','specs'])
for(const [k,v] of Object.entries(spec)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v).slice(0,900)});
for(const ns of ['global','custom'])
for(const [k,v] of Object.entries(ident)) if(v) mfs.push({namespace:ns,key:k,type:'single_line_text_field',value:String(v)});
if(primary_hex) mfs.push({namespace:'custom',key:'color_hex',type:'single_line_text_field',value:primary_hex});
let ok=0,err=0;
for(const m of mfs){ try{ await shop(`/products/${pid}/metafields.json`,{method:'POST',body:JSON.stringify({metafield:m})}); ok++; }catch{ err++; } await sleep(50); }
return {ok,err};
}
// ---- storage guard: confirm a freshly-created product actually has images attached ----
async function verifyImages(pid){
try{ const {product}=await shop(`/products/${pid}.json?fields=id,images`); return (product.images||[]).length; }catch{ return -1; }
}
async function main(){
const done = ledgerMap();
const skus = q1(`select dw_sku from carnegie_catalog order by pattern_name, color_number`).split('\n').filter(Boolean);
const todo = skus.filter((s,i)=> (i%SHARD_M===SHARD_N) && !(done.get(s)&&done.get(s).product_id));
console.log(`[shard ${SHARD_N+1}/${SHARD_M}] catalog ${skus.length} | already created ${[...done.values()].filter(r=>r.product_id).length} | this shard to create ${todo.length} (limit ${LIMIT})`);
let made=0, mfrSkipped=0, sinceCheck=0, imgFailStreak=0;
for(const sku of todo){
if(made>=LIMIT) break;
let row;
try{ row = JSON.parse(q1(`select row_to_json(t) from (select * from carnegie_catalog where dw_sku='${sku.replace(/'/g,"''")}') t`)); }
catch(e){ ledgerAppend({dw_sku:sku, error:'row-fetch: '+e.message, at:new Date().toISOString()}); continue; }
let built;
try{ built = buildPayload(row); }
catch(e){ ledgerAppend({dw_sku:sku, error:'build: '+e.message, at:new Date().toISOString()}); continue; }
let product;
try{ ({product} = await shop('/products.json',{method:'POST',body:JSON.stringify(built.payload)})); }
catch(e){
// storage-cap detection on create
if(/FILE_STORAGE_LIMIT_EXCEEDED|storage limit/i.test(e.message)){ haltStorage(sku, e.message); return; }
ledgerAppend({dw_sku:sku, error:'create: '+e.message.slice(0,200), at:new Date().toISOString()}); continue;
}
// DUPE-GUARD: ledger the create IMMEDIATELY so a crash/kill before metafields can never re-create it.
ledgerAppend({dw_sku:sku, product_id:product.id, handle:built.handle, at:new Date().toISOString(), phase:'created'});
// CRASH-GUARD: metafields/verify must never throw the whole shard down.
let mf={ok:0,err:'skipped'};
let attached = product.images ? product.images.length : 0;
try{
mf = await setMetafields(product.id, built);
if(built.imgCount>0 && attached===0){ attached = await verifyImages(product.id); }
}catch(e){ mf={ok:0,err:('mf/verify: '+String(e.message||e)).slice(0,120)}; }
const imageStarved = built.imgCount>0 && attached<=0;
imgFailStreak = imageStarved ? imgFailStreak+1 : 0;
const wasMfrSkipped = (product.status === 'draft') && !mfrSkuValid(row.mfr_sku);
if (wasMfrSkipped) mfrSkipped++;
const bodyGateHeld = (product.status === 'draft') && !built.bodyValid;
ledgerAppend({ dw_sku:sku, product_id:product.id, handle:built.handle, color:built.colorName,
retail:built.retail, imgExpected:built.imgCount, imgAttached:attached, status:product.status,
mfr_sku:row.mfr_sku, mfr_gate: wasMfrSkipped ? 'DRAFT-no-mfr' : 'ok',
desc_gate: bodyGateHeld ? 'DRAFT-blank-desc' : 'ok',
mf_ok:mf.ok, mf_err:mf.err, at:new Date().toISOString() });
made++; sinceCheck++;
if(imgFailStreak>=3){ haltStorage(sku, `3 consecutive image-starved creates (expected imgs, got 0) — Shopify storage cap suspected`); return; }
if(made%100===0) console.log(` ... ${made} created (last: ${built.handle} $${built.retail} imgs ${attached}/${built.imgCount})`);
// periodic explicit storage check every ~50
if(sinceCheck>=50){
sinceCheck=0;
const cnt = await verifyImages(product.id);
console.log(` [guard @${made}] latest product images=${cnt} status=${product.status}`);
}
await sleep(250); // gentle pace, well under any burst limit
}
// TK-10792: durable skip-count so operators can see the gate's effect without parsing all JSONL
const summary = { type:'run-summary', shard:`${SHARD_N+1}/${SHARD_M}`, made, mfrDraftSkipped:mfrSkipped,
note: mfrSkipped > 0 ? `${mfrSkipped} products held as DRAFT — mfr_sku null/empty/DWAG-*; needs Steve review` : 'all activations had real mfr_sku',
at: new Date().toISOString() };
ledgerAppend(summary);
if (mfrSkipped > 0) console.warn(`\n[TK-10792] ${mfrSkipped} products held as DRAFT (tagged ${NEEDS_MFR_TAG}). Real vendor mfr_sku required before activation. See ledger run-summary.\n`);
console.log(`DONE this run: ${made} created (${mfrSkipped} kept DRAFT due to missing/DWAG mfr_sku — TK-10792). Ledger: ${LEDGER}`);
}
function haltStorage(sku, msg){
const text = `HALTED at ${new Date().toISOString()} on ${sku}: ${msg}`;
fs.writeFileSync(HALT, text+'\n');
console.error('!!! STORAGE HALT !!! '+text);
try{ execFileSync('bash',['-lc',`export TK_AGENT=vp-dw-commerce; tk log TK-10671 ${JSON.stringify('STORAGE HALT: '+text)} 2>/dev/null || true`]); }catch{}
try{ execFileSync('bash',['-lc',`curl -s -X POST http://127.0.0.1:3333/api/wins -H 'Content-Type: application/json' -d ${JSON.stringify(JSON.stringify({project:'carnegie-reprice',title:'Carnegie rollout HALTED: Shopify storage cap',summary:text}))} >/dev/null 2>&1 || true`]); }catch{}
}
main().catch(e=>{ console.error('FATAL', e); process.exit(1); });