[object Object]

← back to Carnegie Reprice

add --shard N/M for parallel disjoint workers (4x throughput on Advanced 80-bucket)

c0e7acb7e81e1671d4a100919f3ef6e9b4d7224c · 2026-08-18 13:22:29 -0700 · steve

Files touched

Diff

commit c0e7acb7e81e1671d4a100919f3ef6e9b4d7224c
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Aug 18 13:22:29 2026 -0700

    add --shard N/M for parallel disjoint workers (4x throughput on Advanced 80-bucket)
---
 rebuild-ledger.jsonl | 4 ++++
 rebuild-line.mjs     | 7 +++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/rebuild-ledger.jsonl b/rebuild-ledger.jsonl
index 92fb854..cf185ac 100644
--- a/rebuild-ledger.jsonl
+++ b/rebuild-ledger.jsonl
@@ -1,3 +1,7 @@
 {"dw_sku":"DWAG-381677","product_id":7923105497139,"handle":"carnegie-abbey-oatmeal-61","color":"Oatmeal","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":116,"mf_err":7,"at":"2026-08-18T20:16:45.230Z"}
 {"dw_sku":"DWAG-381678","product_id":7923105595443,"handle":"carnegie-abbey-blush-62","color":"Blush","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":111,"mf_err":12,"at":"2026-08-18T20:17:22.455Z"}
 {"dw_sku":"DWAG-381679","product_id":7923105660979,"handle":"carnegie-abbey-greige-63","color":"Greige","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":106,"mf_err":17,"at":"2026-08-18T20:17:59.708Z"}
+{"dw_sku":"DWAG-381680","product_id":7923105955891,"handle":"carnegie-abbey-oatmeal-dwag-381680","color":"Oatmeal","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":116,"mf_err":7,"at":"2026-08-18T20:20:11.942Z"}
+{"dw_sku":"DWAG-381681","product_id":7923106021427,"handle":"carnegie-abbey-oatmeal-dwag-381681","color":"Oatmeal","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":116,"mf_err":7,"at":"2026-08-18T20:20:52.297Z"}
+{"dw_sku":"DWAG-381682","product_id":7923106709555,"handle":"carnegie-abbey-oatmeal-dwag-381682","color":"Oatmeal","retail":58,"imgExpected":2,"imgAttached":2,"status":"active","mf_ok":116,"mf_err":7,"at":"2026-08-18T20:21:29.689Z"}
+{"dw_sku":"DWAG-381784","product_id":7923109036083,"handle":"carnegie-abyss-charcoal-dwag-381784","color":"Charcoal","retail":132,"imgExpected":3,"imgAttached":3,"status":"active","mf_ok":127,"mf_err":6,"at":"2026-08-18T20:22:11.601Z"}
diff --git a/rebuild-line.mjs b/rebuild-line.mjs
index e5d77b8..38c0f9c 100644
--- a/rebuild-line.mjs
+++ b/rebuild-line.mjs
@@ -29,6 +29,9 @@ const HALT    = path.join(DIR,'rebuild-HALTED.flag');
 const args = process.argv.slice(2);
 const argVal = f => { const i=args.indexOf(f); return i>=0? args[i+1]:null; };
 const LIMIT = argVal('--limit') ? +argVal('--limit') : Infinity;
+// --shard N/M : this worker handles SKUs where (index % M === N-1). Disjoint slices, ledger-safe.
+let SHARD_N=0, SHARD_M=1;
+if(argVal('--shard')){ const [n,m]=argVal('--shard').split('/').map(Number); SHARD_N=n-1; SHARD_M=m; }
 
 // ---- ledger helpers ----
 function ledgerMap(){ const m=new Map(); if(fs.existsSync(LEDGER)){ for(const line of fs.readFileSync(LEDGER,'utf8').split('\n')){ if(!line.trim())continue; try{const r=JSON.parse(line); if(r.dw_sku) m.set(r.dw_sku, r);}catch{} } } return m; }
@@ -151,8 +154,8 @@ async function verifyImages(pid){
 async function main(){
   const done = ledgerMap();
   const skus = q1(`select dw_sku from carnegie_catalog order by pattern_name, color_number`).split('\n').filter(Boolean);
-  const todo = skus.filter(s=>!(done.get(s)&&done.get(s).product_id));
-  console.log(`catalog ${skus.length} | already created ${[...done.values()].filter(r=>r.product_id).length} | to create ${todo.length} (limit ${LIMIT})`);
+  const todo = skus.filter((s,i)=> (i%SHARD_M===SHARD_N) && !(done.get(s)&&done.get(s).product_id));
+  console.log(`[shard ${SHARD_N+1}/${SHARD_M}] catalog ${skus.length} | already created ${[...done.values()].filter(r=>r.product_id).length} | this shard to create ${todo.length} (limit ${LIMIT})`);
 
   let made=0, sinceCheck=0, imgFailStreak=0;
   for(const sku of todo){

← c05ae91 handle: append dw_sku tail for guaranteed uniqueness (2424 c  ·  back to Carnegie Reprice  ·  add 429 retry-with-backoff in shop(); errored SKUs auto-retr 7ecf829 →