auto-save: 2026-07-28T12:30:33 (1 files) — tmp-kravet-alldata.mjs

1124c80dd1b3653bde3eb111d3a8790c358d6f73 · 2026-07-28 12:30:40 -0700 · Steve Abrams

Files touched

M tmp-kravet-alldata.mjs

Diff

commit 1124c80dd1b3653bde3eb111d3a8790c358d6f73
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jul 28 12:30:40 2026 -0700

    auto-save: 2026-07-28T12:30:33 (1 files) — tmp-kravet-alldata.mjs
---
 tmp-kravet-alldata.mjs | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tmp-kravet-alldata.mjs b/tmp-kravet-alldata.mjs
index d74fc1e..0fd43bb 100644
--- a/tmp-kravet-alldata.mjs
+++ b/tmp-kravet-alldata.mjs
@@ -35,8 +35,13 @@ const rows=execFileSync('psql',['-h','/tmp','-d','dw_unified','-tA','-F','|','-c
     AND s.vendor ILIKE ANY(ARRAY['%brunschwig%','%lee jofa%','%kravet%','%cole%','%baker%','%mulberry%','%clarke%','%groundworks%','%colefax%','%threads%','%andrew martin%','%aerin%','%barclay%','%thom filicia%'])
   ORDER BY s.sku ${LIMIT?`LIMIT ${LIMIT}`:''};`]).toString().trim().split('\n').map(l=>{const[sku,id,mfr]=l.split('|');return{sku,id,mfr};});
 
+// resume ledger (so a watchdog restart continues instead of redoing everything)
+const MF_LEDGER='/Users/macstudio3/kravet-feed-pull/alldata-done.txt';
+const mfDone=new Set(fs.existsSync(MF_LEDGER)?fs.readFileSync(MF_LEDGER,'utf8').trim().split('\n').filter(Boolean):[]);
+const rows2=rows.filter(r=>!mfDone.has(r.sku));
+
 // bulk-load the feed rows we need into a JS map keyed by mfr, using a single psql dump
-const wanted=[...new Set(rows.map(r=>r.mfr))];
+const wanted=[...new Set(rows2.map(r=>r.mfr))];
 const sel=cols.join(',');
 const feed={};
 // chunk the IN() to keep query size sane
@@ -46,12 +51,12 @@ for(let i=0;i<wanted.length;i+=2000){
     `SELECT upper(c1), ${sel} FROM kravet_feed_stage WHERE upper(c1) IN (${chunk});`],{maxBuffer:1<<28}).toString().trim();
   for(const line of out.split('\n')){ if(!line)continue; const parts=line.split('\t'); const key=parts[0]; feed[key]=parts.slice(1); }
 }
-console.log(`targets:${rows.length}  feed rows loaded:${Object.keys(feed).length}`);
+console.log(`targets:${rows2.length} (of ${rows.length}, ${mfDone.size} already done)  feed rows loaded:${Object.keys(feed).length}`);
 
 let ok=0,fail=0,totalMF=0;
-for(const {sku,id,mfr} of rows){
+for(const {sku,id,mfr} of rows2){
   try{
-    const vals=feed[mfr]; if(!vals){fail++;continue;}
+    const vals=feed[mfr]; if(!vals){fail++;fs.appendFileSync(MF_LEDGER,sku+'\n');continue;}
     const mfs=[];
     cols.forEach((c,idx)=>{
       const hdr=hmap[c]; if(SKIP.has(hdr)) return;
@@ -65,8 +70,8 @@ for(const {sku,id,mfr} of rows){
       const errs=d.metafieldsSet.userErrors; if(errs.length) throw new Error(JSON.stringify(errs).slice(0,160));
       await sleep(120);
     }
-    ok++; totalMF+=mfs.length;
-    if(ok%200===0) console.log(`  ...${ok}/${rows.length}  (${totalMF} metafields, ${fail} fail)`);
+    ok++; totalMF+=mfs.length; fs.appendFileSync(MF_LEDGER,sku+'\n');
+    if(ok%200===0) console.log(`  ...${ok}/${rows2.length}  (${totalMF} metafields, ${fail} fail)`);
   }catch(e){ fail++; if(fail<=20)console.error(`  FAIL ${sku}: ${e.message}`); }
   await sleep(90);
 }

← 59e9f3f auto-save: 2026-07-28T11:00:01 (2 files) — tmp-kravet-alldat · back to Contractwallpaper · corner-nav: live as-you-type search with thumbnail previews 7edf9de →