← back to Designer Wallcoverings
auto-save: 2026-06-30T07:29:10 (3 files) — pending-approval/boost-filter-consolidation-2026-06-25 vendor-scrapers/china-seas-refresh DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs
873527ecb2bb362387c0c570c8192fda33e56c49 · 2026-06-30 07:29:15 -0700 · Steve Abrams
Files touched
A DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs
Diff
commit 873527ecb2bb362387c0c570c8192fda33e56c49
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jun 30 07:29:15 2026 -0700
auto-save: 2026-06-30T07:29:10 (3 files) — pending-approval/boost-filter-consolidation-2026-06-25 vendor-scrapers/china-seas-refresh DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs
---
.../ImportNewSkufromURL/_elitis-retry.mjs | 75 ++++++++++++++++++++++
1 file changed, 75 insertions(+)
diff --git a/DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs b/DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs
new file mode 100644
index 00000000..ff1824c3
--- /dev/null
+++ b/DW-Programming/ImportNewSkufromURL/_elitis-retry.mjs
@@ -0,0 +1,75 @@
+import https from 'https';
+import pg from 'pg';
+const pool = new pg.Pool({ host: '/tmp', database: 'dw_unified', max: 6 });
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
+const LOG = '/tmp/elitis-recrawl.log';
+import fs from 'fs';
+const log = (m) => { fs.appendFileSync(LOG, m + '\n'); };
+
+function get(url){ return new Promise((res)=>{ const req=https.get(url,{headers:{'User-Agent':UA},timeout:20000},r=>{
+ if(r.statusCode>=300&&r.statusCode<400&&r.headers.location){ r.resume(); return res(get(r.headers.location)); }
+ if(r.statusCode!==200){ r.resume(); return res(null); }
+ let d=''; r.on('data',c=>d+=c); r.on('end',()=>res(d)); });
+ req.on('timeout',()=>{req.destroy();res(null)}); req.on('error',()=>res(null)); }); }
+
+const nx = (h,k)=>{ const m=h.match(new RegExp('[,{]'+k+':"((?:[^"\\\\]|\\\\.)*)"')); return m?m[1].replace(/\\u([0-9a-f]{4})/gi,(_,c)=>String.fromCharCode(parseInt(c,16))).replace(/\\"/g,'"').replace(/\\\//g,'/').trim():null; };
+
+function extract(html){
+ const o={ sku:null,name:null,description:null,image:null,all_images:[],specs:{} };
+ const m=html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/i);
+ if(m){ try{ const j=JSON.parse(m[1].trim()); o.sku=(j.sku||'').replace(/_/g,'-')||null; o.name=(j.name||'').trim()||null; o.description=(j.description||'').trim()||null; o.image=j.image||null; }catch(e){} }
+ const comp=nx(html,'composition'), laize=nx(html,'laize'), poids=nx(html,'poids'), raccord=nx(html,'raccord'), nonfeu=nx(html,'nonfeu'), plus=nx(html,'plusProduit');
+ let wm=(laize||o.description||'').match(/width[^(]*\(([^)]+)\)/i); const width=wm?wm[1].trim():null;
+ let lm=(laize||o.description||'').match(/length[^(]*\(([^)]+)\)/i); const length=lm?lm[1].trim():null;
+ let im=(width||'').match(/([\d.]+)\s*"/); const width_in=im?parseFloat(im[1]):null;
+ o.specs={ composition:comp, roll:laize, weight:poids, repeat:raccord, fire_rating:nonfeu, features:plus, width, roll_length:length };
+ Object.keys(o.specs).forEach(k=>{ if(!o.specs[k]) delete o.specs[k]; });
+ o.all_images=[...new Set([...html.matchAll(/https:\/\/backoffice\.elitis\.fr\/media\/[^"'\s)]+\.(?:jpg|jpeg|png|webp)/gi)].map(x=>x[0]))].slice(0,12);
+ if(!o.image&&o.all_images[0])o.image=o.all_images[0];
+ o._cols={ composition:comp, fire:nonfeu, material:comp, width, roll_length:length, repeat:raccord, features:plus, width_in };
+ return o;
+}
+
+const { rows } = await pool.query("select mfr_sku, product_url from elitis_catalog where product_url ilike 'http%' and (crawled_at::date < current_date or crawled_at is null) order by mfr_sku");
+fs.writeFileSync(LOG, `START ${rows.length} urls\n`);
+let done=0, ok=0, withSpec=0, fail=0;
+const CONC=6; let idx=0;
+async function worker(){
+ while(idx<rows.length){
+ const r=rows[idx++];
+ const html=await get(r.product_url);
+ done++;
+ if(!html){ fail++; if(done%50===0) log(`progress ${done}/${rows.length} ok=${ok} spec=${withSpec} fail=${fail}`); continue; }
+ const e=extract(html);
+ const hasSpec=Object.keys(e.specs).length>0;
+ if(hasSpec) withSpec++;
+ try{
+ await pool.query(
+ `update elitis_catalog set
+ specs = case when $2::jsonb <> '{}'::jsonb then $2::jsonb else specs end,
+ composition = coalesce(nullif($3,''), composition),
+ fire_rating = coalesce(nullif($4,''), fire_rating),
+ fire_rating_us = coalesce(nullif($4,''), fire_rating_us),
+ material = coalesce(nullif($5,''), material),
+ width = coalesce(nullif($6,''), width),
+ roll_length = coalesce(nullif($7,''), roll_length),
+ pattern_repeat = coalesce(nullif($8,''), pattern_repeat),
+ features = coalesce(nullif($9,''), features),
+ image_url = coalesce(nullif($10,''), image_url),
+ all_images = case when $11<>'[]' then $11 else all_images end,
+ width_inches = coalesce($12, width_inches),
+ crawled_at = now(), updated_at = now()
+ where mfr_sku = $1`,
+ [r.mfr_sku, JSON.stringify(e.specs), e._cols.composition, e._cols.fire, e._cols.material,
+ e._cols.width, e._cols.roll_length, e._cols.repeat, e._cols.features, e.image,
+ JSON.stringify(e.all_images), e._cols.width_in]
+ );
+ ok++;
+ }catch(err){ fail++; if(fail<=5) log(`DBERR ${r.mfr_sku}: ${err.message.slice(0,60)}`); }
+ if(done%50===0) log(`progress ${done}/${rows.length} ok=${ok} spec=${withSpec} fail=${fail}`);
+ }
+}
+await Promise.all(Array.from({length:CONC},()=>worker()));
+log(`DONE total=${done} updated=${ok} withSpecs=${withSpec} fail=${fail}`);
+await pool.end();
+process.exit(0);
← 391d1145 Session wrap: RESUME-abc123A handoff + run-viewer.sh relaunc
·
back to Designer Wallcoverings
·
Adopt dated restart-code convention (MMDDYY+letter): abc123A 0f8c885d →