`enumerate.js`

39 lines
// Phase 0a — enumerate EVERY product image from Shopify itself (ground truth;
// the dw_unified mirror undercounts). Dumps one JSONL row per image to
// inventory.jsonl: {product_id,status,vendor,image_id,position,src,width,height}.
// READ-ONLY (GET only). Resumable-ish: overwrites inventory.jsonl fresh each run.
import fs from 'fs';
import https from 'https';
const ATOK = fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8')
  .split('\n').find(l=>l.startsWith('SHOPIFY_ADMIN_TOKEN=')).split('=').slice(1).join('=').replace(/["' ]/g,'');
const SHOP='designer-laboratory-sandbox.myshopify.com';
const OUT='/Users/macstudio3/Projects/dw-image-shrink/inventory.jsonl';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
function raw(method,path){return new Promise((res,rej)=>{const req=https.request({method,host:SHOP,path,headers:{'X-Shopify-Access-Token':ATOK,'Content-Type':'application/json'}},r=>{let d='';r.on('data',c=>d+=c).on('end',()=>res({status:r.statusCode,retryAfter:r.headers['retry-after'],limit:r.headers['x-shopify-shop-api-call-limit']||'',link:r.headers.link||'',json:(()=>{try{return JSON.parse(d)}catch(e){return d}})()}));});req.on('error',rej);req.end();});}
async function api(path){for(let a=0;a<12;a++){const r=await raw('GET',path);if(r.status===429){await sleep((parseFloat(r.retryAfter)||2)*1000);continue;}const [u,m]=(r.limit||'0/40').split('/').map(Number);await sleep(u>(m*0.5)?400:120);return r;}return{status:429,json:{},link:''};}

fs.writeFileSync(OUT,'');
let url='/admin/api/2024-10/products.json?limit=250&fields=id,status,vendor,images';
let page=0, prods=0, imgs=0, withImg=0;
const t0=Date.now();
while(url){
  const {json,link,status}=await api(url);
  if(status>=400){ console.error('ERR',status,JSON.stringify(json).slice(0,200)); break; }
  const batch=json.products||[];
  let buf='';
  for(const p of batch){
    prods++;
    const arr=p.images||[];
    if(arr.length) withImg++;
    for(const im of arr){
      imgs++;
      buf+=JSON.stringify({product_id:p.id,status:p.status,vendor:p.vendor||'',image_id:im.id,position:im.position,src:im.src,width:im.width,height:im.height})+'\n';
    }
  }
  if(buf) fs.appendFileSync(OUT,buf);
  page++;
  if(page%20===0) console.log(`page ${page} | products ${prods} | images ${imgs} | ${((Date.now()-t0)/1000).toFixed(0)}s`);
  const m=link.match(/<([^>]+)>;\s*rel="next"/); url=m?m[1].replace('https://'+SHOP,''):null;
}
console.log(`DONE: ${page} pages | ${prods} products | ${withImg} with>=1 image | ${imgs} images | ${((Date.now()-t0)/1000).toFixed(0)}s`);