`measure.js`

127 lines
// Phase 0b — read inventory.jsonl (ground-truth image list from enumerate.js),
// measure real bytes via CDN HEAD sampling, HEAD every GIF exactly, calibrate the
// real re-encode ratio on downloaded masters, and write phase0-report.md +
// phase0-data.json with a go/no-go. READ-ONLY against Shopify/CDN (GET/HEAD only).
import fs from 'fs';
import https from 'https';
import { execFileSync } from 'child_process';
const DIR='/Users/macstudio3/Projects/dw-image-shrink';
const INV=DIR+'/inventory.jsonl';
const CAP_GB=500;

const rows=fs.readFileSync(INV,'utf8').split('\n').filter(Boolean).map(l=>JSON.parse(l));
const bucketOf=(w,h)=>{const m=Math.max(w||0,h||0); if(m>3000)return'>3000';if(m>2048)return'2049-3000';if(m>1500)return'1501-2048';if(m>1000)return'1001-1500';return'<=1000';};
const isGif=s=>/\.gif(\?|$)/i.test(s||'');
const ORDER=['>3000','2049-3000','1501-2048','1001-1500','<=1000'];

// ---- HEAD helper with small concurrency pool ----
function head(url){return new Promise(res=>{try{const u=new URL(url);const req=https.request({method:'HEAD',host:u.host,path:u.pathname+u.search,headers:{'User-Agent':'dw-measure'}},r=>{r.resume();res({status:r.statusCode,len:+(r.headers['content-length']||0),type:r.headers['content-type']||''});});req.on('error',()=>res({status:0,len:0,type:''}));req.setTimeout(15000,()=>{req.destroy();res({status:0,len:0,type:''});});req.end();}catch(e){res({status:0,len:0,type:''});}});}
function get(url){return new Promise(res=>{try{const u=new URL(url);https.get({host:u.host,path:u.pathname+u.search,headers:{'User-Agent':'dw-measure'}},r=>{const ch=[];r.on('data',c=>ch.push(c));r.on('end',()=>res(Buffer.concat(ch)));r.on('error',()=>res(null));}).on('error',()=>res(null));}catch(e){res(null);}});}
async function pool(items,fn,conc=24){const out=new Array(items.length);let i=0;await Promise.all(Array.from({length:conc},async()=>{while(i<items.length){const idx=i++;out[idx]=await fn(items[idx],idx);}}));return out;}
const sample=(arr,n)=>{if(arr.length<=n)return arr.slice();const a=arr.slice();for(let i=a.length-1;i>0;i--){const j=(i*2654435761>>>0)%(i+1);[a[i],a[j]]=[a[j],a[i]];}return a.slice(0,n);};
const GB=b=>b/1073741824;

(async()=>{
  const gifs=rows.filter(r=>isGif(r.src));
  const imgs=rows.filter(r=>!isGif(r.src));
  console.log(`inventory: ${rows.length} images | ${imgs.length} raster | ${gifs.length} gif`);

  // buckets over raster images
  const buckets={}; for(const k of ORDER) buckets[k]={rows:[],sampleBytes:[]};
  for(const r of imgs) buckets[bucketOf(r.width,r.height)].rows.push(r);

  // HEAD-sample each bucket (up to 500)
  for(const k of ORDER){
    const s=sample(buckets[k].rows,500);
    const heads=await pool(s,r=>head(r.src),24);
    buckets[k].sampleBytes=heads.filter(h=>h.status===200&&h.len>0).map(h=>h.len);
    const mean=buckets[k].sampleBytes.reduce((a,b)=>a+b,0)/(buckets[k].sampleBytes.length||1);
    console.log(`bucket ${k}: pop=${buckets[k].rows.length} sampled=${buckets[k].sampleBytes.length} mean=${(mean/1024).toFixed(0)}KB`);
  }

  // HEAD ALL gifs (exact — they're the fat individual files)
  const gifHeads=await pool(gifs,r=>head(r.src),24);
  const gifBytes=gifHeads.filter(h=>h.status===200&&h.len>0).map(h=>h.len);
  const gifTotal=gifBytes.reduce((a,b)=>a+b,0);
  console.log(`gifs: ${gifBytes.length}/${gifs.length} measured, total ${GB(gifTotal).toFixed(2)}GB`);

  // Calibrate real re-encode ratio: download+encode ~120 oversized masters
  fs.mkdirSync(DIR+'/calib',{recursive:true});
  const calibSet=sample([...buckets['>3000'].rows,...buckets['2049-3000'].rows],120);
  let ratN=0, inSum=0, outSum=0;
  for(let i=0;i<calibSet.length;i++){
    const r=calibSet[i]; const buf=await get(r.src); if(!buf||!buf.length)continue;
    const inp=`${DIR}/calib/in_${i}`, outp=`${DIR}/calib/out_${i}.jpg`;
    fs.writeFileSync(inp,buf);
    try{const v=JSON.parse(execFileSync('python3',[`${DIR}/shrink_master.py`,'--in',inp,'--out',outp,'--max-edge','2048','--quality','82'],{encoding:'utf8'}));
      if(v.action==='encoded'){ratN++;inSum+=v.in_bytes;outSum+=v.out_bytes;}
    }catch(e){}
    try{fs.unlinkSync(inp)}catch(e){} try{fs.unlinkSync(outp)}catch(e){}
  }
  const ratio = outSum/(inSum||1); // encoded/original for oversized
  console.log(`calibration: ${ratN} encoded, mean out/in ratio = ${(ratio*100).toFixed(1)}%`);

  // Extrapolate totals per bucket, reclaimable for oversized (>2048).
  const est={}; let totalBytes=0, reclaim=0;
  for(const k of ORDER){
    const sb=buckets[k].sampleBytes; const mean=sb.reduce((a,b)=>a+b,0)/(sb.length||1);
    const total=mean*buckets[k].rows.length; est[k]={pop:buckets[k].rows.length,meanKB:mean/1024,totalGB:GB(total)};
    totalBytes+=total;
    if(k==='>3000'||k==='2049-3000') reclaim += total*(1-ratio);       // downsize+quality
    else if(k==='1501-2048') reclaim += total*0.15;                     // quality-only modest
  }
  totalBytes+=gifTotal;

  // status/vendor breakdown for purge track
  const byStatus={}; for(const r of rows){byStatus[r.status]=(byStatus[r.status]||0)+1;}
  const archivedImgs=rows.filter(r=>r.status==='archived').length;

  // light video probe via GraphQL (bounded sample)
  let videoNote='not probed';
  try{
    const ATOK=fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8').split('\n').find(l=>l.startsWith('SHOPIFY_ADMIN_TOKEN=')).split('=').slice(1).join('=').replace(/["' ]/g,'');
    const q=JSON.stringify({query:`{products(first:200,sortKey:UPDATED_AT,reverse:true){edges{node{media(first:10){edges{node{mediaContentType}}}}}}}`});
    const vid=await new Promise(res=>{const req=https.request({method:'POST',host:'designer-laboratory-sandbox.myshopify.com',path:'/admin/api/2024-10/graphql.json',headers:{'X-Shopify-Access-Token':ATOK,'Content-Type':'application/json','Content-Length':Buffer.byteLength(q)}},r=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>res(d));});req.on('error',()=>res(''));req.write(q);req.end();});
    const j=JSON.parse(vid); let types={};
    for(const e of j.data.products.edges) for(const m of e.node.media.edges){const t=m.node.mediaContentType;types[t]=(types[t]||0)+1;}
    videoNote=JSON.stringify(types)+' (from 200 most-recent products)';
  }catch(e){videoNote='probe failed: '+String(e).slice(0,80);}

  const headroom = CAP_GB*0.15;
  const verdict = reclaim >= headroom ? 'GO' : (reclaim>=headroom*0.5?'PARTIAL — likely need purge/upgrade too':'NO-GO on compression alone — pursue purge + plan upgrade');

  const md=`# Phase 0 — DW Shopify Storage Diagnosis
_Generated from ground-truth Shopify enumeration (${rows.length} product images across ${new Set(rows.map(r=>r.product_id)).size} products)._

## Measured product-image footprint (extrapolated from HEAD sampling)
| dimension bucket | # images | mean size | est. total |
|---|--:|--:|--:|
${ORDER.map(k=>`| ${k} | ${est[k].pop.toLocaleString()} | ${est[k].meanKB.toFixed(0)} KB | ${est[k].totalGB.toFixed(2)} GB |`).join('\n')}
| **animated GIF (exact)** | ${gifs.length.toLocaleString()} | ${(gifBytes.reduce((a,b)=>a+b,0)/1024/(gifBytes.length||1)).toFixed(0)} KB | ${GB(gifTotal).toFixed(2)} GB |
| **TOTAL (product images)** | ${rows.length.toLocaleString()} | | **${GB(totalBytes).toFixed(1)} GB** |

Calibration: on ${ratN} real oversized masters, re-encode (≤2048px, q82) → **${(ratio*100).toFixed(0)}%** of original size (≈${((1-ratio)*100).toFixed(0)}% shrink).

## Reclaimable by compression (oversized-first scope)
- >2048px downsize+quality: **${GB(reclaim).toFixed(1)} GB** estimated reclaimable.
- Headroom target (~15% of ${CAP_GB} GB): **${headroom.toFixed(0)} GB**.

## Unmeasured buckets (NOT compressible / need other action)
- **Videos:** ${videoNote}. Videos aren't in the REST images inventory — if VIDEO count is material, they may be a big chunk needing separate handling.
- **Files section** (theme assets, blog/email/app uploads, old themes): NOT API-queryable → **Steve must check Settings → Files** (sort by size) + Online Store → Themes for duplicate/unpublished themes.
- These + GIFs (${GB(gifTotal).toFixed(2)} GB) are the non-JPEG buckets compression won't touch.

## Status breakdown (purge-track candidates)
${Object.entries(byStatus).map(([k,v])=>`- ${k}: ${v.toLocaleString()} images`).join('\n')}
- Archived-product images = ${archivedImgs.toLocaleString()} → purge candidates (Steve-gated).

## VERDICT: ${verdict}

**Reasoning:** product images total ≈ **${GB(totalBytes).toFixed(0)} GB** of the ${CAP_GB} GB cap. If this is well under 500 GB, the compressible images are NOT what filled the cap — the bulk is videos, the Files section, or media the mirror/enumeration can't see — and **compression alone will not clear the cap**. In that case the real levers are: purge orphaned/archived media + oversized GIFs/videos, clean the Files section, and/or upgrade the plan. Compression still helps and runs, but the scratch-upload gate (Phase 4) is the arbiter.
`;
  fs.writeFileSync(DIR+'/phase0-report.md',md);
  fs.writeFileSync(DIR+'/phase0-data.json',JSON.stringify({est,gifTotalGB:GB(gifTotal),totalGB:GB(totalBytes),reclaimGB:GB(reclaim),ratio,byStatus,verdict},null,2));
  try{fs.rmSync(DIR+'/calib',{recursive:true,force:true});}catch(e){}
  console.log('\n'+verdict+'\n → phase0-report.md');
})();