← back to Dw Image Shrink
Image-shrink campaign scaffold: enumerate + measure + encoder + ledger + worker
c9958b6bca1543e306ef511b46ca7dda0a1e32ac · 2026-07-29 07:42:14 -0700 · steve
Files touched
A enqueue.jsA ledger.sqlA measure.jsA package-lock.jsonA package.jsonA shrink_master.pyA shrink_worker.js
Diff
commit c9958b6bca1543e306ef511b46ca7dda0a1e32ac
Author: steve <steve@designerwallcoverings.com>
Date: Wed Jul 29 07:42:14 2026 -0700
Image-shrink campaign scaffold: enumerate + measure + encoder + ledger + worker
---
enqueue.js | 39 +++++++++++++
ledger.sql | 33 +++++++++++
measure.js | 126 ++++++++++++++++++++++++++++++++++++++++++
package-lock.json | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
package.json | 16 ++++++
shrink_master.py | 80 +++++++++++++++++++++++++++
shrink_worker.js | 116 ++++++++++++++++++++++++++++++++++++++
7 files changed, 572 insertions(+)
diff --git a/enqueue.js b/enqueue.js
new file mode 100644
index 0000000..0491893
--- /dev/null
+++ b/enqueue.js
@@ -0,0 +1,39 @@
+// Populate img_shrink_ledger from inventory.jsonl (ground truth).
+// Default scope = oversized (>2048px longest edge), statuses active+draft
+// (archived → purge track, not compression), skips GIFs (never JPEG a spin-gif).
+// Idempotent via ON CONFLICT(orig_image_id) DO NOTHING. Resumable.
+// node enqueue.js [--band oversized|1501] [--status active|active,draft] [--limit N] [--nshards 8]
+import fs from 'fs';
+import pg from 'pg';
+const DIR='/Users/macstudio3/Projects/dw-image-shrink';
+const arg=(k,d)=>{const a=process.argv.find(x=>x.startsWith('--'+k+'='));return a?a.split('=').slice(1).join('='):d;};
+const BAND=arg('band','oversized');
+const STATUS=arg('status','active,draft').split(',');
+const LIMIT=+arg('limit','0')||0;
+const N=+arg('nshards','8');
+const isGif=s=>/\.gif(\?|$)/i.test(s||'');
+const longest=r=>Math.max(r.width||0,r.height||0);
+const minEdge = BAND==='1501'?1501:2049;
+
+const rows=fs.readFileSync(DIR+'/inventory.jsonl','utf8').split('\n').filter(Boolean).map(l=>JSON.parse(l))
+ .filter(r=>STATUS.includes(r.status) && !isGif(r.src) && longest(r)>=minEdge);
+const picked = LIMIT? rows.slice(0,LIMIT) : rows;
+console.log(`enqueue: band>=${minEdge}px, status[${STATUS}] → ${rows.length} candidates${LIMIT?`, taking ${picked.length}`:''}`);
+
+const c=new pg.Client({host:'/tmp',database:'dw_unified'});
+await c.connect();
+let ins=0;
+for(let i=0;i<picked.length;i+=500){
+ const chunk=picked.slice(i,i+500);
+ const vals=[]; const ph=[];
+ chunk.forEach((r,j)=>{const b=j*8; ph.push(`($${b+1},$${b+2},$${b+3},$${b+4},$${b+5},$${b+6},$${b+7},$${b+8})`);
+ vals.push(r.product_id, r.product_id%N, r.image_id, r.src, r.position, r.status==null?null:null, r.width, r.height);});
+ // note: col order below
+ const q=`INSERT INTO img_shrink_ledger
+ (product_id,shard,orig_image_id,orig_src,orig_position,orig_alt,orig_width,orig_height)
+ VALUES ${ph.join(',')} ON CONFLICT (orig_image_id) DO NOTHING`;
+ const res=await c.query(q,vals); ins+=res.rowCount;
+}
+const {rows:[cnt]}=await c.query("SELECT count(*) n, count(*) FILTER(WHERE state='pending') p FROM img_shrink_ledger");
+console.log(`inserted ${ins} new rows | ledger total ${cnt.n} (${cnt.p} pending)`);
+await c.end();
diff --git a/ledger.sql b/ledger.sql
new file mode 100644
index 0000000..0c1bf97
--- /dev/null
+++ b/ledger.sql
@@ -0,0 +1,33 @@
+-- Operational ledger for the image-shrink campaign. Lives in the LOCAL Mac2
+-- dw_unified mirror (host=/tmp) — NOT the canonical Kamatera DB (avoids the
+-- repl_user/pg_dump GRANT-gap issue). One row per source image; resumable.
+CREATE TABLE IF NOT EXISTS img_shrink_ledger (
+ id BIGSERIAL PRIMARY KEY,
+ product_id BIGINT NOT NULL,
+ shard INT NOT NULL, -- product_id % N, for fan-out
+ orig_image_id BIGINT NOT NULL,
+ orig_src TEXT NOT NULL,
+ orig_position INT,
+ orig_alt TEXT,
+ orig_width INT,
+ orig_height INT,
+ orig_bytes BIGINT, -- actual (buf.length at process time)
+ content_type TEXT,
+ product_snapshot JSONB, -- images[] snapshot for rollback
+ new_image_id BIGINT,
+ new_src TEXT,
+ new_bytes BIGINT,
+ new_width INT,
+ new_height INT,
+ state TEXT NOT NULL DEFAULT 'pending', -- pending|claimed|done|failed|skipped|imageless_alert
+ skip_reason TEXT,
+ fail_reason TEXT,
+ attempts INT NOT NULL DEFAULT 0,
+ worker_id TEXT,
+ claimed_at TIMESTAMPTZ,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ CONSTRAINT uq_orig_image UNIQUE (orig_image_id)
+);
+CREATE INDEX IF NOT EXISTS ix_shrink_shard_state ON img_shrink_ledger (shard, state);
+CREATE INDEX IF NOT EXISTS ix_shrink_product ON img_shrink_ledger (product_id);
+CREATE INDEX IF NOT EXISTS ix_shrink_state ON img_shrink_ledger (state);
diff --git a/measure.js b/measure.js
new file mode 100644
index 0000000..0aa39e7
--- /dev/null
+++ b/measure.js
@@ -0,0 +1,126 @@
+// Phase 0b — read inventory.jsonl (ground-truth image list from enumerate.js),
+// measure real bytes via CDN HEAD sampling, HEAD every GIF exactly, calibrate the
+// real re-encode ratio on downloaded masters, and write phase0-report.md +
+// phase0-data.json with a go/no-go. READ-ONLY against Shopify/CDN (GET/HEAD only).
+import fs from 'fs';
+import https from 'https';
+import { execFileSync } from 'child_process';
+const DIR='/Users/macstudio3/Projects/dw-image-shrink';
+const INV=DIR+'/inventory.jsonl';
+const CAP_GB=500;
+
+const rows=fs.readFileSync(INV,'utf8').split('\n').filter(Boolean).map(l=>JSON.parse(l));
+const bucketOf=(w,h)=>{const m=Math.max(w||0,h||0); if(m>3000)return'>3000';if(m>2048)return'2049-3000';if(m>1500)return'1501-2048';if(m>1000)return'1001-1500';return'<=1000';};
+const isGif=s=>/\.gif(\?|$)/i.test(s||'');
+const ORDER=['>3000','2049-3000','1501-2048','1001-1500','<=1000'];
+
+// ---- HEAD helper with small concurrency pool ----
+function head(url){return new Promise(res=>{try{const u=new URL(url);const req=https.request({method:'HEAD',host:u.host,path:u.pathname+u.search,headers:{'User-Agent':'dw-measure'}},r=>{r.resume();res({status:r.statusCode,len:+(r.headers['content-length']||0),type:r.headers['content-type']||''});});req.on('error',()=>res({status:0,len:0,type:''}));req.setTimeout(15000,()=>{req.destroy();res({status:0,len:0,type:''});});req.end();}catch(e){res({status:0,len:0,type:''});}});}
+function get(url){return new Promise(res=>{try{const u=new URL(url);https.get({host:u.host,path:u.pathname+u.search,headers:{'User-Agent':'dw-measure'}},r=>{const ch=[];r.on('data',c=>ch.push(c));r.on('end',()=>res(Buffer.concat(ch)));r.on('error',()=>res(null));}).on('error',()=>res(null));}catch(e){res(null);}});}
+async function pool(items,fn,conc=24){const out=new Array(items.length);let i=0;await Promise.all(Array.from({length:conc},async()=>{while(i<items.length){const idx=i++;out[idx]=await fn(items[idx],idx);}}));return out;}
+const sample=(arr,n)=>{if(arr.length<=n)return arr.slice();const a=arr.slice();for(let i=a.length-1;i>0;i--){const j=(i*2654435761>>>0)%(i+1);[a[i],a[j]]=[a[j],a[i]];}return a.slice(0,n);};
+const GB=b=>b/1073741824;
+
+(async()=>{
+ const gifs=rows.filter(r=>isGif(r.src));
+ const imgs=rows.filter(r=>!isGif(r.src));
+ console.log(`inventory: ${rows.length} images | ${imgs.length} raster | ${gifs.length} gif`);
+
+ // buckets over raster images
+ const buckets={}; for(const k of ORDER) buckets[k]={rows:[],sampleBytes:[]};
+ for(const r of imgs) buckets[bucketOf(r.width,r.height)].rows.push(r);
+
+ // HEAD-sample each bucket (up to 500)
+ for(const k of ORDER){
+ const s=sample(buckets[k].rows,500);
+ const heads=await pool(s,r=>head(r.src),24);
+ buckets[k].sampleBytes=heads.filter(h=>h.status===200&&h.len>0).map(h=>h.len);
+ const mean=buckets[k].sampleBytes.reduce((a,b)=>a+b,0)/(buckets[k].sampleBytes.length||1);
+ console.log(`bucket ${k}: pop=${buckets[k].rows.length} sampled=${buckets[k].sampleBytes.length} mean=${(mean/1024).toFixed(0)}KB`);
+ }
+
+ // HEAD ALL gifs (exact — they're the fat individual files)
+ const gifHeads=await pool(gifs,r=>head(r.src),24);
+ const gifBytes=gifHeads.filter(h=>h.status===200&&h.len>0).map(h=>h.len);
+ const gifTotal=gifBytes.reduce((a,b)=>a+b,0);
+ console.log(`gifs: ${gifBytes.length}/${gifs.length} measured, total ${GB(gifTotal).toFixed(2)}GB`);
+
+ // Calibrate real re-encode ratio: download+encode ~120 oversized masters
+ fs.mkdirSync(DIR+'/calib',{recursive:true});
+ const calibSet=sample([...buckets['>3000'].rows,...buckets['2049-3000'].rows],120);
+ let ratN=0, inSum=0, outSum=0;
+ for(let i=0;i<calibSet.length;i++){
+ const r=calibSet[i]; const buf=await get(r.src); if(!buf||!buf.length)continue;
+ const inp=`${DIR}/calib/in_${i}`, outp=`${DIR}/calib/out_${i}.jpg`;
+ fs.writeFileSync(inp,buf);
+ try{const v=JSON.parse(execFileSync('python3',[`${DIR}/shrink_master.py`,'--in',inp,'--out',outp,'--max-edge','2048','--quality','82'],{encoding:'utf8'}));
+ if(v.action==='encoded'){ratN++;inSum+=v.in_bytes;outSum+=v.out_bytes;}
+ }catch(e){}
+ try{fs.unlinkSync(inp)}catch(e){} try{fs.unlinkSync(outp)}catch(e){}
+ }
+ const ratio = outSum/(inSum||1); // encoded/original for oversized
+ console.log(`calibration: ${ratN} encoded, mean out/in ratio = ${(ratio*100).toFixed(1)}%`);
+
+ // Extrapolate totals per bucket, reclaimable for oversized (>2048).
+ const est={}; let totalBytes=0, reclaim=0;
+ for(const k of ORDER){
+ const sb=buckets[k].sampleBytes; const mean=sb.reduce((a,b)=>a+b,0)/(sb.length||1);
+ const total=mean*buckets[k].rows.length; est[k]={pop:buckets[k].rows.length,meanKB:mean/1024,totalGB:GB(total)};
+ totalBytes+=total;
+ if(k==='>3000'||k==='2049-3000') reclaim += total*(1-ratio); // downsize+quality
+ else if(k==='1501-2048') reclaim += total*0.15; // quality-only modest
+ }
+ totalBytes+=gifTotal;
+
+ // status/vendor breakdown for purge track
+ const byStatus={}; for(const r of rows){byStatus[r.status]=(byStatus[r.status]||0)+1;}
+ const archivedImgs=rows.filter(r=>r.status==='archived').length;
+
+ // light video probe via GraphQL (bounded sample)
+ let videoNote='not probed';
+ try{
+ const ATOK=fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8').split('\n').find(l=>l.startsWith('SHOPIFY_ADMIN_TOKEN=')).split('=').slice(1).join('=').replace(/["' ]/g,'');
+ const q=JSON.stringify({query:`{products(first:200,sortKey:UPDATED_AT,reverse:true){edges{node{media(first:10){edges{node{mediaContentType}}}}}}}`});
+ const vid=await new Promise(res=>{const req=https.request({method:'POST',host:'designer-laboratory-sandbox.myshopify.com',path:'/admin/api/2024-10/graphql.json',headers:{'X-Shopify-Access-Token':ATOK,'Content-Type':'application/json','Content-Length':Buffer.byteLength(q)}},r=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>res(d));});req.on('error',()=>res(''));req.write(q);req.end();});
+ const j=JSON.parse(vid); let types={};
+ for(const e of j.data.products.edges) for(const m of e.node.media.edges){const t=m.node.mediaContentType;types[t]=(types[t]||0)+1;}
+ videoNote=JSON.stringify(types)+' (from 200 most-recent products)';
+ }catch(e){videoNote='probe failed: '+String(e).slice(0,80);}
+
+ const headroom = CAP_GB*0.15;
+ const verdict = reclaim >= headroom ? 'GO' : (reclaim>=headroom*0.5?'PARTIAL — likely need purge/upgrade too':'NO-GO on compression alone — pursue purge + plan upgrade');
+
+ const md=`# Phase 0 — DW Shopify Storage Diagnosis
+_Generated from ground-truth Shopify enumeration (${rows.length} product images across ${new Set(rows.map(r=>r.product_id)).size} products)._
+
+## Measured product-image footprint (extrapolated from HEAD sampling)
+| dimension bucket | # images | mean size | est. total |
+|---|--:|--:|--:|
+${ORDER.map(k=>`| ${k} | ${est[k].pop.toLocaleString()} | ${est[k].meanKB.toFixed(0)} KB | ${est[k].totalGB.toFixed(2)} GB |`).join('\n')}
+| **animated GIF (exact)** | ${gifs.length.toLocaleString()} | ${(gifBytes.reduce((a,b)=>a+b,0)/1024/(gifBytes.length||1)).toFixed(0)} KB | ${GB(gifTotal).toFixed(2)} GB |
+| **TOTAL (product images)** | ${rows.length.toLocaleString()} | | **${GB(totalBytes).toFixed(1)} GB** |
+
+Calibration: on ${ratN} real oversized masters, re-encode (≤2048px, q82) → **${(ratio*100).toFixed(0)}%** of original size (≈${((1-ratio)*100).toFixed(0)}% shrink).
+
+## Reclaimable by compression (oversized-first scope)
+- >2048px downsize+quality: **${GB(reclaim).toFixed(1)} GB** estimated reclaimable.
+- Headroom target (~15% of ${CAP_GB} GB): **${headroom.toFixed(0)} GB**.
+
+## Unmeasured buckets (NOT compressible / need other action)
+- **Videos:** ${videoNote}. Videos aren't in the REST images inventory — if VIDEO count is material, they may be a big chunk needing separate handling.
+- **Files section** (theme assets, blog/email/app uploads, old themes): NOT API-queryable → **Steve must check Settings → Files** (sort by size) + Online Store → Themes for duplicate/unpublished themes.
+- These + GIFs (${GB(gifTotal).toFixed(2)} GB) are the non-JPEG buckets compression won't touch.
+
+## Status breakdown (purge-track candidates)
+${Object.entries(byStatus).map(([k,v])=>`- ${k}: ${v.toLocaleString()} images`).join('\n')}
+- Archived-product images = ${archivedImgs.toLocaleString()} → purge candidates (Steve-gated).
+
+## VERDICT: ${verdict}
+
+**Reasoning:** product images total ≈ **${GB(totalBytes).toFixed(0)} GB** of the ${CAP_GB} GB cap. If this is well under 500 GB, the compressible images are NOT what filled the cap — the bulk is videos, the Files section, or media the mirror/enumeration can't see — and **compression alone will not clear the cap**. In that case the real levers are: purge orphaned/archived media + oversized GIFs/videos, clean the Files section, and/or upgrade the plan. Compression still helps and runs, but the scratch-upload gate (Phase 4) is the arbiter.
+`;
+ fs.writeFileSync(DIR+'/phase0-report.md',md);
+ fs.writeFileSync(DIR+'/phase0-data.json',JSON.stringify({est,gifTotalGB:GB(gifTotal),totalGB:GB(totalBytes),reclaimGB:GB(reclaim),ratio,byStatus,verdict},null,2));
+ try{fs.rmSync(DIR+'/calib',{recursive:true,force:true});}catch(e){}
+ console.log('\n'+verdict+'\n → phase0-report.md');
+})();
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 0000000..8630598
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,162 @@
+{
+ "name": "dw-image-shrink",
+ "version": "1.0.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "dw-image-shrink",
+ "version": "1.0.0",
+ "license": "ISC",
+ "dependencies": {
+ "pg": "^8.22.0"
+ }
+ },
+ "node_modules/pg": {
+ "version": "8.22.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
+ "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-connection-string": "^2.14.0",
+ "pg-pool": "^3.14.0",
+ "pg-protocol": "^1.15.0",
+ "pg-types": "2.2.0",
+ "pgpass": "1.0.5"
+ },
+ "engines": {
+ "node": ">= 16.0.0"
+ },
+ "optionalDependencies": {
+ "pg-cloudflare": "^1.4.0"
+ },
+ "peerDependencies": {
+ "pg-native": ">=3.0.1"
+ },
+ "peerDependenciesMeta": {
+ "pg-native": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/pg-cloudflare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
+ "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
+ "license": "MIT",
+ "optional": true
+ },
+ "node_modules/pg-connection-string": {
+ "version": "2.14.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
+ "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
+ "license": "MIT"
+ },
+ "node_modules/pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+ "license": "ISC",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/pg-pool": {
+ "version": "3.14.0",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
+ "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
+ "license": "MIT",
+ "peerDependencies": {
+ "pg": ">=8.0"
+ }
+ },
+ "node_modules/pg-protocol": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
+ "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
+ "license": "MIT"
+ },
+ "node_modules/pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "license": "MIT",
+ "dependencies": {
+ "split2": "^4.1.0"
+ }
+ },
+ "node_modules/postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/postgres-bytea": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
+ "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "license": "MIT",
+ "dependencies": {
+ "xtend": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/split2": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+ "license": "ISC",
+ "engines": {
+ "node": ">= 10.x"
+ }
+ },
+ "node_modules/xtend": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4"
+ }
+ }
+ }
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..e2f5e3c
--- /dev/null
+++ b/package.json
@@ -0,0 +1,16 @@
+{
+ "name": "dw-image-shrink",
+ "version": "1.0.0",
+ "description": "",
+ "main": "enumerate.js",
+ "scripts": {
+ "test": "echo \"Error: no test specified\" && exit 1"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC",
+ "type": "commonjs",
+ "dependencies": {
+ "pg": "^8.22.0"
+ }
+}
diff --git a/shrink_master.py b/shrink_master.py
new file mode 100644
index 0000000..52132bc
--- /dev/null
+++ b/shrink_master.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Re-encode a product-image master to best-practice web size.
+Rules (see plan): longest edge <= MAX_EDGE (default 2048), JPEG q82, strip EXIF,
+sRGB. Guards: NEVER re-encode an animated GIF or video; keep true-alpha PNGs as
+optimized PNG (don't JPEG them); never upscale; report a 'no_win' if the encoded
+file isn't meaningfully smaller. Prints a JSON verdict on stdout.
+
+Usage: shrink_master.py --in <path> --out <path> [--max-edge 2048] [--quality 82]
+Exit 0 always; read the JSON {action, reason, in_bytes, out_bytes, w,h,new_w,new_h}.
+"""
+import argparse, json, os, sys
+from PIL import Image, ImageOps
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--in', dest='inp', required=True)
+ ap.add_argument('--out', dest='out', required=True)
+ ap.add_argument('--max-edge', type=int, default=2048)
+ ap.add_argument('--quality', type=int, default=82)
+ ap.add_argument('--min-win', type=float, default=0.90,
+ help='out must be < this * in_bytes to count as a win')
+ a = ap.parse_args()
+
+ in_bytes = os.path.getsize(a.inp)
+ try:
+ img = Image.open(a.inp)
+ except Exception as e:
+ return out({'action': 'skip', 'reason': 'unreadable:' + str(e)[:60], 'in_bytes': in_bytes})
+
+ fmt = (img.format or '').upper()
+ w, h = img.size
+
+ # GUARD: animated GIF/webp -> never destroy
+ if getattr(img, 'is_animated', False) or getattr(img, 'n_frames', 1) > 1:
+ return out({'action': 'skip', 'reason': 'animated', 'in_bytes': in_bytes, 'w': w, 'h': h})
+
+ # apply EXIF orientation then drop metadata
+ img = ImageOps.exif_transpose(img)
+ w, h = img.size
+
+ # detect true alpha (not just mode)
+ has_alpha = img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info)
+ real_alpha = False
+ if has_alpha:
+ alpha = img.convert('RGBA').getchannel('A')
+ real_alpha = alpha.getextrema()[0] < 255
+
+ longest = max(w, h)
+ scale = min(1.0, a.max_edge / longest) # never upscale
+ new_w, new_h = (round(w * scale), round(h * scale)) if scale < 1.0 else (w, h)
+ if scale < 1.0:
+ img = img.resize((new_w, new_h), Image.Resampling.LANCZOS)
+
+ try:
+ if real_alpha:
+ # keep PNG, optimize; strip metadata by re-saving without exif
+ img.convert('RGBA').save(a.out, format='PNG', optimize=True)
+ enc_fmt = 'PNG'
+ else:
+ img.convert('RGB').save(a.out, format='JPEG', quality=a.quality,
+ optimize=True, progressive=True)
+ enc_fmt = 'JPEG'
+ except Exception as e:
+ return out({'action': 'skip', 'reason': 'encode_fail:' + str(e)[:60], 'in_bytes': in_bytes, 'w': w, 'h': h})
+
+ out_bytes = os.path.getsize(a.out)
+ win = out_bytes < in_bytes * a.min_win
+ if not win:
+ try: os.remove(a.out)
+ except OSError: pass
+ return out({'action': 'skip', 'reason': 'no_win', 'in_bytes': in_bytes,
+ 'out_bytes': out_bytes, 'w': w, 'h': h})
+ return out({'action': 'encoded', 'reason': enc_fmt, 'in_bytes': in_bytes,
+ 'out_bytes': out_bytes, 'w': w, 'h': h, 'new_w': new_w, 'new_h': new_h})
+
+def out(d):
+ print(json.dumps(d)); sys.exit(0)
+
+if __name__ == '__main__':
+ main()
diff --git a/shrink_worker.js b/shrink_worker.js
new file mode 100644
index 0000000..92fa7d0
--- /dev/null
+++ b/shrink_worker.js
@@ -0,0 +1,116 @@
+// Core worker: atomic-claim a ledger row → GET product snapshot → download the
+// master → re-encode (shrink_master.py) → DELETE-first the old image → base64
+// upload the smaller one → verify → record. Keeps the ORIGINAL bytes in hand and
+// AUTO-RESTORES them if the new upload fails, so a product is never left
+// permanently imageless. Resumable + crash-safe (FOR UPDATE SKIP LOCKED claim).
+// node shrink_worker.js [--shard k] [--allshards] [--limit N] [--wid id] [--writedelay 500] [--dry]
+import fs from 'fs';
+import os from 'os';
+import https from 'https';
+import pg from 'pg';
+import { execFileSync } from 'child_process';
+const DIR='/Users/macstudio3/Projects/dw-image-shrink';
+const SHOP='designer-laboratory-sandbox.myshopify.com';
+const ATOK=fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8').split('\n').find(l=>l.startsWith('SHOPIFY_ADMIN_TOKEN=')).split('=').slice(1).join('=').replace(/["' ]/g,'');
+const arg=(k,d)=>{const a=process.argv.find(x=>x.startsWith('--'+k+'='));return a?a.split('=').slice(1).join('='):d;};
+const ALL=process.argv.includes('--allshards');
+const SHARD=+arg('shard','0');
+const LIMIT=+arg('limit','0')||1e9;
+const WID=arg('wid','w'+process.pid);
+const WRITEDELAY=+arg('writedelay','500'); // ms between Shopify WRITE calls (2/s at 500)
+const DRY=process.argv.includes('--dry');
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+
+// ---- Shopify REST with leaky-bucket backoff ----
+function raw(method,path,body){return new Promise((res,rej)=>{const data=body?JSON.stringify(body):null;const req=https.request({method,host:SHOP,path,headers:{'X-Shopify-Access-Token':ATOK,'Content-Type':'application/json',...(data?{'Content-Length':Buffer.byteLength(data)}:{})}},r=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>res({status:r.statusCode,retryAfter:r.headers['retry-after'],limit:r.headers['x-shopify-shop-api-call-limit']||'',json:(()=>{try{return JSON.parse(d)}catch(e){return d}})()}));});req.on('error',rej);req.setTimeout(60000,()=>{req.destroy(new Error('timeout'))});if(data)req.write(data);req.end();});}
+async function api(method,path,body){for(let a=0;a<10;a++){let r;try{r=await raw(method,path,body);}catch(e){await sleep(1500);continue;}if(r.status===429){await sleep((parseFloat(r.retryAfter)||3)*1000);continue;}const [u,m]=(r.limit||'0/40').split('/').map(Number);await sleep(u>(m*0.5)?800:200);return r;}return{status:429,json:{error:'ratelimited'}};}
+function download(url){return new Promise(res=>{try{const u=new URL(url);https.get({host:u.host,path:u.pathname+u.search,headers:{'User-Agent':'dw-shrink'}},r=>{if(r.statusCode!==200){r.resume();return res(null);}const ch=[];r.on('data',c=>ch.push(c));r.on('end',()=>res(Buffer.concat(ch)));r.on('error',()=>res(null));}).on('error',()=>res(null));}catch(e){res(null);}});}
+function head(url){return new Promise(res=>{try{const u=new URL(url);const req=https.request({method:'HEAD',host:u.host,path:u.pathname+u.search},r=>{r.resume();res(+(r.headers['content-length']||0));});req.on('error',()=>res(0));req.setTimeout(15000,()=>{req.destroy();res(0);});req.end();}catch(e){res(0);}});}
+
+const db=new pg.Client({host:'/tmp',database:'dw_unified'});
+await db.connect();
+
+async function claim(){
+ const where = ALL ? "state='pending'" : "shard=$1 AND state='pending'";
+ const params = ALL ? [] : [SHARD];
+ const q=`UPDATE img_shrink_ledger SET state='claimed', worker_id=$${params.length+1}, claimed_at=now(), attempts=attempts+1, updated_at=now()
+ WHERE id=(SELECT id FROM img_shrink_ledger WHERE ${where} ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`;
+ const r=await db.query(q,[...params,WID]); return r.rows[0]||null;
+}
+async function mark(id,fields){const keys=Object.keys(fields);const set=keys.map((k,i)=>`${k}=$${i+2}`).join(',');await db.query(`UPDATE img_shrink_ledger SET ${set}, updated_at=now() WHERE id=$1`,[id,...keys.map(k=>fields[k])]);}
+
+let done=0,skipped=0,failed=0,alerts=0,processed=0,reclaimed=0;
+const tmp=os.tmpdir();
+while(processed<LIMIT){
+ const row=await claim(); if(!row)break; processed++;
+ const pid=row.product_id, oid=Number(row.orig_image_id);
+ try{
+ // 1) snapshot current product images
+ const pr=await api('GET',`/admin/api/2024-10/products/${pid}.json?fields=id,handle,images`);
+ const imgs=(pr.json.product&&pr.json.product.images)||[];
+ const target=imgs.find(im=>im.id===oid);
+ if(!target){ await mark(row.id,{state:'skipped',skip_reason:'image_gone'}); skipped++; continue; }
+ const origCount=imgs.length, position=target.position, alt=target.alt||null, handle=pr.json.product.handle;
+ await mark(row.id,{product_snapshot:JSON.stringify(imgs),orig_position:position,orig_alt:alt});
+
+ // 2) download original master
+ const origBuf=await download(target.src);
+ if(!origBuf||!origBuf.length){ await mark(row.id,{state:'failed',fail_reason:'download'}); failed++; continue; }
+ const origBytes=origBuf.length;
+ const fin=`${tmp}/shr_${row.id}_in`, fout=`${tmp}/shr_${row.id}_out.jpg`;
+ fs.writeFileSync(fin,origBuf);
+
+ // 3) encode
+ let v; try{ v=JSON.parse(execFileSync('python3',[`${DIR}/shrink_master.py`,'--in',fin,'--out',fout,'--max-edge','2048','--quality','82'],{encoding:'utf8'})); }
+ catch(e){ v={action:'skip',reason:'py_err'}; }
+ if(v.action!=='encoded'){ await mark(row.id,{state:'skipped',skip_reason:v.reason||'no_encode',orig_bytes:origBytes}); skipped++; cleanup(fin,fout); continue; }
+ const encBuf=fs.readFileSync(fout); const newBytesLocal=encBuf.length;
+ const ext = v.reason==='PNG'?'png':'jpg';
+ const filename=`${(handle||'img').slice(0,60)}-${oid}.${ext}`;
+
+ if(DRY){ await mark(row.id,{state:'skipped',skip_reason:'dry',orig_bytes:origBytes,new_bytes:newBytesLocal}); skipped++; cleanup(fin,fout); continue; }
+
+ // 4) DELETE-FIRST (frees storage so the smaller upload won't 413)
+ await sleep(WRITEDELAY);
+ const del=await api('DELETE',`/admin/api/2024-10/products/${pid}/images/${oid}.json`);
+ if(del.status>=400){ await mark(row.id,{state:'failed',fail_reason:'delete_'+del.status,orig_bytes:origBytes}); failed++; cleanup(fin,fout); continue; }
+
+ // 5) UPLOAD smaller (base64 attachment — no external host needed)
+ await sleep(WRITEDELAY);
+ const up=await api('POST',`/admin/api/2024-10/products/${pid}/images.json`,{image:{attachment:encBuf.toString('base64'),filename,position,...(alt?{alt}:{})}});
+ if(up.status>=400||!up.json.image){
+ // RESTORE original — never leave imageless
+ await sleep(WRITEDELAY);
+ const rs=await api('POST',`/admin/api/2024-10/products/${pid}/images.json`,{image:{attachment:origBuf.toString('base64'),filename:`restore-${oid}.jpg`,position,...(alt?{alt}:{})}});
+ if(rs.status>=400||!rs.json.image){ await mark(row.id,{state:'imageless_alert',fail_reason:'upload+restore_failed',orig_bytes:origBytes}); alerts++; }
+ else { await mark(row.id,{state:'failed',fail_reason:'upload_'+up.status+'_restored',orig_bytes:origBytes}); failed++; }
+ cleanup(fin,fout); continue;
+ }
+ const ni=up.json.image;
+
+ // 6) verify: new bytes < orig, dims ok, product image count unchanged
+ const nb=await head(ni.src)||newBytesLocal;
+ const pr2=await api('GET',`/admin/api/2024-10/products/${pid}.json?fields=images`);
+ const count2=((pr2.json.product&&pr2.json.product.images)||[]).length;
+ const ok = nb>0 && nb<origBytes && (ni.width||0)<=2048 && count2===origCount;
+ if(ok){
+ await mark(row.id,{state:'done',new_image_id:ni.id,new_src:ni.src,new_bytes:nb,new_width:ni.width,new_height:ni.height,orig_bytes:origBytes,fail_reason:null});
+ done++; reclaimed+=(origBytes-nb);
+ } else {
+ // regression → restore original, remove the new one
+ await sleep(WRITEDELAY);
+ await api('POST',`/admin/api/2024-10/products/${pid}/images.json`,{image:{attachment:origBuf.toString('base64'),filename:`restore-${oid}.jpg`,position,...(alt?{alt}:{})}});
+ await sleep(WRITEDELAY);
+ await api('DELETE',`/admin/api/2024-10/products/${pid}/images/${ni.id}.json`);
+ await mark(row.id,{state:'failed',fail_reason:`verify(nb=${nb},w=${ni.width},cnt=${count2}/${origCount})`,orig_bytes:origBytes});
+ failed++;
+ }
+ cleanup(fin,fout);
+ }catch(e){
+ await mark(row.id,{state:'failed',fail_reason:('exc:'+String(e.message||e)).slice(0,120)}); failed++;
+ }
+ if(processed%25===0) console.log(`[${WID}] processed ${processed} | done ${done} skip ${skipped} fail ${failed} alert ${alerts} | reclaimed ${(reclaimed/1048576).toFixed(1)}MB`);
+}
+function cleanup(...f){for(const x of f){try{fs.unlinkSync(x)}catch(e){}}}
+console.log(`[${WID}] FINISHED processed=${processed} done=${done} skipped=${skipped} failed=${failed} imageless_alert=${alerts} reclaimed=${(reclaimed/1048576).toFixed(1)}MB`);
+await db.end();
← d2f7cab auto-save: 2026-07-29T07:37:00 (2 files) — .gitignore enumer
·
back to Dw Image Shrink
·
Add reaper + phase4 verify; encoder validated (3290px 1.2MB→ ef8a15e →