Add reaper + phase4 verify; encoder validated (3290px 1.2MB→2048px 533KB)

ef8a15ee13980b9a9dbb6dedbbca179fe25a3b85 · 2026-07-29 07:44:19 -0700 · steve
Files touched

A phase4-verify.js
A reaper.js
Diff

commit ef8a15ee13980b9a9dbb6dedbbca179fe25a3b85
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Jul 29 07:44:19 2026 -0700

    Add reaper + phase4 verify; encoder validated (3290px 1.2MB→2048px 533KB)
---
 phase4-verify.js | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 reaper.js        | 17 +++++++++++++++++
 2 files changed, 69 insertions(+)

diff --git a/phase4-verify.js b/phase4-verify.js
new file mode 100644
index 0000000..e44ad89
--- /dev/null
+++ b/phase4-verify.js
@@ -0,0 +1,52 @@
+// Phase 4 — verification + the DEFINITIVE cap gate.
+// (1) Ledger rollups: counts by state + total bytes reclaimed.
+// (2) Scratch upload gate: create/reuse a DRAFT "ZZ Storage Probe" product, POST
+//     a real ~800px test JPEG, and read the status. 200 = cap CLEARED (uploads
+//     resume). 413 = still AT CAP. Deletes the test image after (keeps the probe
+//     product for reuse). Pass --gate-only to skip rollups.
+import fs from 'fs';
+import os from 'os';
+import https from 'https';
+import pg from 'pg';
+import { execFileSync } from 'child_process';
+const SHOP='designer-laboratory-sandbox.myshopify.com';
+const ATOK=fs.readFileSync('/Users/macstudio3/Projects/secrets-manager/.env','utf8').split('\n').find(l=>l.startsWith('SHOPIFY_ADMIN_TOKEN=')).split('=').slice(1).join('=').replace(/["' ]/g,'');
+const sleep=ms=>new Promise(r=>setTimeout(r,ms));
+function raw(method,path,body){return new Promise((res,rej)=>{const data=body?JSON.stringify(body):null;const req=https.request({method,host:SHOP,path,headers:{'X-Shopify-Access-Token':ATOK,'Content-Type':'application/json',...(data?{'Content-Length':Buffer.byteLength(data)}:{})}},r=>{let d='';r.on('data',c=>d+=c);r.on('end',()=>res({status:r.statusCode,json:(()=>{try{return JSON.parse(d)}catch(e){return d}})()}));});req.on('error',rej);if(data)req.write(data);req.end();});}
+async function api(m,p,b){for(let a=0;a<6;a++){const r=await raw(m,p,b);if(r.status===429){await sleep(3000);continue;}return r;}return{status:429,json:{}};}
+
+if(!process.argv.includes('--gate-only')){
+  const db=new pg.Client({host:'/tmp',database:'dw_unified'}); await db.connect();
+  const {rows}=await db.query("SELECT state,count(*) n, sum(coalesce(orig_bytes,0)-coalesce(new_bytes,0)) FILTER(WHERE state='done') recl FROM img_shrink_ledger GROUP BY state ORDER BY state");
+  console.log('=== LEDGER ROLLUP ===');
+  let recl=0; for(const r of rows){ console.log(`  ${r.state.padEnd(16)} ${r.n}`); if(r.recl) recl=+r.recl; }
+  console.log(`  reclaimed(done): ${(recl/1073741824).toFixed(2)} GB`);
+  const {rows:[a]}=await db.query("SELECT count(*) FILTER(WHERE state='imageless_alert') n FROM img_shrink_ledger");
+  if(+a.n>0) console.log(`  ⚠️ imageless_alert rows: ${a.n} (products left imageless — investigate!)`);
+  await db.end();
+}
+
+// ---- scratch upload gate ----
+// generate a real ~800x800 test JPEG
+const fp=`${os.tmpdir()}/dw_storage_probe.jpg`;
+execFileSync('python3',['-c',`from PIL import Image; import random; im=Image.new('RGB',(800,800),(120,120,120)); [im.putpixel((random.randint(0,799),random.randint(0,799)),(random.randint(0,255),0,0)) for _ in range(4000)]; im.save('${fp}','JPEG',quality=82)`]);
+const b64=fs.readFileSync(fp).toString('base64');
+
+// find or create the probe product
+let probe;
+const search=await api('GET','/admin/api/2024-10/products.json?limit=1&title=ZZ%20Storage%20Probe&fields=id,title');
+if(search.json.products&&search.json.products.length){ probe=search.json.products[0].id; }
+else{ const c=await api('POST','/admin/api/2024-10/products.json',{product:{title:'ZZ Storage Probe',status:'draft',published:false}}); probe=c.json.product&&c.json.product.id; }
+if(!probe){ console.log('GATE: could not create/find probe product'); process.exit(1); }
+
+const up=await api('POST',`/admin/api/2024-10/products/${probe}/images.json`,{image:{attachment:b64,filename:'storage-probe.jpg'}});
+console.log('\n=== CAP GATE (scratch upload) ===');
+if(up.status<300 && up.json.image){
+  console.log('✅ 200 — UPLOAD SUCCEEDED → cap is CLEARED, uploads have resumed.');
+  await api('DELETE',`/admin/api/2024-10/products/${probe}/images/${up.json.image.id}.json`); // clean up test image
+  process.exit(0);
+}else{
+  const msg=JSON.stringify(up.json).slice(0,160);
+  console.log(`❌ ${up.status} — UPLOAD FAILED → still AT CAP. ${msg}`);
+  process.exit(2);
+}
diff --git a/reaper.js b/reaper.js
new file mode 100644
index 0000000..ac30b4b
--- /dev/null
+++ b/reaper.js
@@ -0,0 +1,17 @@
+// Crash-recovery reaper: reset rows stuck in 'claimed' longer than STALE minutes
+// back to 'pending' so another worker re-processes them. Safe to run on a loop
+// alongside the workers.  node reaper.js [--stale 15] [--loop]
+import pg from 'pg';
+const arg=(k,d)=>{const a=process.argv.find(x=>x.startsWith('--'+k+'='));return a?a.split('=').slice(1).join('='):d;};
+const STALE=+arg('stale','15');
+const LOOP=process.argv.includes('--loop');
+const db=new pg.Client({host:'/tmp',database:'dw_unified'});
+await db.connect();
+async function reap(){
+  const r=await db.query(`UPDATE img_shrink_ledger SET state='pending', worker_id=null, updated_at=now()
+    WHERE state='claimed' AND claimed_at < now() - ($1 || ' minutes')::interval RETURNING id`,[STALE]);
+  if(r.rowCount) console.log(new Date().toISOString(),'reaped',r.rowCount,'stale claimed rows');
+  return r.rowCount;
+}
+if(LOOP){ while(true){ await reap(); await new Promise(r=>setTimeout(r,60000)); } }
+else { await reap(); await db.end(); }
← c9958b6 Image-shrink campaign scaffold: enumerate + measure + encode · back to Dw Image Shrink · HALT: delete-first fails at cap (async GC); hold 197 rows; a d25c69d →