← back to Dw Signup Fulfillment
TK-11366: store-wide signal survives cart-mix control - but its direction of harm flips
f050f30d9a12ea8d603bc78e598c4c2d2c3c49a4 · 2026-09-10 10:51:13 -0700 · Steve Abrams
Banded non-sample US orders by subtotal and compared free-shipping share WITHIN each
band, before vs after 2026-08-18 (the raw 22%%->0%% was untrustworthy because median
subtotal moved $294.57 -> $470.60):
$0-100 81%% -> 25%% (-56 pts)
$100-300 41%% -> 0%% (-41 pts)
$300-1000 29%% -> 0%% (-29 pts)
$1000+ 15%% -> 0%% (-15 pts)
Present in every band, so not composition. 'Free Shipping (No Tracking)' appeared on
57 non-sample orders pre, 3 post.
REFRAME: that rate is the SAMPLE-scoped rate. Finding it on full-price orders - 15%% of
$1000+ wallcovering orders - means a sample rate was reaching full-price carts and DW
was eating the freight. So Aug-18 may be a leak in DW's FAVOR being closed, not a
customer-harm regression. Explicitly NOT claiming a second customer-harm incident.
TK-10836 closed (Kelly no-contact, nothing pending). TK-11366 re-scoped from blocked to
doing, with one narrow question: was full-price free shipping intended or leaked?
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Files touched
A verification/tk10836/verify-storewide-signal.js
Diff
commit f050f30d9a12ea8d603bc78e598c4c2d2c3c49a4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 10:51:13 2026 -0700
TK-11366: store-wide signal survives cart-mix control - but its direction of harm flips
Banded non-sample US orders by subtotal and compared free-shipping share WITHIN each
band, before vs after 2026-08-18 (the raw 22%%->0%% was untrustworthy because median
subtotal moved $294.57 -> $470.60):
$0-100 81%% -> 25%% (-56 pts)
$100-300 41%% -> 0%% (-41 pts)
$300-1000 29%% -> 0%% (-29 pts)
$1000+ 15%% -> 0%% (-15 pts)
Present in every band, so not composition. 'Free Shipping (No Tracking)' appeared on
57 non-sample orders pre, 3 post.
REFRAME: that rate is the SAMPLE-scoped rate. Finding it on full-price orders - 15%% of
$1000+ wallcovering orders - means a sample rate was reaching full-price carts and DW
was eating the freight. So Aug-18 may be a leak in DW's FAVOR being closed, not a
customer-harm regression. Explicitly NOT claiming a second customer-harm incident.
TK-10836 closed (Kelly no-contact, nothing pending). TK-11366 re-scoped from blocked to
doing, with one narrow question: was full-price free shipping intended or leaked?
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
verification/tk10836/verify-storewide-signal.js | 55 +++++++++++++++++++++++++
1 file changed, 55 insertions(+)
diff --git a/verification/tk10836/verify-storewide-signal.js b/verification/tk10836/verify-storewide-signal.js
new file mode 100644
index 0000000..9aa9dbd
--- /dev/null
+++ b/verification/tk10836/verify-storewide-signal.js
@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+// TK-11366: is the NON-SAMPLE "store-wide" signal real, or a cart-mix artifact?
+// The raw 22%->0% number is untrustworthy because full-price carts legitimately pay shipping and
+// composition can differ between windows. CONTROL FOR IT: bucket non-sample US orders by subtotal
+// band and compare the free-shipping share WITHIN each band, before vs after 2026-08-18.
+const fs=require('fs'),os=require('os');
+for (const line of fs.readFileSync(os.homedir()+'/Projects/secrets-manager/.env','utf8').split('\n')) {
+ const m=line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)$/); if(!m) continue;
+ let v=m[2].trim().replace(/^['"]|['"]$/g,''); if(!process.env[m[1]]) process.env[m[1]]=v;
+}
+const STORE=process.env.SHOPIFY_STORE_DOMAIN, TOKEN=process.env.SHOPIFY_FULL_ACCESS_TOKEN, API='2024-10';
+async function gql(q,v={}){const r=await fetch(`https://${STORE}/admin/api/${API}/graphql.json`,{method:'POST',
+ headers:{'X-Shopify-Access-Token':TOKEN,'Content-Type':'application/json'},body:JSON.stringify({query:q,variables:v})});
+ const j=await r.json(); if(j.errors)console.error('ERR',JSON.stringify(j.errors).slice(0,300)); return j.data;}
+const Q=`query($c:String){ orders(first:40, after:$c, query:"created_at:>2026-06-01", sortKey:CREATED_AT){
+ pageInfo{hasNextPage endCursor}
+ nodes{ name createdAt
+ subtotalPriceSet{shopMoney{amount}}
+ totalShippingPriceSet{shopMoney{amount}}
+ shippingLine{ title source }
+ shippingAddress{countryCodeV2}
+ lineItems(first:30){ nodes{ sku title quantity } } } } }`;
+(async()=>{
+ let c=null, rows=[];
+ do{ const d=await gql(Q,{c}); if(!d) break;
+ for(const o of d.orders.nodes){
+ const lis=o.lineItems.nodes; if(!lis.length) continue;
+ if(o.shippingAddress?.countryCodeV2!=='US') continue;
+ const isS=li=>/(^|-)sample/i.test(li.sku||'')||/sample|memo|swatch/i.test(li.title||'');
+ if(!lis.every(li=>!isS(li))) continue; // NON-sample only
+ rows.push({date:o.createdAt.slice(0,10),
+ sub:Number(o.subtotalPriceSet?.shopMoney?.amount||0),
+ ship:Number(o.totalShippingPriceSet?.shopMoney?.amount||0),
+ rate:o.shippingLine?.title||'(none)', src:o.shippingLine?.source||'(none)'});
+ } c=d.orders.pageInfo.hasNextPage?d.orders.pageInfo.endCursor:null; }while(c);
+ const BANDS=[[0,100],[100,300],[300,1000],[1000,1e9]];
+ const cut='2026-08-18';
+ console.log(`NON-sample US ORDERS since 2026-06-01: n=${rows.length}`);
+ console.log(`\n### free-shipping share WITHIN each subtotal band (controls for cart mix) ###`);
+ console.log(` band pre free% post free% delta`);
+ for(const [lo,hi] of BANDS){
+ const inb=r=>r.sub>=lo&&r.sub<hi;
+ const pre=rows.filter(r=>inb(r)&&r.date<cut), post=rows.filter(r=>inb(r)&&r.date>=cut);
+ const f=a=>a.length?100*a.filter(r=>r.ship===0).length/a.length:null;
+ const fp=f(pre), fq=f(post);
+ const lbl=hi>1e8?`$${lo}+`:`$${lo}-${hi}`;
+ console.log(` ${lbl.padEnd(16)} n=${String(pre.length).padStart(4)} ${fp===null?' - ':fp.toFixed(0).padStart(4)+'%'} n=${String(post.length).padStart(4)} ${fq===null?' - ':fq.toFixed(0).padStart(4)+'%'} ${fp!==null&&fq!==null?(fq-fp>=0?'+':'')+(fq-fp).toFixed(0)+' pts':'n/a'}`);
+ }
+ const C=(a,k)=>{const m={};for(const r of a)m[r[k]]=(m[r[k]]||0)+1;return Object.fromEntries(Object.entries(m).sort((x,y)=>y[1]-x[1]).slice(0,6));};
+ console.log(`\n### shipping rate mix, non-sample orders ###`);
+ console.log(' PRE :', JSON.stringify(C(rows.filter(r=>r.date<cut),'rate')));
+ console.log(' POST:', JSON.stringify(C(rows.filter(r=>r.date>=cut),'rate')));
+ const med=a=>{a=[...a].sort((x,y)=>x-y);return a.length?a[Math.floor(a.length/2)]:null;};
+ console.log(`\n median SUBTOTAL pre $${med(rows.filter(r=>r.date<cut).map(r=>r.sub))} post $${med(rows.filter(r=>r.date>=cut).map(r=>r.sub))} <-- if this moved, mix changed`);
+})();
← a4afa27 TK: cap guarded auto-approve at 50/day, falling back to the
·
back to Dw Signup Fulfillment
·
TK-10836: park the Kelly remedy — Steve: stop sending to Kel b3b8acb →