`scripts/ai-preamble-leak-sweep/ai-preamble-leak-sweep.mjs`

65 lines
// ai-preamble-leak-sweep — READ-ONLY, $0. Cycle 91. Finds the genuine customer-facing defect: LLM
// scaffolding/preamble leaked into live product descriptions ("Here is a description...", "Certainly!
// Here's...", "As an AI...", "I hope this helps", "Below is a...") = the body-text analogue of a blank PDP.
// Tiered patterns (HIGH-confidence vs WEAK) to control false positives; captures a snippet per hit for
// human verification. ALSO counts the live "Wallpaper" title blast-radius as CONTEXT (NOT a defect, NOT
// reopening GMC). NEGATIVE CONTROL: the bulk of bodies have no preamble -> a leak is a true anomaly. EXCLUDES PJ.
import fs from 'fs';
const ENV=fs.readFileSync('/Users/stevestudio2/Projects/secrets-manager/.env','utf8');
const TOK=((ENV.match(/^SHOPIFY_ADMIN_TOKEN=(.+)$/m)||[])[1]||'').replace(/['"\r]/g,'').trim();
const GQL='https://designer-laboratory-sandbox.myshopify.com/admin/api/2024-10/graphql.json';
const PJ=/phillip[- ]?jeffries/i;
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function gql(q,v){for(let a=0;a<6;a++){const r=await fetch(GQL,{method:'POST',headers:{'X-Shopify-Access-Token':TOK,'Content-Type':'application/json'},body:JSON.stringify({query:q,variables:v})});if(r.status===429||r.status>=500){await sleep(2000*(a+1));continue;}const j=await r.json();if(j.errors&&JSON.stringify(j.errors).includes('Throttled')){await sleep(2500*(a+1));continue;}return j;}return null;}
// HIGH-confidence LLM-preamble patterns (low false-positive)
const HIGH=[
  /\bas an ai\b/i,
  /\bas a language model\b/i,
  /\bi(?:'m| am) (?:sorry|unable|just an ai|happy to (?:help|provide|assist))/i,
  /\bi (?:cannot|can(?:'|no)t) (?:assist|help|provide|fulfill|generate)/i,
  /\bcertainly[!,.]?\s+(?:here|i|below)/i,
  /\bsure[!,]\s+(?:here|i|below)/i,
  /\bhere(?:'s| is) (?:a|an|the|your) (?:description|revised|rewritten|product description|seo|updated|enhanced)/i,
  /\bhere(?:'s| is) (?:a|an|the) (?:possible|sample|suggested)/i,
  /\bbelow is (?:a|an|the) (?:description|revised|rewritten|product)/i,
  /\bi hope this helps\b/i,
  /\b(?:revised|rewritten) (?:product )?description:/i,
  /\bas requested,?\s+(?:here|i|below)/i,
  /\[insert [a-z ]+\]/i,
];
// WEAK patterns (flag separately; higher false-positive)
const WEAK=[ /\bhere(?:'s| is) (?:a|an|the|your)\b/i, /\blet me know if\b/i, /\bfeel free to\b/i ];
const stripHtml=h=>(h||'').replace(/<[^>]+>/g,' ').replace(/&[a-z]+;/g,' ').replace(/\s+/g,' ').trim();
const Q=`query($cursor:String){ products(first:60, after:$cursor, query:"status:active"){ pageInfo{hasNextPage endCursor}
  edges{node{ title vendor descriptionHtml } } } }`;
let cursor=null,pages=0,scanned=0,complete=true;
let high=0, weakOnly=0, wallpaperTitle=0, emptyBody=0;
const highByVendor={}, highEx=[];
while(true){ const j=await gql(Q,{cursor}); if(!j?.data){complete=false;break;} const c=j.data.products;
  for(const e of c.edges){ const n=e.node; if(PJ.test(n.vendor||''))continue; scanned++;
    if(/wallpaper/i.test(n.title||'')) wallpaperTitle++;
    const body=stripHtml(n.descriptionHtml);
    if(!body){ emptyBody++; continue; }
    const hi=HIGH.find(rx=>rx.test(body));
    if(hi){ high++; highByVendor[n.vendor]=(highByVendor[n.vendor]||0)+1;
      if(highEx.length<30){ const m=body.match(hi); const at=Math.max(0,(m.index||0)-20); highEx.push({t:(n.title||'').slice(0,40),vendor:n.vendor,pat:hi.source.slice(0,30),snip:body.slice(at,at+90)}); } }
    else if(WEAK.some(rx=>rx.test(body))) weakOnly++;
  }
  pages++; if(!c.pageInfo.hasNextPage)break; cursor=c.pageInfo.endCursor;
  const cost=j.extensions?.cost?.throttleStatus; if(cost&&cost.currentlyAvailable<500) await sleep(1500); else await sleep(250);
  if(pages%100===0) console.log(`  ${pages}p scanned=${scanned} high-preamble=${high}`);
}
const pct=x=>scanned?(100*x/scanned).toFixed(3):'0';
console.log(`\n=== ai-preamble-leak-sweep (READ-ONLY, $0) ===`);
console.log(`active scanned: ${scanned} (complete=${complete}, ${pages}p) [PJ excluded]`);
console.log(`\nAI-PREAMBLE LEAK (the defect):`);
console.log(`  HIGH-confidence preamble in live body: ${high} (${pct(high)}%)`);
console.log(`  WEAK-only (needs human review, higher false-positive): ${weakOnly} (${pct(weakOnly)}%)`);
console.log(`  empty/no body: ${emptyBody}`);
console.log(`  NEGATIVE CONTROL: ${scanned-high-weakOnly} bodies with no preamble -> a HIGH hit is a true anomaly.`);
console.log(`\nCONTEXT (NOT a defect, NOT GMC): live titles containing 'Wallpaper': ${wallpaperTitle} (${pct(wallpaperTitle)}%) [brand-hygiene debt, Steve-gated bulk rewrite]`);
console.log(`\nHIGH-preamble by vendor (top 15):`); Object.entries(highByVendor).sort((a,b)=>b[1]-a[1]).slice(0,15).forEach(([v,n])=>console.log(`  ${v}: ${n}`));
console.log(`\nHIGH-preamble examples (snippet for verification):`); highEx.forEach(x=>console.log(`  "${x.t}" [${x.vendor}] /${x.pat}/ => "...${x.snip}..."`));
fs.writeFileSync('/tmp/ai-preamble-leak-sweep.json',JSON.stringify({ts:new Date().toISOString(),scanned,complete,high,weakOnly,emptyBody,wallpaperTitle,highByVendor,highEx},null,2));
console.log(`\nwrote /tmp/ai-preamble-leak-sweep.json`);