`full-scan-classify.mjs`

68 lines
// TK-10653 — AUTHORITATIVE full both-ends scan of ALL product-target redirects.
// Liveness-checks every UNIQUE handle (union of sources + targets) ONCE via productByHandle
// (ACTIVE + onlineStoreUrl == live storefront landing, matches verified storefront behavior),
// caches results, then classifies every /products/ redirect:
//   source LIVE                 -> INERT (product served; redirect ignored)   KEEP
//   source DEAD + target LIVE   -> GOOD migration (301 -> live product)        KEEP
//   source DEAD + target DEAD   -> BROKEN (fires -> 404)                       FIX (repoint or prune)
// Emits data/full-scan/{handle-liveness.json, broken-redirects.jsonl, classify-exact.json}.
// READ-ONLY. No writes to Shopify.
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
const HOME=process.env.HOME;
const txt=readFileSync(`${HOME}/Projects/secrets-manager/.env`,'utf8');
const env={}; for(const l of txt.split('\n')){const m=l.match(/^([A-Z0-9_]+)=(.*)$/); if(m) env[m[1]]=m[2].replace(/^["']|["']$/g,'');}
const STORE=env.SHOPIFY_STORE_DOMAIN||'designer-laboratory-sandbox.myshopify.com', TOKEN=env.SHOPIFY_FULL_ACCESS_TOKEN, API='2024-10';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function gql(q,v={}){for(let a=0;a<12;a++){try{
  const res=await fetch(`https://${STORE}/admin/api/${API}/graphql.json`,{method:'POST',headers:{'Content-Type':'application/json','X-Shopify-Access-Token':TOKEN},body:JSON.stringify({query:q,variables:v})});
  const tx=await res.text();let j;try{j=JSON.parse(tx);}catch{await sleep(1200*(a+1));continue;}
  if(j.errors){if(/THROTTLED/i.test(JSON.stringify(j.errors))){await sleep(1200*(a+1));continue;}throw new Error(JSON.stringify(j.errors));}
  return j.data;
}catch(e){if(a<11){await sleep(1200*(a+1));continue;}throw e;}}}
const norm=s=>(s||'').split('?')[0].split('#')[0];
mkdirSync('data/full-scan',{recursive:true});
const CACHE='data/full-scan/handle-liveness.json';
const cache= existsSync(CACHE)? JSON.parse(readFileSync(CACHE,'utf8')) : {};

// build work set
const rows=readFileSync('data/redirects-dump.jsonl','utf8').trim().split('\n').map(JSON.parse)
  .filter(r=>norm(r.target).startsWith('/products/'));
const handles=new Set();
for(const r of rows){
  handles.add(norm(r.target).replace('/products/',''));
  const p=norm(r.path); if(p.startsWith('/products/')) handles.add(p.replace('/products/',''));
}
const todo=[...handles].filter(h=>!(h in cache));
console.error(`total redirects=${rows.length} unique handles=${handles.size} cached=${handles.size-todo.length} todo=${todo.length}`);

// liveness: query 20 handles at once via aliased productByHandle
let done=0;
for(let i=0;i<todo.length;i+=20){
  const batch=todo.slice(i,i+20);
  const q='query{'+batch.map((h,k)=>`p${k}:productByHandle(handle:${JSON.stringify(h)}){status onlineStoreUrl}`).join(' ')+'}';
  let d; try{ d=await gql(q); }catch(e){ // fall back to singles on a batch error
    d={}; for(let k=0;k<batch.length;k++){ try{const dd=await gql(`query($h:String!){productByHandle(handle:$h){status onlineStoreUrl}}`,{h:batch[k]}); d[`p${k}`]=dd.productByHandle;}catch{ d[`p${k}`]=undefined; } }
  }
  batch.forEach((h,k)=>{ const p=d[`p${k}`]; cache[h]= !!(p&&p.status==='ACTIVE'&&p.onlineStoreUrl); });
  done+=batch.length;
  if(i%2000===0 || done>=todo.length){ writeFileSync(CACHE,JSON.stringify(cache)); process.stderr.write(`  liveness ${done}/${todo.length}\n`); }
}
writeFileSync(CACHE,JSON.stringify(cache));

// classify every redirect
let inert=0, good=0, broken=0; const brokenRows=[];
for(const r of rows){
  const p=norm(r.path), th=norm(r.target).replace('/products/','');
  const sh= p.startsWith('/products/')? p.replace('/products/','') : null;
  const srcLive= sh? cache[sh]===true : false;
  if(srcLive){ inert++; continue; }
  const tgtLive= cache[th]===true;
  if(tgtLive){ good++; } else { broken++; brokenRows.push({id:r.id,path:r.path,target:r.target}); }
}
writeFileSync('data/full-scan/broken-redirects.jsonl', brokenRows.map(x=>JSON.stringify(x)).join('\n')+'\n');
const out={ total_product_target:rows.length, unique_handles:handles.size,
  INERT_source_live:inert, GOOD_migration:good, BROKEN_fires_to_404:broken,
  broken_rate:+(broken/rows.length).toFixed(4) };
writeFileSync('data/full-scan/classify-exact.json', JSON.stringify(out,null,2));
console.log(JSON.stringify(out,null,2));