`pull-and-classify.mjs`

80 lines
// TK-10640 — READ-ONLY pull of ALL Shopify url redirects + structural classification.
// Pulls every redirect once (paginated), dumps to data/redirects-dump.jsonl, then
// classifies prune candidates that need NO per-product call:
//   self         : path === target (useless self-redirect)
//   empty_target : target empty or "/" (sends to homepage — usually junk)
//   chain_source : target is itself the PATH of another redirect (redirect chain; Shopify
//                  won't hop twice, so the intermediate is dead weight)
//   dup_target   : >1 redirect whose path differs but all point at same target (informational)
// Dead-target (/products/<handle> gone) is done separately by prune-redirects.mjs.
// NO WRITES. Uses SHOPIFY_FULL_ACCESS_TOKEN (only token with content read scope).
import { readFileSync, writeFileSync, appendFileSync } from 'node:fs';
const txt = readFileSync(`${process.env.HOME}/Projects/secrets-manager/.env`,'utf8');
const env={}; for (const l of txt.split('\n')){const m=l.match(/^([A-Z0-9_]+)=(.*)$/); if(m) env[m[1]]=m[2].replace(/^["']|["']$/g,'');}
const STORE=env.SHOPIFY_STORE_DOMAIN, TOKEN=env.SHOPIFY_FULL_ACCESS_TOKEN;
const API='2024-10';
const sleep=ms=>new Promise(r=>setTimeout(r,ms));
async function gql(query,variables={}){
  for(let a=0;;a++){
    const res=await fetch(`https://${STORE}/admin/api/${API}/graphql.json`,{method:'POST',
      headers:{'Content-Type':'application/json','X-Shopify-Access-Token':TOKEN},
      body:JSON.stringify({query,variables})});
    const j=await res.json();
    if(j.errors){ if(/THROTTLED/i.test(JSON.stringify(j.errors))&&a<8){await sleep(2000*(a+1));continue;} throw new Error(JSON.stringify(j.errors)); }
    return j.data;
  }
}
const DUMP=`${process.env.HOME}/Projects/tk10640-redirect-prune/data/redirects-dump.jsonl`;
writeFileSync(DUMP,'');
let cursor=null, n=0;
const all=[];
while(true){
  const d=await gql(`query($c:String){ urlRedirects(first:250, after:$c){ pageInfo{ hasNextPage endCursor } nodes{ id path target } } }`,{c:cursor});
  const buf=[];
  for(const r of d.urlRedirects.nodes){ all.push(r); buf.push(JSON.stringify(r)); n++; }
  appendFileSync(DUMP, buf.join('\n')+'\n');
  if(n%5000===0) process.stderr.write(`  pulled ${n}\n`);
  if(!d.urlRedirects.pageInfo.hasNextPage) break;
  cursor=d.urlRedirects.pageInfo.endCursor;
}
process.stderr.write(`pulled ${n} total\n`);
// classify
const pathSet=new Set(all.map(r=>r.path));
const norm=s=>(s||'').split('?')[0].split('#')[0];
const targetCount=new Map();
for(const r of all){ const t=norm(r.target); targetCount.set(t,(targetCount.get(t)||0)+1); }
const cat={self:[],empty_target:[],chain_source:[],};
const targetKind={products:0,collections:0,pages:0,blogs:0,other_internal:0,external:0,root_or_empty:0};
for(const r of all){
  const t=norm(r.target);
  if(!t||t==='/') { cat.empty_target.push(r); targetKind.root_or_empty++; continue; }
  if(r.path===r.target){ cat.self.push(r); }
  if(pathSet.has(t) && t!==r.path){ cat.chain_source.push(r); }
  if(/^\/products\//.test(t)) targetKind.products++;
  else if(/^\/collections\//.test(t)) targetKind.collections++;
  else if(/^\/pages\//.test(t)) targetKind.pages++;
  else if(/^\/blogs\//.test(t)) targetKind.blogs++;
  else if(/^https?:\/\//.test(t)) targetKind.external++;
  else if(/^\//.test(t)) targetKind.other_internal++;
}
const dupTargets=[...targetCount.entries()].filter(([,c])=>c>1);
const dupRedirectCount=dupTargets.reduce((s,[,c])=>s+c,0);
const summary={
  total:n,
  target_kind:targetKind,
  prune_candidates_no_apicall:{
    self_redirect:cat.self.length,
    empty_or_root_target:cat.empty_target.length,
    chain_source_intermediate:cat.chain_source.length,
  },
  duplicate_target_groups:dupTargets.length,
  redirects_sharing_a_target:dupRedirectCount,
  note:'dead-target(/products gone) counted separately by prune-redirects.mjs',
};
writeFileSync(`${process.env.HOME}/Projects/tk10640-redirect-prune/data/classify-summary.json`, JSON.stringify(summary,null,2));
// sample dumps for the memo
writeFileSync(`${process.env.HOME}/Projects/tk10640-redirect-prune/data/cand-self.jsonl`, cat.self.slice(0,200).map(r=>JSON.stringify(r)).join('\n'));
writeFileSync(`${process.env.HOME}/Projects/tk10640-redirect-prune/data/cand-empty.jsonl`, cat.empty_target.slice(0,200).map(r=>JSON.stringify(r)).join('\n'));
writeFileSync(`${process.env.HOME}/Projects/tk10640-redirect-prune/data/cand-chain.jsonl`, cat.chain_source.slice(0,200).map(r=>JSON.stringify(r)).join('\n'));
console.log(JSON.stringify(summary,null,2));