`scripts/finalize_morris_manifest.mjs`

76 lines
#!/usr/bin/env node
// TK-10881 — consolidate ALL harvested Morris rows, apply robust title-parse QA, HTTP-200 validate
// every image ($0, media CDN un-challenged), stage the final manifest-morris.json + coverage report.
import https from 'https'; import fs from 'fs';
const DIR = new URL('..', import.meta.url).pathname;
const P = f => `${DIR}pilot/${f}`;
const portal = {};
for (const l of fs.readFileSync(P('morris_feed_harvest.jsonl'), 'utf8').split('\n').filter(Boolean)) { const r = JSON.parse(l); portal[r.base_code] = r; }

// gather harvested rows from every harvest file; keep the best row per base_code (prefer image + non-404)
const toks = s => (s||'').toLowerCase().replace(/&amp;/g,' ').split(/[^a-z0-9]+/).filter(Boolean);
const stem = w => w.replace(/s$/,'');
const stems = arr => new Set(arr.map(stem));
// robust QA: parse Morris title "<Pattern> Wallpaper|Fabric in <Color> by Morris & Co"
function qaPass(portalPattern, portalColor, title) {
  if (!title || /^\s*404/.test(title)) return false;
  const left = title.split(/\bby\b/i)[0];
  const [namePart, colorPart=''] = left.split(/\bin\b/i);
  const nameTok = toks(namePart).filter(w => !['wallpaper','fabric'].includes(w));
  const colorTok = toks(colorPart);
  const pstem = stems([...toks(portalPattern), ...toks(portalColor||'')]);
  const nameOk = nameTok.length ? nameTok.some(w => pstem.has(stem(w))) : false;
  const colorOk = colorTok.length ? colorTok.some(w => pstem.has(stem(w))) : true;
  return nameOk && colorOk;
}

// gather harvested rows; keep the best row per base_code — QA-pass dominates, then image, then non-404
const files = ['morris_image_harvest.jsonl','morris_prob_harvest.jsonl','morris_prob_harvest2.jsonl','morris_prob_harvest3.jsonl','morris_prob_harvest4.jsonl','morris_prob_harvest5.jsonl'];
const best = {};
for (const f of files) { if (!fs.existsSync(P(f))) continue;
  for (const l of fs.readFileSync(P(f),'utf8').split('\n').filter(Boolean)) { const r = JSON.parse(l);
    const p = portal[r.base_code]; if (!p) continue;
    const score = (r.image?2:0) + ((r.title&&!/404/.test(r.title))?1:0) + (r.image && qaPass(p.pattern,p.color,r.title)?8:0);
    const prev = best[r.base_code];
    if (!prev || score > prev._s) best[r.base_code] = { ...r, _s: score };
  }
}

function head(url){ return new Promise(res=>{ const req=https.request(url,{method:'GET',headers:{'User-Agent':'Mozilla/5.0','Range':'bytes=0-0'}},r=>{r.resume();res({status:r.statusCode,ct:r.headers['content-type']||''});}); req.on('error',()=>res({status:0,ct:''})); req.setTimeout(30000,()=>{req.destroy();res({status:0,ct:''});}); req.end(); }); }
const noCache = u => u ? u.replace(/\/cache\/[a-f0-9]{32}\//,'/') : u;

(async () => {
  const all = Object.keys(portal);
  const rows = all.map(s => ({ base_code:s, ...(best[s]||{}) }));
  // QA classify
  const qaPassSet = new Set(), qaFail = [];
  for (const s of all) { const b = best[s];
    if (b && b.image && qaPass(portal[s].pattern, portal[s].color, b.title)) qaPassSet.add(s);
    else if (b) qaFail.push({ base_code:s, pattern:portal[s].pattern, title:b.title||null, has_image:!!b.image });
    else qaFail.push({ base_code:s, pattern:portal[s].pattern, title:null, has_image:false, note:'not-harvested' });
  }

  // validate images for QA-pass rows ($0 parallel)
  const pass = [...qaPassSet];
  let idx=0, ok=0, bad=0; const CONC=12; const valid={};
  async function w(){ while(idx<pass.length){ const s=pass[idx++]; const full=noCache(best[s].image);
    let chosen=full,r=await head(full);
    if(!(r.status>=200&&r.status<400&&/image\//.test(r.ct))){ chosen=best[s].image; r=await head(best[s].image); }
    const good=r.status>=200&&r.status<400&&/image\//.test(r.ct);
    if(good) ok++; else bad++;
    valid[s]={image:chosen,image_full:full,http:r.status,ct:r.ct,ok:good};
  } }
  await Promise.all(Array.from({length:CONC},w));

  const manifest = pass.filter(s=>valid[s].ok).map(s=>{ const p=portal[s]; const v=valid[s];
    return { base_code:s, mfr_sku:p.mfr_sku||s, product_type:p.product_type, pattern:p.pattern, color:p.color,
      collection:p.collection, ssp_usd:p.ssp_usd, trade_usd:p.trade_usd, retail_usd:p.retail_usd,
      image:v.image, image_full:v.image_full, source_url:best[s].url, image_http:v.http }; });
  fs.writeFileSync(P('manifest-morris.json'), JSON.stringify(manifest,null,1));

  const stillMissing = all.filter(s => !manifest.find(m=>m.base_code===s));
  fs.writeFileSync(P('morris_still_missing.json'), JSON.stringify(stillMissing.map(s=>({base_code:s,product_type:portal[s].product_type,pattern:portal[s].pattern,reason: best[s]? (best[s].image? 'qa-or-validate-fail':'no-image-on-page'):'no-url-resolved'})),null,1));

  console.log(`[finalize] portal=${all.length} qa_pass=${pass.length} img_ok=${ok} img_bad=${bad} MANIFEST=${manifest.length} (${(100*manifest.length/all.length).toFixed(1)}%) still_missing=${stillMissing.length}`);
})();