`scripts/validate_stage_morris.mjs`

86 lines
#!/usr/bin/env node
// TK-10881 — HTTP-200 validate every Morris image URL (plain fetch, $0, media CDN un-challenged)
// then stage manifest-morris.json. Also emits a per-image "no-cache full-size" URL (strip the
// Magento cache/<hash>/ token) which serves the ~360KB original; we validate BOTH and prefer the
// full-size when it 200s image/*. QA: flags any row whose page <title> disagrees with the portal
// pattern (guards against a bad SKU->URL join). Read-only against the web; writes staging files.
import https from 'https';
import fs from 'fs';

const DIR = new URL('..', import.meta.url).pathname;
const HARVEST = `${DIR}pilot/morris_image_harvest.jsonl`;
const PORTAL = `${DIR}pilot/morris_feed_harvest.jsonl`;
const OUT_MANIFEST = `${DIR}pilot/manifest-morris.json`;
const OUT_VALID = `${DIR}pilot/morris_image_validated.jsonl`;
const OUT_QA = `${DIR}pilot/morris_image_qa_flags.json`;

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
function head(url) {
  return new Promise(resolve => {
    const req = https.request(url, { method: 'GET', headers: { 'User-Agent': UA, 'Range': 'bytes=0-0' } }, res => {
      res.resume();
      resolve({ status: res.statusCode, ct: res.headers['content-type'] || '', len: res.headers['content-length'] || res.headers['content-range'] || '' });
    });
    req.on('error', () => resolve({ status: 0, ct: '', len: '' }));
    req.setTimeout(30000, () => { req.destroy(); resolve({ status: 0, ct: '', len: '' }); });
    req.end();
  });
}
const noCache = u => u ? u.replace(/\/cache\/[a-f0-9]{32}\//, '/') : u;
const norm = s => (s || '').toLowerCase().replace(/&amp;|&/g, ' ').replace(/[^a-z0-9]+/g, ' ').trim();

(async () => {
  const portal = {};
  for (const l of fs.readFileSync(PORTAL, 'utf8').split('\n').filter(Boolean)) { const r = JSON.parse(l); portal[r.base_code] = r; }
  const rows = fs.readFileSync(HARVEST, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
  console.log(`[validate] ${rows.length} harvested rows`);

  const CONC = 12; let idx = 0, ok = 0, bad = 0;
  const out = [];
  async function worker() {
    while (idx < rows.length) {
      const r = rows[idx++];
      if (!r.image) { out.push({ ...r, img_ok: false, reason: 'no-image-on-page' }); bad++; continue; }
      const full = noCache(r.image);
      let chosen = full, res = await head(full);
      if (!(res.status >= 200 && res.status < 400 && /image\//.test(res.ct))) { chosen = r.image; res = await head(r.image); }
      const good = res.status >= 200 && res.status < 400 && /image\//.test(res.ct);
      if (good) ok++; else bad++;
      out.push({ base_code: r.base_code, url: r.url, image_full: full, image_cache: r.image, image: chosen, http: res.status, content_type: res.ct, img_ok: good, title: r.title });
    }
  }
  await Promise.all(Array.from({ length: CONC }, worker));

  // QA: title vs portal pattern token overlap
  const qa = [];
  for (const r of out) {
    const p = portal[r.base_code]; if (!p) continue;
    const pt = new Set(norm(p.pattern).split(' ').filter(w => w && !['wallpaper','fabric','and','the'].includes(w)));
    const tt = new Set(norm(r.title).split(' '));
    const inter = [...pt].filter(w => tt.has(w));
    // pattern name first word should appear in title; flag if <50% overlap
    if (pt.size && inter.length < Math.max(1, Math.ceil(0.5 * pt.size))) {
      qa.push({ base_code: r.base_code, portal_pattern: p.pattern, page_title: r.title, overlap: inter.length + '/' + pt.size });
    }
  }

  fs.writeFileSync(OUT_VALID, out.map(o => JSON.stringify(o)).join('\n') + '\n');
  fs.writeFileSync(OUT_QA, JSON.stringify(qa, null, 1));

  // Manifest = only rows with a validated image AND passing QA (no title mismatch)
  const qaBad = new Set(qa.map(q => q.base_code));
  const manifest = out.filter(o => o.img_ok && !qaBad.has(o.base_code)).map(o => {
    const p = portal[o.base_code] || {};
    return {
      base_code: o.base_code, mfr_sku: p.mfr_sku || o.base_code, product_type: p.product_type,
      pattern: p.pattern, color: p.color, collection: p.collection,
      ssp_usd: p.ssp_usd, trade_usd: p.trade_usd, retail_usd: p.retail_usd,
      image: o.image, image_full: o.image_full, source_url: o.url, image_http: o.http
    };
  });
  fs.writeFileSync(OUT_MANIFEST, JSON.stringify(manifest, null, 1));

  console.log(`[validate] img_ok=${ok} img_bad=${bad}  QA-flagged=${qa.length}  manifest=${manifest.length}`);
  console.log(`[validate] wrote ${OUT_MANIFEST}, ${OUT_VALID}, ${OUT_QA}`);
})();